From c37421a3b4493c0b0f9afef15a4ea7b74d152067 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 Feb 2026 10:58:41 +0100 Subject: add simple http client tests (#751) * add simple http client tests and fix run loop of http server to not rely on application quit --- src/zenhttp/httpclient.cpp | 164 +++++++++++++++++++++++++++- src/zenhttp/include/zenhttp/httpserver.h | 4 +- src/zenhttp/servers/httpasio.cpp | 17 +-- src/zenhttp/servers/httpasio.h | 2 + src/zenhttp/servers/httpmulti.cpp | 11 +- src/zenhttp/servers/httpnull.cpp | 11 +- src/zenhttp/servers/httpsys.cpp | 18 +-- src/zenhttp/transports/winsocktransport.cpp | 2 +- 8 files changed, 202 insertions(+), 27 deletions(-) diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 43e9fb468..0544bf5c8 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -22,8 +22,13 @@ #include "clients/httpclientcommon.h" #if ZEN_WITH_TESTS +# include # include # include +# include "servers/httpasio.h" +# include "servers/httpsys.h" + +# include #endif // ZEN_WITH_TESTS namespace zen { @@ -388,7 +393,164 @@ TEST_CASE("httpclient") { using namespace std::literals; - SUBCASE("client") {} + struct TestHttpService : public HttpService + { + TestHttpService() = default; + + virtual const char* BaseUri() const override { return "/test/"; } + virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override + { + if (HttpServiceRequest.RelativeUri() == "yo") + { + return HttpServiceRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "hey man"); + } + return HttpServiceRequest.WriteResponse(HttpResponseCode::OK); + } + }; + + TestHttpService TestService; + ScopedTemporaryDirectory TmpDir; + + SUBCASE("asio") + { + Ref AsioServer = CreateHttpAsioServer(AsioConfig{}); + + int Port = AsioServer->Initialize(7575, TmpDir.Path()); + REQUIRE(Port != -1); + + AsioServer->RegisterService(TestService); + + std::thread SeverThread([&]() { AsioServer->Run(false); }); + + { + auto _ = MakeGuard([&]() { + if (SeverThread.joinable()) + { + SeverThread.join(); + } + AsioServer->Close(); + }); + + { + HttpClient Client(fmt::format("127.0.0.1:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response TestResponse = Client.Get("/test/yo"); + CHECK(TestResponse.IsSuccess()); + } + + if (IsIPv6Capable()) + { + HttpClient Client(fmt::format("[::1]:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response TestResponse = Client.Get("/test/yo"); + CHECK(TestResponse.IsSuccess()); + } + + { + HttpClient Client(fmt::format("localhost:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response TestResponse = Client.Get("/test/yo"); + CHECK(TestResponse.IsSuccess()); + } +# if 0 + { + HttpClient Client(fmt::format("10.24.101.77:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response TestResponse = Client.Get("/test/yo"); + CHECK(TestResponse.IsSuccess()); + } +# endif // 0 + AsioServer->RequestExit(); + } + } + +# if ZEN_PLATFORM_WINDOWS + SUBCASE("httpsys") + { + Ref HttpSysServer = CreateHttpSysServer(HttpSysConfig{.ForceLoopback = true}); + + int Port = HttpSysServer->Initialize(7575, TmpDir.Path()); + REQUIRE(Port != -1); + + HttpSysServer->RegisterService(TestService); + + std::thread SeverThread([&]() { HttpSysServer->Run(false); }); + + { + auto _ = MakeGuard([&]() { + if (SeverThread.joinable()) + { + SeverThread.join(); + } + HttpSysServer->Close(); + }); + + if (true) + { + HttpClient Client(fmt::format("127.0.0.1:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response TestResponse = Client.Get("/test/yo"); + CHECK(TestResponse.IsSuccess()); + } + + if (IsIPv6Capable()) + { + HttpClient Client(fmt::format("[::1]:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response TestResponse = Client.Get("/test/yo"); + CHECK(TestResponse.IsSuccess()); + } + + { + HttpClient Client(fmt::format("localhost:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response TestResponse = Client.Get("/test/yo"); + CHECK(TestResponse.IsSuccess()); + } +# if 0 + { + HttpClient Client(fmt::format("10.24.101.77:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response TestResponse = Client.Get("/test"); + CHECK(TestResponse.IsSuccess()); + } +# endif // 0 + HttpSysServer->RequestExit(); + } + } +# endif // ZEN_PLATFORM_WINDOWS } void diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 3438a1471..6660bebf9 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -30,8 +30,10 @@ class HttpService; */ class HttpServerRequest { -public: +protected: explicit HttpServerRequest(HttpService& Service); + +public: ~HttpServerRequest(); // Synchronous operations diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 18a0f6a40..76fea65b3 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -97,7 +97,11 @@ IsIPv6AvailableSysctl(void) return val == 0; } +#endif // ZEN_PLATFORM_LINUX +namespace zen { + +#if ZEN_PLATFORM_LINUX bool IsIPv6Capable() { @@ -121,8 +125,6 @@ IsIPv6Capable() } #endif -namespace zen { - const FLLMTag& GetHttpasioTag() { @@ -1992,7 +1994,8 @@ HttpAsioServer::OnInitialize(int BasePort, std::filesystem::path DataDir) void HttpAsioServer::OnRun(bool IsInteractive) { - const int WaitTimeout = 1000; + const int WaitTimeout = 1000; + bool ShutdownRequested = false; #if ZEN_PLATFORM_WINDOWS if (IsInteractive) @@ -2012,8 +2015,8 @@ HttpAsioServer::OnRun(bool IsInteractive) } } - m_ShutdownEvent.Wait(WaitTimeout); - } while (!IsApplicationExitRequested()); + ShutdownRequested = m_ShutdownEvent.Wait(WaitTimeout); + } while (!ShutdownRequested); #else if (IsInteractive) { @@ -2022,8 +2025,8 @@ HttpAsioServer::OnRun(bool IsInteractive) do { - m_ShutdownEvent.Wait(WaitTimeout); - } while (!IsApplicationExitRequested()); + ShutdownRequested = m_ShutdownEvent.Wait(WaitTimeout); + } while (!ShutdownRequested); #endif } diff --git a/src/zenhttp/servers/httpasio.h b/src/zenhttp/servers/httpasio.h index c483dfc28..3ec1141a7 100644 --- a/src/zenhttp/servers/httpasio.h +++ b/src/zenhttp/servers/httpasio.h @@ -15,4 +15,6 @@ struct AsioConfig Ref CreateHttpAsioServer(const AsioConfig& Config); +bool IsIPv6Capable(); + } // namespace zen diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp index 31cb04be5..95624245f 100644 --- a/src/zenhttp/servers/httpmulti.cpp +++ b/src/zenhttp/servers/httpmulti.cpp @@ -56,7 +56,8 @@ HttpMultiServer::OnInitialize(int BasePort, std::filesystem::path DataDir) void HttpMultiServer::OnRun(bool IsInteractiveSession) { - const int WaitTimeout = 1000; + const int WaitTimeout = 1000; + bool ShutdownRequested = false; #if ZEN_PLATFORM_WINDOWS if (IsInteractiveSession) @@ -76,8 +77,8 @@ HttpMultiServer::OnRun(bool IsInteractiveSession) } } - m_ShutdownEvent.Wait(WaitTimeout); - } while (!IsApplicationExitRequested()); + ShutdownRequested = m_ShutdownEvent.Wait(WaitTimeout); + } while (!ShutdownRequested); #else if (IsInteractiveSession) { @@ -86,8 +87,8 @@ HttpMultiServer::OnRun(bool IsInteractiveSession) do { - m_ShutdownEvent.Wait(WaitTimeout); - } while (!IsApplicationExitRequested()); + ShutdownRequested = m_ShutdownEvent.Wait(WaitTimeout); + } while (!ShutdownRequested); #endif } diff --git a/src/zenhttp/servers/httpnull.cpp b/src/zenhttp/servers/httpnull.cpp index 0ec1cb3c4..b770b97db 100644 --- a/src/zenhttp/servers/httpnull.cpp +++ b/src/zenhttp/servers/httpnull.cpp @@ -34,7 +34,8 @@ HttpNullServer::OnInitialize(int BasePort, std::filesystem::path DataDir) void HttpNullServer::OnRun(bool IsInteractiveSession) { - const int WaitTimeout = 1000; + const int WaitTimeout = 1000; + bool ShutdownRequested = false; #if ZEN_PLATFORM_WINDOWS if (IsInteractiveSession) @@ -54,8 +55,8 @@ HttpNullServer::OnRun(bool IsInteractiveSession) } } - m_ShutdownEvent.Wait(WaitTimeout); - } while (!IsApplicationExitRequested()); + ShutdownRequested = m_ShutdownEvent.Wait(WaitTimeout); + } while (!ShutdownRequested); #else if (IsInteractiveSession) { @@ -64,8 +65,8 @@ HttpNullServer::OnRun(bool IsInteractiveSession) do { - m_ShutdownEvent.Wait(WaitTimeout); - } while (!IsApplicationExitRequested()); + ShutdownRequested = m_ShutdownEvent.Wait(WaitTimeout); + } while (!ShutdownRequested); #endif } diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 54cc0c22d..0d2bb8fbd 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -1128,11 +1128,14 @@ HttpSysServer::InitializeServer(int BasePort) // port for the current user. eg: // netsh http add urlacl url=http://*:8558/ user= - ZEN_WARN( - "Unable to register handler using '{}' - falling back to local-only. " - "Please ensure the appropriate netsh URL reservation configuration " - "is made to allow http.sys access (see https://github.com/EpicGames/zen/blob/main/README.md)", - WideToUtf8(WildcardUrlPath)); + if (!m_InitialConfig.ForceLoopback) + { + ZEN_WARN( + "Unable to register handler using '{}' - falling back to local-only. " + "Please ensure the appropriate netsh URL reservation configuration " + "is made to allow http.sys access (see https://github.com/EpicGames/zen/blob/main/README.md)", + WideToUtf8(WildcardUrlPath)); + } const std::u8string_view Hosts[] = {u8"[::1]"sv, u8"localhost"sv, u8"127.0.0.1"sv}; @@ -1337,6 +1340,7 @@ HttpSysServer::OnRun(bool IsInteractive) ZEN_CONSOLE("Zen Server running (http.sys). Press ESC or Q to quit"); } + bool ShutdownRequested = false; do { // int WaitTimeout = -1; @@ -1357,9 +1361,9 @@ HttpSysServer::OnRun(bool IsInteractive) } } - m_ShutdownEvent.Wait(WaitTimeout); + ShutdownRequested = m_ShutdownEvent.Wait(WaitTimeout); UpdateLofreqTimerValue(); - } while (!IsApplicationExitRequested()); + } while (!ShutdownRequested); } void diff --git a/src/zenhttp/transports/winsocktransport.cpp b/src/zenhttp/transports/winsocktransport.cpp index c06a50c95..0217ed44e 100644 --- a/src/zenhttp/transports/winsocktransport.cpp +++ b/src/zenhttp/transports/winsocktransport.cpp @@ -322,7 +322,7 @@ SocketTransportPluginImpl::Initialize(TransportServer* ServerInterface) else { } - } while (!IsApplicationExitRequested() && m_KeepRunning.test()); + } while (m_KeepRunning.test()); ZEN_INFO("HTTP plugin server accept thread exit"); }); -- cgit v1.2.3 From 3a563f5e8fcabffe686e1deb5862bdf39078ebdf Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 12 Feb 2026 15:25:05 +0100 Subject: add IsLocalMachineRequest to HttpServerRequest (#749) * add IsLocalMachineRequest to HttpServerRequest --- src/zenhttp/httpclient.cpp | 23 ++++++++++++++++++++--- src/zenhttp/httpserver.cpp | 5 ++++- src/zenhttp/include/zenhttp/httpserver.h | 2 ++ src/zenhttp/servers/httpasio.cpp | 23 ++++++++++++++++++++--- src/zenhttp/servers/httpplugin.cpp | 3 +++ src/zenhttp/servers/httpsys.cpp | 29 +++++++++++++++++++++++++++++ 6 files changed, 78 insertions(+), 7 deletions(-) diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 0544bf5c8..c77be8624 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -402,7 +402,14 @@ TEST_CASE("httpclient") { if (HttpServiceRequest.RelativeUri() == "yo") { - return HttpServiceRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "hey man"); + if (HttpServiceRequest.IsLocalMachineRequest()) + { + return HttpServiceRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "hey family"); + } + else + { + return HttpServiceRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "hey stranger"); + } } return HttpServiceRequest.WriteResponse(HttpResponseCode::OK); } @@ -440,6 +447,7 @@ TEST_CASE("httpclient") HttpClient::Response TestResponse = Client.Get("/test/yo"); CHECK(TestResponse.IsSuccess()); + CHECK_EQ(TestResponse.AsText(), "hey family"); } if (IsIPv6Capable()) @@ -452,6 +460,7 @@ TEST_CASE("httpclient") HttpClient::Response TestResponse = Client.Get("/test/yo"); CHECK(TestResponse.IsSuccess()); + CHECK_EQ(TestResponse.AsText(), "hey family"); } { @@ -463,6 +472,7 @@ TEST_CASE("httpclient") HttpClient::Response TestResponse = Client.Get("/test/yo"); CHECK(TestResponse.IsSuccess()); + CHECK_EQ(TestResponse.AsText(), "hey family"); } # if 0 { @@ -474,7 +484,9 @@ TEST_CASE("httpclient") HttpClient::Response TestResponse = Client.Get("/test/yo"); CHECK(TestResponse.IsSuccess()); + CHECK_EQ(TestResponse.AsText(), "hey family"); } + Sleep(20000); # endif // 0 AsioServer->RequestExit(); } @@ -483,7 +495,7 @@ TEST_CASE("httpclient") # if ZEN_PLATFORM_WINDOWS SUBCASE("httpsys") { - Ref HttpSysServer = CreateHttpSysServer(HttpSysConfig{.ForceLoopback = true}); + Ref HttpSysServer = CreateHttpSysServer(HttpSysConfig{.ForceLoopback = false}); int Port = HttpSysServer->Initialize(7575, TmpDir.Path()); REQUIRE(Port != -1); @@ -511,6 +523,7 @@ TEST_CASE("httpclient") HttpClient::Response TestResponse = Client.Get("/test/yo"); CHECK(TestResponse.IsSuccess()); + CHECK_EQ(TestResponse.AsText(), "hey family"); } if (IsIPv6Capable()) @@ -523,6 +536,7 @@ TEST_CASE("httpclient") HttpClient::Response TestResponse = Client.Get("/test/yo"); CHECK(TestResponse.IsSuccess()); + CHECK_EQ(TestResponse.AsText(), "hey family"); } { @@ -534,6 +548,7 @@ TEST_CASE("httpclient") HttpClient::Response TestResponse = Client.Get("/test/yo"); CHECK(TestResponse.IsSuccess()); + CHECK_EQ(TestResponse.AsText(), "hey family"); } # if 0 { @@ -543,9 +558,11 @@ TEST_CASE("httpclient") ZEN_INFO("Request using {}", Client.GetBaseUri()); - HttpClient::Response TestResponse = Client.Get("/test"); + HttpClient::Response TestResponse = Client.Get("/test/yo"); CHECK(TestResponse.IsSuccess()); + CHECK_EQ(TestResponse.AsText(), "hey family"); } + Sleep(20000); # endif // 0 HttpSysServer->RequestExit(); } diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index c4e67d4ed..8985120b0 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -1310,7 +1310,10 @@ TEST_CASE("http.common") { TestHttpServerRequest(HttpService& Service, std::string_view Uri) : HttpServerRequest(Service) { m_Uri = Uri; } virtual IoBuffer ReadPayload() override { return IoBuffer(); } - virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span Blobs) override + + virtual bool IsLocalMachineRequest() const override { return false; } + + virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span Blobs) override { ZEN_UNUSED(ResponseCode, ContentType, Blobs); } diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 6660bebf9..f0a667686 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -89,6 +89,8 @@ public: CbObject ReadPayloadObject(); CbPackage ReadPayloadPackage(); + virtual bool IsLocalMachineRequest() const = 0; + /** Respond with payload No data will have been sent when any of these functions return. Instead, the response will be transmitted diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 76fea65b3..8bfbd8b37 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -529,12 +529,18 @@ public: class HttpAsioServerRequest : public HttpServerRequest { public: - HttpAsioServerRequest(HttpRequestParser& Request, HttpService& Service, IoBuffer PayloadBuffer, uint32_t RequestNumber); + HttpAsioServerRequest(HttpRequestParser& Request, + HttpService& Service, + IoBuffer PayloadBuffer, + uint32_t RequestNumber, + bool IsLocalMachineRequest); ~HttpAsioServerRequest(); virtual Oid ParseSessionId() const override; virtual uint32_t ParseRequestId() const override; + virtual bool IsLocalMachineRequest() const override; + virtual IoBuffer ReadPayload() override; virtual void WriteResponse(HttpResponseCode ResponseCode) override; virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span Blobs) override; @@ -550,6 +556,7 @@ public: HttpRequestParser& m_Request; uint32_t m_RequestNumber = 0; // Note: different to request ID which is derived from headers IoBuffer m_PayloadBuffer; + bool m_IsLocalMachineRequest; std::unique_ptr m_Response; }; @@ -1168,7 +1175,9 @@ HttpServerConnection::HandleRequest() { ZEN_TRACE_CPU("asio::HandleRequest"); - HttpAsioServerRequest Request(m_RequestData, *Service, m_RequestData.Body(), RequestNumber); + bool IsLocalConnection = m_Socket->local_endpoint().address() == m_Socket->remote_endpoint().address(); + + HttpAsioServerRequest Request(m_RequestData, *Service, m_RequestData.Body(), RequestNumber, IsLocalConnection); ZEN_TRACE_VERBOSE("handle request, connection: {}, request: {}'", m_ConnectionId, RequestNumber); @@ -1634,11 +1643,13 @@ private: HttpAsioServerRequest::HttpAsioServerRequest(HttpRequestParser& Request, HttpService& Service, IoBuffer PayloadBuffer, - uint32_t RequestNumber) + uint32_t RequestNumber, + bool IsLocalMachineRequest) : HttpServerRequest(Service) , m_Request(Request) , m_RequestNumber(RequestNumber) , m_PayloadBuffer(std::move(PayloadBuffer)) +, m_IsLocalMachineRequest(IsLocalMachineRequest) { const int PrefixLength = Service.UriPrefixLength(); @@ -1710,6 +1721,12 @@ HttpAsioServerRequest::ParseRequestId() const return m_Request.RequestId(); } +bool +HttpAsioServerRequest::IsLocalMachineRequest() const +{ + return m_IsLocalMachineRequest; +} + IoBuffer HttpAsioServerRequest::ReadPayload() { diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index b9217ed87..1a630c16f 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -143,6 +143,9 @@ public: HttpPluginServerRequest(const HttpPluginServerRequest&) = delete; HttpPluginServerRequest& operator=(const HttpPluginServerRequest&) = delete; + // As this is plugin transport connection used for specialized connections we assume it is not a machine local connection + virtual bool IsLocalMachineRequest() const /* override*/ { return false; } + virtual Oid ParseSessionId() const override; virtual uint32_t ParseRequestId() const override; diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 0d2bb8fbd..01c4559a1 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -313,6 +313,8 @@ public: virtual Oid ParseSessionId() const override; virtual uint32_t ParseRequestId() const override; + virtual bool IsLocalMachineRequest() const; + virtual IoBuffer ReadPayload() override; virtual void WriteResponse(HttpResponseCode ResponseCode) override; virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span Blobs) override; @@ -1814,6 +1816,33 @@ HttpSysServerRequest::ParseRequestId() const return 0; } +bool +HttpSysServerRequest::IsLocalMachineRequest() const +{ + const PSOCKADDR LocalAddress = m_HttpTx.HttpRequest()->Address.pLocalAddress; + const PSOCKADDR RemoteAddress = m_HttpTx.HttpRequest()->Address.pRemoteAddress; + if (LocalAddress->sa_family != RemoteAddress->sa_family) + { + return false; + } + if (LocalAddress->sa_family == AF_INET) + { + const SOCKADDR_IN& LocalAddressv4 = (const SOCKADDR_IN&)(*LocalAddress); + const SOCKADDR_IN& RemoteAddressv4 = (const SOCKADDR_IN&)(*RemoteAddress); + return LocalAddressv4.sin_addr.S_un.S_addr == RemoteAddressv4.sin_addr.S_un.S_addr; + } + else if (LocalAddress->sa_family == AF_INET6) + { + const SOCKADDR_IN6& LocalAddressv6 = (const SOCKADDR_IN6&)(*LocalAddress); + const SOCKADDR_IN6& RemoteAddressv6 = (const SOCKADDR_IN6&)(*RemoteAddress); + return memcmp(&LocalAddressv6.sin6_addr, &RemoteAddressv6.sin6_addr, sizeof(in6_addr)) == 0; + } + else + { + return false; + } +} + IoBuffer HttpSysServerRequest::ReadPayload() { -- cgit v1.2.3 From fb6bf42e0b074e9dd0a3e4f5f3e02782bf780f2c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 12 Feb 2026 15:29:13 +0100 Subject: bump sentry to 0.12.1 (#721) --- repo/packages/s/sentry-native/xmake.lua | 4 +++- scripts/bundle.lua | 3 ++- xmake.lua | 8 ++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/repo/packages/s/sentry-native/xmake.lua b/repo/packages/s/sentry-native/xmake.lua index 7024a174d..ca683ee4c 100644 --- a/repo/packages/s/sentry-native/xmake.lua +++ b/repo/packages/s/sentry-native/xmake.lua @@ -5,6 +5,8 @@ package("sentry-native") set_urls("https://github.com/getsentry/sentry-native/releases/download/$(version)/sentry-native.zip", "https://github.com/getsentry/sentry-native.git") + add_versions("0.12.1", "ae2a37a1c5f86bedb3fbee6dd444e2f47f1ef002050b95ab8339607851944177") + add_versions("0.12.0", "3bf6eebb7dcc9c99267746324734a15164ba0058d67f690e315d47ee0bd8e953") add_versions("0.11.3", "6a4ccd2bf91320ca84169b322cbbfe5a0d13f0b4ee45bb4adf93bd1c4c59a08a") add_versions("0.11.2", "3f6a5ca384096fa1a9cc9624ec24fe5490f0630bb11302d9006cd522f4f6c5a3") add_versions("0.11.1", "04c80503cfaf0904f3adf43f97cea4cc6bdd4c21707c093ee0ed34e7a3f8e3e7") @@ -50,7 +52,7 @@ package("sentry-native") elseif is_plat("macosx") then add_deps("libcurl", {system = false, configs = {openssl3 = true}}) --add_deps("libcurl", {system = false, configs = {mbedtls = true}}) - add_frameworks("CoreText", "CoreGraphics", "SystemConfiguration", "CoreFoundation", "Foundation") + add_frameworks("CoreText", "CoreGraphics", "SystemConfiguration", "CoreFoundation", "Foundation", "IoKit") add_syslinks("bsm") end diff --git a/scripts/bundle.lua b/scripts/bundle.lua index debb1e615..07e120d04 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -176,7 +176,8 @@ local function main_windows(signidentity) "/v", "/as", "build/windows/x64/release/zenserver.exe", - "build/windows/x64/release/zen.exe") + "build/windows/x64/release/zen.exe", + "build/windows/x64/release/crashpad_handler.exe") if ret > 0 then raise("Failed signing zenserver binary") end diff --git a/xmake.lua b/xmake.lua index 18429de61..5d3162e46 100644 --- a/xmake.lua +++ b/xmake.lua @@ -113,11 +113,11 @@ if is_plat("linux") and os.getenv("UE_TOOLCHAIN_DIR") then end if has_config("zensentry") and not use_asan then if is_plat("linux") then - add_requires("sentry-native 0.7.6") + add_requires("sentry-native 0.12.1", {configs = {backend = "crashpad"}}) elseif is_plat("windows") then - add_requires("sentry-native 0.7.6", {debug = is_mode("debug"), configs = {backend = "crashpad"}}) + add_requires("sentry-native 0.12.1", {debug = is_mode("debug"), configs = {backend = "crashpad"}}) else - add_requires("sentry-native 0.7.6", {configs = {backend = "crashpad"}}) + add_requires("sentry-native 0.12.1", {configs = {backend = "crashpad"}}) end end --add_rules("c++.unity_build") @@ -209,7 +209,7 @@ function add_define_by_config(define, config_name) end option("zensentry") - set_default(false) + set_default(true) set_showmenu(true) set_description("Enables Sentry support") option_end() -- cgit v1.2.3 From 3b5b777900d9f59ff32eb7cea79e3a72a08c67a6 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 13 Feb 2026 13:27:08 +0100 Subject: add IHttpRequestFilter to allow server implementation to filter/reject requests (#753) * add IHttpRequestFilter to allow server implementation to filter/reject requests --- src/zenhttp/httpclient.cpp | 104 ++++++++++++++++++++++-- src/zenhttp/httpserver.cpp | 12 ++- src/zenhttp/include/zenhttp/httpserver.h | 22 +++++- src/zenhttp/servers/httpasio.cpp | 131 +++++++++++++++++++++---------- src/zenhttp/servers/httpmulti.cpp | 9 +++ src/zenhttp/servers/httpmulti.h | 1 + src/zenhttp/servers/httpnull.cpp | 6 ++ src/zenhttp/servers/httpnull.h | 1 + src/zenhttp/servers/httpplugin.cpp | 117 ++++++++++++++++++--------- src/zenhttp/servers/httpsys.cpp | 46 ++++++++++- 10 files changed, 353 insertions(+), 96 deletions(-) diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index c77be8624..16729ce38 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -427,13 +427,13 @@ TEST_CASE("httpclient") AsioServer->RegisterService(TestService); - std::thread SeverThread([&]() { AsioServer->Run(false); }); + std::thread ServerThread([&]() { AsioServer->Run(false); }); { auto _ = MakeGuard([&]() { - if (SeverThread.joinable()) + if (ServerThread.joinable()) { - SeverThread.join(); + ServerThread.join(); } AsioServer->Close(); }); @@ -502,13 +502,13 @@ TEST_CASE("httpclient") HttpSysServer->RegisterService(TestService); - std::thread SeverThread([&]() { HttpSysServer->Run(false); }); + std::thread ServerThread([&]() { HttpSysServer->Run(false); }); { auto _ = MakeGuard([&]() { - if (SeverThread.joinable()) + if (ServerThread.joinable()) { - SeverThread.join(); + ServerThread.join(); } HttpSysServer->Close(); }); @@ -570,6 +570,98 @@ TEST_CASE("httpclient") # endif // ZEN_PLATFORM_WINDOWS } +TEST_CASE("httpclient.requestfilter") +{ + using namespace std::literals; + + struct TestHttpService : public HttpService + { + TestHttpService() = default; + + virtual const char* BaseUri() const override { return "/test/"; } + virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override + { + if (HttpServiceRequest.RelativeUri() == "yo") + { + return HttpServiceRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "hey family"); + } + + { + CHECK(HttpServiceRequest.RelativeUri() != "should_filter"); + return HttpServiceRequest.WriteResponse(HttpResponseCode::InternalServerError); + } + + { + CHECK(HttpServiceRequest.RelativeUri() != "should_forbid"); + return HttpServiceRequest.WriteResponse(HttpResponseCode::InternalServerError); + } + } + }; + + TestHttpService TestService; + ScopedTemporaryDirectory TmpDir; + + class MyFilterImpl : public IHttpRequestFilter + { + public: + virtual Result FilterRequest(HttpServerRequest& Request) + { + if (Request.RelativeUri() == "should_filter") + { + Request.WriteResponse(HttpResponseCode::MethodNotAllowed, HttpContentType::kText, "no thank you"); + return Result::ResponseSent; + } + else if (Request.RelativeUri() == "should_forbid") + { + return Result::Forbidden; + } + return Result::Accepted; + } + }; + + MyFilterImpl MyFilter; + + Ref AsioServer = CreateHttpAsioServer(AsioConfig{}); + + AsioServer->SetHttpRequestFilter(&MyFilter); + + int Port = AsioServer->Initialize(7575, TmpDir.Path()); + REQUIRE(Port != -1); + + AsioServer->RegisterService(TestService); + + std::thread ServerThread([&]() { AsioServer->Run(false); }); + + { + auto _ = MakeGuard([&]() { + if (ServerThread.joinable()) + { + ServerThread.join(); + } + AsioServer->Close(); + }); + + HttpClient Client(fmt::format("localhost:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response YoResponse = Client.Get("/test/yo"); + CHECK(YoResponse.IsSuccess()); + CHECK_EQ(YoResponse.AsText(), "hey family"); + + HttpClient::Response ShouldFilterResponse = Client.Get("/test/should_filter"); + CHECK_EQ(ShouldFilterResponse.StatusCode, HttpResponseCode::MethodNotAllowed); + CHECK_EQ(ShouldFilterResponse.AsText(), "no thank you"); + + HttpClient::Response ShouldForbitResponse = Client.Get("/test/should_forbid"); + CHECK_EQ(ShouldForbitResponse.StatusCode, HttpResponseCode::Forbidden); + + AsioServer->RequestExit(); + } +} + void httpclient_forcelink() { diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 8985120b0..d8367fcb2 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -463,7 +463,7 @@ HttpService::HandlePackageRequest(HttpServerRequest& HttpServiceRequest) ////////////////////////////////////////////////////////////////////////// -HttpServerRequest::HttpServerRequest(HttpService& Service) : m_BaseUri(Service.BaseUri()) +HttpServerRequest::HttpServerRequest(HttpService& Service) : m_Service(Service) { } @@ -970,7 +970,7 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan()) { ExtendableStringBuilder<128> RoutePath; - RoutePath.Append(Request.BaseUri()); + RoutePath.Append(Request.Service().BaseUri()); RoutePath.Append(Handler.Pattern); ActiveSpan->AddAttribute("http.route"sv, RoutePath.ToView()); } @@ -994,7 +994,7 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) if (otel::Span* ActiveSpan = otel::Span::GetCurrentSpan()) { ExtendableStringBuilder<128> RoutePath; - RoutePath.Append(Request.BaseUri()); + RoutePath.Append(Request.Service().BaseUri()); RoutePath.Append(Handler.Pattern); ActiveSpan->AddAttribute("http.route"sv, RoutePath.ToView()); } @@ -1052,6 +1052,12 @@ HttpServer::EnumerateServices(std::function&& Callba } } +void +HttpServer::SetHttpRequestFilter(IHttpRequestFilter* RequestFilter) +{ + OnSetHttpRequestFilter(RequestFilter); +} + ////////////////////////////////////////////////////////////////////////// HttpRpcHandler::HttpRpcHandler() diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index f0a667686..60f6bc9f2 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -41,7 +41,7 @@ public: [[nodiscard]] inline std::string_view RelativeUri() const { return m_Uri; } // Returns URI without service prefix [[nodiscard]] std::string_view RelativeUriWithExtension() const { return m_UriWithExtension; } [[nodiscard]] inline std::string_view QueryString() const { return m_QueryString; } - [[nodiscard]] inline std::string_view BaseUri() const { return m_BaseUri; } // Service prefix + [[nodiscard]] inline HttpService& Service() const { return m_Service; } struct QueryParams { @@ -121,13 +121,14 @@ protected: kHaveSessionId = 1 << 3, }; - mutable uint32_t m_Flags = 0; + mutable uint32_t m_Flags = 0; + + HttpService& m_Service; // Service handling this request HttpVerb m_Verb = HttpVerb::kGet; HttpContentType m_ContentType = HttpContentType::kBinary; HttpContentType m_AcceptType = HttpContentType::kUnknownContentType; uint64_t m_ContentLength = ~0ull; - std::string_view m_BaseUri; // Base URI path of the service handling this request - std::string_view m_Uri; // URI without service prefix + std::string_view m_Uri; // URI without service prefix std::string_view m_UriWithExtension; std::string_view m_QueryString; mutable uint32_t m_RequestId = ~uint32_t(0); @@ -148,6 +149,17 @@ public: virtual void OnRequestComplete() = 0; }; +struct IHttpRequestFilter +{ + enum class Result + { + Forbidden, + ResponseSent, + Accepted + }; + virtual Result FilterRequest(HttpServerRequest& Request) = 0; +}; + /** * Base class for implementing an HTTP "service" * @@ -184,6 +196,7 @@ class HttpServer : public RefCounted public: void RegisterService(HttpService& Service); void EnumerateServices(std::function&& Callback); + void SetHttpRequestFilter(IHttpRequestFilter* RequestFilter); int Initialize(int BasePort, std::filesystem::path DataDir); void Run(bool IsInteractiveSession); @@ -195,6 +208,7 @@ private: virtual void OnRegisterService(HttpService& Service) = 0; virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) = 0; + virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) = 0; virtual void OnRun(bool IsInteractiveSession) = 0; virtual void OnRequestExit() = 0; virtual void OnClose() = 0; diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 8bfbd8b37..230aac6a8 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -498,16 +498,19 @@ public: HttpAsioServerImpl(); ~HttpAsioServerImpl(); - void Initialize(std::filesystem::path DataDir); - int Start(uint16_t Port, const AsioConfig& Config); - void Stop(); - void RegisterService(const char* UrlPath, HttpService& Service); - HttpService* RouteRequest(std::string_view Url); + void Initialize(std::filesystem::path DataDir); + int Start(uint16_t Port, const AsioConfig& Config); + void Stop(); + void RegisterService(const char* UrlPath, HttpService& Service); + void SetHttpRequestFilter(IHttpRequestFilter* RequestFilter); + HttpService* RouteRequest(std::string_view Url); + IHttpRequestFilter::Result FilterRequest(HttpServerRequest& Request); asio::io_service m_IoService; asio::io_service::work m_Work{m_IoService}; std::unique_ptr m_Acceptor; std::vector m_ThreadPool; + std::atomic m_HttpRequestFilter = nullptr; LoggerRef m_RequestLog; HttpServerTracer m_RequestTracer; @@ -1199,53 +1202,65 @@ HttpServerConnection::HandleRequest() std::vector{Request.ReadPayload()}); } - if (!HandlePackageOffers(*Service, Request, m_PackageHandler)) + IHttpRequestFilter::Result FilterResult = m_Server.FilterRequest(Request); + if (FilterResult == IHttpRequestFilter::Result::Accepted) { - try - { - Service->HandleRequest(Request); - } - catch (const AssertException& AssertEx) - { - // Drop any partially formatted response - Request.m_Response.reset(); - - ZEN_ERROR("Caught assert exception while handling request: {}", AssertEx.FullDescription()); - Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, AssertEx.FullDescription()); - } - catch (const std::system_error& SystemError) + if (!HandlePackageOffers(*Service, Request, m_PackageHandler)) { - // Drop any partially formatted response - Request.m_Response.reset(); - - if (IsOOM(SystemError.code()) || IsOOD(SystemError.code())) + try { - Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, SystemError.what()); + Service->HandleRequest(Request); } - else + catch (const AssertException& AssertEx) { - ZEN_WARN("Caught system error exception while handling request: {}. ({})", - SystemError.what(), - SystemError.code().value()); - Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, SystemError.what()); + // Drop any partially formatted response + Request.m_Response.reset(); + + ZEN_ERROR("Caught assert exception while handling request: {}", AssertEx.FullDescription()); + Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, AssertEx.FullDescription()); } - } - catch (const std::bad_alloc& BadAlloc) - { - // Drop any partially formatted response - Request.m_Response.reset(); + catch (const std::system_error& SystemError) + { + // Drop any partially formatted response + Request.m_Response.reset(); - Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, BadAlloc.what()); - } - catch (const std::exception& ex) - { - // Drop any partially formatted response - Request.m_Response.reset(); + if (IsOOM(SystemError.code()) || IsOOD(SystemError.code())) + { + Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, SystemError.what()); + } + else + { + ZEN_WARN("Caught system error exception while handling request: {}. ({})", + SystemError.what(), + SystemError.code().value()); + Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, SystemError.what()); + } + } + catch (const std::bad_alloc& BadAlloc) + { + // Drop any partially formatted response + Request.m_Response.reset(); + + Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, BadAlloc.what()); + } + catch (const std::exception& ex) + { + // Drop any partially formatted response + Request.m_Response.reset(); - ZEN_WARN("Caught exception while handling request: {}", ex.what()); - Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, ex.what()); + ZEN_WARN("Caught exception while handling request: {}", ex.what()); + Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, ex.what()); + } } } + else if (FilterResult == IHttpRequestFilter::Result::Forbidden) + { + Request.WriteResponse(HttpResponseCode::Forbidden); + } + else + { + ZEN_ASSERT(FilterResult == IHttpRequestFilter::Result::ResponseSent); + } if (std::unique_ptr Response = std::move(Request.m_Response)) { @@ -1923,6 +1938,31 @@ HttpAsioServerImpl::RouteRequest(std::string_view Url) return CandidateService; } +void +HttpAsioServerImpl::SetHttpRequestFilter(IHttpRequestFilter* RequestFilter) +{ + ZEN_MEMSCOPE(GetHttpasioTag()); + RwLock::ExclusiveLockScope _(m_Lock); + m_HttpRequestFilter.store(RequestFilter); +} + +IHttpRequestFilter::Result +HttpAsioServerImpl::FilterRequest(HttpServerRequest& Request) +{ + if (!m_HttpRequestFilter.load()) + { + return IHttpRequestFilter::Result::Accepted; + } + RwLock::SharedLockScope _(m_Lock); + IHttpRequestFilter* RequestFilter = m_HttpRequestFilter.load(); + if (!RequestFilter) + { + return IHttpRequestFilter::Result::Accepted; + } + IHttpRequestFilter::Result FilterResult = RequestFilter->FilterRequest(Request); + return FilterResult; +} + } // namespace zen::asio_http ////////////////////////////////////////////////////////////////////////// @@ -1937,6 +1977,7 @@ public: virtual void OnRegisterService(HttpService& Service) override; virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; + virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; virtual void OnRun(bool IsInteractiveSession) override; virtual void OnRequestExit() override; virtual void OnClose() override; @@ -1984,6 +2025,12 @@ HttpAsioServer::OnRegisterService(HttpService& Service) m_Impl->RegisterService(Service.BaseUri(), Service); } +void +HttpAsioServer::OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) +{ + m_Impl->SetHttpRequestFilter(RequestFilter); +} + int HttpAsioServer::OnInitialize(int BasePort, std::filesystem::path DataDir) { diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp index 95624245f..850d7d6b9 100644 --- a/src/zenhttp/servers/httpmulti.cpp +++ b/src/zenhttp/servers/httpmulti.cpp @@ -53,6 +53,15 @@ HttpMultiServer::OnInitialize(int BasePort, std::filesystem::path DataDir) return EffectivePort; } +void +HttpMultiServer::OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) +{ + for (auto& Server : m_Servers) + { + Server->SetHttpRequestFilter(RequestFilter); + } +} + void HttpMultiServer::OnRun(bool IsInteractiveSession) { diff --git a/src/zenhttp/servers/httpmulti.h b/src/zenhttp/servers/httpmulti.h index ae0ed74cf..1897587a9 100644 --- a/src/zenhttp/servers/httpmulti.h +++ b/src/zenhttp/servers/httpmulti.h @@ -16,6 +16,7 @@ public: ~HttpMultiServer(); virtual void OnRegisterService(HttpService& Service) override; + virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; virtual void OnRun(bool IsInteractiveSession) override; virtual void OnRequestExit() override; diff --git a/src/zenhttp/servers/httpnull.cpp b/src/zenhttp/servers/httpnull.cpp index b770b97db..db360c5fb 100644 --- a/src/zenhttp/servers/httpnull.cpp +++ b/src/zenhttp/servers/httpnull.cpp @@ -24,6 +24,12 @@ HttpNullServer::OnRegisterService(HttpService& Service) ZEN_UNUSED(Service); } +void +HttpNullServer::OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) +{ + ZEN_UNUSED(RequestFilter); +} + int HttpNullServer::OnInitialize(int BasePort, std::filesystem::path DataDir) { diff --git a/src/zenhttp/servers/httpnull.h b/src/zenhttp/servers/httpnull.h index ce7230938..52838f012 100644 --- a/src/zenhttp/servers/httpnull.h +++ b/src/zenhttp/servers/httpnull.h @@ -18,6 +18,7 @@ public: ~HttpNullServer(); virtual void OnRegisterService(HttpService& Service) override; + virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; virtual void OnRun(bool IsInteractiveSession) override; virtual void OnRequestExit() override; diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index 1a630c16f..4219dc292 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -96,6 +96,7 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer // HttpPluginServer virtual void OnRegisterService(HttpService& Service) override; + virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; virtual void OnRun(bool IsInteractiveSession) override; virtual void OnRequestExit() override; @@ -104,7 +105,8 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer virtual void AddPlugin(Ref Plugin) override; virtual void RemovePlugin(Ref Plugin) override; - HttpService* RouteRequest(std::string_view Url); + HttpService* RouteRequest(std::string_view Url); + IHttpRequestFilter::Result FilterRequest(HttpServerRequest& Request); struct ServiceEntry { @@ -112,7 +114,8 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer HttpService* Service; }; - bool m_IsInitialized = false; + std::atomic m_HttpRequestFilter = nullptr; + bool m_IsInitialized = false; RwLock m_Lock; std::vector m_UriHandlers; std::vector> m_Plugins; @@ -395,53 +398,65 @@ HttpPluginConnectionHandler::HandleRequest() std::vector{Request.ReadPayload()}); } - if (!HandlePackageOffers(*Service, Request, m_PackageHandler)) + IHttpRequestFilter::Result FilterResult = m_Server->FilterRequest(Request); + if (FilterResult == IHttpRequestFilter::Result::Accepted) { - try - { - Service->HandleRequest(Request); - } - catch (const AssertException& AssertEx) + if (!HandlePackageOffers(*Service, Request, m_PackageHandler)) { - // Drop any partially formatted response - Request.m_Response.reset(); - - ZEN_ERROR("Caught assert exception while handling request: {}", AssertEx.FullDescription()); - Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, AssertEx.FullDescription()); - } - catch (const std::system_error& SystemError) - { - // Drop any partially formatted response - Request.m_Response.reset(); - - if (IsOOM(SystemError.code()) || IsOOD(SystemError.code())) + try { - Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, SystemError.what()); + Service->HandleRequest(Request); } - else + catch (const AssertException& AssertEx) { - ZEN_WARN("Caught system error exception while handling request: {}. ({})", - SystemError.what(), - SystemError.code().value()); - Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, SystemError.what()); + // Drop any partially formatted response + Request.m_Response.reset(); + + ZEN_ERROR("Caught assert exception while handling request: {}", AssertEx.FullDescription()); + Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, AssertEx.FullDescription()); } - } - catch (const std::bad_alloc& BadAlloc) - { - // Drop any partially formatted response - Request.m_Response.reset(); + catch (const std::system_error& SystemError) + { + // Drop any partially formatted response + Request.m_Response.reset(); + + if (IsOOM(SystemError.code()) || IsOOD(SystemError.code())) + { + Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, SystemError.what()); + } + else + { + ZEN_WARN("Caught system error exception while handling request: {}. ({})", + SystemError.what(), + SystemError.code().value()); + Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, SystemError.what()); + } + } + catch (const std::bad_alloc& BadAlloc) + { + // Drop any partially formatted response + Request.m_Response.reset(); - Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, BadAlloc.what()); - } - catch (const std::exception& ex) - { - // Drop any partially formatted response - Request.m_Response.reset(); + Request.WriteResponse(HttpResponseCode::InsufficientStorage, HttpContentType::kText, BadAlloc.what()); + } + catch (const std::exception& ex) + { + // Drop any partially formatted response + Request.m_Response.reset(); - ZEN_WARN("Caught exception while handling request: {}", ex.what()); - Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, ex.what()); + ZEN_WARN("Caught exception while handling request: {}", ex.what()); + Request.WriteResponse(HttpResponseCode::InternalServerError, HttpContentType::kText, ex.what()); + } } } + else if (FilterResult == IHttpRequestFilter::Result::Forbidden) + { + Request.WriteResponse(HttpResponseCode::Forbidden); + } + else + { + ZEN_ASSERT(FilterResult == IHttpRequestFilter::Result::ResponseSent); + } if (std::unique_ptr Response = std::move(Request.m_Response)) { @@ -752,6 +767,13 @@ HttpPluginServerImpl::OnInitialize(int InBasePort, std::filesystem::path DataDir return m_BasePort; } +void +HttpPluginServerImpl::OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) +{ + RwLock::ExclusiveLockScope _(m_Lock); + m_HttpRequestFilter.store(RequestFilter); +} + void HttpPluginServerImpl::OnClose() { @@ -897,6 +919,23 @@ HttpPluginServerImpl::RouteRequest(std::string_view Url) return CandidateService; } +IHttpRequestFilter::Result +HttpPluginServerImpl::FilterRequest(HttpServerRequest& Request) +{ + if (!m_HttpRequestFilter.load()) + { + return IHttpRequestFilter::Result::Accepted; + } + RwLock::SharedLockScope _(m_Lock); + IHttpRequestFilter* RequestFilter = m_HttpRequestFilter.load(); + if (!RequestFilter) + { + return IHttpRequestFilter::Result::Accepted; + } + IHttpRequestFilter::Result FilterResult = RequestFilter->FilterRequest(Request); + return FilterResult; +} + ////////////////////////////////////////////////////////////////////////// struct HttpPluginServerImpl; diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 01c4559a1..4df4cd079 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -94,6 +94,7 @@ public: virtual void OnRun(bool TestMode) override; virtual void OnRequestExit() override; virtual void OnRegisterService(HttpService& Service) override; + virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; virtual void OnClose() override; WorkerThreadPool& WorkPool(); @@ -101,6 +102,8 @@ public: inline bool IsOk() const { return m_IsOk; } inline bool IsAsyncResponseEnabled() const { return m_IsAsyncResponseEnabled; } + IHttpRequestFilter::Result FilterRequest(HttpServerRequest& Request); + private: int InitializeServer(int BasePort); void Cleanup(); @@ -137,6 +140,9 @@ private: int32_t m_MaxPendingRequests = 128; Event m_ShutdownEvent; HttpSysConfig m_InitialConfig; + + RwLock m_RequestFilterLock; + std::atomic m_HttpRequestFilter = nullptr; }; } // namespace zen @@ -1672,9 +1678,21 @@ HttpSysTransaction::InvokeRequestHandler(HttpService& Service, IoBuffer Payload) otel::ScopedSpan HttpSpan(SpanNamer, SpanAnnotator); # endif - if (!HandlePackageOffers(Service, ThisRequest, m_PackageHandler)) + IHttpRequestFilter::Result FilterResult = m_HttpServer.FilterRequest(ThisRequest); + if (FilterResult == IHttpRequestFilter::Result::Accepted) + { + if (!HandlePackageOffers(Service, ThisRequest, m_PackageHandler)) + { + Service.HandleRequest(ThisRequest); + } + } + else if (FilterResult == IHttpRequestFilter::Result::Forbidden) { - Service.HandleRequest(ThisRequest); + ThisRequest.WriteResponse(HttpResponseCode::Forbidden); + } + else + { + ZEN_ASSERT(FilterResult == IHttpRequestFilter::Result::ResponseSent); } return ThisRequest; @@ -2244,6 +2262,30 @@ HttpSysServer::OnRegisterService(HttpService& Service) RegisterService(Service.BaseUri(), Service); } +void +HttpSysServer::OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) +{ + RwLock::ExclusiveLockScope _(m_RequestFilterLock); + m_HttpRequestFilter.store(RequestFilter); +} + +IHttpRequestFilter::Result +HttpSysServer::FilterRequest(HttpServerRequest& Request) +{ + if (!m_HttpRequestFilter.load()) + { + return IHttpRequestFilter::Result::Accepted; + } + RwLock::SharedLockScope _(m_RequestFilterLock); + IHttpRequestFilter* RequestFilter = m_HttpRequestFilter.load(); + if (!RequestFilter) + { + return IHttpRequestFilter::Result::Accepted; + } + IHttpRequestFilter::Result FilterResult = RequestFilter->FilterRequest(Request); + return FilterResult; +} + Ref CreateHttpSysServer(HttpSysConfig Config) { -- cgit v1.2.3 From b0a3de5fec8f4da8f9513b02bc2326aa6a0e7bd5 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 13 Feb 2026 13:47:51 +0100 Subject: logging config move to zenutil (#754) made logging config options from zenserver available in zen CLI --- CHANGELOG.md | 1 + src/zen/zen.cpp | 23 +- src/zen/zen.h | 5 +- src/zenserver-test/zenserver-test.cpp | 2 +- src/zenserver/config/config.cpp | 73 ++----- src/zenserver/config/config.h | 31 ++- src/zenserver/config/luaconfig.h | 2 +- src/zenserver/diag/logging.cpp | 12 +- src/zenserver/main.cpp | 2 +- src/zenserver/storage/zenstorageserver.cpp | 2 +- src/zenserver/zenserver.cpp | 12 +- src/zenutil/commandlineoptions.cpp | 239 --------------------- src/zenutil/config/commandlineoptions.cpp | 239 +++++++++++++++++++++ src/zenutil/config/environmentoptions.cpp | 84 ++++++++ src/zenutil/config/loggingconfig.cpp | 77 +++++++ src/zenutil/environmentoptions.cpp | 84 -------- src/zenutil/include/zenutil/commandlineoptions.h | 40 ---- .../include/zenutil/config/commandlineoptions.h | 40 ++++ .../include/zenutil/config/environmentoptions.h | 92 ++++++++ src/zenutil/include/zenutil/config/loggingconfig.h | 37 ++++ src/zenutil/include/zenutil/environmentoptions.h | 92 -------- src/zenutil/zenutil.cpp | 2 +- 22 files changed, 640 insertions(+), 551 deletions(-) delete mode 100644 src/zenutil/commandlineoptions.cpp create mode 100644 src/zenutil/config/commandlineoptions.cpp create mode 100644 src/zenutil/config/environmentoptions.cpp create mode 100644 src/zenutil/config/loggingconfig.cpp delete mode 100644 src/zenutil/environmentoptions.cpp delete mode 100644 src/zenutil/include/zenutil/commandlineoptions.h create mode 100644 src/zenutil/include/zenutil/config/commandlineoptions.h create mode 100644 src/zenutil/include/zenutil/config/environmentoptions.h create mode 100644 src/zenutil/include/zenutil/config/loggingconfig.h delete mode 100644 src/zenutil/include/zenutil/environmentoptions.h diff --git a/CHANGELOG.md b/CHANGELOG.md index aaac59c83..487d45fef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## - Improvement: Reduced time project and project oplogs are locked during GC and Validation +- Improvement: `zen` now supports additional configuration of logging options, such as `--log-warn=...` for configuring log levels, etc (see `zen --help`) ## 5.7.20 - Improvement: When validating cache records read from disk we now do a limited validation of the payload to reduce overhead diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 09a2e4f91..25245c3d2 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include #include #include @@ -538,6 +538,9 @@ main(int argc, char** argv) Options.add_options()("corelimit", "Limit concurrency", cxxopts::value(CoreLimit)); + ZenLoggingCmdLineOptions LoggingCmdLineOptions; + LoggingCmdLineOptions.AddCliOptions(Options, GlobalOptions.LoggingConfig); + #if ZEN_WITH_TRACE // We only have this in options for command line help purposes - we parse these argument separately earlier using // GetTraceOptionsFromCommandline() @@ -624,8 +627,8 @@ main(int argc, char** argv) } LimitHardwareConcurrency(CoreLimit); -#if ZEN_USE_SENTRY +#if ZEN_USE_SENTRY { EnvironmentOptions EnvOptions; @@ -671,12 +674,20 @@ main(int argc, char** argv) } #endif - zen::LoggingOptions LogOptions; - LogOptions.IsDebug = GlobalOptions.IsDebug; - LogOptions.IsVerbose = GlobalOptions.IsVerbose; - LogOptions.AllowAsync = false; + LoggingCmdLineOptions.ApplyOptions(GlobalOptions.LoggingConfig); + + const LoggingOptions LogOptions = {.IsDebug = GlobalOptions.IsDebug, + .IsVerbose = GlobalOptions.IsVerbose, + .IsTest = false, + .AllowAsync = false, + .NoConsoleOutput = GlobalOptions.LoggingConfig.NoConsoleOutput, + .QuietConsole = GlobalOptions.LoggingConfig.QuietConsole, + .AbsLogFile = GlobalOptions.LoggingConfig.AbsLogFile, + .LogId = GlobalOptions.LoggingConfig.LogId}; zen::InitializeLogging(LogOptions); + ApplyLoggingOptions(Options, GlobalOptions.LoggingConfig); + std::set_terminate([]() { void* Frames[8]; uint32_t FrameCount = GetCallstack(2, 8, Frames); diff --git a/src/zen/zen.h b/src/zen/zen.h index 05d1e4ec8..e3481beea 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -5,7 +5,8 @@ #include #include #include -#include +#include +#include namespace zen { @@ -14,6 +15,8 @@ struct ZenCliOptions bool IsDebug = false; bool IsVerbose = false; + ZenLoggingConfig LoggingConfig; + // Arguments after " -- " on command line are passed through and not parsed std::string PassthroughCommandLine; std::string PassthroughArgs; diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 9a42bb73d..4120dec1a 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -17,7 +17,7 @@ # include # include # include -# include +# include # include # include diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp index 07913e891..2b77df642 100644 --- a/src/zenserver/config/config.cpp +++ b/src/zenserver/config/config.cpp @@ -16,8 +16,8 @@ #include #include #include -#include -#include +#include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -119,10 +119,17 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions ZenServerConfig& ServerOptions = m_ServerOptions; + // logging + + LuaOptions.AddOption("server.logid"sv, ServerOptions.LoggingConfig.LogId, "log-id"sv); + LuaOptions.AddOption("server.abslog"sv, ServerOptions.LoggingConfig.AbsLogFile, "abslog"sv); + LuaOptions.AddOption("server.otlpendpoint"sv, ServerOptions.LoggingConfig.OtelEndpointUri, "otlp-endpoint"sv); + LuaOptions.AddOption("server.quiet"sv, ServerOptions.LoggingConfig.QuietConsole, "quiet"sv); + LuaOptions.AddOption("server.noconsole"sv, ServerOptions.LoggingConfig.NoConsoleOutput, "noconsole"sv); + // server LuaOptions.AddOption("server.dedicated"sv, ServerOptions.IsDedicated, "dedicated"sv); - LuaOptions.AddOption("server.logid"sv, ServerOptions.LogId, "log-id"sv); LuaOptions.AddOption("server.sentry.disable"sv, ServerOptions.SentryConfig.Disable, "no-sentry"sv); LuaOptions.AddOption("server.sentry.allowpersonalinfo"sv, ServerOptions.SentryConfig.AllowPII, "sentry-allow-personal-info"sv); LuaOptions.AddOption("server.sentry.dsn"sv, ServerOptions.SentryConfig.Dsn, "sentry-dsn"sv); @@ -131,12 +138,8 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions LuaOptions.AddOption("server.systemrootdir"sv, ServerOptions.SystemRootDir, "system-dir"sv); LuaOptions.AddOption("server.datadir"sv, ServerOptions.DataDir, "data-dir"sv); LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); - LuaOptions.AddOption("server.abslog"sv, ServerOptions.AbsLogFile, "abslog"sv); - LuaOptions.AddOption("server.otlpendpoint"sv, ServerOptions.OtelEndpointUri, "otlp-endpoint"sv); LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); - LuaOptions.AddOption("server.quiet"sv, ServerOptions.QuietConsole, "quiet"sv); - LuaOptions.AddOption("server.noconsole"sv, ServerOptions.NoConsoleOutput, "noconsole"sv); ////// network @@ -182,9 +185,10 @@ struct ZenServerCmdLineOptions std::string SystemRootDir; std::string ContentDir; std::string DataDir; - std::string AbsLogFile; std::string BaseSnapshotDir; + ZenLoggingCmdLineOptions LoggingOptions; + void AddCliOptions(cxxopts::Options& options, ZenServerConfig& ServerOptions); void ApplyOptions(cxxopts::Options& options, ZenServerConfig& ServerOptions); }; @@ -249,22 +253,7 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi cxxopts::value(ServerOptions.ShouldCrash)->default_value("false"), ""); - // clang-format off - options.add_options("logging") - ("abslog", "Path to log file", cxxopts::value(AbsLogFile)) - ("log-id", "Specify id for adding context to log output", cxxopts::value(ServerOptions.LogId)) - ("quiet", "Configure console logger output to level WARN", cxxopts::value(ServerOptions.QuietConsole)->default_value("false")) - ("noconsole", "Disable console logging", cxxopts::value(ServerOptions.NoConsoleOutput)->default_value("false")) - ("log-trace", "Change selected loggers to level TRACE", cxxopts::value(ServerOptions.Loggers[logging::level::Trace])) - ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value(ServerOptions.Loggers[logging::level::Debug])) - ("log-info", "Change selected loggers to level INFO", cxxopts::value(ServerOptions.Loggers[logging::level::Info])) - ("log-warn", "Change selected loggers to level WARN", cxxopts::value(ServerOptions.Loggers[logging::level::Warn])) - ("log-error", "Change selected loggers to level ERROR", cxxopts::value(ServerOptions.Loggers[logging::level::Err])) - ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value(ServerOptions.Loggers[logging::level::Critical])) - ("log-off", "Change selected loggers to level OFF", cxxopts::value(ServerOptions.Loggers[logging::level::Off])) - ("otlp-endpoint", "OpenTelemetry endpoint URI (e.g http://localhost:4318)", cxxopts::value(ServerOptions.OtelEndpointUri)) - ; - // clang-format on + LoggingOptions.AddCliOptions(options, ServerOptions.LoggingConfig); options .add_option("lifetime", "", "owner-pid", "Specify owning process id", cxxopts::value(ServerOptions.OwnerPid), ""); @@ -394,9 +383,10 @@ ZenServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenServerConfig ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); - ServerOptions.AbsLogFile = MakeSafeAbsolutePath(AbsLogFile); ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); + + LoggingOptions.ApplyOptions(ServerOptions.LoggingConfig); } ////////////////////////////////////////////////////////////////////////// @@ -466,34 +456,7 @@ ZenServerConfiguratorBase::Configure(int argc, char* argv[]) } #endif - if (m_ServerOptions.QuietConsole) - { - bool HasExplicitConsoleLevel = false; - for (int i = 0; i < logging::level::LogLevelCount; ++i) - { - if (m_ServerOptions.Loggers[i].find("console") != std::string::npos) - { - HasExplicitConsoleLevel = true; - break; - } - } - - if (!HasExplicitConsoleLevel) - { - std::string& WarnLoggers = m_ServerOptions.Loggers[logging::level::Warn]; - if (!WarnLoggers.empty()) - { - WarnLoggers += ","; - } - WarnLoggers += "console"; - } - } - - for (int i = 0; i < logging::level::LogLevelCount; ++i) - { - logging::ConfigureLogLevels(logging::level::LogLevel(i), m_ServerOptions.Loggers[i]); - } - logging::RefreshLogLevels(); + ApplyLoggingOptions(options, m_ServerOptions.LoggingConfig); BaseOptions.ApplyOptions(options, m_ServerOptions); ApplyOptions(options); @@ -532,9 +495,9 @@ ZenServerConfiguratorBase::Configure(int argc, char* argv[]) m_ServerOptions.DataDir = PickDefaultStateDirectory(m_ServerOptions.SystemRootDir); } - if (m_ServerOptions.AbsLogFile.empty()) + if (m_ServerOptions.LoggingConfig.AbsLogFile.empty()) { - m_ServerOptions.AbsLogFile = m_ServerOptions.DataDir / "logs" / "zenserver.log"; + m_ServerOptions.LoggingConfig.AbsLogFile = m_ServerOptions.DataDir / "logs" / "zenserver.log"; } m_ServerOptions.HttpConfig.IsDedicatedServer = m_ServerOptions.IsDedicated; diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h index 7c3192a1f..32c22cb05 100644 --- a/src/zenserver/config/config.h +++ b/src/zenserver/config/config.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -42,29 +43,25 @@ struct ZenServerConfig HttpServerConfig HttpConfig; ZenSentryConfig SentryConfig; ZenStatsConfig StatsConfig; - int BasePort = 8558; // Service listen port (used for both UDP and TCP) - int OwnerPid = 0; // Parent process id (zero for standalone) - bool IsDebug = false; - bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not - bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization - bool IsTest = false; - bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) - bool NoConsoleOutput = false; // Control default use of stdout for diagnostics - bool QuietConsole = false; // Configure console logger output to level WARN - int CoreLimit = 0; // If set, hardware concurrency queries are capped at this number - bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements - bool ShouldCrash = false; // Option for testing crash handling - bool IsFirstRun = false; + ZenLoggingConfig LoggingConfig; + int BasePort = 8558; // Service listen port (used for both UDP and TCP) + int OwnerPid = 0; // Parent process id (zero for standalone) + bool IsDebug = false; + bool IsCleanStart = false; // Indicates whether all state should be wiped on startup or not + bool IsPowerCycle = false; // When true, the process shuts down immediately after initialization + bool IsTest = false; + bool Detach = true; // Whether zenserver should detach from existing process group (Mac/Linux) + int CoreLimit = 0; // If set, hardware concurrency queries are capped at this number + int LieCpu = 0; + bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements + bool ShouldCrash = false; // Option for testing crash handling + bool IsFirstRun = false; std::filesystem::path ConfigFile; // Path to Lua config file std::filesystem::path SystemRootDir; // System root directory (used for machine level config) std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) std::filesystem::path DataDir; // Root directory for state (used for testing) - std::filesystem::path AbsLogFile; // Absolute path to main log file std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) std::string ChildId; // Id assigned by parent process (used for lifetime management) - std::string LogId; // Id for tagging log output - std::string Loggers[zen::logging::level::LogLevelCount]; - std::string OtelEndpointUri; // OpenTelemetry endpoint URI #if ZEN_WITH_TRACE bool HasTraceCommandlineOptions = false; diff --git a/src/zenserver/config/luaconfig.h b/src/zenserver/config/luaconfig.h index ce7013a9a..e3ac3b343 100644 --- a/src/zenserver/config/luaconfig.h +++ b/src/zenserver/config/luaconfig.h @@ -4,7 +4,7 @@ #include #include -#include +#include ZEN_THIRD_PARTY_INCLUDES_START #include diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp index 4962b9006..75a8efc09 100644 --- a/src/zenserver/diag/logging.cpp +++ b/src/zenserver/diag/logging.cpp @@ -28,10 +28,10 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService) const LoggingOptions LogOptions = {.IsDebug = InOptions.IsDebug, .IsVerbose = false, .IsTest = InOptions.IsTest, - .NoConsoleOutput = InOptions.NoConsoleOutput, - .QuietConsole = InOptions.QuietConsole, - .AbsLogFile = InOptions.AbsLogFile, - .LogId = InOptions.LogId}; + .NoConsoleOutput = InOptions.LoggingConfig.NoConsoleOutput, + .QuietConsole = InOptions.LoggingConfig.QuietConsole, + .AbsLogFile = InOptions.LoggingConfig.AbsLogFile, + .LogId = InOptions.LoggingConfig.LogId}; BeginInitializeLogging(LogOptions); @@ -79,10 +79,10 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService) } #if ZEN_WITH_OTEL - if (!InOptions.OtelEndpointUri.empty()) + if (!InOptions.LoggingConfig.OtelEndpointUri.empty()) { // TODO: Should sanity check that endpoint is reachable? Also, a valid URI? - auto OtelSink = std::make_shared(InOptions.OtelEndpointUri); + auto OtelSink = std::make_shared(InOptions.LoggingConfig.OtelEndpointUri); zen::logging::Default().SpdLogger->sinks().push_back(std::move(OtelSink)); } #endif diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 3a58d1f4a..1a929b026 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include "diag/logging.h" diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index b2cae6482..2b74395c3 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -305,7 +305,7 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions *m_JobQueue, m_CacheStore.Get(), [this]() { Flush(); }, - HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.AbsLogFile, + HttpAdminService::LogPaths{.AbsLogPath = ServerOptions.LoggingConfig.AbsLogFile, .HttpLogPath = ServerOptions.DataDir / "logs" / "http.log", .CacheLogPath = ServerOptions.DataDir / "logs" / "z$.log"}, ServerOptions); diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 2bafeeaa1..d54357368 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -152,7 +152,7 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState:: } m_HealthService.SetHealthInfo({.DataRoot = ServerOptions.DataDir, - .AbsLogPath = ServerOptions.AbsLogFile, + .AbsLogPath = ServerOptions.LoggingConfig.AbsLogFile, .HttpServerClass = std::string(ServerOptions.HttpConfig.ServerClass), .BuildVersion = std::string(ZEN_CFG_VERSION_BUILD_STRING_FULL)}); @@ -387,7 +387,7 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig) // clang-format off std::list> Settings = { {"DataDir"sv, ServerConfig.DataDir.string()}, - {"AbsLogFile"sv, ServerConfig.AbsLogFile.string()}, + {"AbsLogFile"sv, ServerConfig.LoggingConfig.AbsLogFile.string()}, {"SystemRootDir"sv, ServerConfig.SystemRootDir.string()}, {"ContentDir"sv, ServerConfig.ContentDir.string()}, {"BasePort"sv, fmt::to_string(ServerConfig.BasePort)}, @@ -396,13 +396,13 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig) {"IsPowerCycle"sv, fmt::to_string(ServerConfig.IsPowerCycle)}, {"IsTest"sv, fmt::to_string(ServerConfig.IsTest)}, {"Detach"sv, fmt::to_string(ServerConfig.Detach)}, - {"NoConsoleOutput"sv, fmt::to_string(ServerConfig.NoConsoleOutput)}, - {"QuietConsole"sv, fmt::to_string(ServerConfig.QuietConsole)}, + {"NoConsoleOutput"sv, fmt::to_string(ServerConfig.LoggingConfig.NoConsoleOutput)}, + {"QuietConsole"sv, fmt::to_string(ServerConfig.LoggingConfig.QuietConsole)}, {"CoreLimit"sv, fmt::to_string(ServerConfig.CoreLimit)}, {"IsDedicated"sv, fmt::to_string(ServerConfig.IsDedicated)}, {"ShouldCrash"sv, fmt::to_string(ServerConfig.ShouldCrash)}, {"ChildId"sv, ServerConfig.ChildId}, - {"LogId"sv, ServerConfig.LogId}, + {"LogId"sv, ServerConfig.LoggingConfig.LogId}, {"Sentry DSN"sv, ServerConfig.SentryConfig.Dsn.empty() ? "not set" : ServerConfig.SentryConfig.Dsn}, {"Sentry Environment"sv, ServerConfig.SentryConfig.Environment}, {"Statsd Enabled"sv, fmt::to_string(ServerConfig.StatsConfig.Enabled)}, @@ -467,7 +467,7 @@ ZenServerMain::Run() ZEN_OTEL_SPAN("SentryInit"); std::string SentryDatabasePath = (m_ServerOptions.DataDir / ".sentry-native").string(); - std::string SentryAttachmentPath = m_ServerOptions.AbsLogFile.string(); + std::string SentryAttachmentPath = m_ServerOptions.LoggingConfig.AbsLogFile.string(); Sentry.Initialize({.DatabasePath = SentryDatabasePath, .AttachmentsPath = SentryAttachmentPath, diff --git a/src/zenutil/commandlineoptions.cpp b/src/zenutil/commandlineoptions.cpp deleted file mode 100644 index d94564843..000000000 --- a/src/zenutil/commandlineoptions.cpp +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include - -#include -#include - -#include - -#if ZEN_WITH_TESTS -# include -#endif // ZEN_WITH_TESTS - -#ifndef CXXOPTS_HAS_FILESYSTEM -void -cxxopts::values::parse_value(const std::string& text, std::filesystem::path& value) -{ - value = zen::StringToPath(text); -} -#endif - -namespace zen { - -std::vector -ParseCommandLine(std::string_view CommandLine) -{ - auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) { - if (Pos == CommandLine.length()) - { - return true; - } - if (CommandLine[Pos] == ' ') - { - return true; - } - return false; - }; - - bool IsParsingArg = false; - bool IsInQuote = false; - - std::string::size_type Pos = 0; - std::string::size_type ArgStart = 0; - std::vector Args; - while (Pos < CommandLine.length()) - { - if (IsInQuote) - { - if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1)) - { - Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1))); - Pos++; - IsInQuote = false; - IsParsingArg = false; - } - else - { - Pos++; - } - } - else if (IsParsingArg) - { - ZEN_ASSERT(Pos > ArgStart); - if (CommandLine[Pos] == ' ') - { - Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart))); - Pos++; - IsParsingArg = false; - } - else if (CommandLine[Pos] == '"') - { - IsInQuote = true; - Pos++; - } - else - { - Pos++; - } - } - else if (CommandLine[Pos] == '"') - { - IsInQuote = true; - IsParsingArg = true; - ArgStart = Pos; - Pos++; - } - else if (CommandLine[Pos] != ' ') - { - IsParsingArg = true; - ArgStart = Pos; - Pos++; - } - else - { - Pos++; - } - } - if (IsParsingArg) - { - ZEN_ASSERT(Pos > ArgStart); - Args.push_back(std::string(CommandLine.substr(ArgStart))); - } - - return Args; -} - -std::vector -StripCommandlineQuotes(std::vector& InOutArgs) -{ - std::vector RawArgs; - RawArgs.reserve(InOutArgs.size()); - for (std::string& Arg : InOutArgs) - { - std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1); - while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos) - { - Arg.erase(EscapedQuotePos, 1); - EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos); - } - - if (Arg.starts_with("\"")) - { - if (Arg.find('"', 1) == Arg.length() - 1) - { - Arg = Arg.substr(1, Arg.length() - 2); - } - } - else if (Arg.ends_with("\"")) - { - std::string::size_type EqualSign = Arg.find("=", 1); - if (EqualSign != std::string::npos && Arg[EqualSign + 1] == '\"') - { - Arg = Arg.substr(0, EqualSign + 1) + Arg.substr(EqualSign + 2, Arg.length() - (EqualSign + 2) - 1); - } - } - RawArgs.push_back(const_cast(Arg.c_str())); - } - return RawArgs; -} - -std::filesystem::path -StringToPath(const std::string_view& Path) -{ - std::string_view UnquotedPath = RemoveQuotes(Path); - - if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) - { - UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1); - } - - return std::filesystem::path(UnquotedPath).make_preferred(); -} - -std::string_view -RemoveQuotes(const std::string_view& Arg) -{ - if (Arg.length() > 2) - { - if (Arg[0] == '"' && Arg[Arg.length() - 1] == '"') - { - return Arg.substr(1, Arg.length() - 2); - } - } - return Arg; -} - -CommandLineConverter::CommandLineConverter(int& argc, char**& argv) -{ -#if ZEN_PLATFORM_WINDOWS - LPWSTR RawCommandLine = GetCommandLineW(); - std::string CommandLine = WideToUtf8(RawCommandLine); - Args = ParseCommandLine(CommandLine); -#else - Args.reserve(argc); - for (int I = 0; I < argc; I++) - { - std::string Arg(argv[I]); - if ((!Arg.empty()) && (Arg != " ")) - { - Args.emplace_back(std::move(Arg)); - } - } -#endif - RawArgs = StripCommandlineQuotes(Args); - - argc = static_cast(RawArgs.size()); - argv = RawArgs.data(); -} - -#if ZEN_WITH_TESTS - -void -commandlineoptions_forcelink() -{ -} - -TEST_CASE("CommandLine") -{ - std::vector v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\""); - CHECK_EQ(v1[0], "c:\\my\\exe.exe"); - CHECK_EQ(v1[1], "\"quoted arg\""); - CHECK_EQ(v1[2], "\"one\",two,\"three\\\""); - - std::vector v2 = ParseCommandLine( - "--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog " - "--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" " - "--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" " - "--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5"); - - std::vector v2Stripped = StripCommandlineQuotes(v2); - CHECK_EQ(v2Stripped[0], std::string("--tracehost")); - CHECK_EQ(v2Stripped[1], std::string("127.0.0.1")); - CHECK_EQ(v2Stripped[2], std::string("builds")); - CHECK_EQ(v2Stripped[3], std::string("download")); - CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com")); - CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog")); - CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows")); - CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path")); - CHECK_EQ(v2Stripped[8], std::string("--access-token-path=C:\\Users\\dan.engelbrecht\\jupiter-token.json")); - CHECK_EQ(v2Stripped[9], std::string("D:\\Dev\\Spaced Folder\\Target")); - CHECK_EQ(v2Stripped[10], std::string("--alt-path=D:\\Dev\\Spaced Folder2\\Target")); - CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab")); - CHECK_EQ(v2Stripped[12], std::string("--build-part-name")); - CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5")); - - std::vector v3 = ParseCommandLine( - "--tracehost \"127.0.0.1\" builds download --url=\"https://jupiter.devtools.epicgames.com\" --build-part-name=\"win64\""); - std::vector v3Stripped = StripCommandlineQuotes(v3); - - CHECK_EQ(v3Stripped[0], std::string("--tracehost")); - CHECK_EQ(v3Stripped[1], std::string("127.0.0.1")); - CHECK_EQ(v3Stripped[2], std::string("builds")); - CHECK_EQ(v3Stripped[3], std::string("download")); - CHECK_EQ(v3Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com")); - CHECK_EQ(v3Stripped[5], std::string("--build-part-name=win64")); -} - -#endif -} // namespace zen diff --git a/src/zenutil/config/commandlineoptions.cpp b/src/zenutil/config/commandlineoptions.cpp new file mode 100644 index 000000000..84c718ecc --- /dev/null +++ b/src/zenutil/config/commandlineoptions.cpp @@ -0,0 +1,239 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include + +#include + +#if ZEN_WITH_TESTS +# include +#endif // ZEN_WITH_TESTS + +#ifndef CXXOPTS_HAS_FILESYSTEM +void +cxxopts::values::parse_value(const std::string& text, std::filesystem::path& value) +{ + value = zen::StringToPath(text); +} +#endif + +namespace zen { + +std::vector +ParseCommandLine(std::string_view CommandLine) +{ + auto IsWhitespaceOrEnd = [](std::string_view CommandLine, std::string::size_type Pos) { + if (Pos == CommandLine.length()) + { + return true; + } + if (CommandLine[Pos] == ' ') + { + return true; + } + return false; + }; + + bool IsParsingArg = false; + bool IsInQuote = false; + + std::string::size_type Pos = 0; + std::string::size_type ArgStart = 0; + std::vector Args; + while (Pos < CommandLine.length()) + { + if (IsInQuote) + { + if (CommandLine[Pos] == '"' && IsWhitespaceOrEnd(CommandLine, Pos + 1)) + { + Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart + 1))); + Pos++; + IsInQuote = false; + IsParsingArg = false; + } + else + { + Pos++; + } + } + else if (IsParsingArg) + { + ZEN_ASSERT(Pos > ArgStart); + if (CommandLine[Pos] == ' ') + { + Args.push_back(std::string(CommandLine.substr(ArgStart, Pos - ArgStart))); + Pos++; + IsParsingArg = false; + } + else if (CommandLine[Pos] == '"') + { + IsInQuote = true; + Pos++; + } + else + { + Pos++; + } + } + else if (CommandLine[Pos] == '"') + { + IsInQuote = true; + IsParsingArg = true; + ArgStart = Pos; + Pos++; + } + else if (CommandLine[Pos] != ' ') + { + IsParsingArg = true; + ArgStart = Pos; + Pos++; + } + else + { + Pos++; + } + } + if (IsParsingArg) + { + ZEN_ASSERT(Pos > ArgStart); + Args.push_back(std::string(CommandLine.substr(ArgStart))); + } + + return Args; +} + +std::vector +StripCommandlineQuotes(std::vector& InOutArgs) +{ + std::vector RawArgs; + RawArgs.reserve(InOutArgs.size()); + for (std::string& Arg : InOutArgs) + { + std::string::size_type EscapedQuotePos = Arg.find("\\\"", 1); + while (EscapedQuotePos != std::string::npos && Arg.rfind('\"', EscapedQuotePos - 1) != std::string::npos) + { + Arg.erase(EscapedQuotePos, 1); + EscapedQuotePos = Arg.find("\\\"", EscapedQuotePos); + } + + if (Arg.starts_with("\"")) + { + if (Arg.find('"', 1) == Arg.length() - 1) + { + Arg = Arg.substr(1, Arg.length() - 2); + } + } + else if (Arg.ends_with("\"")) + { + std::string::size_type EqualSign = Arg.find("=", 1); + if (EqualSign != std::string::npos && Arg[EqualSign + 1] == '\"') + { + Arg = Arg.substr(0, EqualSign + 1) + Arg.substr(EqualSign + 2, Arg.length() - (EqualSign + 2) - 1); + } + } + RawArgs.push_back(const_cast(Arg.c_str())); + } + return RawArgs; +} + +std::filesystem::path +StringToPath(const std::string_view& Path) +{ + std::string_view UnquotedPath = RemoveQuotes(Path); + + if (UnquotedPath.ends_with('/') || UnquotedPath.ends_with('\\') || UnquotedPath.ends_with(std::filesystem::path::preferred_separator)) + { + UnquotedPath = UnquotedPath.substr(0, UnquotedPath.length() - 1); + } + + return std::filesystem::path(UnquotedPath).make_preferred(); +} + +std::string_view +RemoveQuotes(const std::string_view& Arg) +{ + if (Arg.length() > 2) + { + if (Arg[0] == '"' && Arg[Arg.length() - 1] == '"') + { + return Arg.substr(1, Arg.length() - 2); + } + } + return Arg; +} + +CommandLineConverter::CommandLineConverter(int& argc, char**& argv) +{ +#if ZEN_PLATFORM_WINDOWS + LPWSTR RawCommandLine = GetCommandLineW(); + std::string CommandLine = WideToUtf8(RawCommandLine); + Args = ParseCommandLine(CommandLine); +#else + Args.reserve(argc); + for (int I = 0; I < argc; I++) + { + std::string Arg(argv[I]); + if ((!Arg.empty()) && (Arg != " ")) + { + Args.emplace_back(std::move(Arg)); + } + } +#endif + RawArgs = StripCommandlineQuotes(Args); + + argc = static_cast(RawArgs.size()); + argv = RawArgs.data(); +} + +#if ZEN_WITH_TESTS + +void +commandlineoptions_forcelink() +{ +} + +TEST_CASE("CommandLine") +{ + std::vector v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\""); + CHECK_EQ(v1[0], "c:\\my\\exe.exe"); + CHECK_EQ(v1[1], "\"quoted arg\""); + CHECK_EQ(v1[2], "\"one\",two,\"three\\\""); + + std::vector v2 = ParseCommandLine( + "--tracehost 127.0.0.1 builds download --url=https://jupiter.devtools.epicgames.com --namespace=ue.oplog " + "--bucket=citysample.packaged-build.fortnite-main.windows \"c:\\just\\a\\path\" " + "--access-token-path=\"C:\\Users\\dan.engelbrecht\\jupiter-token.json\" \"D:\\Dev\\Spaced Folder\\Target\\\" " + "--alt-path=\"D:\\Dev\\Spaced Folder2\\Target\\\" 07dn23ifiwesnvoasjncasab --build-part-name win64,linux,ps5"); + + std::vector v2Stripped = StripCommandlineQuotes(v2); + CHECK_EQ(v2Stripped[0], std::string("--tracehost")); + CHECK_EQ(v2Stripped[1], std::string("127.0.0.1")); + CHECK_EQ(v2Stripped[2], std::string("builds")); + CHECK_EQ(v2Stripped[3], std::string("download")); + CHECK_EQ(v2Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com")); + CHECK_EQ(v2Stripped[5], std::string("--namespace=ue.oplog")); + CHECK_EQ(v2Stripped[6], std::string("--bucket=citysample.packaged-build.fortnite-main.windows")); + CHECK_EQ(v2Stripped[7], std::string("c:\\just\\a\\path")); + CHECK_EQ(v2Stripped[8], std::string("--access-token-path=C:\\Users\\dan.engelbrecht\\jupiter-token.json")); + CHECK_EQ(v2Stripped[9], std::string("D:\\Dev\\Spaced Folder\\Target")); + CHECK_EQ(v2Stripped[10], std::string("--alt-path=D:\\Dev\\Spaced Folder2\\Target")); + CHECK_EQ(v2Stripped[11], std::string("07dn23ifiwesnvoasjncasab")); + CHECK_EQ(v2Stripped[12], std::string("--build-part-name")); + CHECK_EQ(v2Stripped[13], std::string("win64,linux,ps5")); + + std::vector v3 = ParseCommandLine( + "--tracehost \"127.0.0.1\" builds download --url=\"https://jupiter.devtools.epicgames.com\" --build-part-name=\"win64\""); + std::vector v3Stripped = StripCommandlineQuotes(v3); + + CHECK_EQ(v3Stripped[0], std::string("--tracehost")); + CHECK_EQ(v3Stripped[1], std::string("127.0.0.1")); + CHECK_EQ(v3Stripped[2], std::string("builds")); + CHECK_EQ(v3Stripped[3], std::string("download")); + CHECK_EQ(v3Stripped[4], std::string("--url=https://jupiter.devtools.epicgames.com")); + CHECK_EQ(v3Stripped[5], std::string("--build-part-name=win64")); +} + +#endif +} // namespace zen diff --git a/src/zenutil/config/environmentoptions.cpp b/src/zenutil/config/environmentoptions.cpp new file mode 100644 index 000000000..fb7f71706 --- /dev/null +++ b/src/zenutil/config/environmentoptions.cpp @@ -0,0 +1,84 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include + +namespace zen { + +EnvironmentOptions::StringOption::StringOption(std::string& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::StringOption::Parse(std::string_view Value) +{ + RefValue = std::string(Value); +} + +EnvironmentOptions::FilePathOption::FilePathOption(std::filesystem::path& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::FilePathOption::Parse(std::string_view Value) +{ + RefValue = MakeSafeAbsolutePath(Value); +} + +EnvironmentOptions::BoolOption::BoolOption(bool& Value) : RefValue(Value) +{ +} +void +EnvironmentOptions::BoolOption::Parse(std::string_view Value) +{ + const std::string Lower = ToLower(Value); + if (Lower == "true" || Lower == "y" || Lower == "yes") + { + RefValue = true; + } + else if (Lower == "false" || Lower == "n" || Lower == "no") + { + RefValue = false; + } +} + +std::shared_ptr +EnvironmentOptions::MakeOption(std::string& Value) +{ + return std::make_shared(Value); +} + +std::shared_ptr +EnvironmentOptions::MakeOption(std::filesystem::path& Value) +{ + return std::make_shared(Value); +} + +std::shared_ptr +EnvironmentOptions::MakeOption(bool& Value) +{ + return std::make_shared(Value); +} + +EnvironmentOptions::EnvironmentOptions() +{ +} + +void +EnvironmentOptions::Parse(const cxxopts::ParseResult& CmdLineResult) +{ + for (auto& It : OptionMap) + { + std::string_view EnvName = It.first; + const Option& Opt = It.second; + if (CmdLineResult.count(Opt.CommandLineOptionName) == 0) + { + std::string EnvValue = GetEnvVariable(EnvName); + if (!EnvValue.empty()) + { + Opt.Value->Parse(EnvValue); + } + } + } +} + +} // namespace zen diff --git a/src/zenutil/config/loggingconfig.cpp b/src/zenutil/config/loggingconfig.cpp new file mode 100644 index 000000000..9ec816b1b --- /dev/null +++ b/src/zenutil/config/loggingconfig.cpp @@ -0,0 +1,77 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenutil/config/loggingconfig.h" + +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +void +ZenLoggingCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenLoggingConfig& LoggingConfig) +{ + // clang-format off + options.add_options("logging") + ("abslog", "Path to log file", cxxopts::value(m_AbsLogFile)) + ("log-id", "Specify id for adding context to log output", cxxopts::value(LoggingConfig.LogId)) + ("quiet", "Configure console logger output to level WARN", cxxopts::value(LoggingConfig.QuietConsole)->default_value("false")) + ("noconsole", "Disable console logging", cxxopts::value(LoggingConfig.NoConsoleOutput)->default_value("false")) + ("log-trace", "Change selected loggers to level TRACE", cxxopts::value(LoggingConfig.Loggers[logging::level::Trace])) + ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value(LoggingConfig.Loggers[logging::level::Debug])) + ("log-info", "Change selected loggers to level INFO", cxxopts::value(LoggingConfig.Loggers[logging::level::Info])) + ("log-warn", "Change selected loggers to level WARN", cxxopts::value(LoggingConfig.Loggers[logging::level::Warn])) + ("log-error", "Change selected loggers to level ERROR", cxxopts::value(LoggingConfig.Loggers[logging::level::Err])) + ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value(LoggingConfig.Loggers[logging::level::Critical])) + ("log-off", "Change selected loggers to level OFF", cxxopts::value(LoggingConfig.Loggers[logging::level::Off])) + ("otlp-endpoint", "OpenTelemetry endpoint URI (e.g http://localhost:4318)", cxxopts::value(LoggingConfig.OtelEndpointUri)) + ; + // clang-format on +} + +void +ZenLoggingCmdLineOptions::ApplyOptions(ZenLoggingConfig& LoggingConfig) +{ + LoggingConfig.AbsLogFile = MakeSafeAbsolutePath(m_AbsLogFile); +} + +void +ApplyLoggingOptions(cxxopts::Options& options, ZenLoggingConfig& LoggingConfig) +{ + ZEN_UNUSED(options); + + if (LoggingConfig.QuietConsole) + { + bool HasExplicitConsoleLevel = false; + for (int i = 0; i < logging::level::LogLevelCount; ++i) + { + if (LoggingConfig.Loggers[i].find("console") != std::string::npos) + { + HasExplicitConsoleLevel = true; + break; + } + } + + if (!HasExplicitConsoleLevel) + { + std::string& WarnLoggers = LoggingConfig.Loggers[logging::level::Warn]; + if (!WarnLoggers.empty()) + { + WarnLoggers += ","; + } + WarnLoggers += "console"; + } + } + + for (int i = 0; i < logging::level::LogLevelCount; ++i) + { + logging::ConfigureLogLevels(logging::level::LogLevel(i), LoggingConfig.Loggers[i]); + } + logging::RefreshLogLevels(); +} + +} // namespace zen diff --git a/src/zenutil/environmentoptions.cpp b/src/zenutil/environmentoptions.cpp deleted file mode 100644 index ee40086c1..000000000 --- a/src/zenutil/environmentoptions.cpp +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include - -#include - -namespace zen { - -EnvironmentOptions::StringOption::StringOption(std::string& Value) : RefValue(Value) -{ -} -void -EnvironmentOptions::StringOption::Parse(std::string_view Value) -{ - RefValue = std::string(Value); -} - -EnvironmentOptions::FilePathOption::FilePathOption(std::filesystem::path& Value) : RefValue(Value) -{ -} -void -EnvironmentOptions::FilePathOption::Parse(std::string_view Value) -{ - RefValue = MakeSafeAbsolutePath(Value); -} - -EnvironmentOptions::BoolOption::BoolOption(bool& Value) : RefValue(Value) -{ -} -void -EnvironmentOptions::BoolOption::Parse(std::string_view Value) -{ - const std::string Lower = ToLower(Value); - if (Lower == "true" || Lower == "y" || Lower == "yes") - { - RefValue = true; - } - else if (Lower == "false" || Lower == "n" || Lower == "no") - { - RefValue = false; - } -} - -std::shared_ptr -EnvironmentOptions::MakeOption(std::string& Value) -{ - return std::make_shared(Value); -} - -std::shared_ptr -EnvironmentOptions::MakeOption(std::filesystem::path& Value) -{ - return std::make_shared(Value); -} - -std::shared_ptr -EnvironmentOptions::MakeOption(bool& Value) -{ - return std::make_shared(Value); -} - -EnvironmentOptions::EnvironmentOptions() -{ -} - -void -EnvironmentOptions::Parse(const cxxopts::ParseResult& CmdLineResult) -{ - for (auto& It : OptionMap) - { - std::string_view EnvName = It.first; - const Option& Opt = It.second; - if (CmdLineResult.count(Opt.CommandLineOptionName) == 0) - { - std::string EnvValue = GetEnvVariable(EnvName); - if (!EnvValue.empty()) - { - Opt.Value->Parse(EnvValue); - } - } - } -} - -} // namespace zen diff --git a/src/zenutil/include/zenutil/commandlineoptions.h b/src/zenutil/include/zenutil/commandlineoptions.h deleted file mode 100644 index 01cceedb1..000000000 --- a/src/zenutil/include/zenutil/commandlineoptions.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include - -ZEN_THIRD_PARTY_INCLUDES_START - -namespace cxxopts::values { -// We declare this specialization before including cxxopts to make it stick -void parse_value(const std::string& text, std::filesystem::path& value); -} // namespace cxxopts::values - -#include -ZEN_THIRD_PARTY_INCLUDES_END - -namespace zen { - -std::vector ParseCommandLine(std::string_view CommandLine); -std::vector StripCommandlineQuotes(std::vector& InOutArgs); -std::filesystem::path StringToPath(const std::string_view& Path); -std::string_view RemoveQuotes(const std::string_view& Arg); - -class CommandLineConverter -{ -public: - CommandLineConverter(int& argc, char**& argv); - - int ArgC = 0; - char** ArgV = nullptr; - -private: - std::vector Args; - std::vector RawArgs; -}; - -void commandlineoptions_forcelink(); // internal - -} // namespace zen diff --git a/src/zenutil/include/zenutil/config/commandlineoptions.h b/src/zenutil/include/zenutil/config/commandlineoptions.h new file mode 100644 index 000000000..01cceedb1 --- /dev/null +++ b/src/zenutil/include/zenutil/config/commandlineoptions.h @@ -0,0 +1,40 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START + +namespace cxxopts::values { +// We declare this specialization before including cxxopts to make it stick +void parse_value(const std::string& text, std::filesystem::path& value); +} // namespace cxxopts::values + +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +std::vector ParseCommandLine(std::string_view CommandLine); +std::vector StripCommandlineQuotes(std::vector& InOutArgs); +std::filesystem::path StringToPath(const std::string_view& Path); +std::string_view RemoveQuotes(const std::string_view& Arg); + +class CommandLineConverter +{ +public: + CommandLineConverter(int& argc, char**& argv); + + int ArgC = 0; + char** ArgV = nullptr; + +private: + std::vector Args; + std::vector RawArgs; +}; + +void commandlineoptions_forcelink(); // internal + +} // namespace zen diff --git a/src/zenutil/include/zenutil/config/environmentoptions.h b/src/zenutil/include/zenutil/config/environmentoptions.h new file mode 100644 index 000000000..1ecdf591a --- /dev/null +++ b/src/zenutil/include/zenutil/config/environmentoptions.h @@ -0,0 +1,92 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +class EnvironmentOptions +{ +public: + class OptionValue + { + public: + virtual void Parse(std::string_view Value) = 0; + + virtual ~OptionValue() {} + }; + + class StringOption : public OptionValue + { + public: + explicit StringOption(std::string& Value); + virtual void Parse(std::string_view Value) override; + std::string& RefValue; + }; + + class FilePathOption : public OptionValue + { + public: + explicit FilePathOption(std::filesystem::path& Value); + virtual void Parse(std::string_view Value) override; + std::filesystem::path& RefValue; + }; + + class BoolOption : public OptionValue + { + public: + explicit BoolOption(bool& Value); + virtual void Parse(std::string_view Value); + bool& RefValue; + }; + + template + class NumberOption : public OptionValue + { + public: + explicit NumberOption(T& Value) : RefValue(Value) {} + virtual void Parse(std::string_view Value) override + { + if (std::optional OptionalValue = ParseInt(Value); OptionalValue.has_value()) + { + RefValue = OptionalValue.value(); + } + } + T& RefValue; + }; + + struct Option + { + std::string CommandLineOptionName; + std::shared_ptr Value; + }; + + std::shared_ptr MakeOption(std::string& Value); + std::shared_ptr MakeOption(std::filesystem::path& Value); + + template + std::shared_ptr MakeOption(T& Value) + { + return std::make_shared>(Value); + }; + + std::shared_ptr MakeOption(bool& Value); + + template + void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "") + { + OptionMap.insert_or_assign(std::string(EnvName), + Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); + }; + + EnvironmentOptions(); + + void Parse(const cxxopts::ParseResult& CmdLineResult); + +private: + std::unordered_map OptionMap; +}; + +} // namespace zen diff --git a/src/zenutil/include/zenutil/config/loggingconfig.h b/src/zenutil/include/zenutil/config/loggingconfig.h new file mode 100644 index 000000000..6d6f64b30 --- /dev/null +++ b/src/zenutil/include/zenutil/config/loggingconfig.h @@ -0,0 +1,37 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +namespace cxxopts { +class Options; +} + +namespace zen { + +struct ZenLoggingConfig +{ + bool NoConsoleOutput = false; // Control default use of stdout for diagnostics + bool QuietConsole = false; // Configure console logger output to level WARN + std::filesystem::path AbsLogFile; // Absolute path to main log file + std::string Loggers[logging::level::LogLevelCount]; + std::string LogId; // Id for tagging log output + std::string OtelEndpointUri; // OpenTelemetry endpoint URI +}; + +void ApplyLoggingOptions(cxxopts::Options& options, ZenLoggingConfig& LoggingConfig); + +class ZenLoggingCmdLineOptions +{ +public: + void AddCliOptions(cxxopts::Options& options, ZenLoggingConfig& LoggingConfig); + void ApplyOptions(ZenLoggingConfig& LoggingConfig); + +private: + std::string m_AbsLogFile; +}; + +} // namespace zen diff --git a/src/zenutil/include/zenutil/environmentoptions.h b/src/zenutil/include/zenutil/environmentoptions.h deleted file mode 100644 index 7418608e4..000000000 --- a/src/zenutil/include/zenutil/environmentoptions.h +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include - -namespace zen { - -class EnvironmentOptions -{ -public: - class OptionValue - { - public: - virtual void Parse(std::string_view Value) = 0; - - virtual ~OptionValue() {} - }; - - class StringOption : public OptionValue - { - public: - explicit StringOption(std::string& Value); - virtual void Parse(std::string_view Value) override; - std::string& RefValue; - }; - - class FilePathOption : public OptionValue - { - public: - explicit FilePathOption(std::filesystem::path& Value); - virtual void Parse(std::string_view Value) override; - std::filesystem::path& RefValue; - }; - - class BoolOption : public OptionValue - { - public: - explicit BoolOption(bool& Value); - virtual void Parse(std::string_view Value); - bool& RefValue; - }; - - template - class NumberOption : public OptionValue - { - public: - explicit NumberOption(T& Value) : RefValue(Value) {} - virtual void Parse(std::string_view Value) override - { - if (std::optional OptionalValue = ParseInt(Value); OptionalValue.has_value()) - { - RefValue = OptionalValue.value(); - } - } - T& RefValue; - }; - - struct Option - { - std::string CommandLineOptionName; - std::shared_ptr Value; - }; - - std::shared_ptr MakeOption(std::string& Value); - std::shared_ptr MakeOption(std::filesystem::path& Value); - - template - std::shared_ptr MakeOption(T& Value) - { - return std::make_shared>(Value); - }; - - std::shared_ptr MakeOption(bool& Value); - - template - void AddOption(std::string_view EnvName, T& Value, std::string_view CommandLineOptionName = "") - { - OptionMap.insert_or_assign(std::string(EnvName), - Option{.CommandLineOptionName = std::string(CommandLineOptionName), .Value = MakeOption(Value)}); - }; - - EnvironmentOptions(); - - void Parse(const cxxopts::ParseResult& CmdLineResult); - -private: - std::unordered_map OptionMap; -}; - -} // namespace zen diff --git a/src/zenutil/zenutil.cpp b/src/zenutil/zenutil.cpp index 51c1ee72e..291dbeadd 100644 --- a/src/zenutil/zenutil.cpp +++ b/src/zenutil/zenutil.cpp @@ -5,7 +5,7 @@ #if ZEN_WITH_TESTS # include -# include +# include # include namespace zen { -- cgit v1.2.3 From 58e1e1ef2deedc49b3e88db57c110b88a39e21da Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 13 Feb 2026 14:47:24 +0100 Subject: spelling fixes (#755) --- src/zen/cmds/builds_cmd.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index f4edb65ab..59b209384 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -3016,7 +3016,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BlobHash.length() != IoHash::StringLength) { throw OptionParseException( - fmt::format("'--blob-hash' ('{}') is malfomed, it must be {} characters long", m_BlobHash, IoHash::StringLength), + fmt::format("'--blob-hash' ('{}') is malformed, it must be {} characters long", m_BlobHash, IoHash::StringLength), SubOption->help()); } @@ -3033,7 +3033,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (m_BuildId.length() != Oid::StringLength) { throw OptionParseException( - fmt::format("'--build-id' ('{}') is malfomed, it must be {} characters long", m_BuildId, Oid::StringLength), + fmt::format("'--build-id' ('{}') is malformed, it must be {} characters long", m_BuildId, Oid::StringLength), SubOption->help()); } else if (Oid BuildId = Oid::FromHexString(m_BuildId); BuildId == Oid::Zero) -- cgit v1.2.3 From df97b6b2abcc8ce13b1d63e3d2cf27c3bd841768 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 13 Feb 2026 15:19:51 +0100 Subject: add foundation for http password protection (#756) --- .../include/zenhttp/security/passwordsecurity.h | 52 +++++ src/zenhttp/security/passwordsecurity.cpp | 221 +++++++++++++++++++++ src/zenhttp/zenhttp.cpp | 2 + 3 files changed, 275 insertions(+) create mode 100644 src/zenhttp/include/zenhttp/security/passwordsecurity.h create mode 100644 src/zenhttp/security/passwordsecurity.cpp diff --git a/src/zenhttp/include/zenhttp/security/passwordsecurity.h b/src/zenhttp/include/zenhttp/security/passwordsecurity.h new file mode 100644 index 000000000..026c2865b --- /dev/null +++ b/src/zenhttp/include/zenhttp/security/passwordsecurity.h @@ -0,0 +1,52 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +struct PasswordSecurityConfiguration +{ + std::string Password; // "password" + bool ProtectMachineLocalRequests = false; // "protect-machine-local-requests" + std::vector UnprotectedUris; // "unprotected-urls" +}; + +class PasswordSecurity +{ +public: + PasswordSecurity(const PasswordSecurityConfiguration& Config); + + [[nodiscard]] inline std::string_view Password() const { return m_Config.Password; } + [[nodiscard]] inline bool ProtectMachineLocalRequests() const { return m_Config.ProtectMachineLocalRequests; } + [[nodiscard]] bool IsUnprotectedUri(std::string_view Uri) const; + + bool IsAllowed(std::string_view Password, std::string_view Uri, bool IsMachineLocalRequest); + +private: + const PasswordSecurityConfiguration m_Config; + tsl::robin_map m_UnprotectedUrlHashes; +}; + +/** + * Expected format (Json) + * { + * "password\": \"1234\", + * "protect-machine-local-requests\": false, + * "unprotected-urls\": [ + * "/health\", + * "/health/info\", + * "/health/version\" + * ] + * } + */ +PasswordSecurityConfiguration ReadPasswordSecurityConfiguration(CbObjectView ConfigObject); + +void passwordsecurity_forcelink(); // internal + +} // namespace zen diff --git a/src/zenhttp/security/passwordsecurity.cpp b/src/zenhttp/security/passwordsecurity.cpp new file mode 100644 index 000000000..37be9a018 --- /dev/null +++ b/src/zenhttp/security/passwordsecurity.cpp @@ -0,0 +1,221 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenhttp/security/passwordsecurity.h" +#include +#include +#include + +#if ZEN_WITH_TESTS +# include +# include +#endif // ZEN_WITH_TESTS + +namespace zen { +using namespace std::literals; + +PasswordSecurity::PasswordSecurity(const PasswordSecurityConfiguration& Config) : m_Config(Config) +{ + m_UnprotectedUrlHashes.reserve(m_Config.UnprotectedUris.size()); + for (uint32_t Index = 0; Index < m_Config.UnprotectedUris.size(); Index++) + { + const std::string& UnprotectedUri = m_Config.UnprotectedUris[Index]; + if (auto Result = m_UnprotectedUrlHashes.insert({HashStringDjb2(UnprotectedUri), Index}); !Result.second) + { + throw std::runtime_error(fmt::format( + "password security unprotected uris does not generate unique hashes. Uri #{} ('{}') collides with uri #{} ('{}')", + Index + 1, + UnprotectedUri, + Result.first->second + 1, + m_Config.UnprotectedUris[Result.first->second])); + } + } +} + +bool +PasswordSecurity::IsUnprotectedUri(std::string_view Uri) const +{ + if (!m_Config.UnprotectedUris.empty()) + { + uint32_t UriHash = HashStringDjb2(Uri); + if (auto It = m_UnprotectedUrlHashes.find(UriHash); It != m_UnprotectedUrlHashes.end()) + { + if (m_Config.UnprotectedUris[It->second] == Uri) + { + return true; + } + } + } + return false; +} + +PasswordSecurityConfiguration +ReadPasswordSecurityConfiguration(CbObjectView ConfigObject) +{ + return PasswordSecurityConfiguration{ + .Password = std::string(ConfigObject["password"sv].AsString()), + .ProtectMachineLocalRequests = ConfigObject["protect-machine-local-requests"sv].AsBool(), + .UnprotectedUris = compactbinary_helpers::ReadArray("unprotected-urls"sv, ConfigObject)}; +} + +bool +PasswordSecurity::IsAllowed(std::string_view InPassword, std::string_view Uri, bool IsMachineLocalRequest) +{ + if (IsUnprotectedUri(Uri)) + { + return true; + } + if (!ProtectMachineLocalRequests() && IsMachineLocalRequest) + { + return true; + } + if (Password().empty()) + { + return true; + } + if (Password() == InPassword) + { + return true; + } + return false; +} + +#if ZEN_WITH_TESTS + +TEST_CASE("passwordsecurity.readconfig") +{ + auto ReadConfigJson = [](std::string_view Json) { + std::string JsonError; + CbObject Config = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + REQUIRE(JsonError.empty()); + return Config; + }; + + { + PasswordSecurityConfiguration EmptyConfig = ReadPasswordSecurityConfiguration(CbObject()); + CHECK(EmptyConfig.Password.empty()); + CHECK(!EmptyConfig.ProtectMachineLocalRequests); + CHECK(EmptyConfig.UnprotectedUris.empty()); + } + + { + const std::string_view SimpleConfigJson = + "{\n" + " \"password\": \"1234\"\n" + "}"; + PasswordSecurityConfiguration SimpleConfig = ReadPasswordSecurityConfiguration(ReadConfigJson(SimpleConfigJson)); + CHECK(SimpleConfig.Password == "1234"); + CHECK(!SimpleConfig.ProtectMachineLocalRequests); + CHECK(SimpleConfig.UnprotectedUris.empty()); + } + + { + const std::string_view ComplexConfigJson = + "{\n" + " \"password\": \"1234\",\n" + " \"protect-machine-local-requests\": true,\n" + " \"unprotected-urls\": [\n" + " \"/health\",\n" + " \"/health/info\",\n" + " \"/health/version\"\n" + " ]\n" + "}"; + PasswordSecurityConfiguration ComplexConfig = ReadPasswordSecurityConfiguration(ReadConfigJson(ComplexConfigJson)); + CHECK(ComplexConfig.Password == "1234"); + CHECK(ComplexConfig.ProtectMachineLocalRequests); + CHECK(ComplexConfig.UnprotectedUris == std::vector({"/health", "/health/info", "/health/version"})); + } +} + +TEST_CASE("passwordsecurity.allowanything") +{ + PasswordSecurity Anything({}); + CHECK(Anything.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(Anything.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(Anything.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(Anything.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); +} + +TEST_CASE("passwordsecurity.allowalllocal") +{ + PasswordSecurity AllLocal({.Password = "123456"}); + CHECK(AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); +} + +TEST_CASE("passwordsecurity.allowonlypassword") +{ + PasswordSecurity AllLocal({.Password = "123456", .ProtectMachineLocalRequests = true}); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); +} + +TEST_CASE("passwordsecurity.allowsomeexternaluris") +{ + PasswordSecurity AllLocal( + {.Password = "123456", .ProtectMachineLocalRequests = false, .UnprotectedUris = std::vector({"/free/access", "/ok"})}); + CHECK(AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed(""sv, "/free/access", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed(""sv, "/ok", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free/access", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed(""sv, "/free/access", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed(""sv, "/ok", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free/access", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); +} + +TEST_CASE("passwordsecurity.allowsomelocaluris") +{ + PasswordSecurity AllLocal( + {.Password = "123456", .ProtectMachineLocalRequests = true, .UnprotectedUris = std::vector({"/free/access", "/ok"})}); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed(""sv, "/free/access", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed(""sv, "/ok", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free/access", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed(""sv, "/free/access", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed(""sv, "/ok", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free/access", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); +} + +TEST_CASE("passwordsecurity.conflictingunprotecteduris") +{ + try + { + PasswordSecurity AllLocal({.Password = "123456", + .ProtectMachineLocalRequests = true, + .UnprotectedUris = std::vector({"/free/access", "/free/access"})}); + CHECK(false); + } + catch (const std::runtime_error& Ex) + { + CHECK_EQ(Ex.what(), + std::string("password security unprotected uris does not generate unique hashes. Uri #2 ('/free/access') collides with " + "uri #1 ('/free/access')")); + } +} +void +passwordsecurity_forcelink() +{ +} +#endif // ZEN_WITH_TESTS + +} // namespace zen diff --git a/src/zenhttp/zenhttp.cpp b/src/zenhttp/zenhttp.cpp index a2679f92e..0b5408453 100644 --- a/src/zenhttp/zenhttp.cpp +++ b/src/zenhttp/zenhttp.cpp @@ -7,6 +7,7 @@ # include # include # include +# include namespace zen { @@ -16,6 +17,7 @@ zenhttp_forcelinktests() http_forcelink(); httpclient_forcelink(); forcelink_packageformat(); + passwordsecurity_forcelink(); } } // namespace zen -- cgit v1.2.3 From 0697a2facd63908b45495fa0a1e94c982e34f052 Mon Sep 17 00:00:00 2001 From: zousar Date: Sat, 14 Feb 2026 23:51:54 -0700 Subject: Enhance dependencies to include soft and hard deps --- src/zenserver/frontend/html/pages/entry.js | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 08589b090..212686e42 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -155,7 +155,7 @@ export class Page extends ZenPage if (Object.keys(tree).length != 0) { - const sub_section = section.add_section("deps"); + const sub_section = section.add_section("dependencies"); this._build_deps(sub_section, tree); } } @@ -271,16 +271,18 @@ export class Page extends ZenPage for (const field of pkgst_entry) { const field_name = field.get_name(); - if (!field_name.endsWith("importedpackageids")) - continue; - - var dep_name = field_name.slice(0, -18); - if (dep_name.length == 0) - dep_name = "imported"; - - var out = tree[dep_name] = []; - for (var item of field.as_array()) - out.push(item.as_value(BigInt)); + if (field_name == "importedpackageids") + { + var out = tree["hard"] = []; + for (var item of field.as_array()) + out.push(item.as_value(BigInt)); + } + else if (field_name == "softpackagereferences") + { + var out = tree["soft"] = []; + for (var item of field.as_array()) + out.push(item.as_value(BigInt)); + } } return tree; -- cgit v1.2.3 From c40e2c7625cf6aab25862c1c18caeb8577884656 Mon Sep 17 00:00:00 2001 From: zousar Date: Sun, 15 Feb 2026 11:55:17 -0700 Subject: Restore handling for hard/soft name prefixes --- src/zenserver/frontend/html/pages/entry.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 212686e42..76afd3e1f 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -271,15 +271,27 @@ export class Page extends ZenPage for (const field of pkgst_entry) { const field_name = field.get_name(); - if (field_name == "importedpackageids") + if (field_name.endsWith("importedpackageids")) { - var out = tree["hard"] = []; + var dep_name = field_name.slice(0, -18); + if (dep_name.length == 0) + dep_name = "hard"; + else + dep_name = "hard." + dep_name; + + var out = tree[dep_name] = []; for (var item of field.as_array()) out.push(item.as_value(BigInt)); } - else if (field_name == "softpackagereferences") + else if (field_name.endsWith("softpackagereferences")) { - var out = tree["soft"] = []; + var dep_name = field_name.slice(0, -21); + if (dep_name.length == 0) + dep_name = "soft"; + else + dep_name = "soft." + dep_name; + + var out = tree[dep_name] = []; for (var item of field.as_array()) out.push(item.as_value(BigInt)); } -- cgit v1.2.3 From 81a6d5e29453db761d058b6418044c8cf04a167e Mon Sep 17 00:00:00 2001 From: zousar Date: Sun, 15 Feb 2026 23:44:17 -0700 Subject: Add support for listing files on oplog entries --- src/zenserver/frontend/html/pages/entry.js | 119 ++++++++++++++++++++++++++--- 1 file changed, 110 insertions(+), 9 deletions(-) diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 76afd3e1f..26ea78142 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -76,6 +76,21 @@ export class Page extends ZenPage return null; } + _is_null_io_hash_string(io_hash) + { + if (!io_hash) + return true; + + for (let char of io_hash) + { + if (char != '0') + { + return false; + } + } + return true; + } + async _build_meta(section, entry) { var tree = {} @@ -142,30 +157,34 @@ export class Page extends ZenPage const name = entry.find("key").as_value(); var section = this.add_section(name); + var has_package_data = false; // tree { var tree = entry.find("$tree"); if (tree == undefined) tree = this._convert_legacy_to_tree(entry); - if (tree == undefined) - return this._display_unsupported(section, entry); - - delete tree["$id"]; - - if (Object.keys(tree).length != 0) + if (tree != undefined) { - const sub_section = section.add_section("dependencies"); - this._build_deps(sub_section, tree); + delete tree["$id"]; + + if (Object.keys(tree).length != 0) + { + const sub_section = section.add_section("dependencies"); + this._build_deps(sub_section, tree); + } + has_package_data = true; } } // meta + if (has_package_data) { this._build_meta(section, entry); } // data + if (has_package_data) { const sub_section = section.add_section("data"); const table = sub_section.add_widget( @@ -181,7 +200,7 @@ export class Page extends ZenPage for (const item of pkg_data.as_array()) { - var io_hash, size, raw_size, file_name; + var io_hash = undefined, size = undefined, raw_size = undefined, file_name = undefined; for (const field of item.as_object()) { if (field.is_named("data")) io_hash = field.as_value(); @@ -219,12 +238,94 @@ export class Page extends ZenPage } } + // files + var has_file_data = false; + { + const sub_section = section.add_section("files"); + const table = sub_section.add_widget( + Table, + ["name", "actions"], Table.Flag_PackRight + ); + table.id("filetable"); + for (const field_name of ["files"]) + { + var file_data = entry.find(field_name); + if (file_data == undefined) + continue; + + has_file_data = true; + + for (const item of file_data.as_array()) + { + var io_hash = undefined, cid = undefined, server_path = undefined, client_path = undefined; + for (const field of item.as_object()) + { + if (field.is_named("data")) io_hash = field.as_value(); + else if (field.is_named("id")) cid = field.as_value(); + else if (field.is_named("serverpath")) server_path = field.as_value(); + else if (field.is_named("clientpath")) client_path = field.as_value(); + } + + if (io_hash instanceof Uint8Array) + { + var ret = ""; + for (var x of io_hash) + ret += x.toString(16).padStart(2, "0"); + io_hash = ret; + } + + if (cid instanceof Uint8Array) + { + var ret = ""; + for (var x of cid) + ret += x.toString(16).padStart(2, "0"); + cid = ret; + } + + const row = table.add_row(server_path); + + var base_name = server_path.split("/").pop().split("\\").pop(); + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + if (this._is_null_io_hash_string(io_hash)) + { + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, cid].join("/") + ); + link.first_child().attr("download", `${cid}_${base_name}`); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-id").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, cid); + } + else + { + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, io_hash].join("/") + ); + link.first_child().attr("download", `${io_hash}_${base_name}`); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, io_hash); + } + + } + } + } + // props + if (has_package_data) { const object = entry.to_js_object(); var sub_section = section.add_section("props"); sub_section.add_widget(PropTable).add_object(object); } + + if (!has_package_data && !has_file_data) + return this._display_unsupported(section, entry); } _display_unsupported(section, entry) -- cgit v1.2.3 From df806dcb92f0b5c9622586460fc86e698ca03ab6 Mon Sep 17 00:00:00 2001 From: zousar Date: Sun, 15 Feb 2026 23:44:54 -0700 Subject: Change breadcrumbs for oplogs to be more descriptive --- src/zenserver/frontend/html/pages/oplog.js | 2 +- src/zenserver/frontend/html/pages/page.js | 16 +++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/zenserver/frontend/html/pages/oplog.js b/src/zenserver/frontend/html/pages/oplog.js index 879fc4c97..a286f8651 100644 --- a/src/zenserver/frontend/html/pages/oplog.js +++ b/src/zenserver/frontend/html/pages/oplog.js @@ -32,7 +32,7 @@ export class Page extends ZenPage this.set_title("oplog - " + oplog); - var section = this.add_section(project + " - " + oplog); + var section = this.add_section(oplog); oplog_info = await oplog_info; this._index_max = oplog_info["opcount"]; diff --git a/src/zenserver/frontend/html/pages/page.js b/src/zenserver/frontend/html/pages/page.js index 9a9541904..2f9643008 100644 --- a/src/zenserver/frontend/html/pages/page.js +++ b/src/zenserver/frontend/html/pages/page.js @@ -97,7 +97,7 @@ export class ZenPage extends PageBase generate_crumbs() { - const auto_name = this.get_param("page") || "start"; + var auto_name = this.get_param("page") || "start"; if (auto_name == "start") return; @@ -114,15 +114,21 @@ export class ZenPage extends PageBase var project = this.get_param("project"); if (project != undefined) { + auto_name = project; var oplog = this.get_param("oplog"); if (oplog != undefined) { - new_crumb("project", `?page=project&project=${project}`); - if (this.get_param("opkey")) - new_crumb("oplog", `?page=oplog&project=${project}&oplog=${oplog}`); + auto_name = oplog; + new_crumb(project, `?page=project&project=${project}`); + var opkey = this.get_param("opkey") + if (opkey != undefined) + { + auto_name = opkey.split("/").pop().split("\\").pop();; + new_crumb(oplog, `?page=oplog&project=${project}&oplog=${oplog}`); + } } } - new_crumb(auto_name.toLowerCase()); + new_crumb(auto_name); } } -- cgit v1.2.3 From 74a5e2fb8dec43682e81a98c9677aef849ca7cc1 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 16 Feb 2026 15:37:13 +0100 Subject: added ResetConsoleLog (#758) also made sure log initialization calls it to ensure the console output format is retained even if the console logger was set up before logging is initialized --- src/zencore/include/zencore/logging.h | 1 + src/zencore/logging.cpp | 8 ++++++++ src/zenutil/logging.cpp | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h index afbbbd3ee..74a44d028 100644 --- a/src/zencore/include/zencore/logging.h +++ b/src/zencore/include/zencore/logging.h @@ -31,6 +31,7 @@ void FlushLogging(); LoggerRef Default(); void SetDefault(std::string_view NewDefaultLoggerId); LoggerRef ConsoleLog(); +void ResetConsoleLog(); void SuppressConsoleLog(); LoggerRef ErrorLog(); void SetErrorLog(std::string_view LoggerId); diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index a6697c443..77e05a909 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -404,6 +404,14 @@ ConsoleLog() return *ConLogger; } +void +ResetConsoleLog() +{ + LoggerRef ConLog = ConsoleLog(); + + ConLog.SpdLogger->set_pattern("%v"); +} + void InitializeLogging() { diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp index 806b96d52..54ac30c5d 100644 --- a/src/zenutil/logging.cpp +++ b/src/zenutil/logging.cpp @@ -233,6 +233,11 @@ FinishInitializeLogging(const LoggingOptions& LogOptions) LogOptions.LogId, std::chrono::system_clock::now() - std::chrono::milliseconds(GetTimeSinceProcessStart()))); // default to duration prefix + // If the console logger was initialized before, the above will change the output format + // so we need to reset it + + logging::ResetConsoleLog(); + if (g_FileSink) { if (LogOptions.AbsLogFile.extension() == ".json") -- cgit v1.2.3 From ccfcb14ef1b837ed6f752ae4f27e0ef88a5b18da Mon Sep 17 00:00:00 2001 From: zousar Date: Mon, 16 Feb 2026 16:39:44 -0700 Subject: Added custom page for cook.artifacts --- src/zenserver/frontend/html/pages/cookartifacts.js | 385 +++++++++++++++++++++ src/zenserver/frontend/html/pages/entry.js | 14 +- src/zenserver/frontend/html/pages/page.js | 15 +- src/zenserver/frontend/html/zen.css | 18 + 4 files changed, 428 insertions(+), 4 deletions(-) create mode 100644 src/zenserver/frontend/html/pages/cookartifacts.js diff --git a/src/zenserver/frontend/html/pages/cookartifacts.js b/src/zenserver/frontend/html/pages/cookartifacts.js new file mode 100644 index 000000000..6c36c7f32 --- /dev/null +++ b/src/zenserver/frontend/html/pages/cookartifacts.js @@ -0,0 +1,385 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +"use strict"; + +import { ZenPage } from "./page.js" +import { Fetcher } from "../util/fetcher.js" +import { Table, Toolbar, PropTable } from "../util/widgets.js" + +//////////////////////////////////////////////////////////////////////////////// +export class Page extends ZenPage +{ + main() + { + this.set_title("cook artifacts"); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const opkey = this.get_param("opkey"); + const artifact_hash = this.get_param("hash"); + + // Fetch the artifact content as JSON + this._artifact = new Fetcher() + .resource("prj", project, "oplog", oplog, artifact_hash + ".json") + .json(); + + // Optionally fetch entry info for display context + if (opkey) + { + this._entry = new Fetcher() + .resource("prj", project, "oplog", oplog, "entries") + .param("opkey", opkey) + .cbo(); + } + + this._build_page(); + } + + // Map CookDependency enum values to display names + _get_dependency_type_name(type_value) + { + const type_names = { + 0: "None", + 1: "File", + 2: "Function", + 3: "TransitiveBuild", + 4: "Package", + 5: "ConsoleVariable", + 6: "Config", + 7: "SettingsObject", + 8: "NativeClass", + 9: "AssetRegistryQuery", + 10: "RedirectionTarget" + }; + return type_names[type_value] || `Unknown (${type_value})`; + } + + // Check if Data content should be expandable + _should_make_expandable(data_string) + { + if (!data_string || data_string.length < 40) + return false; + + // Check if it's JSON array or object + if (!data_string.startsWith('[') && !data_string.startsWith('{')) + return false; + + // Check if formatting would add newlines + try { + const parsed = JSON.parse(data_string); + const formatted = JSON.stringify(parsed, null, 2); + return formatted.includes('\n'); + } catch (e) { + return false; + } + } + + // Get first line of content for collapsed state + _get_first_line(data_string) + { + if (!data_string) + return ""; + + const newline_index = data_string.indexOf('\n'); + if (newline_index === -1) + { + // No newline, truncate if too long + return data_string.length > 80 ? data_string.substring(0, 77) + "..." : data_string; + } + return data_string.substring(0, newline_index) + "..."; + } + + // Format JSON with indentation + _format_json(data_string) + { + try { + const parsed = JSON.parse(data_string); + return JSON.stringify(parsed, null, 2); + } catch (e) { + return data_string; + } + } + + // Toggle expand/collapse state + _toggle_data_cell(cell) + { + const is_expanded = cell.attr("expanded") !== null; + const full_data = cell.attr("data-full"); + + // Find the text wrapper span + const text_wrapper = cell.first_child().next_sibling(); + + if (is_expanded) + { + // Collapse: show first line only + const first_line = this._get_first_line(full_data); + text_wrapper.text(first_line); + cell.attr("expanded", null); + } + else + { + // Expand: show formatted JSON + const formatted = this._format_json(full_data); + text_wrapper.text(formatted); + cell.attr("expanded", ""); + } + } + + // Format dependency data based on its structure + _format_dependency(dep_array) + { + const type = dep_array[0]; + const formatted = {}; + + // Common patterns based on the example data: + // Type 2 (Function): [type, name, array, hash] + // Type 4 (Package): [type, path, hash] + // Type 5 (ConsoleVariable): [type, bool, array, hash] + // Type 8 (NativeClass): [type, path, hash] + // Type 9 (AssetRegistryQuery): [type, bool, object, hash] + // Type 10 (RedirectionTarget): [type, path, hash] + + if (dep_array.length > 1) + { + // Most types have a name/path as second element + if (typeof dep_array[1] === "string") + { + formatted.Name = dep_array[1]; + } + else if (typeof dep_array[1] === "boolean") + { + formatted.Value = dep_array[1].toString(); + } + } + + if (dep_array.length > 2) + { + // Third element varies + if (Array.isArray(dep_array[2])) + { + formatted.Data = JSON.stringify(dep_array[2]); + } + else if (typeof dep_array[2] === "object") + { + formatted.Data = JSON.stringify(dep_array[2]); + } + else if (typeof dep_array[2] === "string") + { + formatted.Hash = dep_array[2]; + } + } + + if (dep_array.length > 3) + { + // Fourth element is usually the hash + if (typeof dep_array[3] === "string") + { + formatted.Hash = dep_array[3]; + } + } + + return formatted; + } + + async _build_page() + { + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + const opkey = this.get_param("opkey"); + const artifact_hash = this.get_param("hash"); + + // Build page title + let title = "Cook Artifacts"; + if (this._entry) + { + try + { + const entry = await this._entry; + const entry_obj = entry.as_object().find("entry").as_object(); + const key = entry_obj.find("key").as_value(); + title = `Cook Artifacts`; + } + catch (e) + { + console.error("Failed to fetch entry:", e); + } + } + + const section = this.add_section(title); + + // Fetch and parse artifact + let artifact; + try + { + artifact = await this._artifact; + } + catch (e) + { + section.text(`Failed to load artifact: ${e.message}`); + return; + } + + // Display artifact info + const info_section = section.add_section("Artifact Info"); + const info_table = info_section.add_widget(Table, ["Property", "Value"], Table.Flag_PackRight); + + if (artifact.Version !== undefined) + info_table.add_row("Version", artifact.Version.toString()); + if (artifact.HasSaveResults !== undefined) + info_table.add_row("HasSaveResults", artifact.HasSaveResults.toString()); + if (artifact.PackageSavedHash !== undefined) + info_table.add_row("PackageSavedHash", artifact.PackageSavedHash); + + // Process SaveBuildDependencies + if (artifact.SaveBuildDependencies && artifact.SaveBuildDependencies.Dependencies) + { + this._build_dependency_section( + section, + "Save Build Dependencies", + artifact.SaveBuildDependencies.Dependencies, + artifact.SaveBuildDependencies.StoredKey + ); + } + + // Process LoadBuildDependencies + if (artifact.LoadBuildDependencies && artifact.LoadBuildDependencies.Dependencies) + { + this._build_dependency_section( + section, + "Load Build Dependencies", + artifact.LoadBuildDependencies.Dependencies, + artifact.LoadBuildDependencies.StoredKey + ); + } + + // Process RuntimeDependencies + if (artifact.RuntimeDependencies && artifact.RuntimeDependencies.length > 0) + { + const runtime_section = section.add_section("Runtime Dependencies"); + const runtime_table = runtime_section.add_widget(Table, ["Path"], Table.Flag_PackRight); + for (const dep of artifact.RuntimeDependencies) + { + const row = runtime_table.add_row(dep); + // Make Path clickable to navigate to entry + row.get_cell(0).text(dep).on_click((opkey) => { + window.location = `?page=entry&project=${project}&oplog=${oplog}&opkey=${opkey.toLowerCase()}`; + }, dep); + } + } + } + + _build_dependency_section(parent_section, title, dependencies, stored_key) + { + const section = parent_section.add_section(title); + + // Add stored key info + if (stored_key) + { + const key_toolbar = section.add_widget(Toolbar); + key_toolbar.left().add(`Key: ${stored_key}`); + } + + // Group dependencies by type + const dependencies_by_type = {}; + + for (const dep_array of dependencies) + { + if (!Array.isArray(dep_array) || dep_array.length === 0) + continue; + + const type = dep_array[0]; + if (!dependencies_by_type[type]) + dependencies_by_type[type] = []; + + dependencies_by_type[type].push(this._format_dependency(dep_array)); + } + + // Sort types numerically + const sorted_types = Object.keys(dependencies_by_type).map(Number).sort((a, b) => a - b); + + for (const type_value of sorted_types) + { + const type_name = this._get_dependency_type_name(type_value); + const deps = dependencies_by_type[type_value]; + + const type_section = section.add_section(type_name); + + // Determine columns based on available fields + const all_fields = new Set(); + for (const dep of deps) + { + for (const field in dep) + all_fields.add(field); + } + let columns = Array.from(all_fields); + + // Remove Hash column for RedirectionTarget as it's not useful + if (type_value === 10) + { + columns = columns.filter(col => col !== "Hash"); + } + + if (columns.length === 0) + { + type_section.text("No data fields"); + continue; + } + + // Create table with dynamic columns + const table = type_section.add_widget(Table, columns, Table.Flag_PackRight); + + // Check if this type should have clickable Name links + const should_link = (type_value === 3 || type_value === 4 || type_value === 10); + const name_col_index = columns.indexOf("Name"); + + for (const dep of deps) + { + const row_values = columns.map(col => dep[col] || ""); + const row = table.add_row(...row_values); + + // Make Name field clickable for Package, TransitiveBuild, and RedirectionTarget + if (should_link && name_col_index >= 0 && dep.Name) + { + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + row.get_cell(name_col_index).text(dep.Name).on_click((opkey) => { + window.location = `?page=entry&project=${project}&oplog=${oplog}&opkey=${opkey.toLowerCase()}`; + }, dep.Name); + } + + // Make Data field expandable/collapsible if needed + const data_col_index = columns.indexOf("Data"); + if (data_col_index >= 0 && dep.Data) + { + const data_cell = row.get_cell(data_col_index); + + if (this._should_make_expandable(dep.Data)) + { + // Store full data in attribute + data_cell.attr("data-full", dep.Data); + + // Clear the cell and rebuild with icon + text + data_cell.inner().innerHTML = ""; + + // Create expand/collapse icon + const icon = data_cell.tag("span").classify("zen_expand_icon").text("+"); + icon.on_click(() => { + this._toggle_data_cell(data_cell); + // Update icon text + const is_expanded = data_cell.attr("expanded") !== null; + icon.text(is_expanded ? "-" : "+"); + }); + + // Add text content wrapper + const text_wrapper = data_cell.tag("span").classify("zen_data_text"); + const first_line = this._get_first_line(dep.Data); + text_wrapper.text(first_line); + + // Store reference to text wrapper for updates + data_cell.attr("data-text-wrapper", "true"); + } + } + } + } + } +} diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 26ea78142..dca3a5c25 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -138,11 +138,23 @@ export class Page extends ZenPage const project = this.get_param("project"); const oplog = this.get_param("oplog"); + const opkey = this.get_param("opkey"); const link = row.get_cell(0).link( - "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") + (key === "cook.artifacts") ? + `?page=cookartifacts&project=${project}&oplog=${oplog}&opkey=${opkey}&hash=${value}` + : "/" + ["prj", project, "oplog", oplog, value+".json"].join("/") ); const action_tb = new Toolbar(row.get_cell(-1), true); + + // Add "view-raw" button for cook.artifacts + if (key === "cook.artifacts") + { + action_tb.left().add("view-raw").on_click(() => { + window.location = "/" + ["prj", project, "oplog", oplog, value+".json"].join("/"); + }); + } + action_tb.left().add("copy-hash").on_click(async (v) => { await navigator.clipboard.writeText(v); }, value); diff --git a/src/zenserver/frontend/html/pages/page.js b/src/zenserver/frontend/html/pages/page.js index 2f9643008..3ec0248cb 100644 --- a/src/zenserver/frontend/html/pages/page.js +++ b/src/zenserver/frontend/html/pages/page.js @@ -118,13 +118,22 @@ export class ZenPage extends PageBase var oplog = this.get_param("oplog"); if (oplog != undefined) { + new_crumb(auto_name, `?page=project&project=${project}`); auto_name = oplog; - new_crumb(project, `?page=project&project=${project}`); var opkey = this.get_param("opkey") if (opkey != undefined) { - auto_name = opkey.split("/").pop().split("\\").pop();; - new_crumb(oplog, `?page=oplog&project=${project}&oplog=${oplog}`); + new_crumb(auto_name, `?page=oplog&project=${project}&oplog=${oplog}`); + auto_name = opkey.split("/").pop().split("\\").pop(); + + // Check if we're viewing cook artifacts + var page = this.get_param("page"); + var hash = this.get_param("hash"); + if (hash != undefined && page == "cookartifacts") + { + new_crumb(auto_name, `?page=entry&project=${project}&oplog=${oplog}&opkey=${opkey}`); + auto_name = "cook artifacts"; + } } } } diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css index cc53c0519..34c265610 100644 --- a/src/zenserver/frontend/html/zen.css +++ b/src/zenserver/frontend/html/zen.css @@ -172,6 +172,24 @@ a { } } +/* expandable cell ---------------------------------------------------------- */ + +.zen_expand_icon { + cursor: pointer; + margin-right: 0.5em; + color: var(--theme_g1); + font-weight: bold; + user-select: none; +} + +.zen_expand_icon:hover { + color: var(--theme_ln); +} + +.zen_data_text { + user-select: text; +} + /* toolbar ------------------------------------------------------------------ */ .zen_toolbar { -- cgit v1.2.3 From 2159b2ce105935ce4d52a726094f9bbb91537d0c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 17 Feb 2026 13:56:33 +0100 Subject: misc fixes brought over from sb/proto (#759) * `RwLock::WithSharedLock` and `RwLock::WithExclusiveLock` can now return a value (which is returned by the passed function) * Comma-separated logger specification now correctly deals with commas * `GetSystemMetrics` properly accounts for cores * cpr response formatter passes arguments in the right order * `HttpServerRequest::SetLogRequest` can be used to selectively log HTTP requests --- src/zencore/include/zencore/thread.h | 8 ++++---- src/zencore/logging.cpp | 2 +- src/zencore/system.cpp | 12 ++++++------ src/zenhttp/include/zenhttp/cprutils.h | 4 ++-- src/zenhttp/include/zenhttp/httpserver.h | 17 +++++++++++++++-- src/zenhttp/servers/httpasio.cpp | 7 ++++++- src/zenhttp/servers/httpsys.cpp | 30 +++++++++++++++--------------- 7 files changed, 49 insertions(+), 31 deletions(-) diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index de8f9399c..a1c68b0b2 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -61,10 +61,10 @@ public: RwLock* m_Lock; }; - inline void WithSharedLock(auto&& Fun) + inline auto WithSharedLock(auto&& Fun) { SharedLockScope $(*this); - Fun(); + return Fun(); } struct ExclusiveLockScope @@ -85,10 +85,10 @@ public: RwLock* m_Lock; }; - inline void WithExclusiveLock(auto&& Fun) + inline auto WithExclusiveLock(auto&& Fun) { ExclusiveLockScope $(*this); - Fun(); + return Fun(); } private: diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index 77e05a909..e79c4b41c 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -251,7 +251,7 @@ RefreshLogLevels(level::LogLevel* DefaultLevel) if (auto CommaPos = Spec.find_first_of(','); CommaPos != std::string_view::npos) { - LoggerName = Spec.substr(CommaPos + 1); + LoggerName = Spec.substr(0, CommaPos); Spec.remove_prefix(CommaPos + 1); } else diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index b9ac3bdee..e92691781 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -66,15 +66,15 @@ GetSystemMetrics() // Determine physical core count DWORD BufferSize = 0; - BOOL Result = GetLogicalProcessorInformation(nullptr, &BufferSize); + BOOL Result = GetLogicalProcessorInformationEx(RelationAll, nullptr, &BufferSize); if (int32_t Error = GetLastError(); Error != ERROR_INSUFFICIENT_BUFFER) { ThrowSystemError(Error, "Failed to get buffer size for logical processor information"); } - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)Memory::Alloc(BufferSize); + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)Memory::Alloc(BufferSize); - Result = GetLogicalProcessorInformation(Buffer, &BufferSize); + Result = GetLogicalProcessorInformationEx(RelationAll, Buffer, &BufferSize); if (!Result) { Memory::Free(Buffer); @@ -84,9 +84,9 @@ GetSystemMetrics() DWORD ProcessorPkgCount = 0; DWORD ProcessorCoreCount = 0; DWORD ByteOffset = 0; - while (ByteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= BufferSize) + while (ByteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) <= BufferSize) { - const SYSTEM_LOGICAL_PROCESSOR_INFORMATION& Slpi = Buffer[ByteOffset / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)]; + const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX& Slpi = Buffer[ByteOffset / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)]; if (Slpi.Relationship == RelationProcessorCore) { ProcessorCoreCount++; @@ -95,7 +95,7 @@ GetSystemMetrics() { ProcessorPkgCount++; } - ByteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); + ByteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX); } Metrics.CoreCount = ProcessorCoreCount; diff --git a/src/zenhttp/include/zenhttp/cprutils.h b/src/zenhttp/include/zenhttp/cprutils.h index a988346e0..c252a5d99 100644 --- a/src/zenhttp/include/zenhttp/cprutils.h +++ b/src/zenhttp/include/zenhttp/cprutils.h @@ -66,10 +66,10 @@ struct fmt::formatter Response.url.str(), Response.status_code, zen::ToString(zen::HttpResponseCode(Response.status_code)), + Response.reason, Response.uploaded_bytes, Response.downloaded_bytes, NiceResponseTime.c_str(), - Response.reason, Json); } else @@ -82,10 +82,10 @@ struct fmt::formatter Response.url.str(), Response.status_code, zen::ToString(zen::HttpResponseCode(Response.status_code)), + Response.reason, Response.uploaded_bytes, Response.downloaded_bytes, NiceResponseTime.c_str(), - Response.reason, Body.GetText()); } } diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 60f6bc9f2..cbac06cb6 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -39,7 +39,7 @@ public: // Synchronous operations [[nodiscard]] inline std::string_view RelativeUri() const { return m_Uri; } // Returns URI without service prefix - [[nodiscard]] std::string_view RelativeUriWithExtension() const { return m_UriWithExtension; } + [[nodiscard]] inline std::string_view RelativeUriWithExtension() const { return m_UriWithExtension; } [[nodiscard]] inline std::string_view QueryString() const { return m_QueryString; } [[nodiscard]] inline HttpService& Service() const { return m_Service; } @@ -81,6 +81,18 @@ public: inline bool IsHandled() const { return !!(m_Flags & kIsHandled); } inline bool SuppressBody() const { return !!(m_Flags & kSuppressBody); } inline void SetSuppressResponseBody() { m_Flags |= kSuppressBody; } + inline void SetLogRequest(bool ShouldLog) + { + if (ShouldLog) + { + m_Flags |= kLogRequest; + } + else + { + m_Flags &= ~kLogRequest; + } + } + inline bool ShouldLogRequest() const { return !!(m_Flags & kLogRequest); } /** Read POST/PUT payload for request body, which is always available without delay */ @@ -119,6 +131,7 @@ protected: kSuppressBody = 1 << 1, kHaveRequestId = 1 << 2, kHaveSessionId = 1 << 3, + kLogRequest = 1 << 4, }; mutable uint32_t m_Flags = 0; @@ -254,7 +267,7 @@ public: inline HttpServerRequest& ServerRequest() { return m_HttpRequest; } private: - HttpRouterRequest(HttpServerRequest& Request) : m_HttpRequest(Request) {} + explicit HttpRouterRequest(HttpServerRequest& Request) : m_HttpRequest(Request) {} ~HttpRouterRequest() = default; HttpRouterRequest(const HttpRouterRequest&) = delete; diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 230aac6a8..1f42b05d2 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -147,7 +147,7 @@ inline LoggerRef InitLogger() { LoggerRef Logger = logging::Get("asio"); - // Logger.set_level(spdlog::level::trace); + // Logger.SetLogLevel(logging::level::Trace); return Logger; } @@ -1264,6 +1264,11 @@ HttpServerConnection::HandleRequest() if (std::unique_ptr Response = std::move(Request.m_Response)) { + if (Request.ShouldLogRequest()) + { + ZEN_INFO("{} {} {} -> {}", ToString(RequestVerb), Uri, Response->ResponseCode(), NiceBytes(Response->ContentLength())); + } + // Transmit the response if (m_RequestData.RequestVerb() == HttpVerb::kHead) diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 4df4cd079..5fed94f1c 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -702,21 +702,22 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) HTTP_CACHE_POLICY CachePolicy; - CachePolicy.Policy = HttpCachePolicyNocache; // HttpCachePolicyUserInvalidates; + CachePolicy.Policy = HttpCachePolicyNocache; CachePolicy.SecondsToLive = 0; // Initial response API call - SendResult = HttpSendHttpResponse(Tx.RequestQueueHandle(), - HttpReq->RequestId, - SendFlags, - &HttpResponse, - &CachePolicy, - NULL, - NULL, - 0, - Tx.Overlapped(), - NULL); + SendResult = HttpSendHttpResponse(Tx.RequestQueueHandle(), // RequestQueueHandle + HttpReq->RequestId, // RequestId + SendFlags, // Flags + &HttpResponse, // HttpResponse + &CachePolicy, // CachePolicy + NULL, // BytesSent + NULL, // Reserved1 + 0, // Reserved2 + Tx.Overlapped(), // Overlapped + NULL // LogData + ); m_IsInitialResponse = false; } @@ -724,9 +725,9 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) { // Subsequent response API calls - SendResult = HttpSendResponseEntityBody(Tx.RequestQueueHandle(), - HttpReq->RequestId, - SendFlags, + SendResult = HttpSendResponseEntityBody(Tx.RequestQueueHandle(), // RequestQueueHandle + HttpReq->RequestId, // RequestId + SendFlags, // Flags (USHORT)ThisRequestChunkCount, // EntityChunkCount &m_HttpDataChunks[ThisRequestChunkOffset], // EntityChunks NULL, // BytesSent @@ -1351,7 +1352,6 @@ HttpSysServer::OnRun(bool IsInteractive) bool ShutdownRequested = false; do { - // int WaitTimeout = -1; int WaitTimeout = 100; if (IsInteractive) -- cgit v1.2.3 From 5e1e23e209eec75a396c18f8eee3d93a9e196bfc Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 17 Feb 2026 14:00:53 +0100 Subject: add http server root password protection (#757) - Feature: Added `--security-config-path` option to zenserver to configure security settings - Expects a path to a .json file - Default is an empty path resulting in no extra security settings and legacy behavior - Current support is a top level filter of incoming http requests restricted to the `password` type - `password` type will check the `Authorization` header and match it to the selected authorization strategy - Currently the security settings is very basic and configured to a fixed username+password at startup { "http" { "root": { "filter": { "type": "password", "config": { "password": { "username": "", "password": "" }, "protect-machine-local-requests": false, "unprotected-uris": [ "/health/", "/health/info", "/health/version" ] } } } } } --- CHANGELOG.md | 29 ++++ src/zencore/include/zencore/string.h | 16 ++ src/zenhttp/httpclient.cpp | 91 ++++++++++++ src/zenhttp/httpserver.cpp | 3 +- src/zenhttp/include/zenhttp/httpserver.h | 7 +- .../include/zenhttp/security/passwordsecurity.h | 38 ++--- .../zenhttp/security/passwordsecurityfilter.h | 51 +++++++ src/zenhttp/security/passwordsecurity.cpp | 164 +++++++-------------- src/zenhttp/security/passwordsecurityfilter.cpp | 56 +++++++ src/zenhttp/servers/httpasio.cpp | 14 +- src/zenhttp/servers/httpmulti.cpp | 1 + src/zenhttp/servers/httpnull.cpp | 1 + src/zenhttp/servers/httpparser.cpp | 6 + src/zenhttp/servers/httpparser.h | 3 + src/zenhttp/servers/httpplugin.cpp | 18 ++- src/zenhttp/servers/httpsys.cpp | 21 ++- src/zenserver/config/config.cpp | 20 ++- src/zenserver/config/config.h | 13 +- src/zenserver/zenserver.cpp | 50 ++++++- src/zenserver/zenserver.h | 8 +- 20 files changed, 444 insertions(+), 166 deletions(-) create mode 100644 src/zenhttp/include/zenhttp/security/passwordsecurityfilter.h create mode 100644 src/zenhttp/security/passwordsecurityfilter.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 487d45fef..c2fe710a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,33 @@ ## +- Feature: Added `--security-config-path` option to zenserver to configure security settings + - Expects a path to a .json file + - Default is an empty path resulting in no extra security settings and legacy behavior + - Current support is a top level filter of incoming http requests restricted to the `password` type + - `password` type will check the `Authorization` header and match it to the selected authorization strategy + - Currently the security settings is very basic and configured to a fixed username+password at startup + + { + "http" { + "root": { + "filter": { + "type": "password", + "config": { + "password": { + "username": "", + "password": "" + }, + "protect-machine-local-requests": false, + "unprotected-uris": [ + "/health/", + "/health/info", + "/health/version" + ] + } + } + } + } + } + - Improvement: Reduced time project and project oplogs are locked during GC and Validation - Improvement: `zen` now supports additional configuration of logging options, such as `--log-warn=...` for configuring log levels, etc (see `zen --help`) diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index cbff6454f..5a12ba5d2 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -796,6 +796,22 @@ HashStringDjb2(const std::string_view& InString) return HashValue; } +constexpr uint32_t +HashStringDjb2(const std::span InStrings) +{ + uint32_t HashValue = 5381; + + for (const std::string_view& String : InStrings) + { + for (int CurChar : String) + { + HashValue = HashValue * 33 + CurChar; + } + } + + return HashValue; +} + constexpr uint32_t HashStringAsLowerDjb2(const std::string_view& InString) { diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 16729ce38..d3b59df2b 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -25,6 +25,7 @@ # include # include # include +# include # include "servers/httpasio.h" # include "servers/httpsys.h" @@ -662,6 +663,96 @@ TEST_CASE("httpclient.requestfilter") } } +TEST_CASE("httpclient.password") +{ + using namespace std::literals; + + struct TestHttpService : public HttpService + { + TestHttpService() = default; + + virtual const char* BaseUri() const override { return "/test/"; } + virtual void HandleRequest(HttpServerRequest& HttpServiceRequest) override + { + if (HttpServiceRequest.RelativeUri() == "yo") + { + return HttpServiceRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "hey family"); + } + + { + CHECK(HttpServiceRequest.RelativeUri() != "should_filter"); + return HttpServiceRequest.WriteResponse(HttpResponseCode::InternalServerError); + } + + { + CHECK(HttpServiceRequest.RelativeUri() != "should_forbid"); + return HttpServiceRequest.WriteResponse(HttpResponseCode::InternalServerError); + } + } + }; + + TestHttpService TestService; + ScopedTemporaryDirectory TmpDir; + + Ref AsioServer = CreateHttpAsioServer(AsioConfig{}); + + int Port = AsioServer->Initialize(7575, TmpDir.Path()); + REQUIRE(Port != -1); + + AsioServer->RegisterService(TestService); + + std::thread ServerThread([&]() { AsioServer->Run(false); }); + + { + auto _ = MakeGuard([&]() { + if (ServerThread.joinable()) + { + ServerThread.join(); + } + AsioServer->Close(); + }); + + SUBCASE("usernamepassword") + { + CbObjectWriter Writer; + { + Writer.BeginObject("basic"); + { + Writer << "username"sv + << "me"; + Writer << "password"sv + << "456123789"; + } + Writer.EndObject(); + Writer << "protect-machine-local-requests" << true; + } + + PasswordHttpFilter::Configuration PasswordFilterOptions = PasswordHttpFilter::ReadConfiguration(Writer.Save()); + + PasswordHttpFilter MyFilter(PasswordFilterOptions); + + AsioServer->SetHttpRequestFilter(&MyFilter); + + HttpClient Client(fmt::format("localhost:{}", Port), + HttpClientSettings{}, + /*CheckIfAbortFunction*/ {}); + + ZEN_INFO("Request using {}", Client.GetBaseUri()); + + HttpClient::Response ForbiddenResponse = Client.Get("/test/yo"); + CHECK(!ForbiddenResponse.IsSuccess()); + CHECK_EQ(ForbiddenResponse.StatusCode, HttpResponseCode::Forbidden); + + HttpClient::Response WithBasicResponse = + Client.Get("/test/yo", + std::pair("Authorization", + fmt::format("Basic {}", PasswordFilterOptions.PasswordConfig.Password))); + CHECK(WithBasicResponse.IsSuccess()); + AsioServer->SetHttpRequestFilter(nullptr); + } + AsioServer->RequestExit(); + } +} void httpclient_forcelink() { diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index d8367fcb2..f2fe4738f 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -1317,7 +1317,8 @@ TEST_CASE("http.common") TestHttpServerRequest(HttpService& Service, std::string_view Uri) : HttpServerRequest(Service) { m_Uri = Uri; } virtual IoBuffer ReadPayload() override { return IoBuffer(); } - virtual bool IsLocalMachineRequest() const override { return false; } + virtual bool IsLocalMachineRequest() const override { return false; } + virtual std::string_view GetAuthorizationHeader() const override { return {}; } virtual void WriteResponse(HttpResponseCode ResponseCode, HttpContentType ContentType, std::span Blobs) override { diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index cbac06cb6..350532126 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -101,7 +101,8 @@ public: CbObject ReadPayloadObject(); CbPackage ReadPayloadPackage(); - virtual bool IsLocalMachineRequest() const = 0; + virtual bool IsLocalMachineRequest() const = 0; + virtual std::string_view GetAuthorizationHeader() const = 0; /** Respond with payload @@ -162,8 +163,10 @@ public: virtual void OnRequestComplete() = 0; }; -struct IHttpRequestFilter +class IHttpRequestFilter { +public: + virtual ~IHttpRequestFilter() {} enum class Result { Forbidden, diff --git a/src/zenhttp/include/zenhttp/security/passwordsecurity.h b/src/zenhttp/include/zenhttp/security/passwordsecurity.h index 026c2865b..6b2b548a6 100644 --- a/src/zenhttp/include/zenhttp/security/passwordsecurity.h +++ b/src/zenhttp/include/zenhttp/security/passwordsecurity.h @@ -10,43 +10,29 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { -struct PasswordSecurityConfiguration -{ - std::string Password; // "password" - bool ProtectMachineLocalRequests = false; // "protect-machine-local-requests" - std::vector UnprotectedUris; // "unprotected-urls" -}; - class PasswordSecurity { public: - PasswordSecurity(const PasswordSecurityConfiguration& Config); + struct Configuration + { + std::string Password; + bool ProtectMachineLocalRequests = false; + std::vector UnprotectedUris; + }; + + explicit PasswordSecurity(const Configuration& Config); [[nodiscard]] inline std::string_view Password() const { return m_Config.Password; } [[nodiscard]] inline bool ProtectMachineLocalRequests() const { return m_Config.ProtectMachineLocalRequests; } - [[nodiscard]] bool IsUnprotectedUri(std::string_view Uri) const; + [[nodiscard]] bool IsUnprotectedUri(std::string_view BaseUri, std::string_view RelativeUri) const; - bool IsAllowed(std::string_view Password, std::string_view Uri, bool IsMachineLocalRequest); + bool IsAllowed(std::string_view Password, std::string_view BaseUri, std::string_view RelativeUri, bool IsMachineLocalRequest); private: - const PasswordSecurityConfiguration m_Config; - tsl::robin_map m_UnprotectedUrlHashes; + const Configuration m_Config; + tsl::robin_map m_UnprotectedUriHashes; }; -/** - * Expected format (Json) - * { - * "password\": \"1234\", - * "protect-machine-local-requests\": false, - * "unprotected-urls\": [ - * "/health\", - * "/health/info\", - * "/health/version\" - * ] - * } - */ -PasswordSecurityConfiguration ReadPasswordSecurityConfiguration(CbObjectView ConfigObject); - void passwordsecurity_forcelink(); // internal } // namespace zen diff --git a/src/zenhttp/include/zenhttp/security/passwordsecurityfilter.h b/src/zenhttp/include/zenhttp/security/passwordsecurityfilter.h new file mode 100644 index 000000000..c098f05ad --- /dev/null +++ b/src/zenhttp/include/zenhttp/security/passwordsecurityfilter.h @@ -0,0 +1,51 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen { + +class PasswordHttpFilter : public IHttpRequestFilter +{ +public: + static constexpr std::string_view TypeName = "password"; + + struct Configuration + { + PasswordSecurity::Configuration PasswordConfig; + std::string AuthenticationTypeString; + }; + + /** + * Expected format (Json) + * { + * "password": { # "Authorization: Basic " style + * "username": "", + * "password": "" + * }, + * "protect-machine-local-requests": false, + * "unprotected-uris": [ + * "/health/", + * "/health/info", + * "/health/version" + * ] + * } + */ + static Configuration ReadConfiguration(CbObjectView Config); + + explicit PasswordHttpFilter(const PasswordHttpFilter::Configuration& Config) + : m_PasswordSecurity(Config.PasswordConfig) + , m_AuthenticationTypeString(Config.AuthenticationTypeString) + { + } + + virtual Result FilterRequest(HttpServerRequest& Request) override; + +private: + PasswordSecurity m_PasswordSecurity; + const std::string m_AuthenticationTypeString; +}; + +} // namespace zen diff --git a/src/zenhttp/security/passwordsecurity.cpp b/src/zenhttp/security/passwordsecurity.cpp index 37be9a018..a8fb9c3f5 100644 --- a/src/zenhttp/security/passwordsecurity.cpp +++ b/src/zenhttp/security/passwordsecurity.cpp @@ -13,13 +13,13 @@ namespace zen { using namespace std::literals; -PasswordSecurity::PasswordSecurity(const PasswordSecurityConfiguration& Config) : m_Config(Config) +PasswordSecurity::PasswordSecurity(const Configuration& Config) : m_Config(Config) { - m_UnprotectedUrlHashes.reserve(m_Config.UnprotectedUris.size()); + m_UnprotectedUriHashes.reserve(m_Config.UnprotectedUris.size()); for (uint32_t Index = 0; Index < m_Config.UnprotectedUris.size(); Index++) { const std::string& UnprotectedUri = m_Config.UnprotectedUris[Index]; - if (auto Result = m_UnprotectedUrlHashes.insert({HashStringDjb2(UnprotectedUri), Index}); !Result.second) + if (auto Result = m_UnprotectedUriHashes.insert({HashStringDjb2(UnprotectedUri), Index}); !Result.second) { throw std::runtime_error(fmt::format( "password security unprotected uris does not generate unique hashes. Uri #{} ('{}') collides with uri #{} ('{}')", @@ -32,35 +32,30 @@ PasswordSecurity::PasswordSecurity(const PasswordSecurityConfiguration& Config) } bool -PasswordSecurity::IsUnprotectedUri(std::string_view Uri) const +PasswordSecurity::IsUnprotectedUri(std::string_view BaseUri, std::string_view RelativeUri) const { if (!m_Config.UnprotectedUris.empty()) { - uint32_t UriHash = HashStringDjb2(Uri); - if (auto It = m_UnprotectedUrlHashes.find(UriHash); It != m_UnprotectedUrlHashes.end()) + uint32_t UriHash = HashStringDjb2(std::array{BaseUri, RelativeUri}); + if (auto It = m_UnprotectedUriHashes.find(UriHash); It != m_UnprotectedUriHashes.end()) { - if (m_Config.UnprotectedUris[It->second] == Uri) + const std::string_view& UnprotectedUri = m_Config.UnprotectedUris[It->second]; + if (UnprotectedUri.length() == BaseUri.length() + RelativeUri.length()) { - return true; + if (UnprotectedUri.substr(0, BaseUri.length()) == BaseUri && UnprotectedUri.substr(BaseUri.length()) == RelativeUri) + { + return true; + } } } } return false; } -PasswordSecurityConfiguration -ReadPasswordSecurityConfiguration(CbObjectView ConfigObject) -{ - return PasswordSecurityConfiguration{ - .Password = std::string(ConfigObject["password"sv].AsString()), - .ProtectMachineLocalRequests = ConfigObject["protect-machine-local-requests"sv].AsBool(), - .UnprotectedUris = compactbinary_helpers::ReadArray("unprotected-urls"sv, ConfigObject)}; -} - bool -PasswordSecurity::IsAllowed(std::string_view InPassword, std::string_view Uri, bool IsMachineLocalRequest) +PasswordSecurity::IsAllowed(std::string_view InPassword, std::string_view BaseUri, std::string_view RelativeUri, bool IsMachineLocalRequest) { - if (IsUnprotectedUri(Uri)) + if (IsUnprotectedUri(BaseUri, RelativeUri)) { return true; } @@ -81,119 +76,74 @@ PasswordSecurity::IsAllowed(std::string_view InPassword, std::string_view Uri, b #if ZEN_WITH_TESTS -TEST_CASE("passwordsecurity.readconfig") -{ - auto ReadConfigJson = [](std::string_view Json) { - std::string JsonError; - CbObject Config = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); - REQUIRE(JsonError.empty()); - return Config; - }; - - { - PasswordSecurityConfiguration EmptyConfig = ReadPasswordSecurityConfiguration(CbObject()); - CHECK(EmptyConfig.Password.empty()); - CHECK(!EmptyConfig.ProtectMachineLocalRequests); - CHECK(EmptyConfig.UnprotectedUris.empty()); - } - - { - const std::string_view SimpleConfigJson = - "{\n" - " \"password\": \"1234\"\n" - "}"; - PasswordSecurityConfiguration SimpleConfig = ReadPasswordSecurityConfiguration(ReadConfigJson(SimpleConfigJson)); - CHECK(SimpleConfig.Password == "1234"); - CHECK(!SimpleConfig.ProtectMachineLocalRequests); - CHECK(SimpleConfig.UnprotectedUris.empty()); - } - - { - const std::string_view ComplexConfigJson = - "{\n" - " \"password\": \"1234\",\n" - " \"protect-machine-local-requests\": true,\n" - " \"unprotected-urls\": [\n" - " \"/health\",\n" - " \"/health/info\",\n" - " \"/health/version\"\n" - " ]\n" - "}"; - PasswordSecurityConfiguration ComplexConfig = ReadPasswordSecurityConfiguration(ReadConfigJson(ComplexConfigJson)); - CHECK(ComplexConfig.Password == "1234"); - CHECK(ComplexConfig.ProtectMachineLocalRequests); - CHECK(ComplexConfig.UnprotectedUris == std::vector({"/health", "/health/info", "/health/version"})); - } -} - TEST_CASE("passwordsecurity.allowanything") { PasswordSecurity Anything({}); - CHECK(Anything.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(Anything.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(Anything.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(Anything.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(Anything.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(Anything.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(Anything.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(Anything.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); } TEST_CASE("passwordsecurity.allowalllocal") { PasswordSecurity AllLocal({.Password = "123456"}); - CHECK(AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); } TEST_CASE("passwordsecurity.allowonlypassword") { PasswordSecurity AllLocal({.Password = "123456", .ProtectMachineLocalRequests = true}); - CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); } TEST_CASE("passwordsecurity.allowsomeexternaluris") { PasswordSecurity AllLocal( {.Password = "123456", .ProtectMachineLocalRequests = false, .UnprotectedUris = std::vector({"/free/access", "/ok"})}); - CHECK(AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed(""sv, "/free/access", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed(""sv, "/ok", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free/access", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed(""sv, "/free/access", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed(""sv, "/ok", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free/access", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed(""sv, "/free", "/access", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed(""sv, "/ok", "", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free", "/access", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", "", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed(""sv, "/free", "/access", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed(""sv, "/ok", "", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free", "/access", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", "", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); } TEST_CASE("passwordsecurity.allowsomelocaluris") { PasswordSecurity AllLocal( {.Password = "123456", .ProtectMachineLocalRequests = true, .UnprotectedUris = std::vector({"/free/access", "/ok"})}); - CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(!AllLocal.IsAllowed(""sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ true)); - CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed(""sv, "/free/access", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed(""sv, "/ok", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free/access", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", /*IsMachineLocalRequest*/ true)); - CHECK(AllLocal.IsAllowed(""sv, "/free/access", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed(""sv, "/ok", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free/access", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", /*IsMachineLocalRequest*/ false)); - CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret/uri", /*IsMachineLocalRequest*/ false)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed(""sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ true)); + CHECK(!AllLocal.IsAllowed("thewrongpassword"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed(""sv, "/free", "/access", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed(""sv, "/ok", "", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free", "/access", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", "", /*IsMachineLocalRequest*/ true)); + CHECK(AllLocal.IsAllowed(""sv, "/free", "/access", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed(""sv, "/ok", "", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/free", "/access", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("thewrongpassword"sv, "/ok", "", /*IsMachineLocalRequest*/ false)); + CHECK(AllLocal.IsAllowed("123456"sv, "/supersecret", "/uri", /*IsMachineLocalRequest*/ false)); } TEST_CASE("passwordsecurity.conflictingunprotecteduris") diff --git a/src/zenhttp/security/passwordsecurityfilter.cpp b/src/zenhttp/security/passwordsecurityfilter.cpp new file mode 100644 index 000000000..87d8cc275 --- /dev/null +++ b/src/zenhttp/security/passwordsecurityfilter.cpp @@ -0,0 +1,56 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zenhttp/security/passwordsecurityfilter.h" + +#include +#include +#include + +namespace zen { + +using namespace std::literals; + +PasswordHttpFilter::Configuration +PasswordHttpFilter::ReadConfiguration(CbObjectView Config) +{ + Configuration Result; + if (CbObjectView PasswordType = Config["basic"sv].AsObjectView(); PasswordType) + { + Result.AuthenticationTypeString = "Basic "; + std::string_view Username = PasswordType["username"sv].AsString(); + std::string_view Password = PasswordType["password"sv].AsString(); + std::string UsernamePassword = fmt::format("{}:{}", Username, Password); + Result.PasswordConfig.Password.resize(Base64::GetEncodedDataSize(uint32_t(UsernamePassword.length()))); + Base64::Encode(reinterpret_cast(UsernamePassword.data()), + uint32_t(UsernamePassword.size()), + const_cast(Result.PasswordConfig.Password.data())); + } + Result.PasswordConfig.ProtectMachineLocalRequests = Config["protect-machine-local-requests"sv].AsBool(); + Result.PasswordConfig.UnprotectedUris = compactbinary_helpers::ReadArray("unprotected-uris"sv, Config); + return Result; +} + +IHttpRequestFilter::Result +PasswordHttpFilter::FilterRequest(HttpServerRequest& Request) +{ + std::string_view Password; + std::string_view AuthorizationHeader = Request.GetAuthorizationHeader(); + size_t AuthorizationHeaderLength = AuthorizationHeader.length(); + if (AuthorizationHeaderLength > m_AuthenticationTypeString.length()) + { + if (StrCaseCompare(AuthorizationHeader.data(), m_AuthenticationTypeString.c_str(), m_AuthenticationTypeString.length()) == 0) + { + Password = AuthorizationHeader.substr(m_AuthenticationTypeString.length()); + } + } + + bool IsAllowed = + m_PasswordSecurity.IsAllowed(Password, Request.Service().BaseUri(), Request.RelativeUri(), Request.IsLocalMachineRequest()); + if (IsAllowed) + { + return Result::Accepted; + } + return Result::Forbidden; +} + +} // namespace zen diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 1f42b05d2..1c0ebef90 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -542,7 +542,8 @@ public: virtual Oid ParseSessionId() const override; virtual uint32_t ParseRequestId() const override; - virtual bool IsLocalMachineRequest() const override; + virtual bool IsLocalMachineRequest() const override; + virtual std::string_view GetAuthorizationHeader() const override; virtual IoBuffer ReadPayload() override; virtual void WriteResponse(HttpResponseCode ResponseCode) override; @@ -1747,6 +1748,12 @@ HttpAsioServerRequest::IsLocalMachineRequest() const return m_IsLocalMachineRequest; } +std::string_view +HttpAsioServerRequest::GetAuthorizationHeader() const +{ + return m_Request.AuthorizationHeader(); +} + IoBuffer HttpAsioServerRequest::ReadPayload() { @@ -1964,8 +1971,8 @@ HttpAsioServerImpl::FilterRequest(HttpServerRequest& Request) { return IHttpRequestFilter::Result::Accepted; } - IHttpRequestFilter::Result FilterResult = RequestFilter->FilterRequest(Request); - return FilterResult; + + return RequestFilter->FilterRequest(Request); } } // namespace zen::asio_http @@ -2080,6 +2087,7 @@ HttpAsioServer::OnRun(bool IsInteractive) if (c == 27 || c == 'Q' || c == 'q') { + m_ShutdownEvent.Set(); RequestApplicationExit(0); } } diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp index 850d7d6b9..310ac9dc0 100644 --- a/src/zenhttp/servers/httpmulti.cpp +++ b/src/zenhttp/servers/httpmulti.cpp @@ -82,6 +82,7 @@ HttpMultiServer::OnRun(bool IsInteractiveSession) if (c == 27 || c == 'Q' || c == 'q') { + m_ShutdownEvent.Set(); RequestApplicationExit(0); } } diff --git a/src/zenhttp/servers/httpnull.cpp b/src/zenhttp/servers/httpnull.cpp index db360c5fb..9bb7ef3bc 100644 --- a/src/zenhttp/servers/httpnull.cpp +++ b/src/zenhttp/servers/httpnull.cpp @@ -57,6 +57,7 @@ HttpNullServer::OnRun(bool IsInteractiveSession) if (c == 27 || c == 'Q' || c == 'q') { + m_ShutdownEvent.Set(); RequestApplicationExit(0); } } diff --git a/src/zenhttp/servers/httpparser.cpp b/src/zenhttp/servers/httpparser.cpp index 93094e21b..be5befcd2 100644 --- a/src/zenhttp/servers/httpparser.cpp +++ b/src/zenhttp/servers/httpparser.cpp @@ -19,6 +19,7 @@ static constinit uint32_t HashExpect = HashStringAsLowerDjb2("Expect"sv); static constinit uint32_t HashSession = HashStringAsLowerDjb2("UE-Session"sv); static constinit uint32_t HashRequest = HashStringAsLowerDjb2("UE-Request"sv); static constinit uint32_t HashRange = HashStringAsLowerDjb2("Range"sv); +static constinit uint32_t HashAuthorization = HashStringAsLowerDjb2("Authorization"sv); ////////////////////////////////////////////////////////////////////////// // @@ -154,6 +155,10 @@ HttpRequestParser::ParseCurrentHeader() { m_ContentTypeHeaderIndex = CurrentHeaderIndex; } + else if (HeaderHash == HashAuthorization) + { + m_AuthorizationHeaderIndex = CurrentHeaderIndex; + } else if (HeaderHash == HashSession) { m_SessionId = Oid::TryFromHexString(HeaderValue); @@ -357,6 +362,7 @@ HttpRequestParser::ResetState() m_AcceptHeaderIndex = -1; m_ContentTypeHeaderIndex = -1; m_RangeHeaderIndex = -1; + m_AuthorizationHeaderIndex = -1; m_Expect100Continue = false; m_BodyBuffer = {}; m_BodyPosition = 0; diff --git a/src/zenhttp/servers/httpparser.h b/src/zenhttp/servers/httpparser.h index 0d2664ec5..ff56ca970 100644 --- a/src/zenhttp/servers/httpparser.h +++ b/src/zenhttp/servers/httpparser.h @@ -46,6 +46,8 @@ struct HttpRequestParser std::string_view RangeHeader() const { return GetHeaderValue(m_RangeHeaderIndex); } + std::string_view AuthorizationHeader() const { return GetHeaderValue(m_AuthorizationHeaderIndex); } + private: struct HeaderRange { @@ -83,6 +85,7 @@ private: int8_t m_AcceptHeaderIndex; int8_t m_ContentTypeHeaderIndex; int8_t m_RangeHeaderIndex; + int8_t m_AuthorizationHeaderIndex; HttpVerb m_RequestVerb; std::atomic_bool m_KeepAlive{false}; bool m_Expect100Continue = false; diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index 4219dc292..8564826d6 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -147,10 +147,10 @@ public: HttpPluginServerRequest& operator=(const HttpPluginServerRequest&) = delete; // As this is plugin transport connection used for specialized connections we assume it is not a machine local connection - virtual bool IsLocalMachineRequest() const /* override*/ { return false; } - - virtual Oid ParseSessionId() const override; - virtual uint32_t ParseRequestId() const override; + virtual bool IsLocalMachineRequest() const /* override*/ { return false; } + virtual std::string_view GetAuthorizationHeader() const override; + virtual Oid ParseSessionId() const override; + virtual uint32_t ParseRequestId() const override; virtual IoBuffer ReadPayload() override; virtual void WriteResponse(HttpResponseCode ResponseCode) override; @@ -636,6 +636,12 @@ HttpPluginServerRequest::~HttpPluginServerRequest() { } +std::string_view +HttpPluginServerRequest::GetAuthorizationHeader() const +{ + return m_Request.AuthorizationHeader(); +} + Oid HttpPluginServerRequest::ParseSessionId() const { @@ -831,6 +837,7 @@ HttpPluginServerImpl::OnRun(bool IsInteractive) if (c == 27 || c == 'Q' || c == 'q') { + m_ShutdownEvent.Set(); RequestApplicationExit(0); } } @@ -932,8 +939,7 @@ HttpPluginServerImpl::FilterRequest(HttpServerRequest& Request) { return IHttpRequestFilter::Result::Accepted; } - IHttpRequestFilter::Result FilterResult = RequestFilter->FilterRequest(Request); - return FilterResult; + return RequestFilter->FilterRequest(Request); } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 5fed94f1c..14896c803 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -72,6 +72,8 @@ GetAddressString(StringBuilderBase& OutString, const SOCKADDR* SockAddr, bool In OutString.Append("unknown"); } +class HttpSysServerRequest; + /** * @brief Windows implementation of HTTP server based on http.sys * @@ -102,7 +104,7 @@ public: inline bool IsOk() const { return m_IsOk; } inline bool IsAsyncResponseEnabled() const { return m_IsAsyncResponseEnabled; } - IHttpRequestFilter::Result FilterRequest(HttpServerRequest& Request); + IHttpRequestFilter::Result FilterRequest(HttpSysServerRequest& Request); private: int InitializeServer(int BasePort); @@ -319,7 +321,8 @@ public: virtual Oid ParseSessionId() const override; virtual uint32_t ParseRequestId() const override; - virtual bool IsLocalMachineRequest() const; + virtual bool IsLocalMachineRequest() const; + virtual std::string_view GetAuthorizationHeader() const override; virtual IoBuffer ReadPayload() override; virtual void WriteResponse(HttpResponseCode ResponseCode) override; @@ -1364,6 +1367,7 @@ HttpSysServer::OnRun(bool IsInteractive) if (c == 27 || c == 'Q' || c == 'q') { + m_ShutdownEvent.Set(); RequestApplicationExit(0); } } @@ -1861,6 +1865,14 @@ HttpSysServerRequest::IsLocalMachineRequest() const } } +std::string_view +HttpSysServerRequest::GetAuthorizationHeader() const +{ + const HTTP_REQUEST* HttpRequestPtr = m_HttpTx.HttpRequest(); + const HTTP_KNOWN_HEADER& AuthorizationHeader = HttpRequestPtr->Headers.KnownHeaders[HttpHeaderAuthorization]; + return std::string_view(AuthorizationHeader.pRawValue, AuthorizationHeader.RawValueLength); +} + IoBuffer HttpSysServerRequest::ReadPayload() { @@ -2270,7 +2282,7 @@ HttpSysServer::OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) } IHttpRequestFilter::Result -HttpSysServer::FilterRequest(HttpServerRequest& Request) +HttpSysServer::FilterRequest(HttpSysServerRequest& Request) { if (!m_HttpRequestFilter.load()) { @@ -2282,8 +2294,7 @@ HttpSysServer::FilterRequest(HttpServerRequest& Request) { return IHttpRequestFilter::Result::Accepted; } - IHttpRequestFilter::Result FilterResult = RequestFilter->FilterRequest(Request); - return FilterResult; + return RequestFilter->FilterRequest(Request); } Ref diff --git a/src/zenserver/config/config.cpp b/src/zenserver/config/config.cpp index 2b77df642..e36352dae 100644 --- a/src/zenserver/config/config.cpp +++ b/src/zenserver/config/config.cpp @@ -140,6 +140,7 @@ ZenServerConfiguratorBase::AddCommonConfigOptions(LuaConfig::Options& LuaOptions LuaOptions.AddOption("server.contentdir"sv, ServerOptions.ContentDir, "content-dir"sv); LuaOptions.AddOption("server.debug"sv, ServerOptions.IsDebug, "debug"sv); LuaOptions.AddOption("server.clean"sv, ServerOptions.IsCleanStart, "clean"sv); + LuaOptions.AddOption("server.security.configpath"sv, ServerOptions.SecurityConfigPath, "security-config-path"sv); ////// network @@ -186,6 +187,7 @@ struct ZenServerCmdLineOptions std::string ContentDir; std::string DataDir; std::string BaseSnapshotDir; + std::string SecurityConfigPath; ZenLoggingCmdLineOptions LoggingOptions; @@ -300,6 +302,13 @@ ZenServerCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenServerConfi cxxopts::value(ServerOptions.HttpConfig.ForceLoopback)->default_value("false"), ""); + options.add_option("network", + "", + "security-config-path", + "Path to http security configuration file", + cxxopts::value(SecurityConfigPath), + ""); + #if ZEN_WITH_HTTPSYS options.add_option("httpsys", "", @@ -380,11 +389,12 @@ ZenServerCmdLineOptions::ApplyOptions(cxxopts::Options& options, ZenServerConfig throw std::runtime_error(fmt::format("'--snapshot-dir' ('{}') must be a directory", ServerOptions.BaseSnapshotDir)); } - ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); - ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); - ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); - ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); - ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); + ServerOptions.SystemRootDir = MakeSafeAbsolutePath(SystemRootDir); + ServerOptions.DataDir = MakeSafeAbsolutePath(DataDir); + ServerOptions.ContentDir = MakeSafeAbsolutePath(ContentDir); + ServerOptions.ConfigFile = MakeSafeAbsolutePath(ConfigFile); + ServerOptions.BaseSnapshotDir = MakeSafeAbsolutePath(BaseSnapshotDir); + ServerOptions.SecurityConfigPath = MakeSafeAbsolutePath(SecurityConfigPath); LoggingOptions.ApplyOptions(ServerOptions.LoggingConfig); } diff --git a/src/zenserver/config/config.h b/src/zenserver/config/config.h index 32c22cb05..55aee07f9 100644 --- a/src/zenserver/config/config.h +++ b/src/zenserver/config/config.h @@ -56,12 +56,13 @@ struct ZenServerConfig bool IsDedicated = false; // Indicates a dedicated/shared instance, with larger resource requirements bool ShouldCrash = false; // Option for testing crash handling bool IsFirstRun = false; - std::filesystem::path ConfigFile; // Path to Lua config file - std::filesystem::path SystemRootDir; // System root directory (used for machine level config) - std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) - std::filesystem::path DataDir; // Root directory for state (used for testing) - std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) - std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::filesystem::path ConfigFile; // Path to Lua config file + std::filesystem::path SystemRootDir; // System root directory (used for machine level config) + std::filesystem::path ContentDir; // Root directory for serving frontend content (experimental) + std::filesystem::path DataDir; // Root directory for state (used for testing) + std::filesystem::path BaseSnapshotDir; // Path to server state snapshot (will be copied into data dir on start) + std::string ChildId; // Id assigned by parent process (used for lifetime management) + std::filesystem::path SecurityConfigPath; // Path to a Json security configuration file #if ZEN_WITH_TRACE bool HasTraceCommandlineOptions = false; diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index d54357368..7f9bf56a9 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -142,6 +143,8 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState:: ZEN_INFO("Effective concurrency: {} (hw: {})", GetHardwareConcurrency(), std::thread::hardware_concurrency()); + InitializeSecuritySettings(ServerOptions); + m_StatusService.RegisterHandler("status", *this); m_Http->RegisterService(m_StatusService); @@ -386,10 +389,10 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig) { // clang-format off std::list> Settings = { - {"DataDir"sv, ServerConfig.DataDir.string()}, - {"AbsLogFile"sv, ServerConfig.LoggingConfig.AbsLogFile.string()}, - {"SystemRootDir"sv, ServerConfig.SystemRootDir.string()}, - {"ContentDir"sv, ServerConfig.ContentDir.string()}, + {"DataDir"sv, fmt::format("{}", ServerConfig.DataDir)}, + {"AbsLogFile"sv, fmt::format("{}", ServerConfig.LoggingConfig.AbsLogFile)}, + {"SystemRootDir"sv, fmt::format("{}", ServerConfig.SystemRootDir)}, + {"ContentDir"sv, fmt::format("{}", ServerConfig.ContentDir)}, {"BasePort"sv, fmt::to_string(ServerConfig.BasePort)}, {"IsDebug"sv, fmt::to_string(ServerConfig.IsDebug)}, {"IsCleanStart"sv, fmt::to_string(ServerConfig.IsCleanStart)}, @@ -406,6 +409,7 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig) {"Sentry DSN"sv, ServerConfig.SentryConfig.Dsn.empty() ? "not set" : ServerConfig.SentryConfig.Dsn}, {"Sentry Environment"sv, ServerConfig.SentryConfig.Environment}, {"Statsd Enabled"sv, fmt::to_string(ServerConfig.StatsConfig.Enabled)}, + {"SecurityConfigPath"sv, fmt::format("{}", ServerConfig.SecurityConfigPath)}, }; // clang-format on @@ -432,6 +436,44 @@ ZenServerBase::LogSettingsSummary(const ZenServerConfig& ServerConfig) } } +void +ZenServerBase::InitializeSecuritySettings(const ZenServerConfig& ServerOptions) +{ + ZEN_ASSERT(m_Http); + + if (!ServerOptions.SecurityConfigPath.empty()) + { + IoBuffer SecurityJson = ReadFile(ServerOptions.SecurityConfigPath).Flatten(); + std::string_view Json(reinterpret_cast(SecurityJson.GetData()), SecurityJson.GetSize()); + std::string JsonError; + CbObject SecurityConfig = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); + if (!JsonError.empty()) + { + throw std::runtime_error( + fmt::format("Invalid security configuration file at {}. '{}'", ServerOptions.SecurityConfigPath, JsonError)); + } + + CbObjectView HttpRootFilterConfig = SecurityConfig["http"sv].AsObjectView()["root"sv].AsObjectView()["filter"sv].AsObjectView(); + if (HttpRootFilterConfig) + { + std::string_view FilterType = HttpRootFilterConfig["type"sv].AsString(); + if (FilterType == PasswordHttpFilter::TypeName) + { + PasswordHttpFilter::Configuration Config = + PasswordHttpFilter::ReadConfiguration(HttpRootFilterConfig["config"].AsObjectView()); + m_HttpRequestFilter = std::make_unique(Config); + m_Http->SetHttpRequestFilter(m_HttpRequestFilter.get()); + } + else + { + throw std::runtime_error(fmt::format("Security configuration file at {} references unknown http root filter type '{}'", + ServerOptions.SecurityConfigPath, + FilterType)); + } + } + } +} + ////////////////////////////////////////////////////////////////////////// ZenServerMain::ZenServerMain(ZenServerConfig& ServerOptions) : m_ServerOptions(ServerOptions) diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index ab7122fcc..efa46f361 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -72,7 +72,10 @@ protected: std::function m_IsReadyFunc; void OnReady(); - Ref m_Http; + Ref m_Http; + + std::unique_ptr m_HttpRequestFilter; + HttpHealthService m_HealthService; HttpStatusService m_StatusService; @@ -107,6 +110,9 @@ protected: // IHttpStatusProvider virtual void HandleStatusRequest(HttpServerRequest& Request) override; + +private: + void InitializeSecuritySettings(const ZenServerConfig& ServerOptions); }; class ZenServerMain -- cgit v1.2.3 From d1324d607e54e2e97d666a2d1ece9ac9495d1eb1 Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 17 Feb 2026 20:21:26 -0700 Subject: Make files table in entry.js paginated and searchable --- src/zenserver/frontend/html/pages/entry.js | 210 +++++++++++++++++++++++------ 1 file changed, 170 insertions(+), 40 deletions(-) diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index dca3a5c25..13d5e44e7 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -26,6 +26,9 @@ export class Page extends ZenPage this._indexer = this.load_indexer(project, oplog); + this._files_index_start = Number(this.get_param("files_start", 0)) || 0; + this._files_index_count = Number(this.get_param("files_count", 50)) || 0; + this._build_page(); } @@ -253,20 +256,13 @@ export class Page extends ZenPage // files var has_file_data = false; { - const sub_section = section.add_section("files"); - const table = sub_section.add_widget( - Table, - ["name", "actions"], Table.Flag_PackRight - ); - table.id("filetable"); - for (const field_name of ["files"]) + var file_data = entry.find("files"); + if (file_data != undefined) { - var file_data = entry.find(field_name); - if (file_data == undefined) - continue; - has_file_data = true; + // Extract files into array + this._files_data = []; for (const item of file_data.as_array()) { var io_hash = undefined, cid = undefined, server_path = undefined, client_path = undefined; @@ -294,37 +290,26 @@ export class Page extends ZenPage cid = ret; } - const row = table.add_row(server_path); + this._files_data.push({ + server_path: server_path, + client_path: client_path, + io_hash: io_hash, + cid: cid + }); + } - var base_name = server_path.split("/").pop().split("\\").pop(); - const project = this.get_param("project"); - const oplog = this.get_param("oplog"); - if (this._is_null_io_hash_string(io_hash)) - { - const link = row.get_cell(0).link( - "/" + ["prj", project, "oplog", oplog, cid].join("/") - ); - link.first_child().attr("download", `${cid}_${base_name}`); - - const action_tb = new Toolbar(row.get_cell(-1), true); - action_tb.left().add("copy-id").on_click(async (v) => { - await navigator.clipboard.writeText(v); - }, cid); - } - else - { - const link = row.get_cell(0).link( - "/" + ["prj", project, "oplog", oplog, io_hash].join("/") - ); - link.first_child().attr("download", `${io_hash}_${base_name}`); - - const action_tb = new Toolbar(row.get_cell(-1), true); - action_tb.left().add("copy-hash").on_click(async (v) => { - await navigator.clipboard.writeText(v); - }, io_hash); - } + this._files_index_max = this._files_data.length; - } + const sub_section = section.add_section("files"); + this._build_files_nav(sub_section); + + this._files_table = sub_section.add_widget( + Table, + ["name", "actions"], Table.Flag_PackRight + ); + this._files_table.id("filetable"); + + this._build_files_table(this._files_index_start); } } @@ -419,4 +404,149 @@ export class Page extends ZenPage params.set("opkey", opkey); window.location.search = params; } + + _build_files_nav(section) + { + const nav = section.add_widget(Toolbar); + const left = nav.left(); + left.add("|<") .on_click(() => this._on_files_next_prev(-10e10)); + left.add("<<") .on_click(() => this._on_files_next_prev(-10)); + left.add("prev").on_click(() => this._on_files_next_prev( -1)); + left.add("next").on_click(() => this._on_files_next_prev( 1)); + left.add(">>") .on_click(() => this._on_files_next_prev( 10)); + left.add(">|") .on_click(() => this._on_files_next_prev( 10e10)); + + left.sep(); + for (var count of [10, 25, 50, 100]) + { + var handler = (n) => this._on_files_change_count(n); + left.add(count).on_click(handler, count); + } + + const right = nav.right(); + right.add(Friendly.sep(this._files_index_max)); + + right.sep(); + var search_input = right.add("search:", "label").tag("input"); + search_input.on("change", (x) => this._search_files(x.inner().value), search_input); + } + + _build_files_table(index) + { + this._files_index_count = Math.max(this._files_index_count, 1); + index = Math.min(index, this._files_index_max - this._files_index_count); + index = Math.max(index, 0); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + + const end_index = Math.min(index + this._files_index_count, this._files_index_max); + + this._files_table.clear(index); + for (var i = index; i < end_index; i++) + { + const file_item = this._files_data[i]; + const row = this._files_table.add_row(file_item.server_path); + + var base_name = file_item.server_path.split("/").pop().split("\\").pop(); + if (this._is_null_io_hash_string(file_item.io_hash)) + { + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, file_item.cid].join("/") + ); + link.first_child().attr("download", `${file_item.cid}_${base_name}`); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-id").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, file_item.cid); + } + else + { + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, file_item.io_hash].join("/") + ); + link.first_child().attr("download", `${file_item.io_hash}_${base_name}`); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, file_item.io_hash); + } + } + + this.set_param("files_start", index); + this.set_param("files_count", this._files_index_count); + this._files_index_start = index; + } + + _on_files_change_count(value) + { + this._files_index_count = parseInt(value); + this._build_files_table(this._files_index_start); + } + + _on_files_next_prev(direction) + { + var index = this._files_index_start + (this._files_index_count * direction); + index = Math.max(0, index); + this._build_files_table(index); + } + + _search_files(needle) + { + if (needle.length == 0) + { + this._build_files_table(this._files_index_start); + return; + } + needle = needle.trim().toLowerCase(); + + this._files_table.clear(this._files_index_start); + + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); + + var added = 0; + const truncate_at = this.get_param("searchmax") || 250; + for (const file_item of this._files_data) + { + if (!file_item.server_path.toLowerCase().includes(needle)) + continue; + + const row = this._files_table.add_row(file_item.server_path); + + var base_name = file_item.server_path.split("/").pop().split("\\").pop(); + if (this._is_null_io_hash_string(file_item.io_hash)) + { + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, file_item.cid].join("/") + ); + link.first_child().attr("download", `${file_item.cid}_${base_name}`); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-id").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, file_item.cid); + } + else + { + const link = row.get_cell(0).link( + "/" + ["prj", project, "oplog", oplog, file_item.io_hash].join("/") + ); + link.first_child().attr("download", `${file_item.io_hash}_${base_name}`); + + const action_tb = new Toolbar(row.get_cell(-1), true); + action_tb.left().add("copy-hash").on_click(async (v) => { + await navigator.clipboard.writeText(v); + }, file_item.io_hash); + } + + if (++added >= truncate_at) + { + this._files_table.add_row("...truncated"); + break; + } + } + } } -- cgit v1.2.3 From 1c8948411e68429f613889c7e278bb0422c172a7 Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 17 Feb 2026 20:46:45 -0700 Subject: Rename the cache section in the web ui --- src/zenserver/frontend/html/pages/start.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zenserver/frontend/html/pages/start.js b/src/zenserver/frontend/html/pages/start.js index 4c8789431..2cf12bf12 100644 --- a/src/zenserver/frontend/html/pages/start.js +++ b/src/zenserver/frontend/html/pages/start.js @@ -46,7 +46,7 @@ export class Page extends ZenPage } // cache - var section = this.add_section("z$"); + var section = this.add_section("cache"); section.tag().classify("dropall").text("drop-all").on_click(() => this.drop_all("z$")); -- cgit v1.2.3 From fbd53c5500d4898be9e2c76646f220dd88a96f36 Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 17 Feb 2026 21:16:38 -0700 Subject: Dependencies table doesn't reflow the entries page --- src/zenserver/frontend/html/pages/entry.js | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index 13d5e44e7..c4746bf52 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -43,25 +43,39 @@ export class Page extends ZenPage return indexer; } - async _build_deps(section, tree) + _build_deps(section, tree) { - const indexer = await this._indexer; + const project = this.get_param("project"); + const oplog = this.get_param("oplog"); for (const dep_name in tree) { const dep_section = section.add_section(dep_name); const table = dep_section.add_widget(Table, ["name", "id"], Table.Flag_PackRight); + for (const dep_id of tree[dep_name]) { - const cell_values = ["", dep_id.toString(16).padStart(16, "0")]; + const hex_id = dep_id.toString(16).padStart(16, "0"); + const cell_values = ["loading...", hex_id]; const row = table.add_row(...cell_values); - var opkey = indexer.lookup_id(dep_id); - row.get_cell(0).text(opkey).on_click((k) => this.view_opkey(k), opkey); + // Asynchronously resolve the name + this._resolve_dep_name(row.get_cell(0), dep_id, project, oplog); } } } + async _resolve_dep_name(cell, dep_id, project, oplog) + { + const indexer = await this._indexer; + const opkey = indexer.lookup_id(dep_id); + + if (opkey) + { + cell.text(opkey).on_click((k) => this.view_opkey(k), opkey); + } + } + _find_iohash_field(container, name) { const found_field = container.find(name); -- cgit v1.2.3 From d8ad42c32f0d754948c7fd1c60beda6dcb8d36c6 Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 17 Feb 2026 21:22:17 -0700 Subject: Changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2fe710a9..1c3d3c970 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ - Improvement: Reduced time project and project oplogs are locked during GC and Validation - Improvement: `zen` now supports additional configuration of logging options, such as `--log-warn=...` for configuring log levels, etc (see `zen --help`) +- Improvement: Web UI shows both hard and soft package dependencies +- Improvement: Web UI presents ops with only files without resorting to json representation +- Improvement: Web UI offers a cook artifacts view to present cook dependencies ## 5.7.20 - Improvement: When validating cache records read from disk we now do a limited validation of the payload to reduce overhead -- cgit v1.2.3 From 425673a0230373a1b91c15c475f8e543ab246bce Mon Sep 17 00:00:00 2001 From: zousar Date: Tue, 17 Feb 2026 21:24:11 -0700 Subject: updatefrontend --- src/zenserver/frontend/html.zip | Bin 163229 -> 182962 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index 5d33302dd..67752fbc2 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ -- cgit v1.2.3 From b55fdf7c1dfe6d3e52b08a160a77472ec1480cf7 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 18 Feb 2026 08:54:05 +0100 Subject: convert ZEN_ASSERTs to exception to handle corrupt data gracefully (#760) * convert ZEN_ASSERTs to exception to handle corrupt data gracefully --- CHANGELOG.md | 1 + .../builds/buildstorageoperations.cpp | 68 ++++++++++++++++++---- .../zenremotestore/builds/buildstorageoperations.h | 3 +- 3 files changed, 59 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2fe710a9..16a6b7fb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ - Improvement: Reduced time project and project oplogs are locked during GC and Validation - Improvement: `zen` now supports additional configuration of logging options, such as `--log-warn=...` for configuring log levels, etc (see `zen --help`) +- Bugfix: If a corrupted block (or partial block) is downloaded, handle it gracefully and end the download instead of causing an assert ## 5.7.20 - Improvement: When validating cache records read from disk we now do a limited validation of the payload to reduce overhead diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 2319ad66d..ade431393 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -4083,7 +4083,8 @@ BuildsOperationUpdateFolder::WriteSequenceChunkToCache(BufferedWriteFileCache::L } bool -BuildsOperationUpdateFolder::GetBlockWriteOps(std::span ChunkRawHashes, +BuildsOperationUpdateFolder::GetBlockWriteOps(const IoHash& BlockRawHash, + std::span ChunkRawHashes, std::span ChunkCompressedLengths, std::span> SequenceIndexChunksLeftToWriteCounters, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, @@ -4115,9 +4116,34 @@ BuildsOperationUpdateFolder::GetBlockWriteOps(std::span ChunkR uint64_t VerifyChunkSize; CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer::MakeView(ChunkMemoryView), VerifyChunkHash, VerifyChunkSize); - ZEN_ASSERT(CompressedChunk); - ZEN_ASSERT(VerifyChunkHash == ChunkHash); - ZEN_ASSERT(VerifyChunkSize == m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + if (!CompressedChunk) + { + throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} is not a valid compressed buffer", + ChunkHash, + OffsetInBlock, + ChunkCompressedSize, + BlockRawHash)); + } + if (VerifyChunkHash != ChunkHash) + { + throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} has a mismatching content hash {}", + ChunkHash, + OffsetInBlock, + ChunkCompressedSize, + BlockRawHash, + VerifyChunkHash)); + } + if (VerifyChunkSize != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]) + { + throw std::runtime_error( + fmt::format("Chunk {} at {}, size {} in block {} has a mismatching raw size {}, expected {}", + ChunkHash, + OffsetInBlock, + ChunkCompressedSize, + BlockRawHash, + VerifyChunkSize, + m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])); + } OodleCompressor ChunkCompressor; OodleCompressionLevel ChunkCompressionLevel; @@ -4138,7 +4164,18 @@ BuildsOperationUpdateFolder::GetBlockWriteOps(std::span ChunkR { Decompressed = CompressedChunk.Decompress().AsIoBuffer(); } - ZEN_ASSERT(Decompressed.GetSize() == m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]); + + if (Decompressed.GetSize() != m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex]) + { + throw std::runtime_error(fmt::format("Chunk {} at {}, size {} in block {} decompressed to size {}, expected {}", + ChunkHash, + OffsetInBlock, + ChunkCompressedSize, + BlockRawHash, + Decompressed.GetSize(), + m_RemoteContent.ChunkedContent.ChunkRawSizes[ChunkIndex])); + } + ZEN_ASSERT_SLOW(ChunkHash == IoHash::HashBuffer(Decompressed)); for (const ChunkedContentLookup::ChunkSequenceLocation* Target : ChunkTargetPtrs) { @@ -4237,7 +4274,8 @@ BuildsOperationUpdateFolder::WriteChunksBlockToCache(const ChunkBlockDescription const std::vector ChunkCompressedLengths = ReadChunkBlockHeader(BlockView.Mid(CompressedBuffer::GetHeaderSizeForNoneEncoder()), HeaderSize); - if (GetBlockWriteOps(BlockDescription.ChunkRawHashes, + if (GetBlockWriteOps(BlockDescription.BlockHash, + BlockDescription.ChunkRawHashes, ChunkCompressedLengths, SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndexNeedsCopyFromSourceFlags, @@ -4252,7 +4290,8 @@ BuildsOperationUpdateFolder::WriteChunksBlockToCache(const ChunkBlockDescription return false; } - if (GetBlockWriteOps(BlockDescription.ChunkRawHashes, + if (GetBlockWriteOps(BlockDescription.BlockHash, + BlockDescription.ChunkRawHashes, BlockDescription.ChunkCompressedLengths, SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndexNeedsCopyFromSourceFlags, @@ -4283,7 +4322,8 @@ BuildsOperationUpdateFolder::WritePartialBlockChunksToCache(const ChunkBlockDesc const MemoryView BlockView = BlockMemoryBuffer.GetView(); BlockWriteOps Ops; - if (GetBlockWriteOps(BlockDescription.ChunkRawHashes, + if (GetBlockWriteOps(BlockDescription.BlockHash, + BlockDescription.ChunkRawHashes, BlockDescription.ChunkCompressedLengths, SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndexNeedsCopyFromSourceFlags, @@ -5334,6 +5374,13 @@ BuildsOperationUploadFolder::FetchChunk(const ChunkedFolderContent& Content, ZEN_ASSERT(!ChunkLocations.empty()); CompositeBuffer Chunk = OpenFileCache.GetRange(ChunkLocations[0].SequenceIndex, ChunkLocations[0].Offset, Content.ChunkedContent.ChunkRawSizes[ChunkIndex]); + if (!Chunk) + { + throw std::runtime_error(fmt::format("Unable to read chunk at {}, size {} from '{}'", + ChunkLocations[0].Offset, + Content.ChunkedContent.ChunkRawSizes[ChunkIndex], + Content.Paths[Lookup.SequenceIndexFirstPathIndex[ChunkLocations[0].SequenceIndex]])); + } ZEN_ASSERT_SLOW(IoHash::HashBuffer(Chunk) == ChunkHash); return Chunk; }; @@ -5362,10 +5409,7 @@ BuildsOperationUploadFolder::GenerateBlock(const ChunkedFolderContent& Content, Content.ChunkedContent.ChunkHashes[ChunkIndex], [this, &Content, &Lookup, &OpenFileCache, ChunkIndex](const IoHash& ChunkHash) -> std::pair { CompositeBuffer Chunk = FetchChunk(Content, Lookup, ChunkHash, OpenFileCache); - if (!Chunk) - { - ZEN_ASSERT(false); - } + ZEN_ASSERT(Chunk); uint64_t RawSize = Chunk.GetSize(); const bool ShouldCompressChunk = RawSize >= m_Options.MinimumSizeForCompressInBlock && diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h index 6304159ae..9e5bf8d91 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h @@ -339,7 +339,8 @@ private: const uint64_t FileOffset, const uint32_t PathIndex); - bool GetBlockWriteOps(std::span ChunkRawHashes, + bool GetBlockWriteOps(const IoHash& BlockRawHash, + std::span ChunkRawHashes, std::span ChunkCompressedLengths, std::span> SequenceIndexChunksLeftToWriteCounters, std::span> RemoteChunkIndexNeedsCopyFromSourceFlags, -- cgit v1.2.3 From ae9c30841074da9226a76c1eb2fb3a3e29086bf6 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 18 Feb 2026 09:40:35 +0100 Subject: add selective request logging support to http.sys (#762) * implemented selective request logging for http.sys for consistency with asio * fixed traversal of GetLogicalProcessorInformationEx to account for variable-sized records * also adds CPU usage metrics --- src/zencore/include/zencore/system.h | 1 + src/zencore/system.cpp | 169 +++++++++++++++++++++++++++-------- src/zenhttp/servers/httpsys.cpp | 25 +++++- 3 files changed, 156 insertions(+), 39 deletions(-) diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h index aec2e0ce4..bf3c15d3d 100644 --- a/src/zencore/include/zencore/system.h +++ b/src/zencore/include/zencore/system.h @@ -25,6 +25,7 @@ struct SystemMetrics uint64_t AvailVirtualMemoryMiB = 0; uint64_t PageFileMiB = 0; uint64_t AvailPageFileMiB = 0; + float CpuUsagePercent = 0.0f; }; SystemMetrics GetSystemMetrics(); diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index e92691781..267c87e12 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -13,6 +13,8 @@ ZEN_THIRD_PARTY_INCLUDES_START # include # include +# include +# pragma comment(lib, "pdh.lib") ZEN_THIRD_PARTY_INCLUDES_END #elif ZEN_PLATFORM_LINUX # include @@ -65,55 +67,98 @@ GetSystemMetrics() // Determine physical core count - DWORD BufferSize = 0; - BOOL Result = GetLogicalProcessorInformationEx(RelationAll, nullptr, &BufferSize); - if (int32_t Error = GetLastError(); Error != ERROR_INSUFFICIENT_BUFFER) { - ThrowSystemError(Error, "Failed to get buffer size for logical processor information"); - } - - PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)Memory::Alloc(BufferSize); + DWORD BufferSize = 0; + BOOL Result = GetLogicalProcessorInformationEx(RelationAll, nullptr, &BufferSize); + if (int32_t Error = GetLastError(); Error != ERROR_INSUFFICIENT_BUFFER) + { + ThrowSystemError(Error, "Failed to get buffer size for logical processor information"); + } - Result = GetLogicalProcessorInformationEx(RelationAll, Buffer, &BufferSize); - if (!Result) - { - Memory::Free(Buffer); - throw std::runtime_error("Failed to get logical processor information"); - } + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX Buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)Memory::Alloc(BufferSize); - DWORD ProcessorPkgCount = 0; - DWORD ProcessorCoreCount = 0; - DWORD ByteOffset = 0; - while (ByteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) <= BufferSize) - { - const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX& Slpi = Buffer[ByteOffset / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX)]; - if (Slpi.Relationship == RelationProcessorCore) + Result = GetLogicalProcessorInformationEx(RelationAll, Buffer, &BufferSize); + if (!Result) { - ProcessorCoreCount++; + Memory::Free(Buffer); + throw std::runtime_error("Failed to get logical processor information"); } - else if (Slpi.Relationship == RelationProcessorPackage) + + DWORD ProcessorPkgCount = 0; + DWORD ProcessorCoreCount = 0; + DWORD LogicalProcessorCount = 0; + + BYTE* Ptr = reinterpret_cast(Buffer); + BYTE* const End = Ptr + BufferSize; + while (Ptr < End) { - ProcessorPkgCount++; + const SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX& Slpi = *reinterpret_cast(Ptr); + if (Slpi.Relationship == RelationProcessorCore) + { + ++ProcessorCoreCount; + + // Count logical processors (threads) across all processor groups for this core. + // Each core entry lists one GROUP_AFFINITY per group it spans; each set bit + // in the Mask represents one logical processor (HyperThreading sibling). + for (WORD g = 0; g < Slpi.Processor.GroupCount; ++g) + { + LogicalProcessorCount += static_cast(__popcnt64(Slpi.Processor.GroupMask[g].Mask)); + } + } + else if (Slpi.Relationship == RelationProcessorPackage) + { + ++ProcessorPkgCount; + } + Ptr += Slpi.Size; } - ByteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX); - } - Metrics.CoreCount = ProcessorCoreCount; - Metrics.CpuCount = ProcessorPkgCount; + Metrics.CoreCount = ProcessorCoreCount; + Metrics.CpuCount = ProcessorPkgCount; + Metrics.LogicalProcessorCount = LogicalProcessorCount; - Memory::Free(Buffer); + Memory::Free(Buffer); + } // Query memory status - MEMORYSTATUSEX MemStatus{.dwLength = sizeof(MEMORYSTATUSEX)}; - GlobalMemoryStatusEx(&MemStatus); + { + MEMORYSTATUSEX MemStatus{.dwLength = sizeof(MEMORYSTATUSEX)}; + GlobalMemoryStatusEx(&MemStatus); + + Metrics.SystemMemoryMiB = MemStatus.ullTotalPhys / 1024 / 1024; + Metrics.AvailSystemMemoryMiB = MemStatus.ullAvailPhys / 1024 / 1024; + Metrics.VirtualMemoryMiB = MemStatus.ullTotalVirtual / 1024 / 1024; + Metrics.AvailVirtualMemoryMiB = MemStatus.ullAvailVirtual / 1024 / 1024; + Metrics.PageFileMiB = MemStatus.ullTotalPageFile / 1024 / 1024; + Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; + } + + // Query CPU usage using PDH + // + // TODO: This should be changed to not require a Sleep, perhaps by using some + // background metrics gathering mechanism. + + { + PDH_HQUERY QueryHandle = nullptr; + PDH_HCOUNTER CounterHandle = nullptr; - Metrics.SystemMemoryMiB = MemStatus.ullTotalPhys / 1024 / 1024; - Metrics.AvailSystemMemoryMiB = MemStatus.ullAvailPhys / 1024 / 1024; - Metrics.VirtualMemoryMiB = MemStatus.ullTotalVirtual / 1024 / 1024; - Metrics.AvailVirtualMemoryMiB = MemStatus.ullAvailVirtual / 1024 / 1024; - Metrics.PageFileMiB = MemStatus.ullTotalPageFile / 1024 / 1024; - Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; + if (PdhOpenQueryW(nullptr, 0, &QueryHandle) == ERROR_SUCCESS) + { + if (PdhAddEnglishCounterW(QueryHandle, L"\\Processor(_Total)\\% Processor Time", 0, &CounterHandle) == ERROR_SUCCESS) + { + PdhCollectQueryData(QueryHandle); + Sleep(100); + PdhCollectQueryData(QueryHandle); + + PDH_FMT_COUNTERVALUE CounterValue; + if (PdhGetFormattedCounterValue(CounterHandle, PDH_FMT_DOUBLE, nullptr, &CounterValue) == ERROR_SUCCESS) + { + Metrics.CpuUsagePercent = static_cast(CounterValue.doubleValue); + } + } + PdhCloseQuery(QueryHandle); + } + } return Metrics; } @@ -190,6 +235,39 @@ GetSystemMetrics() } } + // Query CPU usage + Metrics.CpuUsagePercent = 0.0f; + if (FILE* Stat = fopen("/proc/stat", "r")) + { + char Line[256]; + unsigned long User, Nice, System, Idle, IoWait, Irq, SoftIrq; + static unsigned long PrevUser = 0, PrevNice = 0, PrevSystem = 0, PrevIdle = 0, PrevIoWait = 0, PrevIrq = 0, PrevSoftIrq = 0; + + if (fgets(Line, sizeof(Line), Stat)) + { + if (sscanf(Line, "cpu %lu %lu %lu %lu %lu %lu %lu", &User, &Nice, &System, &Idle, &IoWait, &Irq, &SoftIrq) == 7) + { + unsigned long TotalDelta = (User + Nice + System + Idle + IoWait + Irq + SoftIrq) - + (PrevUser + PrevNice + PrevSystem + PrevIdle + PrevIoWait + PrevIrq + PrevSoftIrq); + unsigned long IdleDelta = Idle - PrevIdle; + + if (TotalDelta > 0) + { + Metrics.CpuUsagePercent = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; + } + + PrevUser = User; + PrevNice = Nice; + PrevSystem = System; + PrevIdle = Idle; + PrevIoWait = IoWait; + PrevIrq = Irq; + PrevSoftIrq = SoftIrq; + } + } + fclose(Stat); + } + // Get memory information long Pages = sysconf(_SC_PHYS_PAGES); long PageSize = sysconf(_SC_PAGE_SIZE); @@ -270,6 +348,25 @@ GetSystemMetrics() sysctlbyname("hw.packages", &Packages, &Size, nullptr, 0); Metrics.CpuCount = Packages > 0 ? Packages : 1; + // Query CPU usage using host_statistics64 + Metrics.CpuUsagePercent = 0.0f; + host_cpu_load_info_data_t CpuLoad; + mach_msg_type_number_t CpuCount = sizeof(CpuLoad) / sizeof(natural_t); + if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (host_info_t)&CpuLoad, &CpuCount) == KERN_SUCCESS) + { + unsigned long TotalTicks = 0; + for (int i = 0; i < CPU_STATE_MAX; ++i) + { + TotalTicks += CpuLoad.cpu_ticks[i]; + } + + if (TotalTicks > 0) + { + unsigned long IdleTicks = CpuLoad.cpu_ticks[CPU_STATE_IDLE]; + Metrics.CpuUsagePercent = 100.0f * (TotalTicks - IdleTicks) / TotalTicks; + } + } + // Get memory information uint64_t MemSize = 0; Size = sizeof(MemSize); diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 14896c803..c640ba90b 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -331,6 +331,8 @@ public: virtual void WriteResponseAsync(std::function&& ContinuationHandler) override; virtual bool TryGetRanges(HttpRanges& Ranges) override; + void LogRequest(HttpMessageResponseRequest* Response); + using HttpServerRequest::WriteResponse; HttpSysServerRequest(const HttpSysServerRequest&) = delete; @@ -429,7 +431,8 @@ public: virtual HttpSysRequestHandler* HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) override; void SuppressResponseBody(); // typically used for HEAD requests - inline int64_t GetResponseBodySize() const { return m_TotalDataSize; } + inline uint16_t GetResponseCode() const { return m_ResponseCode; } + inline int64_t GetResponseBodySize() const { return m_TotalDataSize; } private: eastl::fixed_vector m_HttpDataChunks; @@ -1886,7 +1889,7 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode) ZEN_ASSERT(IsHandled() == false); - auto Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)ResponseCode); + HttpMessageResponseRequest* Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)ResponseCode); if (SuppressBody()) { @@ -1904,6 +1907,7 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode) # endif SetIsHandled(); + LogRequest(Response); } void @@ -1913,7 +1917,7 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentTy ZEN_ASSERT(IsHandled() == false); - auto Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)ResponseCode, ContentType, Blobs); + HttpMessageResponseRequest* Response = new HttpMessageResponseRequest(m_HttpTx, (uint16_t)ResponseCode, ContentType, Blobs); if (SuppressBody()) { @@ -1931,6 +1935,20 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentTy # endif SetIsHandled(); + LogRequest(Response); +} + +void +HttpSysServerRequest::LogRequest(HttpMessageResponseRequest* Response) +{ + if (ShouldLogRequest()) + { + ZEN_INFO("{} {} {} -> {}", + ToString(RequestVerb()), + m_UriUtf8.c_str(), + Response->GetResponseCode(), + NiceBytes(Response->GetResponseBodySize())); + } } void @@ -1959,6 +1977,7 @@ HttpSysServerRequest::WriteResponse(HttpResponseCode ResponseCode, HttpContentTy # endif SetIsHandled(); + LogRequest(Response); } void -- cgit v1.2.3 From 149a5c2faa8d59290b8b44717e504532e906aae2 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 18 Feb 2026 11:28:03 +0100 Subject: structured compute basics (#714) this change adds the `zencompute` component, which can be used to distribute work dispatched from UE using the DDB (Derived Data Build) APIs via zenserver this change also adds a distinct zenserver compute mode (`zenserver compute`) which is intended to be used for leaf compute nodes to exercise the compute functionality without directly involving UE, a `zen exec` subcommand is also added, which can be used to feed replays through the system all new functionality is considered *experimental* and disabled by default at this time, behind the `zencompute` option in xmake config --- docs/compute.md | 152 ++++ src/zen/cmds/exec_cmd.cpp | 654 ++++++++++++++ src/zen/cmds/exec_cmd.h | 97 ++ src/zen/xmake.lua | 5 +- src/zen/zen.cpp | 39 +- src/zencompute-test/xmake.lua | 9 + src/zencompute-test/zencompute-test.cpp | 32 + src/zencompute/actionrecorder.cpp | 258 ++++++ src/zencompute/actionrecorder.h | 91 ++ src/zencompute/functionrunner.cpp | 112 +++ src/zencompute/functionrunner.h | 207 +++++ src/zencompute/functionservice.cpp | 957 ++++++++++++++++++++ src/zencompute/httpfunctionservice.cpp | 709 +++++++++++++++ src/zencompute/httporchestrator.cpp | 81 ++ .../include/zencompute/functionservice.h | 132 +++ .../include/zencompute/httpfunctionservice.h | 73 ++ .../include/zencompute/httporchestrator.h | 44 + .../include/zencompute/recordingreader.h | 127 +++ src/zencompute/include/zencompute/zencompute.h | 11 + src/zencompute/localrunner.cpp | 722 +++++++++++++++ src/zencompute/localrunner.h | 100 +++ src/zencompute/recordingreader.cpp | 335 +++++++ src/zencompute/remotehttprunner.cpp | 457 ++++++++++ src/zencompute/remotehttprunner.h | 80 ++ src/zencompute/xmake.lua | 11 + src/zencompute/zencompute.cpp | 12 + src/zennet/beacon.cpp | 170 ++++ src/zennet/include/zennet/beacon.h | 38 + src/zennet/include/zennet/statsdclient.h | 2 + src/zennet/statsdclient.cpp | 1 + src/zenserver-test/function-tests.cpp | 34 + src/zenserver/compute/computeserver.cpp | 330 +++++++ src/zenserver/compute/computeserver.h | 106 +++ src/zenserver/compute/computeservice.cpp | 100 +++ src/zenserver/compute/computeservice.h | 36 + src/zenserver/frontend/html/compute.html | 991 +++++++++++++++++++++ src/zenserver/main.cpp | 55 +- src/zenserver/storage/storageconfig.cpp | 1 + src/zenserver/storage/storageconfig.h | 1 + src/zenserver/storage/zenstorageserver.cpp | 21 + src/zenserver/storage/zenstorageserver.h | 26 +- src/zenserver/xmake.lua | 4 + src/zenserver/zenserver.cpp | 8 + src/zenserver/zenserver.h | 13 +- src/zentest-appstub/xmake.lua | 3 + src/zentest-appstub/zentest-appstub.cpp | 391 +++++++- thirdparty/xmake.lua | 2 +- xmake.lua | 12 + 48 files changed, 7804 insertions(+), 48 deletions(-) create mode 100644 docs/compute.md create mode 100644 src/zen/cmds/exec_cmd.cpp create mode 100644 src/zen/cmds/exec_cmd.h create mode 100644 src/zencompute-test/xmake.lua create mode 100644 src/zencompute-test/zencompute-test.cpp create mode 100644 src/zencompute/actionrecorder.cpp create mode 100644 src/zencompute/actionrecorder.h create mode 100644 src/zencompute/functionrunner.cpp create mode 100644 src/zencompute/functionrunner.h create mode 100644 src/zencompute/functionservice.cpp create mode 100644 src/zencompute/httpfunctionservice.cpp create mode 100644 src/zencompute/httporchestrator.cpp create mode 100644 src/zencompute/include/zencompute/functionservice.h create mode 100644 src/zencompute/include/zencompute/httpfunctionservice.h create mode 100644 src/zencompute/include/zencompute/httporchestrator.h create mode 100644 src/zencompute/include/zencompute/recordingreader.h create mode 100644 src/zencompute/include/zencompute/zencompute.h create mode 100644 src/zencompute/localrunner.cpp create mode 100644 src/zencompute/localrunner.h create mode 100644 src/zencompute/recordingreader.cpp create mode 100644 src/zencompute/remotehttprunner.cpp create mode 100644 src/zencompute/remotehttprunner.h create mode 100644 src/zencompute/xmake.lua create mode 100644 src/zencompute/zencompute.cpp create mode 100644 src/zennet/beacon.cpp create mode 100644 src/zennet/include/zennet/beacon.h create mode 100644 src/zenserver-test/function-tests.cpp create mode 100644 src/zenserver/compute/computeserver.cpp create mode 100644 src/zenserver/compute/computeserver.h create mode 100644 src/zenserver/compute/computeservice.cpp create mode 100644 src/zenserver/compute/computeservice.h create mode 100644 src/zenserver/frontend/html/compute.html diff --git a/docs/compute.md b/docs/compute.md new file mode 100644 index 000000000..417622f94 --- /dev/null +++ b/docs/compute.md @@ -0,0 +1,152 @@ +# DDC compute interface design documentation + +This is a work in progress + +## General architecture + +The Zen server compute interfaces implement a basic model for distributing compute processes. +Clients can implement [Functions](#functions) in [worker executables](#workers) and dispatch +[actions](#actions) to them via a message based interface. + +The API requires users to describe the actions and the workers explicitly fully up front and the +work is described and submitted as singular objects to the compute service. The model somewhat +resembles Lambda and other stateless compute services but is more tightly constrained to allow +for optimizations and to integrate tightly with the storage components in Zen server. + +This is in contrast with Unreal Build Accelerator in where the worker (remote process) +and the inputs are discovered on-the-fly as the worker progresses and inputs and results +are communicated via relatively high-frequency RPCs. + +### Actions + +An action is described by an action descriptor, which is a compact binary object which +contains a self-contained description of the inputs and the function which should be applied +to generate an output. + +#### Sample Action Descriptor + +``` +work item 4857714dee2383b50b2e7d72afd79848ab5d13f8 (2 attachments): +Function: CompileShaderJobs +FunctionVersion: '83027356-2cf7-41ca-aba5-c81ab0ff2129' +BuildSystemVersion: '17fe280d-ccd8-4be8-a9d1-89c944a70969' +Inputs: + Input: + RawHash: 0c01d9f19033256ca974fced523d1e15b27c1b0a + RawSize: 4482 + Virtual0: + RawHash: dd9bbcb8763badd2f015f94f8f6e360362e2bce0 + RawSize: 3334 +``` + +### Functions + +Functions are identified by a name, and a version specification. For +matching purposes there's also a build system version specification. +When workers are registered with the compute service, they are entered +into a table and as actions stream in the compute subsystem will try to +find a worker which implements the required function using the +`[Function,FunctionVersion,BuildSystemVersion]` tuple. In practice there +may be more than one matching worker and it's up to the compute service +to pick one. + +``` +=== Known functions =========================== +function version build system worker id +CompileShaderJobs 83027356-2cf7-41ca-aba5-c81ab0ff2129 17fe280d-ccd8-4be8-a9d1-89c944a70969 69cb9bb50e9600b5bd5e5ca4ba0f9187b118069a +``` + +### Workers + +A worker is an executable which accepts some command line options which are used to pass the +information required to execute an action. There are two modes, one legacy mode which is +file-based and a streaming mode. + +In the file-based mode the option is simply `-Build=` which points to an action +descriptor in compact binary format (see above). By convention, the referenced inputs are in a folder +named `Inputs` where any input blobs are stored as `CompressedBuffer`-format files named +after the `IoHash` of the uncompressed contents. + +In the streaming mode, the data is provided through a streaming socket interface instead +of using the file system. This eliminates process spawning overheads and enables intra-process +pipelining for greater efficiency. The streaming mode is not yet implemented fully. + +### Worker Descriptors + +Workers are declared by passing a worker descriptor to the compute service. The descriptor +contains information about which executable files are required to execute the worker and how +they need to be laid out. You can optionally also provide additional non-executable files to +go along with the executables. + +The descriptor also lists the functions implemented by the worker. Each function defines +a version which is used when matching actions (the function version is passed in as the +`FunctionVersion` in the action descriptor). + +Each worker links in a small set of common support code which is used to handle the +communication with the invoking program (the 'build system'). To be able to evolve this +interface, each worker also indicates the version of the build system using the +`BuildSystemVersion` attribute. + +#### Sample Worker Descriptor + +``` +worker 69cb9bb50e9600b5bd5e5ca4ba0f9187b118069a: +name: ShaderBuildWorker +path: Engine/Binaries/Win64/ShaderBuildWorker.exe +host: Win64 +buildsystem_version: '17fe280d-ccd8-4be8-a9d1-89c944a70969' +timeout: 300 +cores: 1 +environment: [] +executables: + - name: 'Engine/Binaries/Win64/ShaderBuildWorker-DerivedDataBuildWorker.dll' + hash: f4dbec80e549bae2916288f1b9428c2878d9ae7a + size: 166912 + - name: 'Engine/Binaries/Win64/ShaderBuildWorker-DerivedDataCache.dll' + hash: 8025d561ede05db19b235fc2ef290e2b029c1b8c + size: 4339200 + - name: Engine/Binaries/Win64/ShaderBuildWorker.exe + hash: b85862fca2ce04990470f27bae9ead7f31d9b27e + size: 60928 + - name: Engine/Binaries/Win64/ShaderBuildWorker.modules + hash: 7b05741a69a2ea607c5578668a8de50b04259668 + size: 3739 + - name: Engine/Binaries/Win64/ShaderBuildWorker.version + hash: 8fdfd9f825febf2191b555393e69b32a1d78c24f + size: 259 +files: [] +dirs: + - Engine/Binaries/Win64 +functions: + - name: CompileShaderJobs + version: '83027356-2cf7-41ca-aba5-c81ab0ff2129' +``` + +## API (WIP not final) + +The compute interfaces are currently exposed on the `/apply` endpoint but this +will be subject to change as we adapt the interfaces during development. The LSN +APIs below are intended to replace the action ID oriented APIs. + +The POST APIs typically involve a two-step dance where a descriptor is POSTed and +the service responds with a list of `needs` chunks (identified via `IoHash`) which +it does not have yet. The client can then follow up with a POST of a Compact Binary +Package containing the descriptor along with the needed chunks. + +`/apply/ready` - health check endpoint returns HTTP 200 OK or HTTP 503 + +`/apply/sysinfo` - system information endpoint + +`/apply/record/start`, `/apply/record/stop` - start/stop action recording + +`/apply/workers/{worker}` - GET/POST worker descriptors and payloads + +`/apply/jobs/completed` - GET list of completed actions + +`/apply/jobs/{lsn}` - GET completed action results from LSN, POST action cancellation by LSN, priority changes by LSN + +`/apply/jobs/{worker}/{action}` - GET completed action (job) results by action ID + +`/apply/jobs/{worker}` - GET pending/running jobs for worker, POST requests to schedule action as a job + +`/apply/jobs` - POST request to schedule action as a job diff --git a/src/zen/cmds/exec_cmd.cpp b/src/zen/cmds/exec_cmd.cpp new file mode 100644 index 000000000..2d9d0d12e --- /dev/null +++ b/src/zen/cmds/exec_cmd.cpp @@ -0,0 +1,654 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "exec_cmd.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace std::literals; + +namespace eastl { + +template<> +struct hash : public zen::IoHash::Hasher +{ +}; + +} // namespace eastl + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen { + +ExecCommand::ExecCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName), ""); + m_Options.add_option("", "", "log", "Action log directory", cxxopts::value(m_RecordingLogPath), ""); + m_Options.add_option("", "p", "path", "Recording path (directory or .actionlog file)", cxxopts::value(m_RecordingPath), ""); + m_Options.add_option("", "", "offset", "Recording replay start offset", cxxopts::value(m_Offset), ""); + m_Options.add_option("", "", "stride", "Recording replay stride", cxxopts::value(m_Stride), ""); + m_Options.add_option("", "", "limit", "Recording replay limit", cxxopts::value(m_Limit), ""); + m_Options.add_option("", "", "beacon", "Beacon path", cxxopts::value(m_BeaconPath), ""); + m_Options.add_option("", + "", + "mode", + "Select execution mode (http,inproc,dump,direct,beacon,buildlog)", + cxxopts::value(m_Mode)->default_value("http"), + ""); + m_Options.add_option("", "", "quiet", "Quiet mode (less logging)", cxxopts::value(m_Quiet), ""); + m_Options.parse_positional("mode"); +} + +ExecCommand::~ExecCommand() +{ +} + +void +ExecCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + // Configure + + if (!ParseOptions(argc, argv)) + { + return; + } + + m_HostName = ResolveTargetHostSpec(m_HostName); + + if (m_RecordingPath.empty()) + { + throw OptionParseException("replay path is required!", m_Options.help()); + } + + m_VerboseLogging = GlobalOptions.IsVerbose; + m_QuietLogging = m_Quiet && !m_VerboseLogging; + + enum ExecMode + { + kHttp, + kDirect, + kInproc, + kDump, + kBeacon, + kBuildLog + } Mode; + + if (m_Mode == "http"sv) + { + Mode = kHttp; + } + else if (m_Mode == "direct"sv) + { + Mode = kDirect; + } + else if (m_Mode == "inproc"sv) + { + Mode = kInproc; + } + else if (m_Mode == "dump"sv) + { + Mode = kDump; + } + else if (m_Mode == "beacon"sv) + { + Mode = kBeacon; + } + else if (m_Mode == "buildlog"sv) + { + Mode = kBuildLog; + } + else + { + throw OptionParseException("invalid mode specified!", m_Options.help()); + } + + // Gather information from recording path + + std::unique_ptr Reader; + std::unique_ptr UeReader; + + std::filesystem::path RecordingPath{m_RecordingPath}; + + if (!std::filesystem::is_directory(RecordingPath)) + { + throw OptionParseException("replay path should be a directory path!", m_Options.help()); + } + else + { + if (std::filesystem::is_directory(RecordingPath / "cid")) + { + Reader = std::make_unique(RecordingPath); + m_WorkerMap = Reader->ReadWorkers(); + m_ChunkResolver = Reader.get(); + m_RecordingReader = Reader.get(); + } + else + { + UeReader = std::make_unique(RecordingPath); + m_WorkerMap = UeReader->ReadWorkers(); + m_ChunkResolver = UeReader.get(); + m_RecordingReader = UeReader.get(); + } + } + + ZEN_CONSOLE("found {} workers, {} action items", m_WorkerMap.size(), m_RecordingReader->GetActionCount()); + + for (auto& Kv : m_WorkerMap) + { + CbObject WorkerDesc = Kv.second.GetObject(); + const IoHash& WorkerId = Kv.first; + + RegisterWorkerFunctionsFromDescription(WorkerDesc, WorkerId); + + if (m_VerboseLogging) + { + zen::ExtendableStringBuilder<1024> ObjStr; +# if 0 + zen::CompactBinaryToJson(WorkerDesc, ObjStr); + ZEN_CONSOLE("worker {}: {}", WorkerId, ObjStr); +# else + zen::CompactBinaryToYaml(WorkerDesc, ObjStr); + ZEN_CONSOLE("worker {}:\n{}", WorkerId, ObjStr); +# endif + } + } + + if (m_VerboseLogging) + { + EmitFunctionList(m_FunctionList); + } + + // Iterate over work items and dispatch or log them + + int ReturnValue = 0; + + Stopwatch ExecTimer; + + switch (Mode) + { + case kHttp: + // Forward requests to HTTP function service + ReturnValue = HttpExecute(); + break; + + case kDirect: + // Not currently supported + ReturnValue = LocalMessagingExecute(); + break; + + case kInproc: + // Handle execution in-core (by spawning child processes) + ReturnValue = InProcessExecute(); + break; + + case kDump: + // Dump high level information about actions to console + ReturnValue = DumpWorkItems(); + break; + + case kBeacon: + ReturnValue = BeaconExecute(); + break; + + case kBuildLog: + ReturnValue = BuildActionsLog(); + break; + + default: + ZEN_ERROR("Unknown operating mode! No work submitted"); + + ReturnValue = 1; + } + + ZEN_CONSOLE("complete - took {}", NiceTimeSpanMs(ExecTimer.GetElapsedTimeMs())); + + if (!ReturnValue) + { + ZEN_CONSOLE("all work items completed successfully"); + } + else + { + ZEN_CONSOLE("some work items failed (code {})", ReturnValue); + } +} + +int +ExecCommand::InProcessExecute() +{ + ZEN_ASSERT(m_ChunkResolver); + ChunkResolver& Resolver = *m_ChunkResolver; + + zen::compute::FunctionServiceSession FunctionSession(Resolver); + + std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); + FunctionSession.AddLocalRunner(Resolver, TempPath); + + return ExecUsingSession(FunctionSession); +} + +int +ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSession) +{ + struct JobTracker + { + public: + inline void Insert(int LsnField) + { + RwLock::ExclusiveLockScope _(Lock); + PendingJobs.insert(LsnField); + } + + inline bool IsEmpty() const + { + RwLock::SharedLockScope _(Lock); + return PendingJobs.empty(); + } + + inline void Remove(int CompleteLsn) + { + RwLock::ExclusiveLockScope _(Lock); + PendingJobs.erase(CompleteLsn); + } + + inline size_t GetSize() const + { + RwLock::SharedLockScope _(Lock); + return PendingJobs.size(); + } + + private: + mutable RwLock Lock; + std::unordered_set PendingJobs; + }; + + JobTracker PendingJobs; + + std::atomic IsDraining{0}; + + auto DrainCompletedJobs = [&] { + if (IsDraining.exchange(1)) + { + return; + } + + auto _ = MakeGuard([&] { IsDraining.store(0, std::memory_order_release); }); + + CbObjectWriter Cbo; + FunctionSession.GetCompleted(Cbo); + + if (CbObject Completed = Cbo.Save()) + { + for (auto& It : Completed["completed"sv]) + { + int32_t CompleteLsn = It.AsInt32(); + + CbPackage ResultPackage; + HttpResponseCode Response = FunctionSession.GetActionResult(CompleteLsn, /* out */ ResultPackage); + + if (Response == HttpResponseCode::OK) + { + PendingJobs.Remove(CompleteLsn); + + ZEN_CONSOLE("completed: LSN {} ({} still pending)", CompleteLsn, PendingJobs.GetSize()); + } + } + } + }; + + // Describe workers + + ZEN_CONSOLE("describing {} workers", m_WorkerMap.size()); + + for (auto Kv : m_WorkerMap) + { + CbPackage WorkerDesc = Kv.second; + + FunctionSession.RegisterWorker(WorkerDesc); + } + + // Then submit work items + + int FailedWorkCounter = 0; + size_t RemainingWorkItems = m_RecordingReader->GetActionCount(); + int SubmittedWorkItems = 0; + + ZEN_CONSOLE("submitting {} work items", RemainingWorkItems); + + int OffsetCounter = m_Offset; + int StrideCounter = m_Stride; + + auto ShouldSchedule = [&]() -> bool { + if (m_Limit && SubmittedWorkItems >= m_Limit) + { + // Limit reached, ignore + + return false; + } + + if (OffsetCounter && OffsetCounter--) + { + // Still in offset, ignore + + return false; + } + + if (--StrideCounter == 0) + { + StrideCounter = m_Stride; + + return true; + } + + return false; + }; + + m_RecordingReader->IterateActions( + [&](CbObject ActionObject, const IoHash& ActionId) { + // Enqueue job + + Stopwatch SubmitTimer; + + const int Priority = 0; + + if (ShouldSchedule()) + { + if (m_VerboseLogging) + { + int AttachmentCount = 0; + uint64_t AttachmentBytes = 0; + eastl::hash_set ReferencedChunks; + + ActionObject.IterateAttachments([&](CbFieldView Field) { + IoHash AttachData = Field.AsAttachment(); + + ReferencedChunks.insert(AttachData); + ++AttachmentCount; + + if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachData)) + { + AttachmentBytes += ChunkData.GetSize(); + } + }); + + zen::ExtendableStringBuilder<1024> ObjStr; + zen::CompactBinaryToJson(ActionObject, ObjStr); + ZEN_CONSOLE("work item {} ({} attachments, {} bytes): {}", + ActionId, + AttachmentCount, + NiceBytes(AttachmentBytes), + ObjStr); + } + + if (zen::compute::FunctionServiceSession::EnqueueResult EnqueueResult = + FunctionSession.EnqueueAction(ActionObject, Priority)) + { + const int32_t LsnField = EnqueueResult.Lsn; + + --RemainingWorkItems; + ++SubmittedWorkItems; + + if (!m_QuietLogging) + { + ZEN_CONSOLE("submitted work item #{} - LSN {} - {}. {} remaining", + SubmittedWorkItems, + LsnField, + NiceTimeSpanMs(SubmitTimer.GetElapsedTimeMs()), + RemainingWorkItems); + } + + PendingJobs.Insert(LsnField); + } + else + { + if (!m_QuietLogging) + { + std::string_view FunctionName = ActionObject["Function"sv].AsString(); + const Guid FunctionVersion = ActionObject["FunctionVersion"sv].AsUuid(); + const Guid BuildSystemVersion = ActionObject["BuildSystemVersion"sv].AsUuid(); + + ZEN_ERROR( + "failed to resolve function for work with (Function:{},FunctionVersion:{},BuildSystemVersion:{}). Work " + "descriptor " + "at: 'file://{}'", + std::string(FunctionName), + FunctionVersion, + BuildSystemVersion, + ""); + + EmitFunctionListOnce(m_FunctionList); + } + + ++FailedWorkCounter; + } + } + + // Check for completed work + + DrainCompletedJobs(); + }, + 8); + + // Wait until all pending work is complete + + while (!PendingJobs.IsEmpty()) + { + // TODO: improve this logic + zen::Sleep(500); + + DrainCompletedJobs(); + } + + if (FailedWorkCounter) + { + return 1; + } + + return 0; +} + +int +ExecCommand::LocalMessagingExecute() +{ + // Non-HTTP work submission path + + // To be reimplemented using final transport + + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +int +ExecCommand::HttpExecute() +{ + ZEN_ASSERT(m_ChunkResolver); + ChunkResolver& Resolver = *m_ChunkResolver; + + std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); + + zen::compute::FunctionServiceSession FunctionSession(Resolver); + FunctionSession.AddRemoteRunner(Resolver, TempPath, m_HostName); + + return ExecUsingSession(FunctionSession); +} + +int +ExecCommand::BeaconExecute() +{ + ZEN_ASSERT(m_ChunkResolver); + ChunkResolver& Resolver = *m_ChunkResolver; + std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); + + zen::compute::FunctionServiceSession FunctionSession(Resolver); + FunctionSession.AddRemoteRunner(Resolver, TempPath, "http://localhost:8558"); + // FunctionSession.AddRemoteRunner(Resolver, TempPath, "http://10.99.9.246:8558"); + + return ExecUsingSession(FunctionSession); +} + +////////////////////////////////////////////////////////////////////////// + +void +ExecCommand::RegisterWorkerFunctionsFromDescription(const CbObject& WorkerDesc, const IoHash& WorkerId) +{ + const Guid WorkerBuildSystemVersion = WorkerDesc["buildsystem_version"sv].AsUuid(); + + for (auto& Item : WorkerDesc["functions"sv]) + { + CbObjectView Function = Item.AsObjectView(); + + std::string_view FunctionName = Function["name"sv].AsString(); + const Guid FunctionVersion = Function["version"sv].AsUuid(); + + m_FunctionList.emplace_back(FunctionDefinition{.FunctionName = std::string{FunctionName}, + .FunctionVersion = FunctionVersion, + .BuildSystemVersion = WorkerBuildSystemVersion, + .WorkerId = WorkerId}); + } +} + +void +ExecCommand::EmitFunctionListOnce(const std::vector& FunctionList) +{ + if (m_FunctionListEmittedOnce == false) + { + EmitFunctionList(FunctionList); + + m_FunctionListEmittedOnce = true; + } +} + +int +ExecCommand::DumpWorkItems() +{ + std::atomic EmittedCount{0}; + + eastl::hash_map SeenAttachments; // Attachment CID -> count of references + + m_RecordingReader->IterateActions( + [&](CbObject ActionObject, const IoHash& ActionId) { + eastl::hash_map Attachments; + + uint64_t AttachmentBytes = 0; + uint64_t UncompressedAttachmentBytes = 0; + + ActionObject.IterateAttachments([&](const zen::CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + IoBuffer AttachmentData = m_ChunkResolver->FindChunkByCid(AttachmentCid); + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); + Attachments[AttachmentCid] = CompressedData; + + AttachmentBytes += CompressedData.GetCompressedSize(); + UncompressedAttachmentBytes += CompressedData.DecodeRawSize(); + + if (auto [Iter, Inserted] = SeenAttachments.insert({AttachmentCid, 1}); !Inserted) + { + ++Iter->second; + } + }); + + zen::ExtendableStringBuilder<1024> ObjStr; + +# if 0 + zen::CompactBinaryToJson(ActionObject, ObjStr); + ZEN_CONSOLE("work item {} ({} attachments): {}", ActionId, Attachments.size(), ObjStr); +# else + zen::CompactBinaryToYaml(ActionObject, ObjStr); + ZEN_CONSOLE("work item {} ({} attachments, {}->{} bytes):\n{}", + ActionId, + Attachments.size(), + AttachmentBytes, + UncompressedAttachmentBytes, + ObjStr); +# endif + + ++EmittedCount; + }, + 1); + + ZEN_CONSOLE("emitted: {} actions", EmittedCount.load()); + + eastl::map> ReferenceHistogram; + + for (const auto& [K, V] : SeenAttachments) + { + if (V > 1) + { + ReferenceHistogram[V].push_back(K); + } + } + + for (const auto& [RefCount, Cids] : ReferenceHistogram) + { + ZEN_CONSOLE("{} attachments with {} references", Cids.size(), RefCount); + } + + return 0; +} + +////////////////////////////////////////////////////////////////////////// + +int +ExecCommand::BuildActionsLog() +{ + ZEN_ASSERT(m_ChunkResolver); + ChunkResolver& Resolver = *m_ChunkResolver; + + if (m_RecordingPath.empty()) + { + throw OptionParseException("need to specify recording path", m_Options.help()); + } + + if (std::filesystem::exists(m_RecordingLogPath)) + { + throw OptionParseException(fmt::format("recording log directory '{}' already exists!", m_RecordingLogPath), m_Options.help()); + } + + ZEN_NOT_IMPLEMENTED("build log generation not implemented yet!"); + + std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); + + zen::compute::FunctionServiceSession FunctionSession(Resolver); + FunctionSession.StartRecording(Resolver, m_RecordingLogPath); + + return ExecUsingSession(FunctionSession); +} + +void +ExecCommand::EmitFunctionList(const std::vector& FunctionList) +{ + ZEN_CONSOLE("=== Known functions:\n==========================="); + + ZEN_CONSOLE("{:30} {:36} {:36} {}", "function", "version", "build system", "worker id"); + + for (const FunctionDefinition& Func : FunctionList) + { + ZEN_CONSOLE("{:30} {:36} {:36} {}", Func.FunctionName, Func.FunctionVersion, Func.BuildSystemVersion, Func.WorkerId); + } + + ZEN_CONSOLE("==========================="); +} + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zen/cmds/exec_cmd.h b/src/zen/cmds/exec_cmd.h new file mode 100644 index 000000000..43d092144 --- /dev/null +++ b/src/zen/cmds/exec_cmd.h @@ -0,0 +1,97 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +#include +#include +#include +#include + +#include +#include +#include + +namespace zen { +class CbPackage; +class CbObject; +struct IoHash; +class ChunkResolver; +} // namespace zen + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen::compute { +class FunctionServiceSession; +} + +namespace zen { + +/** + * Zen CLI command for executing functions from a recording + * + * Mostly for testing and debugging purposes + */ + +class ExecCommand : public ZenCmdBase +{ +public: + ExecCommand(); + ~ExecCommand(); + + static constexpr char Name[] = "exec"; + static constexpr char Description[] = "Execute functions from a recording"; + + virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::filesystem::path m_BeaconPath; + std::filesystem::path m_RecordingPath; + std::filesystem::path m_RecordingLogPath; + int m_Offset = 0; + int m_Stride = 1; + int m_Limit = 0; + bool m_Quiet = false; + std::string m_Mode{"http"}; + + struct FunctionDefinition + { + std::string FunctionName; + zen::Guid FunctionVersion; + zen::Guid BuildSystemVersion; + zen::IoHash WorkerId; + }; + + bool m_FunctionListEmittedOnce = false; + void EmitFunctionListOnce(const std::vector& FunctionList); + void EmitFunctionList(const std::vector& FunctionList); + + std::unordered_map m_WorkerMap; + std::vector m_FunctionList; + bool m_VerboseLogging = false; + bool m_QuietLogging = false; + + zen::ChunkResolver* m_ChunkResolver = nullptr; + zen::compute::RecordingReaderBase* m_RecordingReader = nullptr; + + void RegisterWorkerFunctionsFromDescription(const zen::CbObject& WorkerDesc, const zen::IoHash& WorkerId); + + int ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSession); + + // Execution modes + + int DumpWorkItems(); + int HttpExecute(); + int InProcessExecute(); + int LocalMessagingExecute(); + int BeaconExecute(); + int BuildActionsLog(); +}; + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zen/xmake.lua b/src/zen/xmake.lua index ab094fef3..f889c3296 100644 --- a/src/zen/xmake.lua +++ b/src/zen/xmake.lua @@ -6,15 +6,12 @@ target("zen") add_files("**.cpp") add_files("zen.cpp", {unity_ignored = true }) add_deps("zencore", "zenhttp", "zenremotestore", "zenstore", "zenutil") + add_deps("zencompute", "zennet") add_deps("cxxopts", "fmt") add_packages("json11") add_includedirs(".") set_symbols("debug") - if is_mode("release") then - set_optimize("fastest") - end - if is_plat("windows") then add_files("zen.rc") add_ldflags("/subsystem:console,5.02") diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 25245c3d2..018f77738 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -11,6 +11,7 @@ #include "cmds/cache_cmd.h" #include "cmds/copy_cmd.h" #include "cmds/dedup_cmd.h" +#include "cmds/exec_cmd.h" #include "cmds/info_cmd.h" #include "cmds/print_cmd.h" #include "cmds/projectstore_cmd.h" @@ -316,22 +317,25 @@ main(int argc, char** argv) } #endif // ZEN_WITH_TRACE - AttachCommand AttachCmd; - BenchCommand BenchCmd; - BuildsCommand BuildsCmd; - CacheDetailsCommand CacheDetailsCmd; - CacheGetCommand CacheGetCmd; - CacheGenerateCommand CacheGenerateCmd; - CacheInfoCommand CacheInfoCmd; - CacheStatsCommand CacheStatsCmd; - CopyCommand CopyCmd; - CopyStateCommand CopyStateCmd; - CreateOplogCommand CreateOplogCmd; - CreateProjectCommand CreateProjectCmd; - DedupCommand DedupCmd; - DownCommand DownCmd; - DropCommand DropCmd; - DropProjectCommand ProjectDropCmd; + AttachCommand AttachCmd; + BenchCommand BenchCmd; + BuildsCommand BuildsCmd; + CacheDetailsCommand CacheDetailsCmd; + CacheGetCommand CacheGetCmd; + CacheGenerateCommand CacheGenerateCmd; + CacheInfoCommand CacheInfoCmd; + CacheStatsCommand CacheStatsCmd; + CopyCommand CopyCmd; + CopyStateCommand CopyStateCmd; + CreateOplogCommand CreateOplogCmd; + CreateProjectCommand CreateProjectCmd; + DedupCommand DedupCmd; + DownCommand DownCmd; + DropCommand DropCmd; + DropProjectCommand ProjectDropCmd; +#if ZEN_WITH_COMPUTE_SERVICES + ExecCommand ExecCmd; +#endif // ZEN_WITH_COMPUTE_SERVICES ExportOplogCommand ExportOplogCmd; FlushCommand FlushCmd; GcCommand GcCmd; @@ -388,6 +392,9 @@ main(int argc, char** argv) {"dedup", &DedupCmd, "Dedup files"}, {"down", &DownCmd, "Bring zen server down"}, {"drop", &DropCmd, "Drop cache namespace or bucket"}, +#if ZEN_WITH_COMPUTE_SERVICES + {ExecCommand::Name, &ExecCmd, ExecCommand::Description}, +#endif {"gc-status", &GcStatusCmd, "Garbage collect zen storage status check"}, {"gc-stop", &GcStopCmd, "Request cancel of running garbage collection in zen storage"}, {"gc", &GcCmd, "Garbage collect zen storage"}, diff --git a/src/zencompute-test/xmake.lua b/src/zencompute-test/xmake.lua new file mode 100644 index 000000000..64a3c7703 --- /dev/null +++ b/src/zencompute-test/xmake.lua @@ -0,0 +1,9 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +target("zencompute-test") + set_kind("binary") + set_group("tests") + add_headerfiles("**.h") + add_files("*.cpp") + add_deps("zencompute", "zencore") + add_packages("vcpkg::doctest") diff --git a/src/zencompute-test/zencompute-test.cpp b/src/zencompute-test/zencompute-test.cpp new file mode 100644 index 000000000..237812e12 --- /dev/null +++ b/src/zencompute-test/zencompute-test.cpp @@ -0,0 +1,32 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include +#include +#include + +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC +# include +# include +# include +#endif + +#if ZEN_WITH_TESTS +# define ZEN_TEST_WITH_RUNNER 1 +# include +#endif + +int +main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) +{ +#if ZEN_WITH_TESTS + zen::zencompute_forcelinktests(); + + zen::logging::InitializeLogging(); + zen::MaximizeOpenFileCount(); + + return ZEN_RUN_TESTS(argc, argv); +#else + return 0; +#endif +} diff --git a/src/zencompute/actionrecorder.cpp b/src/zencompute/actionrecorder.cpp new file mode 100644 index 000000000..04c4b5141 --- /dev/null +++ b/src/zencompute/actionrecorder.cpp @@ -0,0 +1,258 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "actionrecorder.h" + +#include "functionrunner.h" + +#include +#include +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +# define ZEN_CONCRT_AVAILABLE 1 +#else +# define ZEN_CONCRT_AVAILABLE 0 +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen::compute { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +RecordingFileWriter::RecordingFileWriter() +{ +} + +RecordingFileWriter::~RecordingFileWriter() +{ + Close(); +} + +void +RecordingFileWriter::Open(std::filesystem::path FilePath) +{ + using namespace std::literals; + + m_File.Open(FilePath, BasicFile::Mode::kTruncate); + m_File.Write("----DDC2----DATA", 16, 0); + m_FileOffset = 16; + + std::filesystem::path TocPath = FilePath.replace_extension(".ztoc"); + m_TocFile.Open(TocPath, BasicFile::Mode::kTruncate); + + m_TocWriter << "version"sv << 1; + m_TocWriter.BeginArray("toc"sv); +} + +void +RecordingFileWriter::Close() +{ + m_TocWriter.EndArray(); + CbObject Toc = m_TocWriter.Save(); + + std::error_code Ec; + m_TocFile.WriteAll(Toc.GetBuffer().AsIoBuffer(), Ec); +} + +void +RecordingFileWriter::AppendObject(const CbObject& Object, const IoHash& ObjectHash) +{ + RwLock::ExclusiveLockScope _(m_FileLock); + + MemoryView ObjectView = Object.GetBuffer().GetView(); + + std::error_code Ec; + m_File.Write(ObjectView, m_FileOffset, Ec); + + if (Ec) + { + throw std::system_error(Ec, "failed writing to archive"); + } + + m_TocWriter.BeginArray(); + m_TocWriter.AddHash(ObjectHash); + m_TocWriter.AddInteger(m_FileOffset); + m_TocWriter.AddInteger(gsl::narrow(ObjectView.GetSize())); + m_TocWriter.EndArray(); + + m_FileOffset += ObjectView.GetSize(); +} + +////////////////////////////////////////////////////////////////////////// + +ActionRecorder::ActionRecorder(ChunkResolver& InChunkResolver, const std::filesystem::path& RecordingLogPath) +: m_ChunkResolver(InChunkResolver) +, m_RecordingLogDir(RecordingLogPath) +{ + std::error_code Ec; + CreateDirectories(m_RecordingLogDir, Ec); + + if (Ec) + { + ZEN_WARN("Could not create directory '{}': {}", m_RecordingLogDir, Ec.message()); + } + + CleanDirectory(m_RecordingLogDir, /* ForceRemoveReadOnlyFiles */ true, Ec); + + if (Ec) + { + ZEN_WARN("Could not clean directory '{}': {}", m_RecordingLogDir, Ec.message()); + } + + m_WorkersFile.Open(m_RecordingLogDir / "workers.zdat"); + m_ActionsFile.Open(m_RecordingLogDir / "actions.zdat"); + + CidStoreConfiguration CidConfig; + CidConfig.RootDirectory = m_RecordingLogDir / "cid"; + CidConfig.HugeValueThreshold = 128 * 1024 * 1024; + + m_CidStore.Initialize(CidConfig); +} + +ActionRecorder::~ActionRecorder() +{ + Shutdown(); +} + +void +ActionRecorder::Shutdown() +{ + m_CidStore.Flush(); +} + +void +ActionRecorder::RegisterWorker(const CbPackage& WorkerPackage) +{ + const IoHash WorkerId = WorkerPackage.GetObjectHash(); + + m_WorkersFile.AppendObject(WorkerPackage.GetObject(), WorkerId); + + std::unordered_set AddedChunks; + uint64_t AddedBytes = 0; + + // First add all attachments from the worker package itself + + for (const CbAttachment& Attachment : WorkerPackage.GetAttachments()) + { + CompressedBuffer Buffer = Attachment.AsCompressedBinary(); + IoBuffer Data = Buffer.GetCompressed().Flatten().AsIoBuffer(); + + const IoHash ChunkHash = Buffer.DecodeRawHash(); + + CidStore::InsertResult Result = m_CidStore.AddChunk(Data, ChunkHash, CidStore::InsertMode::kCopyOnly); + + AddedChunks.insert(ChunkHash); + + if (Result.New) + { + AddedBytes += Data.GetSize(); + } + } + + // Not all attachments will be present in the worker package, so we need to add + // all referenced chunks to ensure that the recording is self-contained and not + // referencing data in the main CID store + + CbObject WorkerDescriptor = WorkerPackage.GetObject(); + + WorkerDescriptor.IterateAttachments([&](const CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + + if (!AddedChunks.contains(AttachmentCid)) + { + IoBuffer AttachmentData = m_ChunkResolver.FindChunkByCid(AttachmentCid); + + if (AttachmentData) + { + CidStore::InsertResult Result = m_CidStore.AddChunk(AttachmentData, AttachmentCid, CidStore::InsertMode::kCopyOnly); + + if (Result.New) + { + AddedBytes += AttachmentData.GetSize(); + } + } + else + { + ZEN_WARN("RegisterWorker: could not resolve attachment chunk {} for worker {}", AttachmentCid, WorkerId); + } + + AddedChunks.insert(AttachmentCid); + } + }); + + ZEN_INFO("recorded worker {} with {} attachments ({} bytes)", WorkerId, AddedChunks.size(), AddedBytes); +} + +bool +ActionRecorder::RecordAction(Ref Action) +{ + bool AllGood = true; + + Action->ActionObj.IterateAttachments([&](CbFieldView Field) { + IoHash AttachData = Field.AsHash(); + IoBuffer ChunkData = m_ChunkResolver.FindChunkByCid(AttachData); + + if (ChunkData) + { + if (ChunkData.GetContentType() == ZenContentType::kCompressedBinary) + { + IoHash DecompressedHash; + uint64_t RawSize = 0; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer(ChunkData), /* out */ DecompressedHash, /* out*/ RawSize); + + OodleCompressor Compressor; + OodleCompressionLevel CompressionLevel; + uint64_t BlockSize = 0; + if (Compressed.TryGetCompressParameters(/* out */ Compressor, /* out */ CompressionLevel, /* out */ BlockSize)) + { + if (Compressor == OodleCompressor::NotSet) + { + CompositeBuffer Decompressed = Compressed.DecompressToComposite(); + CompressedBuffer NewCompressed = CompressedBuffer::Compress(std::move(Decompressed), + OodleCompressor::Mermaid, + OodleCompressionLevel::Fast, + BlockSize); + + ChunkData = NewCompressed.GetCompressed().Flatten().AsIoBuffer(); + } + } + } + + const uint64_t ChunkSize = ChunkData.GetSize(); + + m_CidStore.AddChunk(ChunkData, AttachData, CidStore::InsertMode::kCopyOnly); + ++m_ChunkCounter; + m_ChunkBytesCounter.fetch_add(ChunkSize); + } + else + { + AllGood = false; + + ZEN_WARN("could not resolve chunk {}", AttachData); + } + }); + + if (AllGood) + { + m_ActionsFile.AppendObject(Action->ActionObj, Action->ActionId); + ++m_ActionsCounter; + + return true; + } + else + { + return false; + } +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/actionrecorder.h b/src/zencompute/actionrecorder.h new file mode 100644 index 000000000..9cc2b44a2 --- /dev/null +++ b/src/zencompute/actionrecorder.h @@ -0,0 +1,91 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace zen { +class CbObject; +class CbPackage; +struct IoHash; +} // namespace zen + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen::compute { + +////////////////////////////////////////////////////////////////////////// + +struct RecordingFileWriter +{ + RecordingFileWriter(RecordingFileWriter&&) = delete; + RecordingFileWriter& operator=(RecordingFileWriter&&) = delete; + + RwLock m_FileLock; + BasicFile m_File; + uint64_t m_FileOffset = 0; + CbObjectWriter m_TocWriter; + BasicFile m_TocFile; + + RecordingFileWriter(); + ~RecordingFileWriter(); + + void Open(std::filesystem::path FilePath); + void Close(); + void AppendObject(const CbObject& Object, const IoHash& ObjectHash); +}; + +////////////////////////////////////////////////////////////////////////// + +/** + * Recording "runner" implementation + * + * This class writes out all actions and their attachments to a recording directory + * in a format that can be read back by the RecordingReader. + * + * The contents of the recording directory will be self-contained, with all referenced + * attachments stored in the recording directory itself, so that the recording can be + * moved or shared without needing to maintain references to the main CID store. + * + */ + +class ActionRecorder +{ +public: + ActionRecorder(ChunkResolver& InChunkResolver, const std::filesystem::path& RecordingLogPath); + ~ActionRecorder(); + + ActionRecorder(const ActionRecorder&) = delete; + ActionRecorder& operator=(const ActionRecorder&) = delete; + + void Shutdown(); + void RegisterWorker(const CbPackage& WorkerPackage); + bool RecordAction(Ref Action); + +private: + ChunkResolver& m_ChunkResolver; + std::filesystem::path m_RecordingLogDir; + + RecordingFileWriter m_WorkersFile; + RecordingFileWriter m_ActionsFile; + GcManager m_Gc; + CidStore m_CidStore{m_Gc}; + std::atomic m_ChunkCounter{0}; + std::atomic m_ChunkBytesCounter{0}; + std::atomic m_ActionsCounter{0}; +}; + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/functionrunner.cpp b/src/zencompute/functionrunner.cpp new file mode 100644 index 000000000..8e7c12b2b --- /dev/null +++ b/src/zencompute/functionrunner.cpp @@ -0,0 +1,112 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "functionrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include + +# include +# include + +namespace zen::compute { + +FunctionRunner::FunctionRunner(std::filesystem::path BasePath) : m_ActionsPath(BasePath / "actions") +{ +} + +FunctionRunner::~FunctionRunner() = default; + +size_t +FunctionRunner::QueryCapacity() +{ + return 1; +} + +std::vector +FunctionRunner::SubmitActions(const std::vector>& Actions) +{ + std::vector Results; + Results.reserve(Actions.size()); + + for (const Ref& Action : Actions) + { + Results.push_back(SubmitAction(Action)); + } + + return Results; +} + +void +FunctionRunner::MaybeDumpAction(int ActionLsn, const CbObject& ActionObject) +{ + if (m_DumpActions) + { + std::string UniqueId = fmt::format("{}.ddb", ActionLsn); + std::filesystem::path Path = m_ActionsPath / UniqueId; + + zen::WriteFile(Path, IoBuffer(ActionObject.GetBuffer().AsIoBuffer())); + } +} + +////////////////////////////////////////////////////////////////////////// + +RunnerAction::RunnerAction(FunctionServiceSession* OwnerSession) : m_OwnerSession(OwnerSession) +{ + this->Timestamps[static_cast(State::New)] = DateTime::Now().GetTicks(); +} + +RunnerAction::~RunnerAction() +{ +} + +void +RunnerAction::SetActionState(State NewState) +{ + ZEN_ASSERT(NewState < State::_Count); + this->Timestamps[static_cast(NewState)] = DateTime::Now().GetTicks(); + + do + { + if (State CurrentState = m_ActionState.load(); CurrentState == NewState) + { + // No state change + return; + } + else + { + if (NewState <= CurrentState) + { + // Cannot transition to an earlier or same state + return; + } + + if (m_ActionState.compare_exchange_strong(CurrentState, NewState)) + { + // Successful state change + + m_OwnerSession->PostUpdate(this); + + return; + } + } + } while (true); +} + +void +RunnerAction::SetResult(CbPackage&& Result) +{ + m_Result = std::move(Result); +} + +CbPackage& +RunnerAction::GetResult() +{ + ZEN_ASSERT(IsCompleted()); + return m_Result; +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES \ No newline at end of file diff --git a/src/zencompute/functionrunner.h b/src/zencompute/functionrunner.h new file mode 100644 index 000000000..6fd0d84cc --- /dev/null +++ b/src/zencompute/functionrunner.h @@ -0,0 +1,207 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include + +namespace zen::compute { + +struct SubmitResult +{ + bool IsAccepted = false; + std::string Reason; +}; + +/** Base interface for classes implementing a remote execution "runner" + */ +class FunctionRunner : public RefCounted +{ + FunctionRunner(FunctionRunner&&) = delete; + FunctionRunner& operator=(FunctionRunner&&) = delete; + +public: + FunctionRunner(std::filesystem::path BasePath); + virtual ~FunctionRunner() = 0; + + virtual void Shutdown() = 0; + virtual void RegisterWorker(const CbPackage& WorkerPackage) = 0; + + [[nodiscard]] virtual SubmitResult SubmitAction(Ref Action) = 0; + [[nodiscard]] virtual size_t GetSubmittedActionCount() = 0; + [[nodiscard]] virtual bool IsHealthy() = 0; + [[nodiscard]] virtual size_t QueryCapacity(); + [[nodiscard]] virtual std::vector SubmitActions(const std::vector>& Actions); + +protected: + std::filesystem::path m_ActionsPath; + bool m_DumpActions = false; + void MaybeDumpAction(int ActionLsn, const CbObject& ActionObject); +}; + +template +struct RunnerGroup +{ + void AddRunner(RunnerType* Runner) + { + m_RunnersLock.WithExclusiveLock([&] { m_Runners.emplace_back(Runner); }); + } + size_t QueryCapacity() + { + size_t TotalCapacity = 0; + m_RunnersLock.WithSharedLock([&] { + for (const auto& Runner : m_Runners) + { + TotalCapacity += Runner->QueryCapacity(); + } + }); + return TotalCapacity; + } + + SubmitResult SubmitAction(Ref Action) + { + RwLock::SharedLockScope _(m_RunnersLock); + + const int InitialIndex = m_NextSubmitIndex.load(std::memory_order_acquire); + int Index = InitialIndex; + const int RunnerCount = gsl::narrow(m_Runners.size()); + + if (RunnerCount == 0) + { + return {.IsAccepted = false, .Reason = "No runners available"}; + } + + do + { + while (Index >= RunnerCount) + { + Index -= RunnerCount; + } + + auto& Runner = m_Runners[Index++]; + + SubmitResult Result = Runner->SubmitAction(Action); + + if (Result.IsAccepted == true) + { + m_NextSubmitIndex = Index % RunnerCount; + + return Result; + } + + while (Index >= RunnerCount) + { + Index -= RunnerCount; + } + } while (Index != InitialIndex); + + return {.IsAccepted = false}; + } + + size_t GetSubmittedActionCount() + { + RwLock::SharedLockScope _(m_RunnersLock); + + size_t TotalCount = 0; + + for (const auto& Runner : m_Runners) + { + TotalCount += Runner->GetSubmittedActionCount(); + } + + return TotalCount; + } + + void RegisterWorker(CbPackage Worker) + { + RwLock::SharedLockScope _(m_RunnersLock); + + for (auto& Runner : m_Runners) + { + Runner->RegisterWorker(Worker); + } + } + + void Shutdown() + { + RwLock::SharedLockScope _(m_RunnersLock); + + for (auto& Runner : m_Runners) + { + Runner->Shutdown(); + } + } + +private: + RwLock m_RunnersLock; + std::vector> m_Runners; + std::atomic m_NextSubmitIndex{0}; +}; + +/** + * This represents an action going through different stages of scheduling and execution. + */ +struct RunnerAction : public RefCounted +{ + explicit RunnerAction(FunctionServiceSession* OwnerSession); + ~RunnerAction(); + + int ActionLsn = 0; + WorkerDesc Worker; + IoHash ActionId; + CbObject ActionObj; + int Priority = 0; + + enum class State + { + New, + Pending, + Running, + Completed, + Failed, + _Count + }; + + static const char* ToString(State _) + { + switch (_) + { + case State::New: + return "New"; + case State::Pending: + return "Pending"; + case State::Running: + return "Running"; + case State::Completed: + return "Completed"; + case State::Failed: + return "Failed"; + default: + return "Unknown"; + } + } + + uint64_t Timestamps[static_cast(State::_Count)] = {}; + + State ActionState() const { return m_ActionState; } + void SetActionState(State NewState); + + bool IsSuccess() const { return ActionState() == State::Completed; } + bool IsCompleted() const { return ActionState() == State::Completed || ActionState() == State::Failed; } + + void SetResult(CbPackage&& Result); + CbPackage& GetResult(); + +private: + std::atomic m_ActionState = State::New; + FunctionServiceSession* m_OwnerSession = nullptr; + CbPackage m_Result; +}; + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES \ No newline at end of file diff --git a/src/zencompute/functionservice.cpp b/src/zencompute/functionservice.cpp new file mode 100644 index 000000000..0698449e9 --- /dev/null +++ b/src/zencompute/functionservice.cpp @@ -0,0 +1,957 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencompute/functionservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "functionrunner.h" +# include "actionrecorder.h" +# include "localrunner.h" +# include "remotehttprunner.h" + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include +# include +# include +# include +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +ZEN_THIRD_PARTY_INCLUDES_END + +using namespace std::literals; + +namespace zen::compute { + +////////////////////////////////////////////////////////////////////////// + +struct FunctionServiceSession::Impl +{ + FunctionServiceSession* m_FunctionServiceSession; + ChunkResolver& m_ChunkResolver; + LoggerRef m_Log{logging::Get("apply")}; + + Impl(FunctionServiceSession* InFunctionServiceSession, ChunkResolver& InChunkResolver) + : m_FunctionServiceSession(InFunctionServiceSession) + , m_ChunkResolver(InChunkResolver) + { + m_SchedulingThread = std::thread{&Impl::MonitorThreadFunction, this}; + } + + void Shutdown(); + bool IsHealthy(); + + LoggerRef Log() { return m_Log; } + + std::atomic_bool m_AcceptActions = true; + + struct FunctionDefinition + { + std::string FunctionName; + Guid FunctionVersion; + Guid BuildSystemVersion; + IoHash WorkerId; + }; + + void EmitStats(CbObjectWriter& Cbo) + { + m_WorkerLock.WithSharedLock([&] { Cbo << "worker_count"sv << m_WorkerMap.size(); }); + m_ResultsLock.WithSharedLock([&] { Cbo << "actions_complete"sv << m_ResultsMap.size(); }); + m_PendingLock.WithSharedLock([&] { Cbo << "actions_pending"sv << m_PendingActions.size(); }); + Cbo << "actions_submitted"sv << GetSubmittedActionCount(); + EmitSnapshot("actions_retired"sv, m_ResultRate, Cbo); + } + + void RegisterWorker(CbPackage Worker); + WorkerDesc GetWorkerDescriptor(const IoHash& WorkerId); + + std::atomic m_ActionsCounter = 0; // sequence number + + RwLock m_PendingLock; + std::map> m_PendingActions; + + RwLock m_RunningLock; + std::unordered_map> m_RunningMap; + + RwLock m_ResultsLock; + std::unordered_map> m_ResultsMap; + metrics::Meter m_ResultRate; + std::atomic m_RetiredCount{0}; + + HttpResponseCode GetActionResult(int ActionLsn, CbPackage& OutResultPackage); + HttpResponseCode FindActionResult(const IoHash& ActionId, CbPackage& ResultPackage); + + std::atomic m_ShutdownRequested{false}; + + std::thread m_SchedulingThread; + std::atomic m_SchedulingThreadEnabled{true}; + Event m_SchedulingThreadEvent; + + void MonitorThreadFunction(); + void SchedulePendingActions(); + + // Workers + + RwLock m_WorkerLock; + std::unordered_map m_WorkerMap; + std::vector m_FunctionList; + std::vector GetKnownWorkerIds(); + + // Runners + + RunnerGroup m_LocalRunnerGroup; + RunnerGroup m_RemoteRunnerGroup; + + EnqueueResult EnqueueAction(CbObject ActionObject, int Priority); + EnqueueResult EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int RequestPriority); + + void GetCompleted(CbWriter& Cbo); + + // Recording + + void StartRecording(ChunkResolver& InCidStore, const std::filesystem::path& RecordingPath); + void StopRecording(); + + std::unique_ptr m_Recorder; + + // History tracking + + RwLock m_ActionHistoryLock; + std::deque m_ActionHistory; + size_t m_HistoryLimit = 1000; + + std::vector GetActionHistory(int Limit); + + // + + [[nodiscard]] size_t QueryCapacity(); + + [[nodiscard]] SubmitResult SubmitAction(Ref Action); + [[nodiscard]] std::vector SubmitActions(const std::vector>& Actions); + [[nodiscard]] size_t GetSubmittedActionCount(); + + // Updates + + RwLock m_UpdatedActionsLock; + std::vector> m_UpdatedActions; + + void HandleActionUpdates(); + void PostUpdate(RunnerAction* Action); + + void ShutdownRunners(); +}; + +bool +FunctionServiceSession::Impl::IsHealthy() +{ + return true; +} + +void +FunctionServiceSession::Impl::Shutdown() +{ + m_AcceptActions = false; + m_ShutdownRequested = true; + + m_SchedulingThreadEnabled = false; + m_SchedulingThreadEvent.Set(); + if (m_SchedulingThread.joinable()) + { + m_SchedulingThread.join(); + } + + ShutdownRunners(); +} + +void +FunctionServiceSession::Impl::ShutdownRunners() +{ + m_LocalRunnerGroup.Shutdown(); + m_RemoteRunnerGroup.Shutdown(); +} + +void +FunctionServiceSession::Impl::StartRecording(ChunkResolver& InCidStore, const std::filesystem::path& RecordingPath) +{ + ZEN_INFO("starting recording to '{}'", RecordingPath); + + m_Recorder = std::make_unique(InCidStore, RecordingPath); + + ZEN_INFO("started recording to '{}'", RecordingPath); +} + +void +FunctionServiceSession::Impl::StopRecording() +{ + ZEN_INFO("stopping recording"); + + m_Recorder = nullptr; + + ZEN_INFO("stopped recording"); +} + +std::vector +FunctionServiceSession::Impl::GetActionHistory(int Limit) +{ + RwLock::SharedLockScope _(m_ActionHistoryLock); + + if (Limit > 0 && static_cast(Limit) < m_ActionHistory.size()) + { + return std::vector(m_ActionHistory.end() - Limit, m_ActionHistory.end()); + } + + return std::vector(m_ActionHistory.begin(), m_ActionHistory.end()); +} + +void +FunctionServiceSession::Impl::RegisterWorker(CbPackage Worker) +{ + RwLock::ExclusiveLockScope _(m_WorkerLock); + + const IoHash& WorkerId = Worker.GetObject().GetHash(); + + if (m_WorkerMap.insert_or_assign(WorkerId, Worker).second) + { + // Note that since the convention currently is that WorkerId is equal to the hash + // of the worker descriptor there is no chance that we get a second write with a + // different descriptor. Thus we only need to call this the first time, when the + // worker is added + + m_LocalRunnerGroup.RegisterWorker(Worker); + m_RemoteRunnerGroup.RegisterWorker(Worker); + + if (m_Recorder) + { + m_Recorder->RegisterWorker(Worker); + } + + CbObject WorkerObj = Worker.GetObject(); + + // Populate worker database + + const Guid WorkerBuildSystemVersion = WorkerObj["buildsystem_version"sv].AsUuid(); + + for (auto& Item : WorkerObj["functions"sv]) + { + CbObjectView Function = Item.AsObjectView(); + + std::string_view FunctionName = Function["name"sv].AsString(); + const Guid FunctionVersion = Function["version"sv].AsUuid(); + + m_FunctionList.emplace_back(FunctionDefinition{.FunctionName = std::string{FunctionName}, + .FunctionVersion = FunctionVersion, + .BuildSystemVersion = WorkerBuildSystemVersion, + .WorkerId = WorkerId}); + } + } +} + +WorkerDesc +FunctionServiceSession::Impl::GetWorkerDescriptor(const IoHash& WorkerId) +{ + RwLock::SharedLockScope _(m_WorkerLock); + + if (auto It = m_WorkerMap.find(WorkerId); It != m_WorkerMap.end()) + { + const CbPackage& Desc = It->second; + return {Desc, WorkerId}; + } + + return {}; +} + +std::vector +FunctionServiceSession::Impl::GetKnownWorkerIds() +{ + std::vector WorkerIds; + WorkerIds.reserve(m_WorkerMap.size()); + + m_WorkerLock.WithSharedLock([&] { + for (const auto& [WorkerId, _] : m_WorkerMap) + { + WorkerIds.push_back(WorkerId); + } + }); + + return WorkerIds; +} + +FunctionServiceSession::EnqueueResult +FunctionServiceSession::Impl::EnqueueAction(CbObject ActionObject, int Priority) +{ + // Resolve function to worker + + IoHash WorkerId{IoHash::Zero}; + + std::string_view FunctionName = ActionObject["Function"sv].AsString(); + const Guid FunctionVersion = ActionObject["FunctionVersion"sv].AsUuid(); + const Guid BuildSystemVersion = ActionObject["BuildSystemVersion"sv].AsUuid(); + + for (const FunctionDefinition& FuncDef : m_FunctionList) + { + if (FuncDef.FunctionName == FunctionName && FuncDef.FunctionVersion == FunctionVersion && + FuncDef.BuildSystemVersion == BuildSystemVersion) + { + WorkerId = FuncDef.WorkerId; + + break; + } + } + + if (WorkerId == IoHash::Zero) + { + CbObjectWriter Writer; + + Writer << "Function"sv << FunctionName << "FunctionVersion"sv << FunctionVersion << "BuildSystemVersion" << BuildSystemVersion; + Writer << "error" + << "no worker matches the action specification"; + + return {0, Writer.Save()}; + } + + if (auto It = m_WorkerMap.find(WorkerId); It != m_WorkerMap.end()) + { + CbPackage WorkerPackage = It->second; + + return EnqueueResolvedAction(WorkerDesc{WorkerPackage, WorkerId}, ActionObject, Priority); + } + + CbObjectWriter Writer; + + Writer << "Function"sv << FunctionName << "FunctionVersion"sv << FunctionVersion << "BuildSystemVersion" << BuildSystemVersion; + Writer << "error" + << "no worker found despite match"; + + return {0, Writer.Save()}; +} + +FunctionServiceSession::EnqueueResult +FunctionServiceSession::Impl::EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int RequestPriority) +{ + const int ActionLsn = ++m_ActionsCounter; + + Ref Pending{new RunnerAction(m_FunctionServiceSession)}; + + Pending->ActionLsn = ActionLsn; + Pending->Worker = Worker; + Pending->ActionId = ActionObj.GetHash(); + Pending->ActionObj = ActionObj; + Pending->Priority = RequestPriority; + + SubmitResult SubResult = SubmitAction(Pending); + + if (SubResult.IsAccepted) + { + // Great, the job is being taken care of by the runner + ZEN_DEBUG("direct schedule LSN {}", Pending->ActionLsn); + } + else + { + ZEN_DEBUG("action {} ({}) PENDING", Pending->ActionId, Pending->ActionLsn); + + Pending->SetActionState(RunnerAction::State::Pending); + } + + if (m_Recorder) + { + m_Recorder->RecordAction(Pending); + } + + CbObjectWriter Writer; + Writer << "lsn" << Pending->ActionLsn; + Writer << "worker" << Pending->Worker.WorkerId; + Writer << "action" << Pending->ActionId; + + return {Pending->ActionLsn, Writer.Save()}; +} + +SubmitResult +FunctionServiceSession::Impl::SubmitAction(Ref Action) +{ + // Loosely round-robin scheduling of actions across runners. + // + // It's not entirely clear what this means given that submits + // can come in across multiple threads, but it's probably better + // than always starting with the first runner. + // + // Longer term we should track the state of the individual + // runners and make decisions accordingly. + + SubmitResult Result = m_LocalRunnerGroup.SubmitAction(Action); + if (Result.IsAccepted) + { + return Result; + } + + return m_RemoteRunnerGroup.SubmitAction(Action); +} + +size_t +FunctionServiceSession::Impl::GetSubmittedActionCount() +{ + return m_LocalRunnerGroup.GetSubmittedActionCount() + m_RemoteRunnerGroup.GetSubmittedActionCount(); +} + +HttpResponseCode +FunctionServiceSession::Impl::GetActionResult(int ActionLsn, CbPackage& OutResultPackage) +{ + // This lock is held for the duration of the function since we need to + // be sure that the action doesn't change state while we are checking the + // different data structures + + RwLock::ExclusiveLockScope _(m_ResultsLock); + + if (auto It = m_ResultsMap.find(ActionLsn); It != m_ResultsMap.end()) + { + OutResultPackage = std::move(It->second->GetResult()); + + m_ResultsMap.erase(It); + + return HttpResponseCode::OK; + } + + { + RwLock::SharedLockScope __(m_PendingLock); + + if (auto FindIt = m_PendingActions.find(ActionLsn); FindIt != m_PendingActions.end()) + { + return HttpResponseCode::Accepted; + } + } + + // Lock order is important here to avoid deadlocks, RwLock m_RunningLock must + // always be taken after m_ResultsLock if both are needed + + { + RwLock::SharedLockScope __(m_RunningLock); + + if (m_RunningMap.find(ActionLsn) != m_RunningMap.end()) + { + return HttpResponseCode::Accepted; + } + } + + return HttpResponseCode::NotFound; +} + +HttpResponseCode +FunctionServiceSession::Impl::FindActionResult(const IoHash& ActionId, CbPackage& OutResultPackage) +{ + // This lock is held for the duration of the function since we need to + // be sure that the action doesn't change state while we are checking the + // different data structures + + RwLock::ExclusiveLockScope _(m_ResultsLock); + + for (auto It = begin(m_ResultsMap), End = end(m_ResultsMap); It != End; ++It) + { + if (It->second->ActionId == ActionId) + { + OutResultPackage = std::move(It->second->GetResult()); + + m_ResultsMap.erase(It); + + return HttpResponseCode::OK; + } + } + + { + RwLock::SharedLockScope __(m_PendingLock); + + for (const auto& [K, Pending] : m_PendingActions) + { + if (Pending->ActionId == ActionId) + { + return HttpResponseCode::Accepted; + } + } + } + + // Lock order is important here to avoid deadlocks, RwLock m_RunningLock must + // always be taken after m_ResultsLock if both are needed + + { + RwLock::SharedLockScope __(m_RunningLock); + + for (const auto& [K, v] : m_RunningMap) + { + if (v->ActionId == ActionId) + { + return HttpResponseCode::Accepted; + } + } + } + + return HttpResponseCode::NotFound; +} + +void +FunctionServiceSession::Impl::GetCompleted(CbWriter& Cbo) +{ + Cbo.BeginArray("completed"); + + m_ResultsLock.WithSharedLock([&] { + for (auto& Kv : m_ResultsMap) + { + Cbo << Kv.first; + } + }); + + Cbo.EndArray(); +} + +# define ZEN_BATCH_SCHEDULER 1 + +void +FunctionServiceSession::Impl::SchedulePendingActions() +{ + int ScheduledCount = 0; + size_t RunningCount = m_RunningLock.WithSharedLock([&] { return m_RunningMap.size(); }); + size_t PendingCount = m_PendingLock.WithSharedLock([&] { return m_PendingActions.size(); }); + size_t ResultCount = m_ResultsLock.WithSharedLock([&] { return m_ResultsMap.size(); }); + + static Stopwatch DumpRunningTimer; + + auto _ = MakeGuard([&] { + ZEN_INFO("scheduled {} pending actions. {} running ({} retired), {} still pending, {} results", + ScheduledCount, + RunningCount, + m_RetiredCount.load(), + PendingCount, + ResultCount); + + if (DumpRunningTimer.GetElapsedTimeMs() > 30000) + { + DumpRunningTimer.Reset(); + + std::set RunningList; + m_RunningLock.WithSharedLock([&] { + for (auto& [K, V] : m_RunningMap) + { + RunningList.insert(K); + } + }); + + ExtendableStringBuilder<1024> RunningString; + for (int i : RunningList) + { + if (RunningString.Size()) + { + RunningString << ", "; + } + + RunningString.Append(IntNum(i)); + } + + ZEN_INFO("running: {}", RunningString); + } + }); + +# if ZEN_BATCH_SCHEDULER + size_t Capacity = QueryCapacity(); + + if (!Capacity) + { + _.Dismiss(); + + return; + } + + std::vector> ActionsToSchedule; + + // Pull actions to schedule from the pending queue, we will try to submit these to the runner outside of the lock + + m_PendingLock.WithExclusiveLock([&] { + if (m_ShutdownRequested) + { + return; + } + + if (m_PendingActions.empty()) + { + return; + } + + size_t NumActionsToSchedule = std::min(Capacity, m_PendingActions.size()); + + auto PendingIt = m_PendingActions.begin(); + const auto PendingEnd = m_PendingActions.end(); + + while (NumActionsToSchedule && PendingIt != PendingEnd) + { + const Ref& Pending = PendingIt->second; + + switch (Pending->ActionState()) + { + case RunnerAction::State::Pending: + ActionsToSchedule.push_back(Pending); + break; + + case RunnerAction::State::Running: + case RunnerAction::State::Completed: + case RunnerAction::State::Failed: + break; + + default: + case RunnerAction::State::New: + ZEN_WARN("unexpected state {} for pending action {}", static_cast(Pending->ActionState()), Pending->ActionLsn); + break; + } + + ++PendingIt; + --NumActionsToSchedule; + } + + PendingCount = m_PendingActions.size(); + }); + + if (ActionsToSchedule.empty()) + { + _.Dismiss(); + return; + } + + ZEN_INFO("attempting schedule of {} pending actions", ActionsToSchedule.size()); + + auto SubmitResults = SubmitActions(ActionsToSchedule); + + // Move successfully scheduled actions to the running map and remove + // from pending queue. It's actually possible that by the time we get + // to this stage some of the actions may have already completed, so + // they should not always be added to the running map + + eastl::hash_set ScheduledActions; + + for (size_t i = 0; i < ActionsToSchedule.size(); ++i) + { + const Ref& Pending = ActionsToSchedule[i]; + const SubmitResult& SubResult = SubmitResults[i]; + + if (SubResult.IsAccepted) + { + ScheduledActions.insert(Pending->ActionLsn); + } + } + + ScheduledCount += (int)ActionsToSchedule.size(); + +# else + m_PendingLock.WithExclusiveLock([&] { + while (!m_PendingActions.empty()) + { + if (m_ShutdownRequested) + { + return; + } + + // Here it would be good if we could decide to pop immediately to avoid + // holding the lock while creating processes etc + const Ref& Pending = m_PendingActions.begin()->second; + FunctionRunner::SubmitResult SubResult = SubmitAction(Pending); + + if (SubResult.IsAccepted) + { + // Great, the job is being taken care of by the runner + + ZEN_DEBUG("action {} ({}) PENDING -> RUNNING", Pending->ActionId, Pending->ActionLsn); + + m_RunningLock.WithExclusiveLock([&] { + m_RunningMap.insert({Pending->ActionLsn, Pending}); + + RunningCount = m_RunningMap.size(); + }); + + m_PendingActions.pop_front(); + + PendingCount = m_PendingActions.size(); + ++ScheduledCount; + } + else + { + // Runner could not accept the job, leave it on the pending queue + + return; + } + } + }); +# endif +} + +void +FunctionServiceSession::Impl::MonitorThreadFunction() +{ + SetCurrentThreadName("FunctionServiceSession_Monitor"); + + auto _ = MakeGuard([&] { ZEN_INFO("monitor thread exiting"); }); + + do + { + int TimeoutMs = 1000; + + if (m_PendingLock.WithSharedLock([&] { return m_PendingActions.size(); })) + { + TimeoutMs = 100; + } + + const bool Timedout = m_SchedulingThreadEvent.Wait(TimeoutMs); + + if (m_SchedulingThreadEnabled == false) + { + return; + } + + HandleActionUpdates(); + + // Schedule pending actions + + SchedulePendingActions(); + + if (!Timedout) + { + m_SchedulingThreadEvent.Reset(); + } + } while (m_SchedulingThreadEnabled); +} + +void +FunctionServiceSession::Impl::PostUpdate(RunnerAction* Action) +{ + m_UpdatedActionsLock.WithExclusiveLock([&] { m_UpdatedActions.emplace_back(Action); }); +} + +void +FunctionServiceSession::Impl::HandleActionUpdates() +{ + std::vector> UpdatedActions; + + m_UpdatedActionsLock.WithExclusiveLock([&] { std::swap(UpdatedActions, m_UpdatedActions); }); + + std::unordered_set SeenLsn; + std::unordered_set RunningLsn; + + for (Ref& Action : UpdatedActions) + { + const int ActionLsn = Action->ActionLsn; + + if (auto [It, Inserted] = SeenLsn.insert(ActionLsn); Inserted) + { + switch (Action->ActionState()) + { + case RunnerAction::State::Pending: + m_PendingLock.WithExclusiveLock([&] { m_PendingActions.insert({ActionLsn, Action}); }); + break; + + case RunnerAction::State::Running: + m_PendingLock.WithExclusiveLock([&] { + m_RunningLock.WithExclusiveLock([&] { + m_RunningMap.insert({ActionLsn, Action}); + m_PendingActions.erase(ActionLsn); + }); + }); + ZEN_DEBUG("action {} ({}) RUNNING", Action->ActionId, ActionLsn); + break; + + case RunnerAction::State::Completed: + case RunnerAction::State::Failed: + m_ResultsLock.WithExclusiveLock([&] { + m_ResultsMap[ActionLsn] = Action; + + m_PendingLock.WithExclusiveLock([&] { + m_RunningLock.WithExclusiveLock([&] { + if (auto FindIt = m_RunningMap.find(ActionLsn); FindIt == m_RunningMap.end()) + { + m_PendingActions.erase(ActionLsn); + } + else + { + m_RunningMap.erase(FindIt); + } + }); + }); + + m_ActionHistoryLock.WithExclusiveLock([&] { + ActionHistoryEntry Entry{.Lsn = ActionLsn, + .ActionId = Action->ActionId, + .WorkerId = Action->Worker.WorkerId, + .ActionDescriptor = Action->ActionObj, + .Succeeded = Action->ActionState() == RunnerAction::State::Completed}; + + std::copy(std::begin(Action->Timestamps), std::end(Action->Timestamps), std::begin(Entry.Timestamps)); + + m_ActionHistory.push_back(std::move(Entry)); + + if (m_ActionHistory.size() > m_HistoryLimit) + { + m_ActionHistory.pop_front(); + } + }); + }); + m_RetiredCount.fetch_add(1); + m_ResultRate.Mark(1); + ZEN_DEBUG("action {} ({}) RUNNING -> COMPLETED with {}", + Action->ActionId, + ActionLsn, + Action->ActionState() == RunnerAction::State::Completed ? "SUCCESS" : "FAILURE"); + break; + } + } + } +} + +size_t +FunctionServiceSession::Impl::QueryCapacity() +{ + return m_LocalRunnerGroup.QueryCapacity() + m_RemoteRunnerGroup.QueryCapacity(); +} + +std::vector +FunctionServiceSession::Impl::SubmitActions(const std::vector>& Actions) +{ + std::vector Results; + + for (const Ref& Action : Actions) + { + Results.push_back(SubmitAction(Action)); + } + + return Results; +} + +////////////////////////////////////////////////////////////////////////// + +FunctionServiceSession::FunctionServiceSession(ChunkResolver& InChunkResolver) +{ + m_Impl = std::make_unique(this, InChunkResolver); +} + +FunctionServiceSession::~FunctionServiceSession() +{ + Shutdown(); +} + +bool +FunctionServiceSession::IsHealthy() +{ + return m_Impl->IsHealthy(); +} + +void +FunctionServiceSession::Shutdown() +{ + m_Impl->Shutdown(); +} + +void +FunctionServiceSession::StartRecording(ChunkResolver& InResolver, const std::filesystem::path& RecordingPath) +{ + m_Impl->StartRecording(InResolver, RecordingPath); +} + +void +FunctionServiceSession::StopRecording() +{ + m_Impl->StopRecording(); +} + +void +FunctionServiceSession::EmitStats(CbObjectWriter& Cbo) +{ + m_Impl->EmitStats(Cbo); +} + +std::vector +FunctionServiceSession::GetKnownWorkerIds() +{ + return m_Impl->GetKnownWorkerIds(); +} + +WorkerDesc +FunctionServiceSession::GetWorkerDescriptor(const IoHash& WorkerId) +{ + return m_Impl->GetWorkerDescriptor(WorkerId); +} + +void +FunctionServiceSession::AddLocalRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath) +{ + m_Impl->m_LocalRunnerGroup.AddRunner(new LocalProcessRunner(InChunkResolver, BasePath)); +} + +void +FunctionServiceSession::AddRemoteRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath, std::string_view HostName) +{ + m_Impl->m_RemoteRunnerGroup.AddRunner(new RemoteHttpRunner(InChunkResolver, BasePath, HostName)); +} + +FunctionServiceSession::EnqueueResult +FunctionServiceSession::EnqueueAction(CbObject ActionObject, int Priority) +{ + return m_Impl->EnqueueAction(ActionObject, Priority); +} + +FunctionServiceSession::EnqueueResult +FunctionServiceSession::EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int RequestPriority) +{ + return m_Impl->EnqueueResolvedAction(Worker, ActionObj, RequestPriority); +} + +void +FunctionServiceSession::RegisterWorker(CbPackage Worker) +{ + m_Impl->RegisterWorker(Worker); +} + +HttpResponseCode +FunctionServiceSession::GetActionResult(int ActionLsn, CbPackage& OutResultPackage) +{ + return m_Impl->GetActionResult(ActionLsn, OutResultPackage); +} + +HttpResponseCode +FunctionServiceSession::FindActionResult(const IoHash& ActionId, CbPackage& OutResultPackage) +{ + return m_Impl->FindActionResult(ActionId, OutResultPackage); +} + +std::vector +FunctionServiceSession::GetActionHistory(int Limit) +{ + return m_Impl->GetActionHistory(Limit); +} + +void +FunctionServiceSession::GetCompleted(CbWriter& Cbo) +{ + m_Impl->GetCompleted(Cbo); +} + +void +FunctionServiceSession::PostUpdate(RunnerAction* Action) +{ + m_Impl->PostUpdate(Action); +} + +////////////////////////////////////////////////////////////////////////// + +void +function_forcelink() +{ +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/httpfunctionservice.cpp b/src/zencompute/httpfunctionservice.cpp new file mode 100644 index 000000000..09a9684a7 --- /dev/null +++ b/src/zencompute/httpfunctionservice.cpp @@ -0,0 +1,709 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencompute/httpfunctionservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "functionrunner.h" + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include + +using namespace std::literals; + +namespace zen::compute { + +constinit AsciiSet g_DecimalSet("0123456789"); +auto DecimalMatcher = [](std::string_view Str) { return AsciiSet::HasOnly(Str, g_DecimalSet); }; + +constinit AsciiSet g_HexSet("0123456789abcdefABCDEF"); +auto IoHashMatcher = [](std::string_view Str) { return Str.size() == 40 && AsciiSet::HasOnly(Str, g_HexSet); }; + +HttpFunctionService::HttpFunctionService(CidStore& InCidStore, + IHttpStatsService& StatsService, + [[maybe_unused]] const std::filesystem::path& BaseDir) +: m_CidStore(InCidStore) +, m_StatsService(StatsService) +, m_Log(logging::Get("apply")) +, m_BaseDir(BaseDir) +, m_FunctionService(InCidStore) +{ + m_FunctionService.AddLocalRunner(InCidStore, m_BaseDir / "local"); + + m_StatsService.RegisterHandler("apply", *this); + + m_Router.AddMatcher("lsn", DecimalMatcher); + m_Router.AddMatcher("worker", IoHashMatcher); + m_Router.AddMatcher("action", IoHashMatcher); + + m_Router.RegisterRoute( + "ready", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (m_FunctionService.IsHealthy()) + { + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "ok"); + } + + return HttpReq.WriteResponse(HttpResponseCode::ServiceUnavailable); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "workers", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + CbObjectWriter Cbo; + Cbo.BeginArray("workers"sv); + for (const IoHash& WorkerId : m_FunctionService.GetKnownWorkerIds()) + { + Cbo << WorkerId; + } + Cbo.EndArray(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "workers/{worker}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const IoHash WorkerId = IoHash::FromHexString(Req.GetCapture(1)); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + if (WorkerDesc Desc = m_FunctionService.GetWorkerDescriptor(WorkerId)) + { + return HttpReq.WriteResponse(HttpResponseCode::OK, Desc.Descriptor.GetObject()); + } + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + + case HttpVerb::kPost: + { + switch (HttpReq.RequestContentType()) + { + case HttpContentType::kCbObject: + { + CbObject WorkerSpec = HttpReq.ReadPayloadObject(); + + // Determine which pieces are missing and need to be transmitted + + HashKeySet ChunkSet; + + WorkerSpec.IterateAttachments([&](CbFieldView Field) { + const IoHash Hash = Field.AsHash(); + ChunkSet.AddHashToSet(Hash); + }); + + CbPackage WorkerPackage; + WorkerPackage.SetObject(WorkerSpec); + + m_CidStore.FilterChunks(ChunkSet); + + if (ChunkSet.IsEmpty()) + { + ZEN_DEBUG("worker {}: all attachments already available", WorkerId); + m_FunctionService.RegisterWorker(WorkerPackage); + return HttpReq.WriteResponse(HttpResponseCode::NoContent); + } + + CbObjectWriter ResponseWriter; + ResponseWriter.BeginArray("need"); + + ChunkSet.IterateHashes([&](const IoHash& Hash) { + ZEN_DEBUG("worker {}: need chunk {}", WorkerId, Hash); + ResponseWriter.AddHash(Hash); + }); + + ResponseWriter.EndArray(); + + ZEN_DEBUG("worker {}: need {} attachments", WorkerId, ChunkSet.GetSize()); + + return HttpReq.WriteResponse(HttpResponseCode::NotFound, ResponseWriter.Save()); + } + break; + + case HttpContentType::kCbPackage: + { + CbPackage WorkerSpecPackage = HttpReq.ReadPayloadPackage(); + CbObject WorkerSpec = WorkerSpecPackage.GetObject(); + + std::span Attachments = WorkerSpecPackage.GetAttachments(); + + int AttachmentCount = 0; + int NewAttachmentCount = 0; + uint64_t TotalAttachmentBytes = 0; + uint64_t TotalNewBytes = 0; + + for (const CbAttachment& Attachment : Attachments) + { + ZEN_ASSERT(Attachment.IsCompressedBinary()); + + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer Buffer = Attachment.AsCompressedBinary(); + + ZEN_UNUSED(DataHash); + TotalAttachmentBytes += Buffer.GetCompressedSize(); + ++AttachmentCount; + + const CidStore::InsertResult InsertResult = + m_CidStore.AddChunk(Buffer.GetCompressed().Flatten().AsIoBuffer(), DataHash); + + if (InsertResult.New) + { + TotalNewBytes += Buffer.GetCompressedSize(); + ++NewAttachmentCount; + } + } + + ZEN_DEBUG("worker {}: {} in {} attachments, {} in {} new attachments", + WorkerId, + zen::NiceBytes(TotalAttachmentBytes), + AttachmentCount, + zen::NiceBytes(TotalNewBytes), + NewAttachmentCount); + + m_FunctionService.RegisterWorker(WorkerSpecPackage); + + return HttpReq.WriteResponse(HttpResponseCode::NoContent); + } + break; + + default: + break; + } + } + break; + + default: + break; + } + }, + HttpVerb::kGet | HttpVerb::kPost); + + m_Router.RegisterRoute( + "jobs/completed", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + CbObjectWriter Cbo; + m_FunctionService.GetCompleted(Cbo); + + SystemMetrics Sm = GetSystemMetricsForReporting(); + Cbo.BeginObject("metrics"); + Describe(Sm, Cbo); + Cbo.EndObject(); + + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/history", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto QueryParams = HttpReq.GetQueryParams(); + + int QueryLimit = 50; + + if (auto LimitParam = QueryParams.GetValue("limit"); LimitParam.empty() == false) + { + QueryLimit = ParseInt(LimitParam).value_or(50); + } + + CbObjectWriter Cbo; + Cbo.BeginArray("history"); + for (const auto& Entry : m_FunctionService.GetActionHistory(QueryLimit)) + { + Cbo.BeginObject(); + Cbo << "lsn"sv << Entry.Lsn; + Cbo << "actionId"sv << Entry.ActionId; + Cbo << "workerId"sv << Entry.WorkerId; + Cbo << "succeeded"sv << Entry.Succeeded; + Cbo << "actionDescriptor"sv << Entry.ActionDescriptor; + + for (const auto& Timestamp : Entry.Timestamps) + { + Cbo.AddInteger( + fmt::format("time_{}"sv, RunnerAction::ToString(static_cast(&Timestamp - Entry.Timestamps))), + Timestamp); + } + Cbo.EndObject(); + } + Cbo.EndArray(); + + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/{lsn}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int ActionLsn = std::stoi(std::string{Req.GetCapture(1)}); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + CbPackage Output; + HttpResponseCode ResponseCode = m_FunctionService.GetActionResult(ActionLsn, Output); + + if (ResponseCode == HttpResponseCode::OK) + { + return HttpReq.WriteResponse(HttpResponseCode::OK, Output); + } + + return HttpReq.WriteResponse(ResponseCode); + } + break; + + case HttpVerb::kPost: + { + // Add support for cancellation, priority changes + } + break; + + default: + break; + } + }, + HttpVerb::kGet | HttpVerb::kPost); + + m_Router.RegisterRoute( + "jobs/{worker}/{action}", // This route is inefficient, and is only here for backwards compatibility. The preferred path is the + // one which uses the scheduled action lsn for lookups + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const IoHash ActionId = IoHash::FromHexString(Req.GetCapture(2)); + + CbPackage Output; + if (HttpResponseCode ResponseCode = m_FunctionService.FindActionResult(ActionId, /* out */ Output); + ResponseCode != HttpResponseCode::OK) + { + ZEN_TRACE("jobs/{}/{}: {}", Req.GetCapture(1), Req.GetCapture(2), ToString(ResponseCode)) + + if (ResponseCode == HttpResponseCode::NotFound) + { + return HttpReq.WriteResponse(ResponseCode); + } + + return HttpReq.WriteResponse(ResponseCode); + } + + ZEN_DEBUG("jobs/{}/{}: OK", Req.GetCapture(1), Req.GetCapture(2)) + + return HttpReq.WriteResponse(HttpResponseCode::OK, Output); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/{worker}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const IoHash WorkerId = IoHash::FromHexString(Req.GetCapture(1)); + + WorkerDesc Worker = m_FunctionService.GetWorkerDescriptor(WorkerId); + + if (!Worker) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + const auto QueryParams = Req.ServerRequest().GetQueryParams(); + + int RequestPriority = -1; + + if (auto PriorityParam = QueryParams.GetValue("priority"); PriorityParam.empty() == false) + { + RequestPriority = ParseInt(PriorityParam).value_or(-1); + } + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + // TODO: return status of all pending or executing jobs + break; + + case HttpVerb::kPost: + switch (HttpReq.RequestContentType()) + { + case HttpContentType::kCbObject: + { + // This operation takes the proposed job spec and identifies which + // chunks are not present on this server. This list is then returned in + // the "need" list in the response + + IoBuffer Payload = HttpReq.ReadPayload(); + CbObject ActionObj = LoadCompactBinaryObject(Payload); + + std::vector NeedList; + + ActionObj.IterateAttachments([&](CbFieldView Field) { + const IoHash FileHash = Field.AsHash(); + + if (!m_CidStore.ContainsChunk(FileHash)) + { + NeedList.push_back(FileHash); + } + }); + + if (NeedList.empty()) + { + // We already have everything, enqueue the action for execution + + if (FunctionServiceSession::EnqueueResult Result = + m_FunctionService.EnqueueResolvedAction(Worker, ActionObj, RequestPriority)) + { + ZEN_DEBUG("action {} accepted (lsn {})", ActionObj.GetHash(), Result.Lsn); + + HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + + return; + } + + CbObjectWriter Cbo; + Cbo.BeginArray("need"); + + for (const IoHash& Hash : NeedList) + { + Cbo << Hash; + } + + Cbo.EndArray(); + CbObject Response = Cbo.Save(); + + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Response); + } + break; + + case HttpContentType::kCbPackage: + { + CbPackage Action = HttpReq.ReadPayloadPackage(); + CbObject ActionObj = Action.GetObject(); + + std::span Attachments = Action.GetAttachments(); + + int AttachmentCount = 0; + int NewAttachmentCount = 0; + uint64_t TotalAttachmentBytes = 0; + uint64_t TotalNewBytes = 0; + + for (const CbAttachment& Attachment : Attachments) + { + ZEN_ASSERT(Attachment.IsCompressedBinary()); + + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer DataView = Attachment.AsCompressedBinary(); + + ZEN_UNUSED(DataHash); + + const uint64_t CompressedSize = DataView.GetCompressedSize(); + + TotalAttachmentBytes += CompressedSize; + ++AttachmentCount; + + const CidStore::InsertResult InsertResult = + m_CidStore.AddChunk(DataView.GetCompressed().Flatten().AsIoBuffer(), DataHash); + + if (InsertResult.New) + { + TotalNewBytes += CompressedSize; + ++NewAttachmentCount; + } + } + + if (FunctionServiceSession::EnqueueResult Result = + m_FunctionService.EnqueueResolvedAction(Worker, ActionObj, RequestPriority)) + { + ZEN_DEBUG("accepted action {} (lsn {}): {} in {} attachments. {} new ({} attachments)", + ActionObj.GetHash(), + Result.Lsn, + zen::NiceBytes(TotalAttachmentBytes), + AttachmentCount, + zen::NiceBytes(TotalNewBytes), + NewAttachmentCount); + + HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + + return; + } + break; + + default: + break; + } + break; + + default: + break; + } + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "jobs", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto QueryParams = HttpReq.GetQueryParams(); + + int RequestPriority = -1; + + if (auto PriorityParam = QueryParams.GetValue("priority"); PriorityParam.empty() == false) + { + RequestPriority = ParseInt(PriorityParam).value_or(-1); + } + + // Resolve worker + + // + + switch (HttpReq.RequestContentType()) + { + case HttpContentType::kCbObject: + { + // This operation takes the proposed job spec and identifies which + // chunks are not present on this server. This list is then returned in + // the "need" list in the response + + IoBuffer Payload = HttpReq.ReadPayload(); + CbObject ActionObj = LoadCompactBinaryObject(Payload); + + std::vector NeedList; + + ActionObj.IterateAttachments([&](CbFieldView Field) { + const IoHash FileHash = Field.AsHash(); + + if (!m_CidStore.ContainsChunk(FileHash)) + { + NeedList.push_back(FileHash); + } + }); + + if (NeedList.empty()) + { + // We already have everything, enqueue the action for execution + + if (FunctionServiceSession::EnqueueResult Result = m_FunctionService.EnqueueAction(ActionObj, RequestPriority)) + { + ZEN_DEBUG("action accepted (lsn {})", Result.Lsn); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + else + { + // Could not resolve? + return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); + } + } + + CbObjectWriter Cbo; + Cbo.BeginArray("need"); + + for (const IoHash& Hash : NeedList) + { + Cbo << Hash; + } + + Cbo.EndArray(); + CbObject Response = Cbo.Save(); + + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Response); + } + + case HttpContentType::kCbPackage: + { + CbPackage Action = HttpReq.ReadPayloadPackage(); + CbObject ActionObj = Action.GetObject(); + + std::span Attachments = Action.GetAttachments(); + + int AttachmentCount = 0; + int NewAttachmentCount = 0; + uint64_t TotalAttachmentBytes = 0; + uint64_t TotalNewBytes = 0; + + for (const CbAttachment& Attachment : Attachments) + { + ZEN_ASSERT(Attachment.IsCompressedBinary()); + + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer DataView = Attachment.AsCompressedBinary(); + + ZEN_UNUSED(DataHash); + + const uint64_t CompressedSize = DataView.GetCompressedSize(); + + TotalAttachmentBytes += CompressedSize; + ++AttachmentCount; + + const CidStore::InsertResult InsertResult = + m_CidStore.AddChunk(DataView.GetCompressed().Flatten().AsIoBuffer(), DataHash); + + if (InsertResult.New) + { + TotalNewBytes += CompressedSize; + ++NewAttachmentCount; + } + } + + if (FunctionServiceSession::EnqueueResult Result = m_FunctionService.EnqueueAction(ActionObj, RequestPriority)) + { + ZEN_DEBUG("accepted action (lsn {}): {} in {} attachments. {} new ({} attachments)", + Result.Lsn, + zen::NiceBytes(TotalAttachmentBytes), + AttachmentCount, + zen::NiceBytes(TotalNewBytes), + NewAttachmentCount); + + HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + else + { + // Could not resolve? + return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); + } + } + return; + } + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "workers/all", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + std::vector WorkerIds = m_FunctionService.GetKnownWorkerIds(); + + CbObjectWriter Cbo; + Cbo.BeginArray("workers"); + + for (const IoHash& WorkerId : WorkerIds) + { + Cbo.BeginObject(); + + Cbo << "id" << WorkerId; + + const auto& Descriptor = m_FunctionService.GetWorkerDescriptor(WorkerId); + + Cbo << "descriptor" << Descriptor.Descriptor.GetObject(); + + Cbo.EndObject(); + } + + Cbo.EndArray(); + + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "sysinfo", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + SystemMetrics Sm = GetSystemMetricsForReporting(); + + CbObjectWriter Cbo; + Describe(Sm, Cbo); + + Cbo << "cpu_usage" << Sm.CpuUsagePercent; + Cbo << "memory_total" << Sm.SystemMemoryMiB * 1024 * 1024; + Cbo << "memory_used" << (Sm.SystemMemoryMiB - Sm.AvailSystemMemoryMiB) * 1024 * 1024; + Cbo << "disk_used" << 100 * 1024; + Cbo << "disk_total" << 100 * 1024 * 1024; + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "record/start", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + m_FunctionService.StartRecording(m_CidStore, m_BaseDir / "recording"); + + return HttpReq.WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "record/stop", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + m_FunctionService.StopRecording(); + + return HttpReq.WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kPost); +} + +HttpFunctionService::~HttpFunctionService() +{ + m_StatsService.UnregisterHandler("apply", *this); +} + +void +HttpFunctionService::Shutdown() +{ + m_FunctionService.Shutdown(); +} + +const char* +HttpFunctionService::BaseUri() const +{ + return "/apply/"; +} + +void +HttpFunctionService::HandleRequest(HttpServerRequest& Request) +{ + metrics::OperationTiming::Scope $(m_HttpRequests); + + if (m_Router.HandleRequest(Request) == false) + { + ZEN_WARN("No route found for {0}", Request.RelativeUri()); + } +} + +void +HttpFunctionService::HandleStatsRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + m_FunctionService.EmitStats(Cbo); + + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +////////////////////////////////////////////////////////////////////////// + +void +httpfunction_forcelink() +{ +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/httporchestrator.cpp b/src/zencompute/httporchestrator.cpp new file mode 100644 index 000000000..39e7e60d7 --- /dev/null +++ b/src/zencompute/httporchestrator.cpp @@ -0,0 +1,81 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencompute/httporchestrator.h" + +#include +#include + +namespace zen::compute { + +HttpOrchestratorService::HttpOrchestratorService() : m_Log(logging::Get("orch")) +{ + m_Router.RegisterRoute( + "provision", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + CbObjectWriter Cbo; + Cbo.BeginArray("workers"); + + m_KnownWorkersLock.WithSharedLock([&] { + for (const auto& [WorkerId, Worker] : m_KnownWorkers) + { + Cbo.BeginObject(); + Cbo << "uri" << Worker.BaseUri; + Cbo << "dt" << Worker.LastSeen.GetElapsedTimeMs(); + Cbo.EndObject(); + } + }); + + Cbo.EndArray(); + + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "announce", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + CbObject Data = HttpReq.ReadPayloadObject(); + + std::string_view WorkerId = Data["id"].AsString(""); + std::string_view WorkerUri = Data["uri"].AsString(""); + + if (WorkerId.empty() || WorkerUri.empty()) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + + m_KnownWorkersLock.WithExclusiveLock([&] { + auto& Worker = m_KnownWorkers[std::string(WorkerId)]; + Worker.BaseUri = WorkerUri; + Worker.LastSeen.Reset(); + }); + + HttpReq.WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kPost); +} + +HttpOrchestratorService::~HttpOrchestratorService() +{ +} + +const char* +HttpOrchestratorService::BaseUri() const +{ + return "/orch/"; +} + +void +HttpOrchestratorService::HandleRequest(HttpServerRequest& Request) +{ + if (m_Router.HandleRequest(Request) == false) + { + ZEN_WARN("No route found for {0}", Request.RelativeUri()); + } +} + +} // namespace zen::compute diff --git a/src/zencompute/include/zencompute/functionservice.h b/src/zencompute/include/zencompute/functionservice.h new file mode 100644 index 000000000..1deb99fd5 --- /dev/null +++ b/src/zencompute/include/zencompute/functionservice.h @@ -0,0 +1,132 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if !defined(ZEN_WITH_COMPUTE_SERVICES) +# define ZEN_WITH_COMPUTE_SERVICES 1 +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include + +# include + +namespace zen { +class ChunkResolver; +class CbObjectWriter; +} // namespace zen + +namespace zen::compute { + +class ActionRecorder; +class FunctionServiceSession; +class IActionResultHandler; +class LocalProcessRunner; +class RemoteHttpRunner; +struct RunnerAction; +struct SubmitResult; + +struct WorkerDesc +{ + CbPackage Descriptor; + IoHash WorkerId{IoHash::Zero}; + + inline operator bool() const { return WorkerId != IoHash::Zero; } +}; + +/** + * Lambda style compute function service + * + * The responsibility of this class is to accept function execution requests, and + * schedule them using one or more FunctionRunner instances. It will basically always + * accept requests, queueing them if necessary, and then hand them off to runners + * as they become available. + * + * This is typically fronted by an API service that handles communication with clients. + */ +class FunctionServiceSession final +{ +public: + FunctionServiceSession(ChunkResolver& InChunkResolver); + ~FunctionServiceSession(); + + void Shutdown(); + bool IsHealthy(); + + // Worker registration and discovery + + void RegisterWorker(CbPackage Worker); + [[nodiscard]] WorkerDesc GetWorkerDescriptor(const IoHash& WorkerId); + [[nodiscard]] std::vector GetKnownWorkerIds(); + + // Action runners + + void AddLocalRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath); + void AddRemoteRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath, std::string_view HostName); + + // Action submission + + struct EnqueueResult + { + int Lsn; + CbObject ResponseMessage; + + inline operator bool() const { return Lsn != 0; } + }; + + [[nodiscard]] EnqueueResult EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int Priority); + [[nodiscard]] EnqueueResult EnqueueAction(CbObject ActionObject, int Priority); + + // Completed action tracking + + [[nodiscard]] HttpResponseCode GetActionResult(int ActionLsn, CbPackage& OutResultPackage); + [[nodiscard]] HttpResponseCode FindActionResult(const IoHash& ActionId, CbPackage& ResultPackage); + + void GetCompleted(CbWriter&); + + // Action history tracking (note that this is separate from completed action tracking, and + // will include actions which have been retired and no longer have their results available) + + struct ActionHistoryEntry + { + int Lsn; + IoHash ActionId; + IoHash WorkerId; + CbObject ActionDescriptor; + bool Succeeded; + uint64_t Timestamps[5] = {}; + }; + + [[nodiscard]] std::vector GetActionHistory(int Limit = 100); + + // Stats reporting + + void EmitStats(CbObjectWriter& Cbo); + + // Recording + + void StartRecording(ChunkResolver& InResolver, const std::filesystem::path& RecordingPath); + void StopRecording(); + +private: + void PostUpdate(RunnerAction* Action); + + friend class FunctionRunner; + friend struct RunnerAction; + + struct Impl; + std::unique_ptr m_Impl; +}; + +void function_forcelink(); + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/include/zencompute/httpfunctionservice.h b/src/zencompute/include/zencompute/httpfunctionservice.h new file mode 100644 index 000000000..6e2344ae6 --- /dev/null +++ b/src/zencompute/include/zencompute/httpfunctionservice.h @@ -0,0 +1,73 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if !defined(ZEN_WITH_COMPUTE_SERVICES) +# define ZEN_WITH_COMPUTE_SERVICES 1 +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "zencompute/functionservice.h" + +# include +# include +# include +# include +# include +# include + +# include +# include +# include + +namespace zen { +class CidStore; +} + +namespace zen::compute { + +class HttpFunctionService; +class FunctionService; + +/** + * HTTP interface for compute function service + */ +class HttpFunctionService : public HttpService, public IHttpStatsProvider +{ +public: + HttpFunctionService(CidStore& InCidStore, IHttpStatsService& StatsService, const std::filesystem::path& BaseDir); + ~HttpFunctionService(); + + void Shutdown(); + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + + // IHttpStatsProvider + + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + +protected: + CidStore& m_CidStore; + IHttpStatsService& m_StatsService; + LoggerRef Log() { return m_Log; } + +private: + LoggerRef m_Log; + std::filesystem ::path m_BaseDir; + HttpRequestRouter m_Router; + FunctionServiceSession m_FunctionService; + + // Metrics + + metrics::OperationTiming m_HttpRequests; +}; + +void httpfunction_forcelink(); + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/include/zencompute/httporchestrator.h b/src/zencompute/include/zencompute/httporchestrator.h new file mode 100644 index 000000000..168c6d7fe --- /dev/null +++ b/src/zencompute/include/zencompute/httporchestrator.h @@ -0,0 +1,44 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include + +#include + +namespace zen::compute { + +/** + * Mock orchestrator service, for testing dynamic provisioning + */ + +class HttpOrchestratorService : public HttpService +{ +public: + HttpOrchestratorService(); + ~HttpOrchestratorService(); + + HttpOrchestratorService(const HttpOrchestratorService&) = delete; + HttpOrchestratorService& operator=(const HttpOrchestratorService&) = delete; + + virtual const char* BaseUri() const override; + virtual void HandleRequest(HttpServerRequest& Request) override; + +private: + HttpRequestRouter m_Router; + LoggerRef m_Log; + + struct KnownWorker + { + std::string_view BaseUri; + Stopwatch LastSeen; + }; + + RwLock m_KnownWorkersLock; + std::unordered_map m_KnownWorkers; +}; + +} // namespace zen::compute diff --git a/src/zencompute/include/zencompute/recordingreader.h b/src/zencompute/include/zencompute/recordingreader.h new file mode 100644 index 000000000..bf1aff125 --- /dev/null +++ b/src/zencompute/include/zencompute/recordingreader.h @@ -0,0 +1,127 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace zen { +class CbObject; +class CbPackage; +struct IoHash; +} // namespace zen + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen::compute { + +////////////////////////////////////////////////////////////////////////// + +class RecordingReaderBase +{ + RecordingReaderBase(const RecordingReaderBase&) = delete; + RecordingReaderBase& operator=(const RecordingReaderBase&) = delete; + +public: + RecordingReaderBase() = default; + virtual ~RecordingReaderBase() = 0; + virtual std::unordered_map ReadWorkers() = 0; + virtual void IterateActions(std::function&& Callback, int TargetParallelism) = 0; + virtual size_t GetActionCount() const = 0; +}; + +////////////////////////////////////////////////////////////////////////// + +/** + * Reader for recordings done via the zencompute recording system, which + * have a shared chunk store and a log of actions with pointers into the + * chunk store for their data. + */ +class RecordingReader : public RecordingReaderBase, public ChunkResolver +{ +public: + explicit RecordingReader(const std::filesystem::path& RecordingPath); + ~RecordingReader(); + + virtual std::unordered_map ReadWorkers() override; + + virtual void IterateActions(std::function&& Callback, + int TargetParallelism) override; + virtual size_t GetActionCount() const override; + +private: + std::filesystem::path m_RecordingLogDir; + BasicFile m_WorkerDataFile; + BasicFile m_ActionDataFile; + GcManager m_Gc; + CidStore m_CidStore{m_Gc}; + + // ChunkResolver interface + virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; + + struct ActionEntry + { + IoHash ActionId; + uint64_t Offset; + uint64_t Size; + }; + + std::vector m_Actions; + + void ScanActions(); +}; + +////////////////////////////////////////////////////////////////////////// + +struct LocalResolver : public ChunkResolver +{ + LocalResolver(const LocalResolver&) = delete; + LocalResolver& operator=(const LocalResolver&) = delete; + + LocalResolver() = default; + ~LocalResolver() = default; + + virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; + void Add(const IoHash& Cid, IoBuffer Data); + +private: + RwLock MapLock; + std::unordered_map Attachments; +}; + +/** + * This is a reader for UE/DDB recordings, which have a different layout on + * disk (no shared chunk store) + */ +class UeRecordingReader : public RecordingReaderBase, public ChunkResolver +{ +public: + explicit UeRecordingReader(const std::filesystem::path& RecordingPath); + ~UeRecordingReader(); + + virtual std::unordered_map ReadWorkers() override; + virtual void IterateActions(std::function&& Callback, + int TargetParallelism) override; + virtual size_t GetActionCount() const override; + virtual IoBuffer FindChunkByCid(const IoHash& DecompressedId) override; + +private: + std::filesystem::path m_RecordingDir; + LocalResolver m_LocalResolver; + std::vector m_WorkDirs; + + CbPackage ReadAction(std::filesystem::path WorkDir); +}; + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/include/zencompute/zencompute.h b/src/zencompute/include/zencompute/zencompute.h new file mode 100644 index 000000000..6dc32eeea --- /dev/null +++ b/src/zencompute/include/zencompute/zencompute.h @@ -0,0 +1,11 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen { + +void zencompute_forcelinktests(); + +} diff --git a/src/zencompute/localrunner.cpp b/src/zencompute/localrunner.cpp new file mode 100644 index 000000000..9a27f3f3d --- /dev/null +++ b/src/zencompute/localrunner.cpp @@ -0,0 +1,722 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "localrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include + +namespace zen::compute { + +using namespace std::literals; + +LocalProcessRunner::LocalProcessRunner(ChunkResolver& Resolver, const std::filesystem::path& BaseDir) +: FunctionRunner(BaseDir) +, m_Log(logging::Get("local_exec")) +, m_ChunkResolver(Resolver) +, m_WorkerPath(std::filesystem::weakly_canonical(BaseDir / "workers")) +, m_SandboxPath(std::filesystem::weakly_canonical(BaseDir / "scratch")) +{ + SystemMetrics Sm = GetSystemMetricsForReporting(); + + m_MaxRunningActions = Sm.LogicalProcessorCount * 2; + + ZEN_INFO("Max concurrent action count: {}", m_MaxRunningActions); + + bool DidCleanup = false; + + if (std::filesystem::is_directory(m_ActionsPath)) + { + ZEN_INFO("Cleaning '{}'", m_ActionsPath); + + std::error_code Ec; + CleanDirectory(m_ActionsPath, /* ForceRemoveReadOnlyFiles */ true, Ec); + + if (Ec) + { + ZEN_WARN("Unable to clean '{}': {}", m_ActionsPath, Ec.message()); + } + + DidCleanup = true; + } + + if (std::filesystem::is_directory(m_SandboxPath)) + { + ZEN_INFO("Cleaning '{}'", m_SandboxPath); + std::error_code Ec; + CleanDirectory(m_SandboxPath, /* ForceRemoveReadOnlyFiles */ true, Ec); + + if (Ec) + { + ZEN_WARN("Unable to clean '{}': {}", m_SandboxPath, Ec.message()); + } + + DidCleanup = true; + } + + // We clean out all workers on startup since we can't know they are good. They could be bad + // due to tampering, malware (which I also mean to include AV and antimalware software) or + // other processes we have no control over + if (std::filesystem::is_directory(m_WorkerPath)) + { + ZEN_INFO("Cleaning '{}'", m_WorkerPath); + std::error_code Ec; + CleanDirectory(m_WorkerPath, /* ForceRemoveReadOnlyFiles */ true, Ec); + + if (Ec) + { + ZEN_WARN("Unable to clean '{}': {}", m_WorkerPath, Ec.message()); + } + + DidCleanup = true; + } + + if (DidCleanup) + { + ZEN_INFO("Cleanup complete"); + } + + m_MonitorThread = std::thread{&LocalProcessRunner::MonitorThreadFunction, this}; + +# if ZEN_PLATFORM_WINDOWS + // Suppress any error dialogs caused by missing dependencies + UINT OldMode = ::SetErrorMode(0); + ::SetErrorMode(OldMode | SEM_FAILCRITICALERRORS); +# endif + + m_AcceptNewActions = true; +} + +LocalProcessRunner::~LocalProcessRunner() +{ + try + { + Shutdown(); + } + catch (std::exception& Ex) + { + ZEN_WARN("exception during local process runner shutdown: {}", Ex.what()); + } +} + +void +LocalProcessRunner::Shutdown() +{ + m_AcceptNewActions = false; + + m_MonitorThreadEnabled = false; + m_MonitorThreadEvent.Set(); + if (m_MonitorThread.joinable()) + { + m_MonitorThread.join(); + } + + CancelRunningActions(); +} + +std::filesystem::path +LocalProcessRunner::CreateNewSandbox() +{ + std::string UniqueId = std::to_string(++m_SandboxCounter); + std::filesystem::path Path = m_SandboxPath / UniqueId; + zen::CreateDirectories(Path); + + return Path; +} + +void +LocalProcessRunner::RegisterWorker(const CbPackage& WorkerPackage) +{ + if (m_DumpActions) + { + CbObject WorkerDescriptor = WorkerPackage.GetObject(); + const IoHash& WorkerId = WorkerPackage.GetObjectHash(); + + std::string UniqueId = fmt::format("worker_{}"sv, WorkerId); + std::filesystem::path Path = m_ActionsPath / UniqueId; + + zen::WriteFile(Path / "worker.ucb", WorkerDescriptor.GetBuffer().AsIoBuffer()); + + ManifestWorker(WorkerPackage, Path / "tree", [&](const IoHash& Cid, CompressedBuffer& ChunkBuffer) { + std::filesystem::path ChunkPath = Path / "chunks" / Cid.ToHexString(); + zen::WriteFile(ChunkPath, ChunkBuffer.GetCompressed()); + }); + + ZEN_INFO("dumped worker '{}' to 'file://{}'", WorkerId, Path); + } +} + +size_t +LocalProcessRunner::QueryCapacity() +{ + // Estimate how much more work we're ready to accept + + RwLock::SharedLockScope _{m_RunningLock}; + + if (!m_AcceptNewActions) + { + return 0; + } + + size_t RunningCount = m_RunningMap.size(); + + if (RunningCount >= size_t(m_MaxRunningActions)) + { + return 0; + } + + return m_MaxRunningActions - RunningCount; +} + +std::vector +LocalProcessRunner::SubmitActions(const std::vector>& Actions) +{ + std::vector Results; + + for (const Ref& Action : Actions) + { + Results.push_back(SubmitAction(Action)); + } + + return Results; +} + +SubmitResult +LocalProcessRunner::SubmitAction(Ref Action) +{ + // Verify whether we can accept more work + + { + RwLock::SharedLockScope _{m_RunningLock}; + + if (!m_AcceptNewActions) + { + return SubmitResult{.IsAccepted = false}; + } + + if (m_RunningMap.size() >= size_t(m_MaxRunningActions)) + { + return SubmitResult{.IsAccepted = false}; + } + } + + using namespace std::literals; + + // Each enqueued action is assigned an integer index (logical sequence number), + // which we use as a key for tracking data structures and as an opaque id which + // may be used by clients to reference the scheduled action + + const int32_t ActionLsn = Action->ActionLsn; + const CbObject& ActionObj = Action->ActionObj; + const IoHash ActionId = ActionObj.GetHash(); + + MaybeDumpAction(ActionLsn, ActionObj); + + std::filesystem::path SandboxPath = CreateNewSandbox(); + + CbPackage WorkerPackage = Action->Worker.Descriptor; + + std::filesystem::path WorkerPath = ManifestWorker(Action->Worker); + + // Write out action + + zen::WriteFile(SandboxPath / "build.action", ActionObj.GetBuffer().AsIoBuffer()); + + // Manifest inputs in sandbox + + ActionObj.IterateAttachments([&](CbFieldView Field) { + const IoHash Cid = Field.AsHash(); + std::filesystem::path FilePath{SandboxPath / "Inputs"sv / Cid.ToHexString()}; + IoBuffer DataBuffer = m_ChunkResolver.FindChunkByCid(Cid); + + if (!DataBuffer) + { + throw std::runtime_error(fmt::format("input CID chunk '{}' missing", Cid)); + } + + zen::WriteFile(FilePath, DataBuffer); + }); + +# if ZEN_PLATFORM_WINDOWS + // Set up environment variables + + StringBuilder<1024> EnvironmentBlock; + + CbObject WorkerDescription = WorkerPackage.GetObject(); + + for (auto& It : WorkerDescription["environment"sv]) + { + EnvironmentBlock.Append(It.AsString()); + EnvironmentBlock.Append('\0'); + } + EnvironmentBlock.Append('\0'); + EnvironmentBlock.Append('\0'); + + // Execute process - this spawns the child process immediately without waiting + // for completion + + std::string_view ExecPath = WorkerDescription["path"sv].AsString(); + std::filesystem::path ExePath = WorkerPath / std::filesystem::path(ExecPath).make_preferred(); + + ExtendableWideStringBuilder<512> CommandLine; + CommandLine.Append(L'"'); + CommandLine.Append(ExePath.c_str()); + CommandLine.Append(L'"'); + CommandLine.Append(L" -Build=build.action"); + + LPSECURITY_ATTRIBUTES lpProcessAttributes = nullptr; + LPSECURITY_ATTRIBUTES lpThreadAttributes = nullptr; + BOOL bInheritHandles = FALSE; + DWORD dwCreationFlags = 0; + + STARTUPINFO StartupInfo{}; + StartupInfo.cb = sizeof StartupInfo; + + PROCESS_INFORMATION ProcessInformation{}; + + ZEN_DEBUG("Executing: {}", WideToUtf8(CommandLine.c_str())); + + CommandLine.EnsureNulTerminated(); + + BOOL Success = CreateProcessW(nullptr, + CommandLine.Data(), + lpProcessAttributes, + lpThreadAttributes, + bInheritHandles, + dwCreationFlags, + (LPVOID)EnvironmentBlock.Data(), // Environment block + SandboxPath.c_str(), // Current directory + &StartupInfo, + /* out */ &ProcessInformation); + + if (!Success) + { + // TODO: this is probably not the best way to report failure. The return + // object should include a failure state and context + + zen::ThrowLastError("Unable to launch process" /* TODO: Add context */); + } + + CloseHandle(ProcessInformation.hThread); + + Ref NewAction{new RunningAction()}; + NewAction->Action = Action; + NewAction->ProcessHandle = ProcessInformation.hProcess; + NewAction->SandboxPath = std::move(SandboxPath); + + { + RwLock::ExclusiveLockScope _(m_RunningLock); + + m_RunningMap[ActionLsn] = std::move(NewAction); + } + + Action->SetActionState(RunnerAction::State::Running); +# else + ZEN_UNUSED(ActionId); + + ZEN_NOT_IMPLEMENTED(); + + int ExitCode = 0; +# endif + + return SubmitResult{.IsAccepted = true}; +} + +size_t +LocalProcessRunner::GetSubmittedActionCount() +{ + RwLock::SharedLockScope _(m_RunningLock); + return m_RunningMap.size(); +} + +std::filesystem::path +LocalProcessRunner::ManifestWorker(const WorkerDesc& Worker) +{ + RwLock::SharedLockScope _(m_WorkerLock); + + std::filesystem::path WorkerDir = m_WorkerPath / fmt::format("runner_{}", Worker.WorkerId); + + if (!std::filesystem::exists(WorkerDir)) + { + _.ReleaseNow(); + + RwLock::ExclusiveLockScope $(m_WorkerLock); + + if (!std::filesystem::exists(WorkerDir)) + { + ManifestWorker(Worker.Descriptor, WorkerDir, [](const IoHash&, CompressedBuffer&) {}); + } + } + + return WorkerDir; +} + +void +LocalProcessRunner::DecompressAttachmentToFile(const CbPackage& FromPackage, + CbObjectView FileEntry, + const std::filesystem::path& SandboxRootPath, + std::function& ChunkReferenceCallback) +{ + std::string_view Name = FileEntry["name"sv].AsString(); + const IoHash ChunkHash = FileEntry["hash"sv].AsHash(); + const uint64_t Size = FileEntry["size"sv].AsUInt64(); + + CompressedBuffer Compressed; + + if (const CbAttachment* Attachment = FromPackage.FindAttachment(ChunkHash)) + { + Compressed = Attachment->AsCompressedBinary(); + } + else + { + IoBuffer DataBuffer = m_ChunkResolver.FindChunkByCid(ChunkHash); + + if (!DataBuffer) + { + throw std::runtime_error(fmt::format("worker chunk '{}' missing", ChunkHash)); + } + + uint64_t DataRawSize = 0; + IoHash DataRawHash; + Compressed = CompressedBuffer::FromCompressed(SharedBuffer{DataBuffer}, DataRawHash, DataRawSize); + + if (DataRawSize != Size) + { + throw std::runtime_error( + fmt::format("worker chunk '{}' size: {}, action spec expected {}", ChunkHash, DataBuffer.Size(), Size)); + } + } + + ChunkReferenceCallback(ChunkHash, Compressed); + + std::filesystem::path FilePath{SandboxRootPath / std::filesystem::path(Name).make_preferred()}; + + SharedBuffer Decompressed = Compressed.Decompress(); + zen::WriteFile(FilePath, Decompressed.AsIoBuffer()); +} + +void +LocalProcessRunner::ManifestWorker(const CbPackage& WorkerPackage, + const std::filesystem::path& SandboxPath, + std::function&& ChunkReferenceCallback) +{ + CbObject WorkerDescription = WorkerPackage.GetObject(); + + // Manifest worker in Sandbox + + for (auto& It : WorkerDescription["executables"sv]) + { + DecompressAttachmentToFile(WorkerPackage, It.AsObjectView(), SandboxPath, ChunkReferenceCallback); + } + + for (auto& It : WorkerDescription["dirs"sv]) + { + std::string_view Name = It.AsString(); + std::filesystem::path DirPath{SandboxPath / std::filesystem::path(Name).make_preferred()}; + zen::CreateDirectories(DirPath); + } + + for (auto& It : WorkerDescription["files"sv]) + { + DecompressAttachmentToFile(WorkerPackage, It.AsObjectView(), SandboxPath, ChunkReferenceCallback); + } + + WriteFile(SandboxPath / "worker.zcb", WorkerDescription.GetBuffer().AsIoBuffer()); +} + +CbPackage +LocalProcessRunner::GatherActionOutputs(std::filesystem::path SandboxPath) +{ + std::filesystem::path OutputFile = SandboxPath / "build.output"; + FileContents OutputData = zen::ReadFile(OutputFile); + + if (OutputData.ErrorCode) + { + throw std::system_error(OutputData.ErrorCode, fmt::format("Failed to read build output file '{}'", OutputFile)); + } + + CbPackage OutputPackage; + CbObject Output = zen::LoadCompactBinaryObject(OutputData.Flatten()); + + uint64_t TotalAttachmentBytes = 0; + uint64_t TotalRawAttachmentBytes = 0; + + Output.IterateAttachments([&](CbFieldView Field) { + IoHash Hash = Field.AsHash(); + std::filesystem::path OutputPath{SandboxPath / "Outputs" / Hash.ToHexString()}; + FileContents ChunkData = zen::ReadFile(OutputPath); + + if (ChunkData.ErrorCode) + { + throw std::system_error(ChunkData.ErrorCode, fmt::format("Failed to read build output file '{}'", OutputPath)); + } + + uint64_t ChunkDataRawSize = 0; + IoHash ChunkDataHash; + CompressedBuffer AttachmentBuffer = + CompressedBuffer::FromCompressed(SharedBuffer(ChunkData.Flatten()), ChunkDataHash, ChunkDataRawSize); + + if (!AttachmentBuffer) + { + throw std::runtime_error("Invalid output encountered (not valid CompressedBuffer format)"); + } + + TotalAttachmentBytes += AttachmentBuffer.GetCompressedSize(); + TotalRawAttachmentBytes += ChunkDataRawSize; + + CbAttachment Attachment(std::move(AttachmentBuffer), ChunkDataHash); + OutputPackage.AddAttachment(Attachment); + }); + + OutputPackage.SetObject(Output); + + ZEN_DEBUG("Action completed with {} attachments ({} compressed, {} uncompressed)", + OutputPackage.GetAttachments().size(), + NiceBytes(TotalAttachmentBytes), + NiceBytes(TotalRawAttachmentBytes)); + + return OutputPackage; +} + +void +LocalProcessRunner::MonitorThreadFunction() +{ + SetCurrentThreadName("LocalProcessRunner_Monitor"); + + auto _ = MakeGuard([&] { ZEN_INFO("monitor thread exiting"); }); + + do + { + // On Windows it's possible to wait on process handles, so we wait for either a process to exit + // or for the monitor event to be signaled (which indicates we should check for cancellation + // or shutdown). This could be further improved by using a completion port and registering process + // handles with it, but this is a reasonable first implementation given that we shouldn't be dealing + // with an enormous number of concurrent processes. + // + // On other platforms we just wait on the monitor event and poll for process exits at intervals. +# if ZEN_PLATFORM_WINDOWS + auto WaitOnce = [&] { + HANDLE WaitHandles[MAXIMUM_WAIT_OBJECTS]; + + uint32_t NumHandles = 0; + + WaitHandles[NumHandles++] = m_MonitorThreadEvent.GetWindowsHandle(); + + m_RunningLock.WithSharedLock([&] { + for (auto It = begin(m_RunningMap), ItEnd = end(m_RunningMap); It != ItEnd && NumHandles < MAXIMUM_WAIT_OBJECTS; ++It) + { + Ref Action = It->second; + + WaitHandles[NumHandles++] = Action->ProcessHandle; + } + }); + + DWORD WaitResult = WaitForMultipleObjects(NumHandles, WaitHandles, FALSE, 1000); + + // return true if a handle was signaled + return (WaitResult <= NumHandles); + }; +# else + auto WaitOnce = [&] { return m_MonitorThreadEvent.Wait(1000); }; +# endif + + while (!WaitOnce()) + { + if (m_MonitorThreadEnabled == false) + { + return; + } + + SweepRunningActions(); + } + + // Signal received + + SweepRunningActions(); + } while (m_MonitorThreadEnabled); +} + +void +LocalProcessRunner::CancelRunningActions() +{ + Stopwatch Timer; + std::unordered_map> RunningMap; + + m_RunningLock.WithExclusiveLock([&] { std::swap(RunningMap, m_RunningMap); }); + + if (RunningMap.empty()) + { + return; + } + + ZEN_INFO("cancelling all running actions"); + + // For expedience we initiate the process termination for all known + // processes before attempting to wait for them to exit. + + std::vector TerminatedLsnList; + + for (const auto& Kv : RunningMap) + { + Ref Action = Kv.second; + + // Terminate running process + +# if ZEN_PLATFORM_WINDOWS + BOOL Success = TerminateProcess(Action->ProcessHandle, 222); + + if (Success) + { + TerminatedLsnList.push_back(Kv.first); + } + else + { + DWORD LastError = GetLastError(); + + if (LastError != ERROR_ACCESS_DENIED) + { + ZEN_WARN("TerminateProcess for LSN {} not successful: {}", Action->Action->ActionLsn, GetSystemErrorAsString(LastError)); + } + } +# else + ZEN_NOT_IMPLEMENTED("need to implement process termination"); +# endif + } + + // We only post results for processes we have terminated, in order + // to avoid multiple results getting posted for the same action + + for (int Lsn : TerminatedLsnList) + { + if (auto It = RunningMap.find(Lsn); It != RunningMap.end()) + { + Ref Running = It->second; + +# if ZEN_PLATFORM_WINDOWS + if (Running->ProcessHandle != INVALID_HANDLE_VALUE) + { + DWORD WaitResult = WaitForSingleObject(Running->ProcessHandle, 2000); + + if (WaitResult != WAIT_OBJECT_0) + { + ZEN_WARN("wait for LSN {}: process exit did not succeed, result = {}", Running->Action->ActionLsn, WaitResult); + } + else + { + ZEN_DEBUG("LSN {}: process exit OK", Running->Action->ActionLsn); + } + } +# endif + + // Clean up and post error result + + DeleteDirectories(Running->SandboxPath); + Running->Action->SetActionState(RunnerAction::State::Failed); + } + } + + ZEN_INFO("DONE - cancelled {} running processes (took {})", TerminatedLsnList.size(), NiceTimeSpanMs(Timer.GetElapsedTimeMs())); +} + +void +LocalProcessRunner::SweepRunningActions() +{ + std::vector> CompletedActions; + + m_RunningLock.WithExclusiveLock([&] { + // TODO: It would be good to not hold the exclusive lock while making + // system calls and other expensive operations. + + for (auto It = begin(m_RunningMap), ItEnd = end(m_RunningMap); It != ItEnd;) + { + Ref Action = It->second; + +# if ZEN_PLATFORM_WINDOWS + DWORD ExitCode = 0; + BOOL IsSuccess = GetExitCodeProcess(Action->ProcessHandle, &ExitCode); + + if (IsSuccess && ExitCode != STILL_ACTIVE) + { + CloseHandle(Action->ProcessHandle); + Action->ProcessHandle = INVALID_HANDLE_VALUE; + + CompletedActions.push_back(std::move(Action)); + It = m_RunningMap.erase(It); + } + else + { + ++It; + } +# else + // TODO: implement properly for Mac/Linux + + ZEN_UNUSED(Action); +# endif + } + }); + + // Notify outer. Note that this has to be done without holding any local locks + // otherwise we may end up with deadlocks. + + for (Ref Running : CompletedActions) + { + const int ActionLsn = Running->Action->ActionLsn; + + if (Running->ExitCode == 0) + { + try + { + // Gather outputs + + CbPackage OutputPackage = GatherActionOutputs(Running->SandboxPath); + + Running->Action->SetResult(std::move(OutputPackage)); + Running->Action->SetActionState(RunnerAction::State::Completed); + + // We can delete the files at this point + if (!DeleteDirectories(Running->SandboxPath)) + { + ZEN_WARN("Unable to delete directory '{}', this will continue to exist until service restart", Running->SandboxPath); + } + + // Success -- continue with next iteration of the loop + continue; + } + catch (std::exception& Ex) + { + ZEN_ERROR("Encountered failure while gathering outputs for action lsn {}, '{}'", ActionLsn, Ex.what()); + } + } + + // Failed - for now this is indicated with an empty package in + // the results map. We can clean out the sandbox directory immediately. + + std::error_code Ec; + DeleteDirectories(Running->SandboxPath, Ec); + + if (Ec) + { + ZEN_WARN("Unable to delete sandbox directory '{}': {}", Running->SandboxPath, Ec.message()); + } + + Running->Action->SetActionState(RunnerAction::State::Failed); + } +} + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/localrunner.h b/src/zencompute/localrunner.h new file mode 100644 index 000000000..35f464805 --- /dev/null +++ b/src/zencompute/localrunner.h @@ -0,0 +1,100 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencompute/functionservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "functionrunner.h" + +# include +# include +# include +# include +# include + +# include +# include +# include + +namespace zen { +class CbPackage; +} + +namespace zen::compute { + +/** Direct process spawner + + This runner simply sets up a directory structure for each job and + creates a process to perform the computation in it. It is not very + efficient and is intended mostly for testing. + + */ + +class LocalProcessRunner : public FunctionRunner +{ + LocalProcessRunner(LocalProcessRunner&&) = delete; + LocalProcessRunner& operator=(LocalProcessRunner&&) = delete; + +public: + LocalProcessRunner(ChunkResolver& Resolver, const std::filesystem::path& BaseDir); + ~LocalProcessRunner(); + + virtual void Shutdown() override; + virtual void RegisterWorker(const CbPackage& WorkerPackage) override; + [[nodiscard]] virtual SubmitResult SubmitAction(Ref Action) override; + [[nodiscard]] virtual bool IsHealthy() override { return true; } + [[nodiscard]] virtual size_t GetSubmittedActionCount() override; + [[nodiscard]] virtual size_t QueryCapacity() override; + [[nodiscard]] virtual std::vector SubmitActions(const std::vector>& Actions) override; + +protected: + LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + struct RunningAction : public RefCounted + { + Ref Action; + void* ProcessHandle = nullptr; + int ExitCode = 0; + std::filesystem::path SandboxPath; + }; + + std::atomic_bool m_AcceptNewActions; + ChunkResolver& m_ChunkResolver; + RwLock m_WorkerLock; + std::filesystem::path m_WorkerPath; + std::atomic m_SandboxCounter = 0; + std::filesystem::path m_SandboxPath; + int32_t m_MaxRunningActions = 64; // arbitrary limit for testing + + // if used in conjuction with m_ResultsLock, this lock must be taken *after* + // m_ResultsLock to avoid deadlocks + RwLock m_RunningLock; + std::unordered_map> m_RunningMap; + + std::thread m_MonitorThread; + std::atomic m_MonitorThreadEnabled{true}; + Event m_MonitorThreadEvent; + void MonitorThreadFunction(); + void SweepRunningActions(); + void CancelRunningActions(); + + std::filesystem::path CreateNewSandbox(); + void ManifestWorker(const CbPackage& WorkerPackage, + const std::filesystem::path& SandboxPath, + std::function&& ChunkReferenceCallback); + std::filesystem::path ManifestWorker(const WorkerDesc& Worker); + CbPackage GatherActionOutputs(std::filesystem::path SandboxPath); + + void DecompressAttachmentToFile(const CbPackage& FromPackage, + CbObjectView FileEntry, + const std::filesystem::path& SandboxRootPath, + std::function& ChunkReferenceCallback); +}; + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/recordingreader.cpp b/src/zencompute/recordingreader.cpp new file mode 100644 index 000000000..1c1a119cf --- /dev/null +++ b/src/zencompute/recordingreader.cpp @@ -0,0 +1,335 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencompute/recordingreader.h" + +#include +#include +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +# define ZEN_CONCRT_AVAILABLE 1 +#else +# define ZEN_CONCRT_AVAILABLE 0 +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen::compute { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +# if ZEN_PLATFORM_WINDOWS +# define ZEN_BUILD_ACTION L"Build.action" +# define ZEN_WORKER_UCB L"worker.ucb" +# else +# define ZEN_BUILD_ACTION "Build.action" +# define ZEN_WORKER_UCB "worker.ucb" +# endif + +////////////////////////////////////////////////////////////////////////// + +struct RecordingTreeVisitor : public FileSystemTraversal::TreeVisitor +{ + virtual void VisitFile(const std::filesystem::path& Parent, + const path_view& File, + uint64_t FileSize, + uint32_t NativeModeOrAttributes, + uint64_t NativeModificationTick) + { + ZEN_UNUSED(Parent, File, FileSize, NativeModeOrAttributes, NativeModificationTick); + + if (File.compare(path_view(ZEN_BUILD_ACTION)) == 0) + { + WorkDirs.push_back(Parent); + } + else if (File.compare(path_view(ZEN_WORKER_UCB)) == 0) + { + WorkerDirs.push_back(Parent); + } + } + + virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName, uint32_t NativeModeOrAttributes) + { + ZEN_UNUSED(Parent, DirectoryName, NativeModeOrAttributes); + + return true; + } + + std::vector WorkerDirs; + std::vector WorkDirs; +}; + +////////////////////////////////////////////////////////////////////////// + +void +IterateOverArray(auto Array, auto Func, int TargetParallelism) +{ +# if ZEN_CONCRT_AVAILABLE + if (TargetParallelism > 1) + { + concurrency::simple_partitioner Chunker(Array.size() / TargetParallelism); + concurrency::parallel_for_each(begin(Array), end(Array), [&](const auto& Item) { Func(Item); }); + + return; + } +# else + ZEN_UNUSED(TargetParallelism); +# endif + + for (const auto& Item : Array) + { + Func(Item); + } +} + +////////////////////////////////////////////////////////////////////////// + +RecordingReaderBase::~RecordingReaderBase() = default; + +////////////////////////////////////////////////////////////////////////// + +RecordingReader::RecordingReader(const std::filesystem::path& RecordingPath) : m_RecordingLogDir(RecordingPath) +{ + CidStoreConfiguration CidConfig; + CidConfig.RootDirectory = m_RecordingLogDir / "cid"; + CidConfig.HugeValueThreshold = 128 * 1024 * 1024; + + m_CidStore.Initialize(CidConfig); +} + +RecordingReader::~RecordingReader() +{ + m_CidStore.Flush(); +} + +size_t +RecordingReader::GetActionCount() const +{ + return m_Actions.size(); +} + +IoBuffer +RecordingReader::FindChunkByCid(const IoHash& DecompressedId) +{ + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(DecompressedId)) + { + return Chunk; + } + + ZEN_ERROR("failed lookup of chunk with CID '{}'", DecompressedId); + + return {}; +} + +std::unordered_map +RecordingReader::ReadWorkers() +{ + std::unordered_map WorkerMap; + + { + CbObjectFromFile TocFile = LoadCompactBinaryObject(m_RecordingLogDir / "workers.ztoc"); + CbObject Toc = TocFile.Object; + + m_WorkerDataFile.Open(m_RecordingLogDir / "workers.zdat", BasicFile::Mode::kRead); + + ZEN_ASSERT(Toc["version"sv].AsInt32() == 1); + + for (auto& It : Toc["toc"]) + { + CbArrayView Entry = It.AsArrayView(); + CbFieldViewIterator Vit = Entry.CreateViewIterator(); + + const IoHash WorkerId = Vit++->AsHash(); + const uint64_t Offset = Vit++->AsInt64(0); + const uint64_t Size = Vit++->AsInt64(0); + + IoBuffer WorkerRange = m_WorkerDataFile.ReadRange(Offset, Size); + CbObject WorkerDesc = LoadCompactBinaryObject(WorkerRange); + CbPackage& WorkerPkg = WorkerMap[WorkerId]; + WorkerPkg.SetObject(WorkerDesc); + + WorkerDesc.IterateAttachments([&](const zen::CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + IoBuffer AttachmentData = m_CidStore.FindChunkByCid(AttachmentCid); + + if (AttachmentData) + { + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); + WorkerPkg.AddAttachment(CbAttachment(CompressedData, RawHash)); + } + }); + } + } + + // Scan actions as well (this should be called separately, ideally) + + ScanActions(); + + return WorkerMap; +} + +void +RecordingReader::ScanActions() +{ + CbObjectFromFile TocFile = LoadCompactBinaryObject(m_RecordingLogDir / "actions.ztoc"); + CbObject Toc = TocFile.Object; + + m_ActionDataFile.Open(m_RecordingLogDir / "actions.zdat", BasicFile::Mode::kRead); + + ZEN_ASSERT(Toc["version"sv].AsInt32() == 1); + + for (auto& It : Toc["toc"]) + { + CbArrayView ArrayEntry = It.AsArrayView(); + CbFieldViewIterator Vit = ArrayEntry.CreateViewIterator(); + + ActionEntry Entry; + Entry.ActionId = Vit++->AsHash(); + Entry.Offset = Vit++->AsInt64(0); + Entry.Size = Vit++->AsInt64(0); + + m_Actions.push_back(Entry); + } +} + +void +RecordingReader::IterateActions(std::function&& Callback, int TargetParallelism) +{ + IterateOverArray( + m_Actions, + [&](const ActionEntry& Entry) { + CbObject ActionDesc = LoadCompactBinaryObject(m_ActionDataFile.ReadRange(Entry.Offset, Entry.Size)); + + Callback(ActionDesc, Entry.ActionId); + }, + TargetParallelism); +} + +////////////////////////////////////////////////////////////////////////// + +IoBuffer +LocalResolver::FindChunkByCid(const IoHash& DecompressedId) +{ + RwLock::SharedLockScope _(MapLock); + if (auto It = Attachments.find(DecompressedId); It != Attachments.end()) + { + return It->second; + } + + return {}; +} + +void +LocalResolver::Add(const IoHash& Cid, IoBuffer Data) +{ + RwLock::ExclusiveLockScope _(MapLock); + Data.SetContentType(ZenContentType::kCompressedBinary); + Attachments[Cid] = Data; +} + +/// + +UeRecordingReader::UeRecordingReader(const std::filesystem::path& RecordingPath) : m_RecordingDir(RecordingPath) +{ +} + +UeRecordingReader::~UeRecordingReader() +{ +} + +size_t +UeRecordingReader::GetActionCount() const +{ + return m_WorkDirs.size(); +} + +IoBuffer +UeRecordingReader::FindChunkByCid(const IoHash& DecompressedId) +{ + return m_LocalResolver.FindChunkByCid(DecompressedId); +} + +std::unordered_map +UeRecordingReader::ReadWorkers() +{ + std::unordered_map WorkerMap; + + FileSystemTraversal Traversal; + RecordingTreeVisitor Visitor; + Traversal.TraverseFileSystem(m_RecordingDir, Visitor); + + m_WorkDirs = std::move(Visitor.WorkDirs); + + for (const std::filesystem::path& WorkerDir : Visitor.WorkerDirs) + { + CbObjectFromFile WorkerFile = LoadCompactBinaryObject(WorkerDir / "worker.ucb"); + CbObject WorkerDesc = WorkerFile.Object; + const IoHash& WorkerId = WorkerFile.Hash; + CbPackage& WorkerPkg = WorkerMap[WorkerId]; + WorkerPkg.SetObject(WorkerDesc); + + WorkerDesc.IterateAttachments([&](const zen::CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + IoBuffer AttachmentData = ReadFile(WorkerDir / "chunks" / AttachmentCid.ToHexString()).Flatten(); + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); + WorkerPkg.AddAttachment(CbAttachment(CompressedData, RawHash)); + }); + } + + return WorkerMap; +} + +void +UeRecordingReader::IterateActions(std::function&& Callback, int ParallelismTarget) +{ + IterateOverArray( + m_WorkDirs, + [&](const std::filesystem::path& WorkDir) { + CbPackage WorkPackage = ReadAction(WorkDir); + CbObject ActionObject = WorkPackage.GetObject(); + const IoHash& ActionId = WorkPackage.GetObjectHash(); + + Callback(ActionObject, ActionId); + }, + ParallelismTarget); +} + +CbPackage +UeRecordingReader::ReadAction(std::filesystem::path WorkDir) +{ + CbPackage WorkPackage; + std::filesystem::path WorkDescPath = WorkDir / "Build.action"; + CbObjectFromFile ActionFile = LoadCompactBinaryObject(WorkDescPath); + CbObject& ActionObject = ActionFile.Object; + + WorkPackage.SetObject(ActionObject); + + ActionObject.IterateAttachments([&](const zen::CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + IoBuffer AttachmentData = ReadFile(WorkDir / "inputs" / AttachmentCid.ToHexString()).Flatten(); + + m_LocalResolver.Add(AttachmentCid, AttachmentData); + + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); + ZEN_ASSERT(AttachmentCid == RawHash); + WorkPackage.AddAttachment(CbAttachment(CompressedData, RawHash)); + }); + + return WorkPackage; +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/remotehttprunner.cpp b/src/zencompute/remotehttprunner.cpp new file mode 100644 index 000000000..98ced5fe8 --- /dev/null +++ b/src/zencompute/remotehttprunner.cpp @@ -0,0 +1,457 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "remotehttprunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include + +////////////////////////////////////////////////////////////////////////// + +namespace zen::compute { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +RemoteHttpRunner::RemoteHttpRunner(ChunkResolver& InChunkResolver, const std::filesystem::path& BaseDir, std::string_view HostName) +: FunctionRunner(BaseDir) +, m_Log(logging::Get("http_exec")) +, m_ChunkResolver{InChunkResolver} +, m_BaseUrl{fmt::format("{}/apply", HostName)} +, m_Http(m_BaseUrl) +{ + m_MonitorThread = std::thread{&RemoteHttpRunner::MonitorThreadFunction, this}; +} + +RemoteHttpRunner::~RemoteHttpRunner() +{ + Shutdown(); +} + +void +RemoteHttpRunner::Shutdown() +{ + // TODO: should cleanly drain/cancel pending work + + m_MonitorThreadEnabled = false; + m_MonitorThreadEvent.Set(); + if (m_MonitorThread.joinable()) + { + m_MonitorThread.join(); + } +} + +void +RemoteHttpRunner::RegisterWorker(const CbPackage& WorkerPackage) +{ + const IoHash WorkerId = WorkerPackage.GetObjectHash(); + CbPackage WorkerDesc = WorkerPackage; + + std::string WorkerUrl = fmt::format("/workers/{}", WorkerId); + + HttpClient::Response WorkerResponse = m_Http.Get(WorkerUrl); + + if (WorkerResponse.StatusCode == HttpResponseCode::NotFound) + { + HttpClient::Response DescResponse = m_Http.Post(WorkerUrl, WorkerDesc.GetObject()); + + if (DescResponse.StatusCode == HttpResponseCode::NotFound) + { + CbPackage Pkg = WorkerDesc; + + // Build response package by sending only the attachments + // the other end needs. We start with the full package and + // remove the attachments which are not needed. + + { + std::unordered_set Needed; + + CbObject Response = DescResponse.AsObject(); + + for (auto& Item : Response["need"sv]) + { + const IoHash NeedHash = Item.AsHash(); + + Needed.insert(NeedHash); + } + + std::unordered_set ToRemove; + + for (const CbAttachment& Attachment : Pkg.GetAttachments()) + { + const IoHash& Hash = Attachment.GetHash(); + + if (Needed.find(Hash) == Needed.end()) + { + ToRemove.insert(Hash); + } + } + + for (const IoHash& Hash : ToRemove) + { + int RemovedCount = Pkg.RemoveAttachment(Hash); + + ZEN_ASSERT(RemovedCount == 1); + } + } + + // Post resulting package + + HttpClient::Response PayloadResponse = m_Http.Post(WorkerUrl, Pkg); + + if (!IsHttpSuccessCode(PayloadResponse.StatusCode)) + { + ZEN_ERROR("ERROR: unable to register payloads for worker {} at {}{}", WorkerId, m_Http.GetBaseUri(), WorkerUrl); + + // TODO: propagate error + } + } + else if (!IsHttpSuccessCode(DescResponse.StatusCode)) + { + ZEN_ERROR("ERROR: unable to register worker {} at {}{}", WorkerId, m_Http.GetBaseUri(), WorkerUrl); + + // TODO: propagate error + } + else + { + ZEN_ASSERT(DescResponse.StatusCode == HttpResponseCode::NoContent); + } + } + else if (WorkerResponse.StatusCode == HttpResponseCode::OK) + { + // Already known from a previous run + } + else if (!IsHttpSuccessCode(WorkerResponse.StatusCode)) + { + ZEN_ERROR("ERROR: unable to look up worker {} at {}{} (error: {} {})", + WorkerId, + m_Http.GetBaseUri(), + WorkerUrl, + (int)WorkerResponse.StatusCode, + ToString(WorkerResponse.StatusCode)); + + // TODO: propagate error + } +} + +size_t +RemoteHttpRunner::QueryCapacity() +{ + // Estimate how much more work we're ready to accept + + RwLock::SharedLockScope _{m_RunningLock}; + + size_t RunningCount = m_RemoteRunningMap.size(); + + if (RunningCount >= size_t(m_MaxRunningActions)) + { + return 0; + } + + return m_MaxRunningActions - RunningCount; +} + +std::vector +RemoteHttpRunner::SubmitActions(const std::vector>& Actions) +{ + std::vector Results; + + for (const Ref& Action : Actions) + { + Results.push_back(SubmitAction(Action)); + } + + return Results; +} + +SubmitResult +RemoteHttpRunner::SubmitAction(Ref Action) +{ + // Verify whether we can accept more work + + { + RwLock::SharedLockScope _{m_RunningLock}; + if (m_RemoteRunningMap.size() >= size_t(m_MaxRunningActions)) + { + return SubmitResult{.IsAccepted = false}; + } + } + + using namespace std::literals; + + // Each enqueued action is assigned an integer index (logical sequence number), + // which we use as a key for tracking data structures and as an opaque id which + // may be used by clients to reference the scheduled action + + const int32_t ActionLsn = Action->ActionLsn; + const CbObject& ActionObj = Action->ActionObj; + const IoHash ActionId = ActionObj.GetHash(); + + MaybeDumpAction(ActionLsn, ActionObj); + + // Enqueue job + + CbObject Result; + + HttpClient::Response WorkResponse = m_Http.Post("/jobs", ActionObj); + HttpResponseCode WorkResponseCode = WorkResponse.StatusCode; + + if (WorkResponseCode == HttpResponseCode::OK) + { + Result = WorkResponse.AsObject(); + } + else if (WorkResponseCode == HttpResponseCode::NotFound) + { + // Not all attachments are present + + // Build response package including all required attachments + + CbPackage Pkg; + Pkg.SetObject(ActionObj); + + CbObject Response = WorkResponse.AsObject(); + + for (auto& Item : Response["need"sv]) + { + const IoHash NeedHash = Item.AsHash(); + + if (IoBuffer Chunk = m_ChunkResolver.FindChunkByCid(NeedHash)) + { + uint64_t DataRawSize = 0; + IoHash DataRawHash; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer{Chunk}, /* out */ DataRawHash, /* out */ DataRawSize); + + ZEN_ASSERT(DataRawHash == NeedHash); + + Pkg.AddAttachment(CbAttachment(Compressed, NeedHash)); + } + else + { + // No such attachment + + return {.IsAccepted = false, .Reason = fmt::format("missing attachment {}", NeedHash)}; + } + } + + // Post resulting package + + HttpClient::Response PayloadResponse = m_Http.Post("/jobs", Pkg); + + if (!PayloadResponse) + { + ZEN_WARN("unable to register payloads for action {} at {}/jobs", ActionId, m_Http.GetBaseUri()); + + // TODO: include more information about the failure in the response + + return {.IsAccepted = false, .Reason = "HTTP request failed"}; + } + else if (PayloadResponse.StatusCode == HttpResponseCode::OK) + { + Result = PayloadResponse.AsObject(); + } + else + { + // Unexpected response + + const int ResponseStatusCode = (int)PayloadResponse.StatusCode; + + ZEN_WARN("unable to register payloads for action {} at {}/jobs (error: {} {})", + ActionId, + m_Http.GetBaseUri(), + ResponseStatusCode, + ToString(ResponseStatusCode)); + + return {.IsAccepted = false, + .Reason = fmt::format("unexpected response code {} {} from {}/jobs", + ResponseStatusCode, + ToString(ResponseStatusCode), + m_Http.GetBaseUri())}; + } + } + + if (Result) + { + if (const int32_t LsnField = Result["lsn"].AsInt32(0)) + { + HttpRunningAction NewAction; + NewAction.Action = Action; + NewAction.RemoteActionLsn = LsnField; + + { + RwLock::ExclusiveLockScope _(m_RunningLock); + + m_RemoteRunningMap[LsnField] = std::move(NewAction); + } + + ZEN_DEBUG("scheduled action {} with remote LSN {} (local LSN {})", ActionId, LsnField, ActionLsn); + + Action->SetActionState(RunnerAction::State::Running); + + return SubmitResult{.IsAccepted = true}; + } + } + + return {}; +} + +bool +RemoteHttpRunner::IsHealthy() +{ + if (HttpClient::Response Ready = m_Http.Get("/ready")) + { + return true; + } + else + { + // TODO: use response to propagate context + return false; + } +} + +size_t +RemoteHttpRunner::GetSubmittedActionCount() +{ + RwLock::SharedLockScope _(m_RunningLock); + return m_RemoteRunningMap.size(); +} + +void +RemoteHttpRunner::MonitorThreadFunction() +{ + SetCurrentThreadName("RemoteHttpRunner_Monitor"); + + do + { + const int NormalWaitingTime = 1000; + int WaitTimeMs = NormalWaitingTime; + auto WaitOnce = [&] { return m_MonitorThreadEvent.Wait(WaitTimeMs); }; + auto SweepOnce = [&] { + const size_t RetiredCount = SweepRunningActions(); + + m_RunningLock.WithSharedLock([&] { + if (m_RemoteRunningMap.size() > 16) + { + WaitTimeMs = NormalWaitingTime / 4; + } + else + { + if (RetiredCount) + { + WaitTimeMs = NormalWaitingTime / 2; + } + else + { + WaitTimeMs = NormalWaitingTime; + } + } + }); + }; + + while (!WaitOnce()) + { + SweepOnce(); + } + + // Signal received - this may mean we should quit + + SweepOnce(); + } while (m_MonitorThreadEnabled); +} + +size_t +RemoteHttpRunner::SweepRunningActions() +{ + std::vector CompletedActions; + + // Poll remote for list of completed actions + + HttpClient::Response ResponseCompleted = m_Http.Get("/jobs/completed"sv); + + if (CbObject Completed = ResponseCompleted.AsObject()) + { + for (auto& FieldIt : Completed["completed"sv]) + { + const int32_t CompleteLsn = FieldIt.AsInt32(); + + if (HttpClient::Response ResponseJob = m_Http.Get(fmt::format("/jobs/{}"sv, CompleteLsn))) + { + m_RunningLock.WithExclusiveLock([&] { + if (auto CompleteIt = m_RemoteRunningMap.find(CompleteLsn); CompleteIt != m_RemoteRunningMap.end()) + { + HttpRunningAction CompletedAction = std::move(CompleteIt->second); + CompletedAction.ActionResults = ResponseJob.AsPackage(); + CompletedAction.Success = true; + + CompletedActions.push_back(std::move(CompletedAction)); + m_RemoteRunningMap.erase(CompleteIt); + } + else + { + // we received a completion notice for an action we don't know about, + // this can happen if the runner is used by multiple upstream schedulers, + // or if this compute node was recently restarted and lost track of + // previously scheduled actions + } + }); + } + } + + if (CbObjectView Metrics = Completed["metrics"sv].AsObjectView()) + { + // if (const size_t CpuCount = Metrics["core_count"].AsInt32(0)) + if (const int32_t CpuCount = Metrics["lp_count"].AsInt32(0)) + { + const int32_t NewCap = zen::Max(4, CpuCount); + + if (m_MaxRunningActions > NewCap) + { + ZEN_DEBUG("capping {} to {} actions (was {})", m_BaseUrl, NewCap, m_MaxRunningActions); + + m_MaxRunningActions = NewCap; + } + } + } + } + + // Notify outer. Note that this has to be done without holding any local locks + // otherwise we may end up with deadlocks. + + for (HttpRunningAction& HttpAction : CompletedActions) + { + const int ActionLsn = HttpAction.Action->ActionLsn; + + if (HttpAction.Success) + { + ZEN_DEBUG("completed: {} LSN {} (remote LSN {})", HttpAction.Action->ActionId, ActionLsn, HttpAction.RemoteActionLsn); + + HttpAction.Action->SetActionState(RunnerAction::State::Completed); + + HttpAction.Action->SetResult(std::move(HttpAction.ActionResults)); + } + else + { + HttpAction.Action->SetActionState(RunnerAction::State::Failed); + } + } + + return CompletedActions.size(); +} + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/remotehttprunner.h b/src/zencompute/remotehttprunner.h new file mode 100644 index 000000000..1e885da3d --- /dev/null +++ b/src/zencompute/remotehttprunner.h @@ -0,0 +1,80 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencompute/functionservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "functionrunner.h" + +# include +# include +# include +# include + +# include +# include +# include + +namespace zen { +class CidStore; +} + +namespace zen::compute { + +/** HTTP-based runner + + This implements a DDC remote compute execution strategy via REST API + + */ + +class RemoteHttpRunner : public FunctionRunner +{ + RemoteHttpRunner(RemoteHttpRunner&&) = delete; + RemoteHttpRunner& operator=(RemoteHttpRunner&&) = delete; + +public: + RemoteHttpRunner(ChunkResolver& InChunkResolver, const std::filesystem::path& BaseDir, std::string_view HostName); + ~RemoteHttpRunner(); + + virtual void Shutdown() override; + virtual void RegisterWorker(const CbPackage& WorkerPackage) override; + [[nodiscard]] virtual SubmitResult SubmitAction(Ref Action) override; + [[nodiscard]] virtual bool IsHealthy() override; + [[nodiscard]] virtual size_t GetSubmittedActionCount() override; + [[nodiscard]] virtual size_t QueryCapacity() override; + [[nodiscard]] virtual std::vector SubmitActions(const std::vector>& Actions) override; + +protected: + LoggerRef Log() { return m_Log; } + +private: + LoggerRef m_Log; + ChunkResolver& m_ChunkResolver; + std::string m_BaseUrl; + HttpClient m_Http; + + int32_t m_MaxRunningActions = 256; // arbitrary limit for testing + + struct HttpRunningAction + { + Ref Action; + int RemoteActionLsn = 0; // Remote LSN + bool Success = false; + CbPackage ActionResults; + }; + + RwLock m_RunningLock; + std::unordered_map m_RemoteRunningMap; // Note that this is keyed on the *REMOTE* lsn + + std::thread m_MonitorThread; + std::atomic m_MonitorThreadEnabled{true}; + Event m_MonitorThreadEvent; + void MonitorThreadFunction(); + size_t SweepRunningActions(); +}; + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/xmake.lua b/src/zencompute/xmake.lua new file mode 100644 index 000000000..c710b662d --- /dev/null +++ b/src/zencompute/xmake.lua @@ -0,0 +1,11 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +target('zencompute') + set_kind("static") + set_group("libs") + add_headerfiles("**.h") + add_files("**.cpp") + add_includedirs("include", {public=true}) + add_deps("zencore", "zenstore", "zenutil", "zennet", "zenhttp") + add_packages("vcpkg::gsl-lite") + add_packages("vcpkg::spdlog", "vcpkg::cxxopts") diff --git a/src/zencompute/zencompute.cpp b/src/zencompute/zencompute.cpp new file mode 100644 index 000000000..633250f4e --- /dev/null +++ b/src/zencompute/zencompute.cpp @@ -0,0 +1,12 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencompute/zencompute.h" + +namespace zen { + +void +zencompute_forcelinktests() +{ +} + +} // namespace zen diff --git a/src/zennet/beacon.cpp b/src/zennet/beacon.cpp new file mode 100644 index 000000000..394a4afbb --- /dev/null +++ b/src/zennet/beacon.cpp @@ -0,0 +1,170 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +struct FsBeacon::Impl +{ + Impl(std::filesystem::path ShareRoot); + ~Impl(); + + void EnsureValid(); + + void AddGroup(std::string_view GroupId, CbObject Metadata); + void ScanGroup(std::string_view GroupId, std::vector& OutSessions); + void ReadMetadata(std::string_view GroupId, const std::vector& InSessions, std::vector& OutMetadata); + +private: + std::filesystem::path m_ShareRoot; + zen::Oid m_SessionId; + + struct GroupData + { + CbObject Metadata; + BasicFile LockFile; + }; + + std::map m_Registration; + + std::filesystem::path GetSessionMarkerPath(std::string_view GroupId, const Oid& SessionId) + { + Oid::String_t SessionIdString; + SessionId.ToString(SessionIdString); + + return m_ShareRoot / GroupId / SessionIdString; + } +}; + +FsBeacon::Impl::Impl(std::filesystem::path ShareRoot) : m_ShareRoot(ShareRoot), m_SessionId(GetSessionId()) +{ +} + +FsBeacon::Impl::~Impl() +{ +} + +void +FsBeacon::Impl::EnsureValid() +{ +} + +void +FsBeacon::Impl::AddGroup(std::string_view GroupId, CbObject Metadata) +{ + zen::CreateDirectories(m_ShareRoot / GroupId); + std::filesystem::path MarkerFile = GetSessionMarkerPath(GroupId, m_SessionId); + + GroupData& Group = m_Registration[std::string(GroupId)]; + + Group.Metadata = Metadata; + + std::error_code Ec; + Group.LockFile.Open(MarkerFile, + BasicFile::Mode::kTruncate | BasicFile::Mode::kPreventDelete | + BasicFile::Mode::kPreventWrite /* | BasicFile::Mode::kDeleteOnClose */, + Ec); + + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to open beacon marker file '{}' for write", MarkerFile)); + } + + Group.LockFile.WriteAll(Metadata.GetBuffer().AsIoBuffer(), Ec); + + if (Ec) + { + throw std::system_error(Ec, fmt::format("failed to write to beacon marker file '{}'", MarkerFile)); + } + + Group.LockFile.Flush(); +} + +void +FsBeacon::Impl::ScanGroup(std::string_view GroupId, std::vector& OutSessions) +{ + DirectoryContent Dc; + zen::GetDirectoryContent(m_ShareRoot / GroupId, zen::DirectoryContentFlags::IncludeFiles, /* out */ Dc); + + for (const std::filesystem::path& FilePath : Dc.Files) + { + std::filesystem::path File = FilePath.filename(); + + std::error_code Ec; + if (std::filesystem::remove(FilePath, Ec) == false) + { + auto FileString = File.generic_string(); + + if (FileString.length() != Oid::StringLength) + continue; + + if (const Oid SessionId = Oid::FromHexString(FileString)) + { + if (std::filesystem::file_size(File, Ec) > 0) + { + OutSessions.push_back(SessionId); + } + } + } + } +} + +void +FsBeacon::Impl::ReadMetadata(std::string_view GroupId, const std::vector& InSessions, std::vector& OutMetadata) +{ + for (const Oid& SessionId : InSessions) + { + const std::filesystem::path MarkerFile = GetSessionMarkerPath(GroupId, SessionId); + + if (CbObject Metadata = LoadCompactBinaryObject(MarkerFile).Object) + { + OutMetadata.push_back(std::move(Metadata)); + } + } +} + +////////////////////////////////////////////////////////////////////////// + +FsBeacon::FsBeacon(std::filesystem::path ShareRoot) : m_Impl(std::make_unique(ShareRoot)) +{ +} + +FsBeacon::~FsBeacon() +{ +} + +void +FsBeacon::AddGroup(std::string_view GroupId, CbObject Metadata) +{ + m_Impl->AddGroup(GroupId, Metadata); +} + +void +FsBeacon::ScanGroup(std::string_view GroupId, std::vector& OutSessions) +{ + m_Impl->ScanGroup(GroupId, OutSessions); +} + +void +FsBeacon::ReadMetadata(std::string_view GroupId, const std::vector& InSessions, std::vector& OutMetadata) +{ + m_Impl->ReadMetadata(GroupId, InSessions, OutMetadata); +} + +////////////////////////////////////////////////////////////////////////// + +} // namespace zen diff --git a/src/zennet/include/zennet/beacon.h b/src/zennet/include/zennet/beacon.h new file mode 100644 index 000000000..a8d4805cb --- /dev/null +++ b/src/zennet/include/zennet/beacon.h @@ -0,0 +1,38 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +#include +#include +#include +#include + +namespace zen { + +class CbObject; + +/** File-system based peer discovery + + Intended to be used with an SMB file share as the root. + */ + +class FsBeacon +{ +public: + FsBeacon(std::filesystem::path ShareRoot); + ~FsBeacon(); + + void AddGroup(std::string_view GroupId, CbObject Metadata); + void ScanGroup(std::string_view GroupId, std::vector& OutSessions); + void ReadMetadata(std::string_view GroupId, const std::vector& InSessions, std::vector& OutMetadata); + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +} // namespace zen diff --git a/src/zennet/include/zennet/statsdclient.h b/src/zennet/include/zennet/statsdclient.h index c378e49ce..7688c132c 100644 --- a/src/zennet/include/zennet/statsdclient.h +++ b/src/zennet/include/zennet/statsdclient.h @@ -8,6 +8,8 @@ #include #include +#undef SendMessage + namespace zen { class StatsTransportBase diff --git a/src/zennet/statsdclient.cpp b/src/zennet/statsdclient.cpp index fe5ca4dda..a0e8cb6ce 100644 --- a/src/zennet/statsdclient.cpp +++ b/src/zennet/statsdclient.cpp @@ -12,6 +12,7 @@ ZEN_THIRD_PARTY_INCLUDES_START #include #include +#undef SendMessage ZEN_THIRD_PARTY_INCLUDES_END namespace zen { diff --git a/src/zenserver-test/function-tests.cpp b/src/zenserver-test/function-tests.cpp new file mode 100644 index 000000000..559387fa2 --- /dev/null +++ b/src/zenserver-test/function-tests.cpp @@ -0,0 +1,34 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#if ZEN_WITH_TESTS + +# include +# include +# include +# include +# include + +# include "zenserver-test.h" + +namespace zen::tests { + +using namespace std::literals; + +TEST_CASE("function.run") +{ + std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + Instance.SpawnServer(13337); + + ZEN_INFO("Waiting..."); + + Instance.WaitUntilReady(); +} + +} // namespace zen::tests + +#endif diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp new file mode 100644 index 000000000..173f56386 --- /dev/null +++ b/src/zenserver/compute/computeserver.cpp @@ -0,0 +1,330 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "computeserver.h" +#include +#include "computeservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +void +ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options) +{ + Options.add_option("compute", + "", + "upstream-notification-endpoint", + "Endpoint URL for upstream notifications", + cxxopts::value(m_ServerOptions.UpstreamNotificationEndpoint)->default_value(""), + ""); + + Options.add_option("compute", + "", + "instance-id", + "Instance ID for use in notifications", + cxxopts::value(m_ServerOptions.InstanceId)->default_value(""), + ""); +} + +void +ZenComputeServerConfigurator::AddConfigOptions(LuaConfig::Options& Options) +{ + ZEN_UNUSED(Options); +} + +void +ZenComputeServerConfigurator::ApplyOptions(cxxopts::Options& Options) +{ + ZEN_UNUSED(Options); +} + +void +ZenComputeServerConfigurator::OnConfigFileParsed(LuaConfig::Options& LuaOptions) +{ + ZEN_UNUSED(LuaOptions); +} + +void +ZenComputeServerConfigurator::ValidateOptions() +{ +} + +/////////////////////////////////////////////////////////////////////////// + +ZenComputeServer::ZenComputeServer() +{ +} + +ZenComputeServer::~ZenComputeServer() +{ + Cleanup(); +} + +int +ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServerState::ZenServerEntry* ServerEntry) +{ + ZEN_TRACE_CPU("ZenComputeServer::Initialize"); + ZEN_MEMSCOPE(GetZenserverTag()); + + ZEN_INFO(ZEN_APP_NAME " initializing in HUB server mode"); + + const int EffectiveBasePort = ZenServerBase::Initialize(ServerConfig, ServerEntry); + if (EffectiveBasePort < 0) + { + return EffectiveBasePort; + } + + // This is a workaround to make sure we can have automated tests. Without + // this the ranges for different child zen hub processes could overlap with + // the main test range. + ZenServerEnvironment::SetBaseChildId(1000); + + m_DebugOptionForcedCrash = ServerConfig.ShouldCrash; + + InitializeState(ServerConfig); + InitializeServices(ServerConfig); + RegisterServices(ServerConfig); + + ZenServerBase::Finalize(); + + return EffectiveBasePort; +} + +void +ZenComputeServer::Cleanup() +{ + ZEN_TRACE_CPU("ZenStorageServer::Cleanup"); + ZEN_INFO(ZEN_APP_NAME " cleaning up"); + try + { + m_IoContext.stop(); + if (m_IoRunner.joinable()) + { + m_IoRunner.join(); + } + + if (m_Http) + { + m_Http->Close(); + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("exception thrown during Cleanup() in {}: '{}'", ZEN_APP_NAME, Ex.what()); + } +} + +void +ZenComputeServer::InitializeState(const ZenComputeServerConfig& ServerConfig) +{ + ZEN_UNUSED(ServerConfig); +} + +void +ZenComputeServer::InitializeServices(const ZenComputeServerConfig& ServerConfig) +{ + ZEN_INFO("initializing storage"); + + CidStoreConfiguration Config; + Config.RootDirectory = m_DataRoot / "cas"; + + m_CidStore = std::make_unique(m_GcManager); + m_CidStore->Initialize(Config); + + ZEN_INFO("instantiating API service"); + m_ApiService = std::make_unique(*m_Http); + + ZEN_INFO("instantiating compute service"); + m_ComputeService = std::make_unique(ServerConfig.DataDir / "compute"); + + // Ref Runner; + // Runner = zen::compute::CreateLocalRunner(*m_CidStore, ServerConfig.DataDir / "runner"); + + // TODO: (re)implement default configuration here + + ZEN_INFO("instantiating function service"); + m_FunctionService = + std::make_unique(*m_CidStore, m_StatsService, ServerConfig.DataDir / "functions"); +} + +void +ZenComputeServer::RegisterServices(const ZenComputeServerConfig& ServerConfig) +{ + ZEN_UNUSED(ServerConfig); + + if (m_ComputeService) + { + m_Http->RegisterService(*m_ComputeService); + } + + if (m_ApiService) + { + m_Http->RegisterService(*m_ApiService); + } + + if (m_FunctionService) + { + m_Http->RegisterService(*m_FunctionService); + } +} + +void +ZenComputeServer::Run() +{ + if (m_ProcessMonitor.IsActive()) + { + CheckOwnerPid(); + } + + if (!m_TestMode) + { + // clang-format off + ZEN_INFO( R"(__________ _________ __ )" "\n" + R"(\____ /____ ____ \_ ___ \ ____ _____ ______ __ ___/ |_ ____ )" "\n" + R"( / // __ \ / \/ \ \/ / _ \ / \\____ \| | \ __\/ __ \ )" "\n" + R"( / /\ ___/| | \ \___( <_> ) Y Y \ |_> > | /| | \ ___/ )" "\n" + R"(/_______ \___ >___| /\______ /\____/|__|_| / __/|____/ |__| \___ >)" "\n" + R"( \/ \/ \/ \/ \/|__| \/ )"); + // clang-format on + + ExtendableStringBuilder<256> BuildOptions; + GetBuildOptions(BuildOptions, '\n'); + ZEN_INFO("Build options ({}/{}):\n{}", GetOperatingSystemName(), GetCpuName(), BuildOptions); + } + + ZEN_INFO(ZEN_APP_NAME " now running as COMPUTE (pid: {})", GetCurrentProcessId()); + +# if ZEN_PLATFORM_WINDOWS + if (zen::windows::IsRunningOnWine()) + { + ZEN_INFO("detected Wine session - " ZEN_APP_NAME " is not formally tested on Wine and may therefore not work or perform well"); + } +# endif + +# if ZEN_USE_SENTRY + ZEN_INFO("sentry crash handler {}", m_UseSentry ? "ENABLED" : "DISABLED"); + if (m_UseSentry) + { + SentryIntegration::ClearCaches(); + } +# endif + + if (m_DebugOptionForcedCrash) + { + ZEN_DEBUG_BREAK(); + } + + const bool IsInteractiveMode = IsInteractiveSession(); // &&!m_TestMode; + + SetNewState(kRunning); + + OnReady(); + + m_Http->Run(IsInteractiveMode); + + SetNewState(kShuttingDown); + + ZEN_INFO(ZEN_APP_NAME " exiting"); +} + +////////////////////////////////////////////////////////////////////////////////// + +ZenComputeServerMain::ZenComputeServerMain(ZenComputeServerConfig& ServerOptions) +: ZenServerMain(ServerOptions) +, m_ServerOptions(ServerOptions) +{ +} + +void +ZenComputeServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) +{ + ZenComputeServer Server; + Server.SetDataRoot(m_ServerOptions.DataDir); + Server.SetContentRoot(m_ServerOptions.ContentDir); + Server.SetTestMode(m_ServerOptions.IsTest); + Server.SetDedicatedMode(m_ServerOptions.IsDedicated); + + const int EffectiveBasePort = Server.Initialize(m_ServerOptions, Entry); + if (EffectiveBasePort == -1) + { + // Server.Initialize has already logged what the issue is - just exit with failure code here. + std::exit(1); + } + + Entry->EffectiveListenPort = uint16_t(EffectiveBasePort); + if (EffectiveBasePort != m_ServerOptions.BasePort) + { + ZEN_INFO(ZEN_APP_NAME " - relocated to base port {}", EffectiveBasePort); + m_ServerOptions.BasePort = EffectiveBasePort; + } + + std::unique_ptr ShutdownThread; + std::unique_ptr ShutdownEvent; + + ExtendableStringBuilder<64> ShutdownEventName; + ShutdownEventName << "Zen_" << m_ServerOptions.BasePort << "_Shutdown"; + ShutdownEvent.reset(new NamedEvent{ShutdownEventName}); + + // Monitor shutdown signals + + ShutdownThread.reset(new std::thread{[&] { + SetCurrentThreadName("shutdown_mon"); + + ZEN_INFO("shutdown monitor thread waiting for shutdown signal '{}' for process {}", ShutdownEventName, zen::GetCurrentProcessId()); + + if (ShutdownEvent->Wait()) + { + ZEN_INFO("shutdown signal for pid {} received", zen::GetCurrentProcessId()); + Server.RequestExit(0); + } + else + { + ZEN_INFO("shutdown signal wait() failed"); + } + }}); + + auto CleanupShutdown = MakeGuard([&ShutdownEvent, &ShutdownThread] { + ReportServiceStatus(ServiceStatus::Stopping); + + if (ShutdownEvent) + { + ShutdownEvent->Set(); + } + if (ShutdownThread && ShutdownThread->joinable()) + { + ShutdownThread->join(); + } + }); + + // If we have a parent process, establish the mechanisms we need + // to be able to communicate readiness with the parent + + Server.SetIsReadyFunc([&] { + std::error_code Ec; + m_LockFile.Update(MakeLockData(true), Ec); + ReportServiceStatus(ServiceStatus::Running); + NotifyReady(); + }); + + Server.Run(); +} + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zenserver/compute/computeserver.h b/src/zenserver/compute/computeserver.h new file mode 100644 index 000000000..625140b23 --- /dev/null +++ b/src/zenserver/compute/computeserver.h @@ -0,0 +1,106 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zenserver.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include + +namespace cxxopts { +class Options; +} +namespace zen::LuaConfig { +struct Options; +} + +namespace zen::compute { +class HttpFunctionService; +} + +namespace zen { + +class CidStore; +class HttpApiService; +class HttpComputeService; + +struct ZenComputeServerConfig : public ZenServerConfig +{ + std::string UpstreamNotificationEndpoint; + std::string InstanceId; // For use in notifications +}; + +struct ZenComputeServerConfigurator : public ZenServerConfiguratorBase +{ + ZenComputeServerConfigurator(ZenComputeServerConfig& ServerOptions) + : ZenServerConfiguratorBase(ServerOptions) + , m_ServerOptions(ServerOptions) + { + } + + ~ZenComputeServerConfigurator() = default; + +private: + virtual void AddCliOptions(cxxopts::Options& Options) override; + virtual void AddConfigOptions(LuaConfig::Options& Options) override; + virtual void ApplyOptions(cxxopts::Options& Options) override; + virtual void OnConfigFileParsed(LuaConfig::Options& LuaOptions) override; + virtual void ValidateOptions() override; + + ZenComputeServerConfig& m_ServerOptions; +}; + +class ZenComputeServerMain : public ZenServerMain +{ +public: + ZenComputeServerMain(ZenComputeServerConfig& ServerOptions); + virtual void DoRun(ZenServerState::ZenServerEntry* Entry) override; + + ZenComputeServerMain(const ZenComputeServerMain&) = delete; + ZenComputeServerMain& operator=(const ZenComputeServerMain&) = delete; + + typedef ZenComputeServerConfig Config; + typedef ZenComputeServerConfigurator Configurator; + +private: + ZenComputeServerConfig& m_ServerOptions; +}; + +/** + * The compute server handles DDC build function execution requests + * only. It's intended to be used on a pure compute resource and does + * not handle any storage tasks. The actual scheduling happens upstream + * in a storage server instance. + */ + +class ZenComputeServer : public ZenServerBase +{ + ZenComputeServer& operator=(ZenComputeServer&&) = delete; + ZenComputeServer(ZenComputeServer&&) = delete; + +public: + ZenComputeServer(); + ~ZenComputeServer(); + + int Initialize(const ZenComputeServerConfig& ServerConfig, ZenServerState::ZenServerEntry* ServerEntry); + void Run(); + void Cleanup(); + +private: + HttpStatsService m_StatsService; + GcManager m_GcManager; + GcScheduler m_GcScheduler{m_GcManager}; + std::unique_ptr m_CidStore; + std::unique_ptr m_ComputeService; + std::unique_ptr m_ApiService; + std::unique_ptr m_FunctionService; + + void InitializeState(const ZenComputeServerConfig& ServerConfig); + void InitializeServices(const ZenComputeServerConfig& ServerConfig); + void RegisterServices(const ZenComputeServerConfig& ServerConfig); +}; + +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zenserver/compute/computeservice.cpp b/src/zenserver/compute/computeservice.cpp new file mode 100644 index 000000000..2c0bc0ae9 --- /dev/null +++ b/src/zenserver/compute/computeservice.cpp @@ -0,0 +1,100 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "computeservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +# include +ZEN_THIRD_PARTY_INCLUDES_END + +# include + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +struct ResourceMetrics +{ + uint64_t DiskUsageBytes = 0; + uint64_t MemoryUsageBytes = 0; +}; + +////////////////////////////////////////////////////////////////////////// + +struct HttpComputeService::Impl +{ + Impl(const Impl&) = delete; + Impl& operator=(const Impl&) = delete; + + Impl(); + ~Impl(); + + void Initialize(std::filesystem::path BaseDir) { ZEN_UNUSED(BaseDir); } + + void Cleanup() {} + +private: +}; + +HttpComputeService::Impl::Impl() +{ +} + +HttpComputeService::Impl::~Impl() +{ +} + +/////////////////////////////////////////////////////////////////////////// + +HttpComputeService::HttpComputeService(std::filesystem::path BaseDir) : m_Impl(std::make_unique()) +{ + using namespace std::literals; + + m_Impl->Initialize(BaseDir); + + m_Router.RegisterRoute( + "status", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.BeginArray("modules"); + Obj.EndArray(); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "stats", + [this](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); +} + +HttpComputeService::~HttpComputeService() +{ +} + +const char* +HttpComputeService::BaseUri() const +{ + return "/compute/"; +} + +void +HttpComputeService::HandleRequest(zen::HttpServerRequest& Request) +{ + m_Router.HandleRequest(Request); +} + +} // namespace zen +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zenserver/compute/computeservice.h b/src/zenserver/compute/computeservice.h new file mode 100644 index 000000000..339200dd8 --- /dev/null +++ b/src/zenserver/compute/computeservice.h @@ -0,0 +1,36 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if ZEN_WITH_COMPUTE_SERVICES +namespace zen { + +/** ZenServer Compute Service + * + * Manages a set of compute workers for use in UEFN content worker + * + */ +class HttpComputeService : public zen::HttpService +{ +public: + HttpComputeService(std::filesystem::path BaseDir); + ~HttpComputeService(); + + HttpComputeService(const HttpComputeService&) = delete; + HttpComputeService& operator=(const HttpComputeService&) = delete; + + virtual const char* BaseUri() const override; + virtual void HandleRequest(zen::HttpServerRequest& Request) override; + +private: + HttpRequestRouter m_Router; + + struct Impl; + + std::unique_ptr m_Impl; +}; + +} // namespace zen +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zenserver/frontend/html/compute.html b/src/zenserver/frontend/html/compute.html new file mode 100644 index 000000000..668189fe5 --- /dev/null +++ b/src/zenserver/frontend/html/compute.html @@ -0,0 +1,991 @@ + + + + + + Zen Compute Dashboard + + + + +
+
+
+

Zen Compute Dashboard

+
Last updated: Never
+
+
+
+ Checking... +
+
+ +
+ + +
Action Queue
+
+
+
Pending Actions
+
-
+
Waiting to be scheduled
+
+
+
Running Actions
+
-
+
Currently executing
+
+
+
Completed Actions
+
-
+
Results available
+
+
+ + +
+
Action Queue History
+
+ +
+
+ + +
Performance Metrics
+
+
Completion Rate
+
+
+
-
+
1 min rate
+
+
+
-
+
5 min rate
+
+
+
-
+
15 min rate
+
+
+
+
+ Total Retired + - +
+
+ Mean Rate + - +
+
+
+ + +
Workers
+
+
Worker Status
+
+ Registered Workers + - +
+ +
+ + +
Recent Actions
+
+
Action History
+
No actions recorded yet.
+ +
+ + +
System Resources
+
+
+
CPU Usage
+
-
+
Percent
+
+
+
+
+ +
+
+
+ Packages + - +
+
+ Physical Cores + - +
+
+ Logical Processors + - +
+
+
+
+
Memory
+
+ Used + - +
+
+ Total + - +
+
+
+
+
+
+
Disk
+
+ Used + - +
+
+ Total + - +
+
+
+
+
+
+
+ + + + diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 1a929b026..ee783d2a6 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -23,6 +23,9 @@ #include #include "diag/logging.h" + +#include "compute/computeserver.h" + #include "storage/storageconfig.h" #include "storage/zenstorageserver.h" @@ -61,11 +64,19 @@ namespace zen { #if ZEN_PLATFORM_WINDOWS -template +/** Windows Service wrapper for Zen servers + * + * This class wraps a Zen server main entry point (the Main template parameter) + * into a Windows Service by implementing the WindowsService interface. + * + * The Main type needs to implement the virtual functions from the ZenServerMain + * base class, which provides the actual server logic. + */ +template class ZenWindowsService : public WindowsService { public: - ZenWindowsService(typename T::Config& ServerOptions) : m_EntryPoint(ServerOptions) {} + ZenWindowsService(typename Main::Config& ServerOptions) : m_EntryPoint(ServerOptions) {} ZenWindowsService(const ZenWindowsService&) = delete; ZenWindowsService& operator=(const ZenWindowsService&) = delete; @@ -73,7 +84,7 @@ public: virtual int Run() override { return m_EntryPoint.Run(); } private: - T m_EntryPoint; + Main m_EntryPoint; }; #endif // ZEN_PLATFORM_WINDOWS @@ -84,6 +95,23 @@ private: namespace zen { +/** Application main entry point template + * + * This function handles common application startup tasks while allowing + * different server types to be plugged in via the Main template parameter. + * + * On Windows, this function also handles platform-specific service + * installation and uninstallation. + * + * The Main type needs to implement the virtual functions from the ZenServerMain + * base class, which provides the actual server logic. + * + * The Main type is also expected to provide the following members: + * + * typedef Config -- Server configuration type, derived from ZenServerConfig + * typedef Configurator -- Server configuration handler type, implements ZenServerConfiguratorBase + * + */ template int AppMain(int argc, char* argv[]) @@ -241,7 +269,12 @@ main(int argc, char* argv[]) auto _ = zen::MakeGuard([] { // Allow some time for worker threads to unravel, in an effort - // to prevent shutdown races in TLS object destruction + // to prevent shutdown races in TLS object destruction, mainly due to + // threads which we don't directly control (Windows thread pool) and + // therefore can't join. + // + // This isn't a great solution, but for now it seems to help reduce + // shutdown crashes observed in some situations. WaitForThreads(1000); }); @@ -249,6 +282,7 @@ main(int argc, char* argv[]) { kHub, kStore, + kCompute, kTest } ServerMode = kStore; @@ -258,10 +292,14 @@ main(int argc, char* argv[]) { ServerMode = kHub; } - else if (argv[1] == "store"sv) + else if ((argv[1] == "store"sv) || (argv[1] == "storage"sv)) { ServerMode = kStore; } + else if (argv[1] == "compute"sv) + { + ServerMode = kCompute; + } else if (argv[1] == "test"sv) { ServerMode = kTest; @@ -280,6 +318,13 @@ main(int argc, char* argv[]) break; case kHub: return AppMain(argc, argv); + case kCompute: +#if ZEN_WITH_COMPUTE_SERVICES + return AppMain(argc, argv); +#else + fprintf(stderr, "compute services are not compiled in!\n"); + exit(5); +#endif default: case kStore: return AppMain(argc, argv); diff --git a/src/zenserver/storage/storageconfig.cpp b/src/zenserver/storage/storageconfig.cpp index 0f8ab1e98..089b6b572 100644 --- a/src/zenserver/storage/storageconfig.cpp +++ b/src/zenserver/storage/storageconfig.cpp @@ -797,6 +797,7 @@ ZenStorageServerCmdLineOptions::AddCacheOptions(cxxopts::Options& options, ZenSt cxxopts::value(ServerOptions.StructuredCacheConfig.MemMaxAgeSeconds)->default_value("86400"), ""); + options.add_option("compute", "", "lie-cpus", "Lie to upstream about CPU capabilities", cxxopts::value(ServerOptions.LieCpu), ""); options.add_option("cache", "", "cache-bucket-maxblocksize", diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h index d59d05cf6..b408b0c26 100644 --- a/src/zenserver/storage/storageconfig.h +++ b/src/zenserver/storage/storageconfig.h @@ -156,6 +156,7 @@ struct ZenStorageServerConfig : public ZenServerConfig ZenWorkspacesConfig WorksSpacesConfig; std::filesystem::path PluginsConfigFile; // Path to plugins config file bool ObjectStoreEnabled = false; + bool ComputeEnabled = true; std::string ScrubOptions; }; diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index 2b74395c3..ff854b72d 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -182,6 +182,13 @@ ZenStorageServer::RegisterServices() #endif // ZEN_WITH_VFS m_Http->RegisterService(*m_AdminService); + +#if ZEN_WITH_COMPUTE_SERVICES + if (m_HttpFunctionService) + { + m_Http->RegisterService(*m_HttpFunctionService); + } +#endif } void @@ -267,6 +274,16 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions m_BuildStoreService = std::make_unique(m_StatusService, m_StatsService, *m_BuildStore); } +#if ZEN_WITH_COMPUTE_SERVICES + if (ServerOptions.ComputeEnabled) + { + ZEN_OTEL_SPAN("InitializeComputeService"); + + m_HttpFunctionService = + std::make_unique(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions"); + } +#endif + #if ZEN_WITH_VFS m_VfsServiceImpl = std::make_unique(); m_VfsServiceImpl->AddService(Ref(m_ProjectStore)); @@ -805,6 +822,10 @@ ZenStorageServer::Cleanup() Flush(); +#if ZEN_WITH_COMPUTE_SERVICES + m_HttpFunctionService.reset(); +#endif + m_AdminService.reset(); m_VfsService.reset(); m_VfsServiceImpl.reset(); diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h index 5ccb587d6..456447a2a 100644 --- a/src/zenserver/storage/zenstorageserver.h +++ b/src/zenserver/storage/zenstorageserver.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,10 @@ #include "vfs/vfsservice.h" #include "workspaces/httpworkspaces.h" +#if ZEN_WITH_COMPUTE_SERVICES +# include +#endif + namespace zen { class ZenStorageServer : public ZenServerBase @@ -34,11 +39,6 @@ public: ZenStorageServer(); ~ZenStorageServer(); - void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } - void SetTestMode(bool State) { m_TestMode = State; } - void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } - void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } - int Initialize(const ZenStorageServerConfig& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry); void Run(); void Cleanup(); @@ -48,14 +48,9 @@ private: void InitializeStructuredCache(const ZenStorageServerConfig& ServerOptions); void Flush(); - bool m_IsDedicatedMode = false; - bool m_TestMode = false; - bool m_DebugOptionForcedCrash = false; - std::string m_StartupScrubOptions; - CbObject m_RootManifest; - std::filesystem::path m_DataRoot; - std::filesystem::path m_ContentRoot; - asio::steady_timer m_StateMarkerTimer{m_IoContext}; + std::string m_StartupScrubOptions; + CbObject m_RootManifest; + asio::steady_timer m_StateMarkerTimer{m_IoContext}; void EnqueueStateMarkerTimer(); void CheckStateMarker(); @@ -95,6 +90,11 @@ private: std::unique_ptr m_BuildStoreService; std::unique_ptr m_VfsService; std::unique_ptr m_AdminService; + std::unique_ptr m_ApiService; + +#if ZEN_WITH_COMPUTE_SERVICES + std::unique_ptr m_HttpFunctionService; +#endif }; struct ZenStorageServerConfigurator; diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index 6ee80dc62..9ab51beb2 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -2,7 +2,11 @@ target("zenserver") set_kind("binary") + if enable_unity then + add_rules("c++.unity_build", {batchsize = 4}) + end add_deps("zencore", + "zencompute", "zenhttp", "zennet", "zenremotestore", diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 7f9bf56a9..7bf6126df 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -145,6 +146,13 @@ ZenServerBase::Initialize(const ZenServerConfig& ServerOptions, ZenServerState:: InitializeSecuritySettings(ServerOptions); + if (ServerOptions.LieCpu) + { + SetCpuCountForReporting(ServerOptions.LieCpu); + + ZEN_INFO("Reporting concurrency: {}", ServerOptions.LieCpu); + } + m_StatusService.RegisterHandler("status", *this); m_Http->RegisterService(m_StatusService); diff --git a/src/zenserver/zenserver.h b/src/zenserver/zenserver.h index efa46f361..5a8a079c0 100644 --- a/src/zenserver/zenserver.h +++ b/src/zenserver/zenserver.h @@ -43,6 +43,11 @@ public: void SetIsReadyFunc(std::function&& IsReadyFunc) { m_IsReadyFunc = std::move(IsReadyFunc); } + void SetDataRoot(std::filesystem::path Root) { m_DataRoot = Root; } + void SetContentRoot(std::filesystem::path Root) { m_ContentRoot = Root; } + void SetDedicatedMode(bool State) { m_IsDedicatedMode = State; } + void SetTestMode(bool State) { m_TestMode = State; } + protected: int Initialize(const ZenServerConfig& ServerOptions, ZenServerState::ZenServerEntry* ServerEntry); void Finalize(); @@ -55,6 +60,10 @@ protected: bool m_UseSentry = false; bool m_IsPowerCycle = false; + bool m_IsDedicatedMode = false; + bool m_TestMode = false; + bool m_DebugOptionForcedCrash = false; + std::thread m_IoRunner; asio::io_context m_IoContext; void EnsureIoRunner(); @@ -72,6 +81,9 @@ protected: std::function m_IsReadyFunc; void OnReady(); + std::filesystem::path m_DataRoot; // Root directory for server state + std::filesystem::path m_ContentRoot; // Root directory for frontend content + Ref m_Http; std::unique_ptr m_HttpRequestFilter; @@ -114,7 +126,6 @@ protected: private: void InitializeSecuritySettings(const ZenServerConfig& ServerOptions); }; - class ZenServerMain { public: diff --git a/src/zentest-appstub/xmake.lua b/src/zentest-appstub/xmake.lua index 97615e322..db3ff2e2d 100644 --- a/src/zentest-appstub/xmake.lua +++ b/src/zentest-appstub/xmake.lua @@ -5,6 +5,9 @@ target("zentest-appstub") set_group("tests") add_headerfiles("**.h") add_files("*.cpp") + add_deps("zencore") + add_packages("vcpkg::gsl-lite") -- this should ideally be propagated by the zencore dependency + add_packages("vcpkg::mimalloc") if is_os("linux") then add_syslinks("pthread") diff --git a/src/zentest-appstub/zentest-appstub.cpp b/src/zentest-appstub/zentest-appstub.cpp index 24cf21e97..926580d96 100644 --- a/src/zentest-appstub/zentest-appstub.cpp +++ b/src/zentest-appstub/zentest-appstub.cpp @@ -1,33 +1,408 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#include +#include +#include +#include +#include +#include +#include + +#if ZEN_WITH_TESTS +# define ZEN_TEST_WITH_RUNNER 1 +# include +#endif + +#include + #include +#include #include #include #include +#include +#include +#include #include -using namespace std::chrono_literals; +using namespace std::literals; +using namespace zen; + +#if !defined(_MSC_VER) +# define _strnicmp strncasecmp // TEMPORARY WORKAROUND - should not be using this +#endif + +// Some basic functions to implement some test "compute" functions + +std::string +Rot13Function(std::string_view InputString) +{ + std::string OutputString{InputString}; + + std::transform(OutputString.begin(), + OutputString.end(), + OutputString.begin(), + [](std::string::value_type c) -> std::string::value_type { + if (c >= 'a' && c <= 'z') + { + return 'a' + (c - 'a' + 13) % 26; + } + else if (c >= 'A' && c <= 'Z') + { + return 'A' + (c - 'A' + 13) % 26; + } + else + { + return c; + } + }); + + return OutputString; +} + +std::string +ReverseFunction(std::string_view InputString) +{ + std::string OutputString{InputString}; + std::reverse(OutputString.begin(), OutputString.end()); + return OutputString; +} + +std::string +IdentityFunction(std::string_view InputString) +{ + return std::string{InputString}; +} + +std::string +NullFunction(std::string_view) +{ + return {}; +} + +zen::CbObject +DescribeFunctions() +{ + CbObjectWriter Versions; + Versions << "BuildSystemVersion" << Guid::FromString("17fe280d-ccd8-4be8-a9d1-89c944a70969"sv); + + Versions.BeginArray("Functions"sv); + Versions.BeginObject(); + Versions << "Name"sv + << "Null"sv; + Versions << "Version"sv << Guid::FromString("00000000-0000-0000-0000-000000000000"sv); + Versions.EndObject(); + Versions.BeginObject(); + Versions << "Name"sv + << "Identity"sv; + Versions << "Version"sv << Guid::FromString("11111111-1111-1111-1111-111111111111"sv); + Versions.EndObject(); + Versions.BeginObject(); + Versions << "Name"sv + << "Rot13"sv; + Versions << "Version"sv << Guid::FromString("13131313-1313-1313-1313-131313131313"sv); + Versions.EndObject(); + Versions.BeginObject(); + Versions << "Name"sv + << "Reverse"sv; + Versions << "Version"sv << Guid::FromString("31313131-3131-3131-3131-313131313131"sv); + Versions.EndObject(); + Versions.EndArray(); + + return Versions.Save(); +} + +struct ContentResolver +{ + std::filesystem::path InputsRoot; + + CompressedBuffer ResolveChunk(IoHash Hash, uint64_t ExpectedSize) + { + std::filesystem::path ChunkPath = InputsRoot / Hash.ToHexString(); + IoBuffer ChunkBuffer = IoBufferBuilder::MakeFromFile(ChunkPath); + + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer AsCompressed = CompressedBuffer::FromCompressed(SharedBuffer(ChunkBuffer), RawHash, RawSize); + + if (RawSize != ExpectedSize) + { + throw std::runtime_error( + fmt::format("chunk size mismatch - expected {}, got {} for '{}'", ExpectedSize, ChunkBuffer.Size(), ChunkPath)); + } + if (RawHash != Hash) + { + throw std::runtime_error(fmt::format("chunk hash mismatch - expected {}, got {} for '{}'", Hash, RawHash, ChunkPath)); + } + + return AsCompressed; + } +}; + +zen::CbPackage +ExecuteFunction(CbObject Action, ContentResolver ChunkResolver) +{ + auto Apply = [&](auto Func) { + zen::CbPackage Result; + auto Source = Action["Inputs"sv].AsObjectView()["Source"sv].AsObjectView(); + + IoHash InputRawHash = Source["RawHash"sv].AsHash(); + uint64_t InputRawSize = Source["RawSize"sv].AsUInt64(); + + zen::CompressedBuffer InputData = ChunkResolver.ResolveChunk(InputRawHash, InputRawSize); + SharedBuffer Input = InputData.Decompress(); + + std::string Output = Func(std::string_view(static_cast(Input.GetData()), Input.GetSize())); + zen::CompressedBuffer OutputData = + zen::CompressedBuffer::Compress(SharedBuffer::MakeView(Output), OodleCompressor::Selkie, OodleCompressionLevel::HyperFast4); + IoHash OutputRawHash = OutputData.DecodeRawHash(); + + CbAttachment OutputAttachment(std::move(OutputData), OutputRawHash); + + CbObjectWriter Cbo; + Cbo.BeginArray("Values"sv); + Cbo.BeginObject(); + Cbo << "Id" << Oid{1, 2, 3}; + Cbo.AddAttachment("RawHash", OutputAttachment); + Cbo << "RawSize" << Output.size(); + Cbo.EndObject(); + Cbo.EndArray(); + + Result.SetObject(Cbo.Save()); + Result.AddAttachment(std::move(OutputAttachment)); + return Result; + }; + + std::string_view Function = Action["Function"sv].AsString(); + + if (Function == "Rot13"sv) + { + return Apply(Rot13Function); + } + else if (Function == "Reverse"sv) + { + return Apply(ReverseFunction); + } + else if (Function == "Identity"sv) + { + return Apply(IdentityFunction); + } + else if (Function == "Null"sv) + { + return Apply(NullFunction); + } + else + { + return {}; + } +} + +/* This implements a minimal application to help testing of process launch-related + functionality + + It also mimics the DDC2 worker command line interface, so it may be used to + exercise compute infrastructure. + */ int main(int argc, char* argv[]) { int ExitCode = 0; - for (int i = 0; i < argc; ++i) + try { - if (std::strncmp(argv[i], "-t=", 3) == 0) + std::filesystem::path BasePath = std::filesystem::current_path(); + std::filesystem::path InputPath = std::filesystem::current_path() / "Inputs"; + std::filesystem::path OutputPath = std::filesystem::current_path() / "Outputs"; + std::filesystem::path VersionPath = std::filesystem::current_path() / "Versions"; + std::vector ActionPaths; + + /* + GetSwitchValues(TEXT("-B="), ActionPathPatterns); + GetSwitchValues(TEXT("-Build="), ActionPathPatterns); + + GetSwitchValues(TEXT("-I="), InputDirectoryPaths); + GetSwitchValues(TEXT("-Input="), InputDirectoryPaths); + + GetSwitchValues(TEXT("-O="), OutputDirectoryPaths); + GetSwitchValues(TEXT("-Output="), OutputDirectoryPaths); + + GetSwitchValues(TEXT("-V="), VersionPaths); + GetSwitchValues(TEXT("-Version="), VersionPaths); + */ + + auto SplitArg = [](const char* Arg) -> std::string_view { + std::string_view ArgView{Arg}; + if (auto SplitPos = ArgView.find_first_of('='); SplitPos != std::string_view::npos) + { + return ArgView.substr(SplitPos + 1); + } + else + { + return {}; + } + }; + + auto ParseIntArg = [](std::string_view Arg) -> int { + int Rv = 0; + const auto Result = std::from_chars(Arg.data(), Arg.data() + Arg.size(), Rv); + + if (Result.ec != std::errc{}) + { + throw std::invalid_argument(fmt::format("bad argument (not an integer): {}", Arg).c_str()); + } + + return Rv; + }; + + for (int i = 1; i < argc; ++i) + { + std::string_view Arg = argv[i]; + + if (Arg.compare(0, 1, "-")) + { + continue; + } + + if (std::strncmp(argv[i], "-t=", 3) == 0) + { + const int SleepTime = std::atoi(argv[i] + 3); + + printf("[zentest] sleeping for %ds...\n", SleepTime); + + std::this_thread::sleep_for(SleepTime * 1s); + } + else if (std::strncmp(argv[i], "-f=", 3) == 0) + { + // Force a "failure" process exit code to return to the invoker + + // This may throw for invalid arguments, which makes this useful for + // testing exception handling + std::string_view ErrorArg = SplitArg(argv[i]); + ExitCode = ParseIntArg(ErrorArg); + } + else if ((_strnicmp(argv[i], "-input=", 7) == 0) || (_strnicmp(argv[i], "-i=", 3) == 0)) + { + /* mimic DDC2 + + GetSwitchValues(TEXT("-I="), InputDirectoryPaths); + GetSwitchValues(TEXT("-Input="), InputDirectoryPaths); + */ + + std::string_view InputArg = SplitArg(argv[i]); + InputPath = InputArg; + } + else if ((_strnicmp(argv[i], "-output=", 8) == 0) || (_strnicmp(argv[i], "-o=", 3) == 0)) + { + /* mimic DDC2 handling of where files storing output chunk files are directed + + GetSwitchValues(TEXT("-O="), OutputDirectoryPaths); + GetSwitchValues(TEXT("-Output="), OutputDirectoryPaths); + */ + + std::string_view OutputArg = SplitArg(argv[i]); + OutputPath = OutputArg; + } + else if ((_strnicmp(argv[i], "-version=", 8) == 0) || (_strnicmp(argv[i], "-v=", 3) == 0)) + { + /* mimic DDC2 + + GetSwitchValues(TEXT("-V="), VersionPaths); + GetSwitchValues(TEXT("-Version="), VersionPaths); + */ + + std::string_view VersionArg = SplitArg(argv[i]); + VersionPath = VersionArg; + } + else if ((_strnicmp(argv[i], "-build=", 7) == 0) || (_strnicmp(argv[i], "-b=", 3) == 0)) + { + /* mimic DDC2 + + GetSwitchValues(TEXT("-B="), ActionPathPatterns); + GetSwitchValues(TEXT("-Build="), ActionPathPatterns); + */ + + std::string_view BuildActionArg = SplitArg(argv[i]); + std::filesystem::path ActionPath{BuildActionArg}; + ActionPaths.push_back(ActionPath); + + ExitCode = 0; + } + } + + // Emit version information + + if (!VersionPath.empty()) { - const int SleepTime = std::atoi(argv[i] + 3); + CbObjectWriter Version; + + Version << "BuildSystemVersion" << Guid::FromString("17fe280d-ccd8-4be8-a9d1-89c944a70969"sv); + + Version.BeginArray("Functions"); + + Version.BeginObject(); + Version << "Name" + << "Rot13" + << "Version" << Guid::FromString("13131313-1313-1313-1313-131313131313"sv); + Version.EndObject(); - printf("[zentest] sleeping for %ds...\n", SleepTime); + Version.BeginObject(); + Version << "Name" + << "Reverse" + << "Version" << Guid::FromString("98765432-1000-0000-0000-000000000000"sv); + Version.EndObject(); - std::this_thread::sleep_for(SleepTime * 1s); + Version.BeginObject(); + Version << "Name" + << "Identity" + << "Version" << Guid::FromString("11111111-1111-1111-1111-111111111111"sv); + Version.EndObject(); + + Version.BeginObject(); + Version << "Name" + << "Null" + << "Version" << Guid::FromString("00000000-0000-0000-0000-000000000000"sv); + Version.EndObject(); + + Version.EndArray(); + CbObject VersionObject = Version.Save(); + + BinaryWriter Writer; + zen::SaveCompactBinary(Writer, VersionObject); + zen::WriteFile(VersionPath, IoBufferBuilder::MakeFromMemory(Writer.GetView())); } - else if (std::strncmp(argv[i], "-f=", 3) == 0) + + // Evaluate actions + + ContentResolver Resolver; + Resolver.InputsRoot = InputPath; + + for (std::filesystem::path ActionPath : ActionPaths) { - ExitCode = std::atoi(argv[i] + 3); + IoBuffer ActionDescBuffer = ReadFile(ActionPath).Flatten(); + CbObject ActionDesc = LoadCompactBinaryObject(ActionDescBuffer); + CbPackage Result = ExecuteFunction(ActionDesc, Resolver); + CbObject ResultObject = Result.GetObject(); + + BinaryWriter Writer; + zen::SaveCompactBinary(Writer, ResultObject); + zen::WriteFile(ActionPath.replace_extension(".output"), IoBufferBuilder::MakeFromMemory(Writer.GetView())); + + // Also marshal outputs + + for (const auto& Attachment : Result.GetAttachments()) + { + const CompositeBuffer& AttachmentBuffer = Attachment.AsCompressedBinary().GetCompressed(); + zen::WriteFile(OutputPath / Attachment.GetHash().ToHexString(), AttachmentBuffer.Flatten().AsIoBuffer()); + } } } + catch (std::exception& Ex) + { + printf("[zentest] exception caught in main: '%s'\n", Ex.what()); + + ExitCode = 99; + } printf("[zentest] exiting with exit code: %d\n", ExitCode); diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua index f079d803d..07605a016 100644 --- a/thirdparty/xmake.lua +++ b/thirdparty/xmake.lua @@ -86,7 +86,7 @@ target("blake3") if is_os("windows") then add_cflags("/experimental:c11atomics") - add_cflags("/wd4245") -- conversion from 'type1' to 'type2', possible loss of data + add_cflags("/wd4245", {force = true}) -- conversion from 'type1' to 'type2', possible loss of data elseif is_os("macosx") then add_cflags("-Wno-unused-function") end diff --git a/xmake.lua b/xmake.lua index 5d3162e46..3537c618d 100644 --- a/xmake.lua +++ b/xmake.lua @@ -120,6 +120,9 @@ if has_config("zensentry") and not use_asan then add_requires("sentry-native 0.12.1", {configs = {backend = "crashpad"}}) end end + +enable_unity = false + --add_rules("c++.unity_build") if is_mode("release") then @@ -240,6 +243,14 @@ else add_defines("ZEN_WITH_HTTPSYS=0") end +option("zencompute") + set_default(false) + set_showmenu(true) + set_description("Enable compute services endpoint") +option_end() +add_define_by_config("ZEN_WITH_COMPUTE_SERVICES", "zencompute") + + if is_os("windows") then add_defines("UE_MEMORY_TRACE_AVAILABLE=1") option("zenmemtrack") @@ -272,6 +283,7 @@ includes("src/zencore", "src/zencore-test") includes("src/zenhttp", "src/zenhttp-test") includes("src/zennet", "src/zennet-test") includes("src/zenremotestore", "src/zenremotestore-test") +includes("src/zencompute", "src/zencompute-test") includes("src/zenstore", "src/zenstore-test") includes("src/zentelemetry", "src/zentelemetry-test") includes("src/zenutil", "src/zenutil-test") -- cgit v1.2.3 From a948ff9570a5a9d8ec424639cba6f973247a0372 Mon Sep 17 00:00:00 2001 From: zousar Date: Wed, 18 Feb 2026 23:15:09 -0700 Subject: entry.js handles missing/native items more gracefully --- src/zenserver/frontend/html/pages/cookartifacts.js | 20 ++++++++++++++++---- src/zenserver/frontend/html/pages/entry.js | 16 ++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/src/zenserver/frontend/html/pages/cookartifacts.js b/src/zenserver/frontend/html/pages/cookartifacts.js index 6c36c7f32..f2ae094b9 100644 --- a/src/zenserver/frontend/html/pages/cookartifacts.js +++ b/src/zenserver/frontend/html/pages/cookartifacts.js @@ -261,13 +261,25 @@ export class Page extends ZenPage { const row = runtime_table.add_row(dep); // Make Path clickable to navigate to entry - row.get_cell(0).text(dep).on_click((opkey) => { - window.location = `?page=entry&project=${project}&oplog=${oplog}&opkey=${opkey.toLowerCase()}`; - }, dep); + if (this._should_link_dependency(dep)) + { + row.get_cell(0).text(dep).on_click((opkey) => { + window.location = `?page=entry&project=${project}&oplog=${oplog}&opkey=${opkey.toLowerCase()}`; + }, dep); + } } } } + _should_link_dependency(name) + { + // Exclude dependencies starting with /Script/ (code-defined entries) - case insensitive + if (name && name.toLowerCase().startsWith("/script/")) + return false; + + return true; + } + _build_dependency_section(parent_section, title, dependencies, stored_key) { const section = parent_section.add_section(title); @@ -338,7 +350,7 @@ export class Page extends ZenPage const row = table.add_row(...row_values); // Make Name field clickable for Package, TransitiveBuild, and RedirectionTarget - if (should_link && name_col_index >= 0 && dep.Name) + if (should_link && name_col_index >= 0 && dep.Name && this._should_link_dependency(dep.Name)) { const project = this.get_param("project"); const oplog = this.get_param("oplog"); diff --git a/src/zenserver/frontend/html/pages/entry.js b/src/zenserver/frontend/html/pages/entry.js index c4746bf52..f418b17ba 100644 --- a/src/zenserver/frontend/html/pages/entry.js +++ b/src/zenserver/frontend/html/pages/entry.js @@ -181,6 +181,22 @@ export class Page extends ZenPage async _build_page() { var entry = await this._entry; + + // Check if entry exists + if (!entry || entry.as_object().find("entry") == null) + { + const opkey = this.get_param("opkey"); + var section = this.add_section("Entry Not Found"); + section.tag("p").text(`The entry "${opkey}" is not present in this dataset.`); + section.tag("p").text("This could mean:"); + const list = section.tag("ul"); + list.tag("li").text("The entry is for an instance defined in code"); + list.tag("li").text("The entry has not been added to the oplog yet"); + list.tag("li").text("The entry key is misspelled"); + list.tag("li").text("The entry was removed or never existed"); + return; + } + entry = entry.as_object().find("entry").as_object(); const name = entry.find("key").as_value(); -- cgit v1.2.3 From a1f158e14761767f83469e9e522cf542f9ad91e2 Mon Sep 17 00:00:00 2001 From: zousar Date: Wed, 18 Feb 2026 23:17:56 -0700 Subject: updatefrontend --- src/zenserver/frontend/html.zip | Bin 182962 -> 183939 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index 67752fbc2..d70a5a62b 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ -- cgit v1.2.3 From 83f3c5a53316c240b7f068195ddd00772a514af9 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 Feb 2026 07:44:53 +0100 Subject: 5.7.21-pre0 --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index 4b5a784f6..e018afd57 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -5.7.20 \ No newline at end of file +5.7.21-pre0 \ No newline at end of file -- cgit v1.2.3 From 2d2781a03e3ab476dc45193b2bec72d8fd74d00d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 19 Feb 2026 15:06:17 +0100 Subject: 5.7.21 --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index e018afd57..cf3060b3b 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -5.7.21-pre0 \ No newline at end of file +5.7.21 \ No newline at end of file -- cgit v1.2.3 From cae12611580c6c28b1362fa28181b8f388516a47 Mon Sep 17 00:00:00 2001 From: zousar Date: Thu, 19 Feb 2026 13:55:44 -0700 Subject: icon and header logo changes --- src/UnrealEngine.ico | Bin 65288 -> 0 bytes src/zen.ico | Bin 0 -> 12957 bytes src/zen/zen.rc | 2 +- src/zenserver/frontend/html/epicgames.ico | Bin 0 -> 65288 bytes src/zenserver/frontend/html/favicon.ico | Bin 65288 -> 12957 bytes src/zenserver/frontend/html/pages/page.js | 24 ++++++------------------ src/zenserver/frontend/html/zen.css | 16 +++++++++++++++- src/zenserver/zenserver.rc | 2 +- 8 files changed, 23 insertions(+), 21 deletions(-) delete mode 100644 src/UnrealEngine.ico create mode 100644 src/zen.ico create mode 100644 src/zenserver/frontend/html/epicgames.ico diff --git a/src/UnrealEngine.ico b/src/UnrealEngine.ico deleted file mode 100644 index 1cfa301a2..000000000 Binary files a/src/UnrealEngine.ico and /dev/null differ diff --git a/src/zen.ico b/src/zen.ico new file mode 100644 index 000000000..f7fb251b5 Binary files /dev/null and b/src/zen.ico differ diff --git a/src/zen/zen.rc b/src/zen/zen.rc index 661d75011..0617681a7 100644 --- a/src/zen/zen.rc +++ b/src/zen/zen.rc @@ -7,7 +7,7 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US #pragma code_page(1252) -101 ICON "..\\UnrealEngine.ico" +101 ICON "..\\zen.ico" VS_VERSION_INFO VERSIONINFO FILEVERSION ZEN_CFG_VERSION_MAJOR,ZEN_CFG_VERSION_MINOR,ZEN_CFG_VERSION_ALTER,0 diff --git a/src/zenserver/frontend/html/epicgames.ico b/src/zenserver/frontend/html/epicgames.ico new file mode 100644 index 000000000..1cfa301a2 Binary files /dev/null and b/src/zenserver/frontend/html/epicgames.ico differ diff --git a/src/zenserver/frontend/html/favicon.ico b/src/zenserver/frontend/html/favicon.ico index 1cfa301a2..f7fb251b5 100644 Binary files a/src/zenserver/frontend/html/favicon.ico and b/src/zenserver/frontend/html/favicon.ico differ diff --git a/src/zenserver/frontend/html/pages/page.js b/src/zenserver/frontend/html/pages/page.js index 3ec0248cb..3c2d3619a 100644 --- a/src/zenserver/frontend/html/pages/page.js +++ b/src/zenserver/frontend/html/pages/page.js @@ -70,24 +70,12 @@ export class ZenPage extends PageBase { var root = parent.tag().id("branding"); - const zen_store = root.tag("pre").id("logo").text( - "_________ _______ __\n" + - "\\____ /___ ___ / ___// |__ ___ ______ ____\n" + - " / __/ __ \\ / \\ \\___ \\\\_ __// \\\\_ \\/ __ \\\n" + - " / \\ __// | \\/ \\| | ( - )| |\\/\\ __/\n" + - "/______/\\___/\\__|__/\\______/|__| \\___/ |__| \\___|" - ); - zen_store.tag().id("go_home").on_click(() => window.location.search = ""); - - root.tag("img").attr("src", "favicon.ico").id("ue_logo"); - - /* - _________ _______ __ - \____ /___ ___ / ___// |__ ___ ______ ____ - / __/ __ \ / \ \___ \\_ __// \\_ \/ __ \ - / \ __// | \/ \| | ( - )| |\/\ __/ - /______/\___/\__|__/\______/|__| \___/ |__| \___| - */ + const logo_container = root.tag("div").id("logo"); + logo_container.tag("img").attr("src", "favicon.ico").id("zen_icon"); + logo_container.tag("span").id("zen_text").text("zenserver"); + logo_container.tag().id("go_home").on_click(() => window.location.search = ""); + + root.tag("img").attr("src", "epicgames.ico").id("epic_logo"); } set_title(...args) diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css index 34c265610..702bf9aa6 100644 --- a/src/zenserver/frontend/html/zen.css +++ b/src/zenserver/frontend/html/zen.css @@ -365,6 +365,20 @@ a { margin: auto; user-select: none; position: relative; + display: flex; + align-items: center; + gap: 0.8em; + + #zen_icon { + width: 3em; + height: 3em; + } + + #zen_text { + font-size: 2em; + font-weight: bold; + letter-spacing: 0.05em; + } #go_home { width: 100%; @@ -379,7 +393,7 @@ a { filter: drop-shadow(0 0.15em 0.1em var(--theme_p2)); } - #ue_logo { + #epic_logo { position: absolute; top: 1em; right: 0; diff --git a/src/zenserver/zenserver.rc b/src/zenserver/zenserver.rc index e0003ea8f..f353bd9cc 100644 --- a/src/zenserver/zenserver.rc +++ b/src/zenserver/zenserver.rc @@ -28,7 +28,7 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US // Icon with lowest ID value placed first to ensure application icon // remains consistent on all systems. -IDI_ICON1 ICON "..\\UnrealEngine.ico" +IDI_ICON1 ICON "..\\zen.ico" #endif // English (United States) resources ///////////////////////////////////////////////////////////////////////////// -- cgit v1.2.3 From ee26e5af2ced0987fbdf666dc6bce7c2074e925f Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 20 Feb 2026 09:05:23 +0100 Subject: GC - fix handling of attachment ranges, http access token expiration, lock file retry logic (#766) * GC - fix handling of attachment ranges * fix trace/log strings * fix HTTP access token expiration time logic * added missing lock retry in zenserver startup --- src/zenhttp/httpclientauth.cpp | 2 +- src/zenhttp/servers/httpparser.cpp | 9 ++++++--- src/zenserver/compute/computeserver.cpp | 6 +++--- src/zenserver/hub/zenhubserver.cpp | 2 +- src/zenserver/zenserver.cpp | 2 ++ src/zenstore/gc.cpp | 7 ++++--- src/zenstore/include/zenstore/gc.h | 2 +- 7 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/zenhttp/httpclientauth.cpp b/src/zenhttp/httpclientauth.cpp index 72df12d02..02e1b57e2 100644 --- a/src/zenhttp/httpclientauth.cpp +++ b/src/zenhttp/httpclientauth.cpp @@ -170,7 +170,7 @@ namespace zen { namespace httpclientauth { time_t UTCTime = timegm(&Time); HttpClientAccessToken::TimePoint ExpireTime = std::chrono::system_clock::from_time_t(UTCTime); - ExpireTime += std::chrono::microseconds(Millisecond); + ExpireTime += std::chrono::milliseconds(Millisecond); return HttpClientAccessToken{.Value = fmt::format("Bearer {}"sv, Token), .ExpireTime = ExpireTime}; } diff --git a/src/zenhttp/servers/httpparser.cpp b/src/zenhttp/servers/httpparser.cpp index be5befcd2..f0485aa25 100644 --- a/src/zenhttp/servers/httpparser.cpp +++ b/src/zenhttp/servers/httpparser.cpp @@ -226,6 +226,8 @@ NormalizeUrlPath(std::string_view InUrl, std::string& NormalizedUrl) NormalizedUrl.append(Url, UrlIndex); } + // NOTE: this check is redundant given the enclosing if, + // need to verify the intent of this code if (!LastCharWasSeparator) { NormalizedUrl.push_back('/'); @@ -310,6 +312,7 @@ HttpRequestParser::OnHeadersComplete() if (ContentLength) { + // TODO: should sanity-check content length here m_BodyBuffer = IoBuffer(ContentLength); } @@ -329,9 +332,9 @@ HttpRequestParser::OnHeadersComplete() int HttpRequestParser::OnBody(const char* Data, size_t Bytes) { - if (m_BodyPosition + Bytes > m_BodyBuffer.Size()) + if ((m_BodyPosition + Bytes) > m_BodyBuffer.Size()) { - ZEN_WARN("HTTP parser incoming body is larger than content size, need {} more bytes", + ZEN_WARN("HTTP parser incoming body is larger than content size, need {} more buffer bytes", (m_BodyPosition + Bytes) - m_BodyBuffer.Size()); return 1; } @@ -342,7 +345,7 @@ HttpRequestParser::OnBody(const char* Data, size_t Bytes) { if (m_BodyPosition != m_BodyBuffer.Size()) { - ZEN_WARN("Body mismatch! {} != {}", m_BodyPosition, m_BodyBuffer.Size()); + ZEN_WARN("Body size mismatch! {} != {}", m_BodyPosition, m_BodyBuffer.Size()); return 1; } } diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp index 173f56386..0f9ef0287 100644 --- a/src/zenserver/compute/computeserver.cpp +++ b/src/zenserver/compute/computeserver.cpp @@ -82,7 +82,7 @@ ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServ ZEN_TRACE_CPU("ZenComputeServer::Initialize"); ZEN_MEMSCOPE(GetZenserverTag()); - ZEN_INFO(ZEN_APP_NAME " initializing in HUB server mode"); + ZEN_INFO(ZEN_APP_NAME " initializing in COMPUTE server mode"); const int EffectiveBasePort = ZenServerBase::Initialize(ServerConfig, ServerEntry); if (EffectiveBasePort < 0) @@ -91,7 +91,7 @@ ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServ } // This is a workaround to make sure we can have automated tests. Without - // this the ranges for different child zen hub processes could overlap with + // this the ranges for different child zen compute processes could overlap with // the main test range. ZenServerEnvironment::SetBaseChildId(1000); @@ -109,7 +109,7 @@ ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServ void ZenComputeServer::Cleanup() { - ZEN_TRACE_CPU("ZenStorageServer::Cleanup"); + ZEN_TRACE_CPU("ZenComputeServer::Cleanup"); ZEN_INFO(ZEN_APP_NAME " cleaning up"); try { diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp index 7a4ba951d..d0a0db417 100644 --- a/src/zenserver/hub/zenhubserver.cpp +++ b/src/zenserver/hub/zenhubserver.cpp @@ -105,7 +105,7 @@ ZenHubServer::Initialize(const ZenHubServerConfig& ServerConfig, ZenServerState: void ZenHubServer::Cleanup() { - ZEN_TRACE_CPU("ZenStorageServer::Cleanup"); + ZEN_TRACE_CPU("ZenHubServer::Cleanup"); ZEN_INFO(ZEN_APP_NAME " cleaning up"); try { diff --git a/src/zenserver/zenserver.cpp b/src/zenserver/zenserver.cpp index 7bf6126df..5fd35d9b4 100644 --- a/src/zenserver/zenserver.cpp +++ b/src/zenserver/zenserver.cpp @@ -617,6 +617,8 @@ ZenServerMain::Run() { ZEN_INFO(ZEN_APP_NAME " unable to grab lock at '{}' (reason: '{}'), retrying", LockFilePath, Ec.message()); Sleep(500); + + m_LockFile.Create(LockFilePath, MakeLockData(false), Ec); if (Ec) { ZEN_WARN(ZEN_APP_NAME " exiting, unable to grab lock at '{}' (reason: '{}')", LockFilePath, Ec.message()); diff --git a/src/zenstore/gc.cpp b/src/zenstore/gc.cpp index 14caa5abf..c3bdc59f0 100644 --- a/src/zenstore/gc.cpp +++ b/src/zenstore/gc.cpp @@ -1494,7 +1494,8 @@ GcManager::CollectGarbage(const GcSettings& Settings) GcReferenceValidatorStats& Stats = Result.ReferenceValidatorStats[It.second].second; try { - // Go through all the ReferenceCheckers to see if the list of Cids the collector selected are referenced or + // Go through all the ReferenceCheckers to see if the list of Cids the collector selected + // are referenced or not SCOPED_TIMER(Stats.ElapsedMS = std::chrono::milliseconds(Timer.GetElapsedTimeMs());); ReferenceValidator->Validate(Ctx, Stats); } @@ -1952,7 +1953,7 @@ GcScheduler::AppendGCLog(std::string_view Id, GcClock::TimePoint StartTime, cons Writer << "SingleThread"sv << Settings.SingleThread; Writer << "CompactBlockUsageThresholdPercent"sv << Settings.CompactBlockUsageThresholdPercent; Writer << "AttachmentRangeMin"sv << Settings.AttachmentRangeMin; - Writer << "AttachmentRangeMax"sv << Settings.AttachmentRangeMin; + Writer << "AttachmentRangeMax"sv << Settings.AttachmentRangeMax; Writer << "ForceStoreCacheAttachmentMetaData"sv << Settings.StoreCacheAttachmentMetaData; Writer << "ForceStoreProjectAttachmentMetaData"sv << Settings.StoreProjectAttachmentMetaData; Writer << "EnableValidation"sv << Settings.EnableValidation; @@ -2893,7 +2894,7 @@ GcScheduler::CollectGarbage(const GcClock::TimePoint& CacheExpireTime, { m_LastFullGCV2Result = Result; m_LastFullAttachmentRangeMin = AttachmentRangeMin; - m_LastFullAttachmentRangeMin = AttachmentRangeMax; + m_LastFullAttachmentRangeMax = AttachmentRangeMax; } Diff.DiskSize = Result.CompactStoresStatSum.RemovedDisk; Diff.MemorySize = Result.ReferencerStatSum.RemoveExpiredDataStats.FreedMemory; diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h index 734d2e5a7..794f50d96 100644 --- a/src/zenstore/include/zenstore/gc.h +++ b/src/zenstore/include/zenstore/gc.h @@ -238,7 +238,7 @@ bool FilterReferences(GcCtx& Ctx, std::string_view Context, std::vector Date: Fri, 20 Feb 2026 09:07:00 +0100 Subject: fix MakeSafeAbsolutePathInPlace mis-spelling (#765) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (was MakeSafeAbsolutePathÍnPlace - note accent) Also fixed misleading comments on multiple functions in filesystem.h --- src/zen/authutils.cpp | 2 +- src/zen/cmds/builds_cmd.cpp | 44 ++++++++++++++++---------------- src/zen/cmds/print_cmd.cpp | 4 +-- src/zen/cmds/projectstore_cmd.cpp | 2 +- src/zen/cmds/wipe_cmd.cpp | 2 +- src/zen/cmds/workspaces_cmd.cpp | 2 +- src/zencore/filesystem.cpp | 4 +-- src/zencore/include/zencore/filesystem.h | 38 +++++++++++++-------------- 8 files changed, 49 insertions(+), 49 deletions(-) diff --git a/src/zen/authutils.cpp b/src/zen/authutils.cpp index 31db82efd..16427acf5 100644 --- a/src/zen/authutils.cpp +++ b/src/zen/authutils.cpp @@ -233,7 +233,7 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, } else if (!m_AccessTokenPath.empty()) { - MakeSafeAbsolutePathÍnPlace(m_AccessTokenPath); + MakeSafeAbsolutePathInPlace(m_AccessTokenPath); std::string ResolvedAccessToken = ReadAccessTokenFromJsonFile(m_AccessTokenPath); if (!ResolvedAccessToken.empty()) { diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 59b209384..8dfe1093f 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2680,7 +2680,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + MakeSafeAbsolutePathInPlace(m_SystemRootDir); }; ParseSystemOptions(); @@ -2729,7 +2729,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--host', '--url', '--override-host' or '--storage-path' is required", SubOption->help()); } - MakeSafeAbsolutePathÍnPlace(m_StoragePath); + MakeSafeAbsolutePathInPlace(m_StoragePath); }; auto ParseOutputOptions = [&]() { @@ -2947,7 +2947,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--local-path' is required", SubOption->help()); } - MakeSafeAbsolutePathÍnPlace(m_Path); + MakeSafeAbsolutePathInPlace(m_Path); }; auto ParseFileFilters = [&](std::vector& OutIncludeWildcards, std::vector& OutExcludeWildcards) { @@ -3004,7 +3004,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { throw OptionParseException("'--compare-path' is required", SubOption->help()); } - MakeSafeAbsolutePathÍnPlace(m_DiffPath); + MakeSafeAbsolutePathInPlace(m_DiffPath); }; auto ParseBlobHash = [&]() -> IoHash { @@ -3105,7 +3105,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!m_BuildMetadataPath.empty()) { - MakeSafeAbsolutePathÍnPlace(m_BuildMetadataPath); + MakeSafeAbsolutePathInPlace(m_BuildMetadataPath); IoBuffer MetaDataJson = ReadFile(m_BuildMetadataPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; @@ -3202,8 +3202,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ListOptions) { - MakeSafeAbsolutePathÍnPlace(m_ListQueryPath); - MakeSafeAbsolutePathÍnPlace(m_ListResultPath); + MakeSafeAbsolutePathInPlace(m_ListQueryPath); + MakeSafeAbsolutePathInPlace(m_ListResultPath); if (!m_ListResultPath.empty()) { @@ -3255,7 +3255,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); }); @@ -3294,7 +3294,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (SubOption == &m_ListBlocksOptions) { - MakeSafeAbsolutePathÍnPlace(m_ListResultPath); + MakeSafeAbsolutePathInPlace(m_ListResultPath); if (!m_ListResultPath.empty()) { @@ -3316,7 +3316,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this]() { CleanAndRemoveDirectory(GetSmallWorkerPool(EWorkloadType::Burst), m_ZenFolderPath); }); @@ -3387,8 +3387,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3532,7 +3532,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -3632,7 +3632,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); BuildStorageBase::Statistics StorageStats; BuildStorageCache::Statistics StorageCacheStats; @@ -3652,7 +3652,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) std::unique_ptr StructuredOutput; if (!m_LsResultPath.empty()) { - MakeSafeAbsolutePathÍnPlace(m_LsResultPath); + MakeSafeAbsolutePathInPlace(m_LsResultPath); StructuredOutput = std::make_unique(); } @@ -3696,7 +3696,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ParsePath(); ParseDiffPath(); - MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); std::vector ExcludeFolders = DefaultExcludeFolders; std::vector ExcludeExtensions = DefaultExcludeExtensions; @@ -3745,7 +3745,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3828,7 +3828,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3883,7 +3883,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = std::filesystem::current_path() / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); CreateDirectories(m_ZenFolderPath); auto _ = MakeGuard([this, &Workers]() { CleanAndRemoveDirectory(Workers.GetIOWorkerPool(), m_ZenFolderPath); }); @@ -3933,7 +3933,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); EPartialBlockRequestMode PartialBlockRequestMode = ParseAllowPartialBlockRequests(); @@ -4083,8 +4083,8 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { m_ZenFolderPath = m_Path / ZenFolderName; } - MakeSafeAbsolutePathÍnPlace(m_ZenFolderPath); - MakeSafeAbsolutePathÍnPlace(m_ChunkingCachePath); + MakeSafeAbsolutePathInPlace(m_ZenFolderPath); + MakeSafeAbsolutePathInPlace(m_ChunkingCachePath); StorageInstance Storage = CreateBuildStorage(StorageStats, StorageCacheStats, diff --git a/src/zen/cmds/print_cmd.cpp b/src/zen/cmds/print_cmd.cpp index 030cc8b66..c6b250fdf 100644 --- a/src/zen/cmds/print_cmd.cpp +++ b/src/zen/cmds/print_cmd.cpp @@ -84,7 +84,7 @@ PrintCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) } else { - MakeSafeAbsolutePathÍnPlace(m_Filename); + MakeSafeAbsolutePathInPlace(m_Filename); Fc = ReadFile(m_Filename); } @@ -244,7 +244,7 @@ PrintPackageCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** ar if (m_Filename.empty()) throw OptionParseException("'--source' is required", m_Options.help()); - MakeSafeAbsolutePathÍnPlace(m_Filename); + MakeSafeAbsolutePathInPlace(m_Filename); FileContents Fc = ReadFile(m_Filename); IoBuffer Data = Fc.Flatten(); CbPackage Package; diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 4885fd363..4de6ad25c 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -2430,7 +2430,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a { m_SystemRootDir = PickDefaultSystemRootDirectory(); } - MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + MakeSafeAbsolutePathInPlace(m_SystemRootDir); }; ParseSystemOptions(); diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index adf0e61f0..a5029e1c5 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -549,7 +549,7 @@ WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) ProgressMode = (IsVerbose || m_PlainProgress) ? ProgressBar::Mode::Plain : ProgressBar::Mode::Pretty; BoostWorkerThreads = m_BoostWorkerThreads; - MakeSafeAbsolutePathÍnPlace(m_Directory); + MakeSafeAbsolutePathInPlace(m_Directory); if (!IsDir(m_Directory)) { diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 6e6f5d863..2661ac9da 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -398,7 +398,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** } else { - MakeSafeAbsolutePathÍnPlace(m_SystemRootDir); + MakeSafeAbsolutePathInPlace(m_SystemRootDir); } std::filesystem::path StatePath = m_SystemRootDir / "workspaces"; diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 92a065707..1a4ee4b9b 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -3069,7 +3069,7 @@ SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly) } void -MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path) +MakeSafeAbsolutePathInPlace(std::filesystem::path& Path) { if (!Path.empty()) { @@ -3091,7 +3091,7 @@ std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path) { std::filesystem::path Tmp(Path); - MakeSafeAbsolutePathÍnPlace(Tmp); + MakeSafeAbsolutePathInPlace(Tmp); return Tmp; } diff --git a/src/zencore/include/zencore/filesystem.h b/src/zencore/include/zencore/filesystem.h index f28863679..16e2b59f8 100644 --- a/src/zencore/include/zencore/filesystem.h +++ b/src/zencore/include/zencore/filesystem.h @@ -64,80 +64,80 @@ std::filesystem::path PathFromHandle(void* NativeHandle, std::error_code& Ec); */ std::filesystem::path CanonicalPath(std::filesystem::path InPath, std::error_code& Ec); -/** Query file size +/** Check if a path exists and is a regular file (throws) */ bool IsFile(const std::filesystem::path& Path); -/** Query file size +/** Check if a path exists and is a regular file (does not throw) */ bool IsFile(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Check if a path exists and is a directory (throws) */ bool IsDir(const std::filesystem::path& Path); -/** Query file size +/** Check if a path exists and is a directory (does not throw) */ bool IsDir(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Delete file at path, if it exists (throws) */ bool RemoveFile(const std::filesystem::path& Path); -/** Query file size +/** Delete file at path, if it exists (does not throw) */ bool RemoveFile(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Delete directory at path, if it exists (throws) */ bool RemoveDir(const std::filesystem::path& Path); -/** Query file size +/** Delete directory at path, if it exists (does not throw) */ bool RemoveDir(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size +/** Query file size (throws) */ uint64_t FileSizeFromPath(const std::filesystem::path& Path); -/** Query file size +/** Query file size (does not throw) */ uint64_t FileSizeFromPath(const std::filesystem::path& Path, std::error_code& Ec); -/** Query file size from native file handle +/** Query file size from native file handle (throws) */ uint64_t FileSizeFromHandle(void* NativeHandle); -/** Query file size from native file handle +/** Query file size from native file handle (does not throw) */ uint64_t FileSizeFromHandle(void* NativeHandle, std::error_code& Ec); /** Get a native time tick of last modification time */ -uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); +uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); /** Get a native time tick of last modification time */ -uint64_t GetModificationTickFromPath(const std::filesystem::path& Filename); +uint64_t GetModificationTickFromHandle(void* NativeHandle, std::error_code& Ec); bool TryGetFileProperties(const std::filesystem::path& Path, uint64_t& OutSize, uint64_t& OutModificationTick, uint32_t& OutNativeModeOrAttributes); -/** Move a file, if the files are not on the same drive the function will fail +/** Move/rename a file, if the files are not on the same drive the function will fail (throws) */ void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); -/** Move a file, if the files are not on the same drive the function will fail +/** Move/rename a file, if the files are not on the same drive the function will fail */ void RenameFile(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); -/** Move a directory, if the files are not on the same drive the function will fail +/** Move/rename a directory, if the files are not on the same drive the function will fail (throws) */ void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath); -/** Move a directory, if the files are not on the same drive the function will fail +/** Move/rename a directory, if the files are not on the same drive the function will fail */ void RenameDirectory(const std::filesystem::path& SourcePath, const std::filesystem::path& TargetPath, std::error_code& Ec); @@ -421,7 +421,7 @@ uint32_t MakeFileModeReadOnly(uint32_t FileMode, bool ReadOnly); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly, std::error_code& Ec); bool SetFileReadOnly(const std::filesystem::path& Filename, bool ReadOnly); -void MakeSafeAbsolutePathÍnPlace(std::filesystem::path& Path); +void MakeSafeAbsolutePathInPlace(std::filesystem::path& Path); [[nodiscard]] std::filesystem::path MakeSafeAbsolutePath(const std::filesystem::path& Path); class SharedMemory -- cgit v1.2.3 From 4032ceb35dbfc0ffb71789fb9ff3dfcf854ca2d3 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 20 Feb 2026 09:14:26 +0100 Subject: initial CLAUDE.md (#764) --- CLAUDE.md | 271 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..757ce5d2a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,271 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Unreal Zen Storage Service (zenserver) is a high-performance local storage service for UE5, managing secondary data such as cooker output and local caches (DDC - Derived Data Cache). Can be deployed as a daemon on user machines or as a shared cache instance for build farms. + +## Build System + +This project uses **xmake** as its build system. xmake is stateful - run configuration before building. + +### Common Build Commands + +```bash +# Configure for debug (includes tests) +xmake config -m debug + +# Configure for release +xmake config -m release + +# Build everything +xmake + +# Build specific target +xmake build zenserver +xmake build zen + +# Run targets +xmake run zenserver +xmake run zen + +# Generate IDE project files +xmake sln # Windows: Visual Studio 2022 +xmake project -k xcode # macOS: Xcode +``` + +### Build Configuration Options + +- `--zensentry=yes|no` - Enable/disable Sentry crash reporting (default: yes) +- `--zenmimalloc=yes|no` - Use mimalloc for memory management (default: yes, disabled with ASAN) +- `--zenrpmalloc=yes|no` - Use rpmalloc for memory management (default: yes) +- `--httpsys=yes|no` - Enable http.sys server (Windows only, default: yes) +- `--zencompute=yes|no` - Enable compute services endpoint (default: yes) +- `--zenmemtrack=yes|no` - Enable UE Memory Trace support (Windows only, default: yes) +- `--zentrace=yes|no` - Enable UE Trace support (default: yes) + +### Testing + +```bash +# Tests are only built in debug mode +xmake config -m debug +xmake + +# Run all tests +xmake test --run=all + +# Run specific test suites +xmake test --run=core # zencore-test +xmake test --run=store # zenstore-test +xmake test --run=server # zenserver test command +xmake test --run=integration # zenserver-test +xmake test --run=http # zenhttp-test +xmake test --run=util # zenutil-test +xmake test --run=remotestore # zenremotestore-test + +# Run tests with JUnit reporting +xmake test --run=all --junit +``` + +Tests use the **doctest** framework. To run server tests: `xmake run zenserver test` + +### Code Formatting + +```bash +# Format all files (requires pre-commit) +xmake precommit + +# Or use pre-commit directly +pre-commit run --all-files +``` + +Install pre-commit hooks with `pre-commit install` to auto-format on commit. + +### Cleaning Build State + +When encountering build issues, clean all build state: + +**Windows:** +```cmd +rmdir /s /q .xmake build %LOCALAPPDATA%\.xmake %TEMP%\.xmake +``` + +**Linux/macOS:** +```bash +rm -rf .xmake build ~/.xmake +``` + +## Architecture + +### Module Structure + +The codebase is organized into layered modules with clear dependencies: + +**Core Libraries (Foundation Layer):** +- `zenbase` - Platform abstraction primitives +- `zencore` - Core utilities, memory management, logging, filesystem, crypto, threading +- `zenutil` - Higher-level utilities and common patterns + +**Protocol & Network Layer:** +- `zenhttp` - HTTP server implementations (http.sys on Windows, ASIO-based on Linux/Mac) +- `zennet` - Network utilities and protocols +- `transports` - Transport SDK and implementations + +**Storage Layer:** +- `zenstore` - Core storage engine and key-value store +- `zenvfs` - Virtual filesystem implementation +- `zenremotestore` - Remote storage client + +**Service Layer:** +- `zenserver` - Main server binary integrating all services +- `zencompute` - Compute services endpoint +- `zentelemetry` - Telemetry and metrics collection + +**Client Tools:** +- `zen` - CLI client utility for interacting with zenserver + - Commands are in `src/zen/cmds/` (e.g., `cache_cmd.h`, `status_cmd.h`, `ui_cmd.h`) + - Each command inherits from base command classes and registers with the CLI dispatcher + +**Test Projects:** +- `*-test` modules contain tests for their corresponding module (only built in debug mode) +- `zenserver-test` contains integration tests + +### Key Architectural Patterns + +**HTTP Server Implementations:** +- Windows: Uses http.sys kernel driver for highest throughput (requires elevation or URL reservation) +- Linux/macOS: ASIO-based HTTP server (also available on Windows) +- Server selection is compile-time based on platform but can be overridden on the command line via `--http=asio/httpsys/etc` + +**Storage Architecture:** +- zenserver manages multiple storage services (build store, cache, project store, object store) +- Content-addressable storage with deduplication on the lowest storage layer but this is mostly an internal implementation detail. + - Clients generally do not request content via content address. Instead they interface with the higher level storage services + - CAS content is garbage collected based on what is referenced from of higher level services + +**Frontend:** +- Web UI bundled as ZIP in `src/zenserver/frontend/*.zip` +- Dashboards for hub, orchestrator, and compute services are located in `src/zenserver/frontent/html/` + - These are the files which end up being bundled into the front-end zip mentioned above +- Update with `xmake updatefrontend` after modifying HTML/JS, and check in the resulting zip + +**Memory Management:** +- Can use mimalloc or rpmalloc for performance +- UE-style LLM (Low-Level Memory) tracking available on Windows +- Memory tracing support integrated with UE trace system + +**Tracing:** +- UE Trace integration for profiling and diagnostics +- Trace files use .utrace format +- Trace analysis tools in zentrace module (if available) +- Traces can be visualized using Unreal Insights, available in the UE5 code base + +## Coding Standards + +**Naming Conventions (UE-inspired with modifications):** +- Classes/Structs: `PascalCase` +- Functions/Methods: `PascalCase()` +- Member variables: `m_PascalCase` +- Global variables: `g_PascalCase` +- Static variables: `s_PascalCase` +- Thread-local variables: `t_PascalCase` + +**Note:** Unlike UE, no `F` prefix on structs/classes is required or encouraged. Also, no `b` prefix on booleans. + +**Code Style:** +- C++20 standard +- clang-format enforced via pre-commit hooks +- Use `std` containers; `eastl::fixed_vector` etc. for performance-critical paths +- Exceptions used only for unexpected errors, not flow control +- Logging is done via `ZEN_DEBUG(...)`, `ZEN_INFO(...)`, `ZEN_WARN`, `ZEN_ERROR` macros + - Logging macros use `fmt::format` for message formatting. The first argument is the format string, which + must be a string literal (turned into `std::string_view` in the macros) + - Logging channels can be overridden on a scope-by-scope basis (per-class or even per-function) by implementing + a `LoggerRef Log()` function + +**Includes:** +- Wrap third-party includes with `ZEN_THIRD_PARTY_INCLUDES_START` / `ZEN_THIRD_PARTY_INCLUDES_END` + - This helps avoid spurious compiler warnings due to the use of strict warnings-as-error policy + +**Platform Macros:** +- `ZEN_PLATFORM_WINDOWS`, `ZEN_PLATFORM_LINUX`, `ZEN_PLATFORM_MAC` +- Use `is_plat("windows")` etc. in xmake.lua files + +## Security Considerations + +**XSS in Web UI:** +Sanitize all dynamic data before inserting into innerHTML. Use `escapeHtml()` helper functions in JavaScript. + +**http.sys URL Reservation:** +On Windows, zenserver requires elevation or a URL reservation to bind http.sys to network interfaces: +```cmd +netsh http add urlacl url=http://*:8558/ user= +``` + +## Platform-Specific Notes + +**Windows:** +- Primary development platform +- Visual Studio 2022+ required (C++20 features) +- Static CRT linking by default (`/MT` in release, `/MTd` in debug) +- Recommended profilers: Superluminal Performance, Visual Studio profiler + +**Linux:** +- Requires GCC-11+ or Clang-12+ with libstdc++-11+ +- Set `CXX=g++-11` before running xmake if needed +- UE cross-compile toolchain support via `UE_TOOLCHAIN_DIR` environment variable + +**macOS:** +- Requires Xcode or Xcode command-line tools +- Supports both x86_64 and arm64 architectures + +## Dependencies + +Key third-party libraries (managed by xmake): +- EASTL - High-performance containers +- ASIO - Async I/O +- spdlog/fmt - Logging and formatting +- doctest - Testing framework +- cxxopts - Command-line parsing +- json11 - JSON parsing +- lz4 - Compression +- xxhash - Hashing +- OpenSSL (Linux/Mac) - Crypto operations +- Sentry - Crash reporting (optional) + +## Adding New Commands to `zen` CLI + +1. Create `src/zen/cmds/mycommand_cmd.h` and `src/zen/cmds/mycommand_cmd.cpp` +2. Inherit from appropriate base command class +3. Implement `Execute()` method +4. Add `#include "cmds/mycommand_cmd.h"` to `src/zen/zen.cpp` +5. Register command in `zen.cpp` command dispatcher. These should be ordered in alphabetical identifier order for consistent merging + +## Adding New Modules + +1. Create `src/mymodule/` directory structure +2. Add `include/` subdirectory for public headers +3. Create `xmake.lua` defining the target +4. Add `includes("src/mymodule")` to root `xmake.lua` +5. Add dependencies via `add_deps()` in the module's xmake.lua + +## Debugging + +**Visual Studio:** +- Use Debug configuration (includes test code) +- For http.sys debugging, run Visual Studio as Administrator or add URL reservation +- Enable child process debugging with Microsoft Child Process Debugging Power Tool + +**Multi-process Scenarios:** +When debugging zenserver-test or other multi-process scenarios, use child process debugging tools to attach to spawned processes automatically. + +## Bundle Creation + +```bash +# Create deployable ZIP bundle +xmake bundle + +# Update frontend ZIP after HTML changes +xmake updatefrontend +``` -- cgit v1.2.3 From 17898ec8a7ce42c0da27ac50c5c65aeb447c6374 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 20 Feb 2026 10:29:42 +0100 Subject: fix plain progress bar (#768) * fix plain progress not updating current state --- CHANGELOG.md | 3 +++ src/zen/progressbar.cpp | 1 + 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fa4fe031..0098f08b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ ## +- Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly + +## 5.7.21 - Feature: Added `--security-config-path` option to zenserver to configure security settings - Expects a path to a .json file - Default is an empty path resulting in no extra security settings and legacy behavior diff --git a/src/zen/progressbar.cpp b/src/zen/progressbar.cpp index 83606df67..1ee1d1e71 100644 --- a/src/zen/progressbar.cpp +++ b/src/zen/progressbar.cpp @@ -245,6 +245,7 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) const std::string Details = (!NewState.Details.empty()) ? fmt::format(": {}", NewState.Details) : ""; const std::string Output = fmt::format("{} {}% ({}){}\n", Task, PercentDone, NiceTimeSpanMs(ElapsedTimeMS), Details); OutputToConsoleRaw(Output); + m_State = NewState; } else if (m_Mode == Mode::Pretty) { -- cgit v1.2.3 From 80bc5a53fe9077bc20d287b912f6476db233110c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 20 Feb 2026 10:31:31 +0100 Subject: fix builds download indexing timer (#769) * fix build download indexing timer log --- CHANGELOG.md | 1 + src/zen/cmds/builds_cmd.cpp | 7 +++++++ src/zenremotestore/builds/buildstorageoperations.cpp | 7 ------- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0098f08b4..af2414682 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## - Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly +- Bugfix: Time spent indexing local and remote state during `zen builds download` now show the correct time ## 5.7.21 - Feature: Added `--security-config-path` option to zenserver to configure security settings diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 8dfe1093f..849259013 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1467,9 +1467,16 @@ namespace { ZEN_CONSOLE("Downloading build {}, parts:{} to '{}' ({})", BuildId, BuildPartString.ToView(), Path, NiceBytes(RawSize)); } + Stopwatch IndexTimer; + const ChunkedContentLookup LocalLookup = BuildChunkedContentLookup(LocalState.State.ChunkedContent); const ChunkedContentLookup RemoteLookup = BuildChunkedContentLookup(RemoteContent); + if (!IsQuiet) + { + ZEN_OPERATION_LOG_INFO(Output, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); + } + ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Download, TaskSteps::StepCount); BuildsOperationUpdateFolder Updater( diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index ade431393..72e06767a 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -579,13 +579,6 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) CreateDirectories(m_TempDownloadFolderPath); CreateDirectories(m_TempBlockFolderPath); - Stopwatch IndexTimer; - - if (!m_Options.IsQuiet) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, "Indexed local and remote content in {}", NiceTimeSpanMs(IndexTimer.GetElapsedTimeMs())); - } - Stopwatch CacheMappingTimer; std::vector> SequenceIndexChunksLeftToWriteCounters(m_RemoteContent.ChunkedContent.SequenceRawHashes.size()); -- cgit v1.2.3 From da4826d560a66b8a5f09158a93c83caa12348c7b Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 20 Feb 2026 10:32:32 +0100 Subject: move partial chunk block anailsys to chunkblock.h/cpp (#767) --- .../builds/buildstorageoperations.cpp | 882 ++++++--------------- src/zenremotestore/chunking/chunkblock.cpp | 540 ++++++++++++- .../zenremotestore/builds/buildstorageoperations.h | 50 +- .../include/zenremotestore/chunking/chunkblock.h | 94 +++ 4 files changed, 887 insertions(+), 679 deletions(-) diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 72e06767a..4f1b07c37 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -899,343 +899,213 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) CheckRequiredDiskSpace(RemotePathToRemoteIndex); + BlobsExistsResult ExistsResult; { - ZEN_TRACE_CPU("WriteChunks"); - - m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::WriteChunks, (uint32_t)TaskSteps::StepCount); + ChunkBlockAnalyser BlockAnalyser(m_LogOutput, + m_BlockDescriptions, + ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet, .IsVerbose = m_Options.IsVerbose}); - Stopwatch WriteTimer; + std::vector NeededBlocks = BlockAnalyser.GetNeeded( + m_RemoteLookup.ChunkHashToChunkIndex, + [&](uint32_t RemoteChunkIndex) -> bool { return RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]; }); - FilteredRate FilteredDownloadedBytesPerSecond; - FilteredRate FilteredWrittenBytesPerSecond; - - std::unique_ptr WriteProgressBarPtr( - m_LogOutput.CreateProgressBar(m_Options.PrimeCacheOnly ? "Downloading" : "Writing")); - OperationLogOutput::ProgressBar& WriteProgressBar(*WriteProgressBarPtr); - ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); - - struct LooseChunkHashWorkData - { - std::vector ChunkTargetPtrs; - uint32_t RemoteChunkIndex = (uint32_t)-1; - }; - - std::vector LooseChunkHashWorks; - TotalPartWriteCount += CopyChunkDatas.size(); - TotalPartWriteCount += ScavengedSequenceCopyOperations.size(); + std::vector FetchBlockIndexes; + std::vector CachedChunkBlockIndexes; - for (const IoHash ChunkHash : m_LooseChunkHashes) { - auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); - ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end()); - const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; - if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) + ZEN_TRACE_CPU("BlockCacheFileExists"); + for (const ChunkBlockAnalyser::NeededBlock& NeededBlock : NeededBlocks) { - if (m_Options.IsVerbose) + if (m_Options.PrimeCacheOnly) { - ZEN_OPERATION_LOG_INFO(m_LogOutput, "Skipping chunk {} due to cache reuse", ChunkHash); - } - continue; - } - bool NeedsCopy = true; - if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) - { - std::vector ChunkTargetPtrs = - GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); - - if (ChunkTargetPtrs.empty()) - { - if (m_Options.IsVerbose) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, "Skipping chunk {} due to cache reuse", ChunkHash); - } + FetchBlockIndexes.push_back(NeededBlock.BlockIndex); } else { - TotalRequestCount++; - TotalPartWriteCount++; - LooseChunkHashWorks.push_back( - LooseChunkHashWorkData{.ChunkTargetPtrs = ChunkTargetPtrs, .RemoteChunkIndex = RemoteChunkIndex}); - } - } - } - - uint32_t BlockCount = gsl::narrow(m_BlockDescriptions.size()); - - std::vector ChunkIsPickedUpByBlock(m_RemoteContent.ChunkedContent.ChunkHashes.size(), false); - auto GetNeededChunkBlockIndexes = [this, &RemoteChunkIndexNeedsCopyFromSourceFlags, &ChunkIsPickedUpByBlock]( - const ChunkBlockDescription& BlockDescription) { - ZEN_TRACE_CPU("GetNeededChunkBlockIndexes"); - std::vector NeededBlockChunkIndexes; - for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++) - { - const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; - if (auto It = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); It != m_RemoteLookup.ChunkHashToChunkIndex.end()) - { - const uint32_t RemoteChunkIndex = It->second; - if (!ChunkIsPickedUpByBlock[RemoteChunkIndex]) + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[NeededBlock.BlockIndex]; + bool UsingCachedBlock = false; + if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) { - if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex]) + TotalPartWriteCount++; + + std::filesystem::path BlockPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(); + if (IsFile(BlockPath)) { - ChunkIsPickedUpByBlock[RemoteChunkIndex] = true; - NeededBlockChunkIndexes.push_back(ChunkBlockIndex); + CachedChunkBlockIndexes.push_back(NeededBlock.BlockIndex); + UsingCachedBlock = true; } } - } - else - { - ZEN_DEBUG("Chunk {} not found in block {}", ChunkHash, BlockDescription.BlockHash); + if (!UsingCachedBlock) + { + FetchBlockIndexes.push_back(NeededBlock.BlockIndex); + } } } - return NeededBlockChunkIndexes; - }; + } - std::vector CachedChunkBlockIndexes; - std::vector FetchBlockIndexes; - std::vector> AllBlockChunkIndexNeeded; + std::vector NeededLooseChunkIndexes; - for (uint32_t BlockIndex = 0; BlockIndex < BlockCount; BlockIndex++) { - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - - std::vector BlockChunkIndexNeeded = GetNeededChunkBlockIndexes(BlockDescription); - if (!BlockChunkIndexNeeded.empty()) + NeededLooseChunkIndexes.reserve(m_LooseChunkHashes.size()); + for (uint32_t LooseChunkIndex = 0; LooseChunkIndex < m_LooseChunkHashes.size(); LooseChunkIndex++) { - if (m_Options.PrimeCacheOnly) + const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex]; + auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end()); + const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; + + if (RemoteChunkIndexNeedsCopyFromLocalFileFlags[RemoteChunkIndex]) { - FetchBlockIndexes.push_back(BlockIndex); + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Skipping chunk {} due to cache reuse", + m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]); + } + continue; } - else + + bool NeedsCopy = true; + if (RemoteChunkIndexNeedsCopyFromSourceFlags[RemoteChunkIndex].compare_exchange_strong(NeedsCopy, false)) { - bool UsingCachedBlock = false; - if (auto It = CachedBlocksFound.find(BlockDescription.BlockHash); It != CachedBlocksFound.end()) + uint64_t WriteCount = GetChunkWriteCount(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); + if (WriteCount == 0) { - TotalPartWriteCount++; - - std::filesystem::path BlockPath = m_TempBlockFolderPath / BlockDescription.BlockHash.ToHexString(); - if (IsFile(BlockPath)) + if (m_Options.IsVerbose) { - CachedChunkBlockIndexes.push_back(BlockIndex); - UsingCachedBlock = true; + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Skipping chunk {} due to cache reuse", + m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]); } } - if (!UsingCachedBlock) + else { - FetchBlockIndexes.push_back(BlockIndex); + NeededLooseChunkIndexes.push_back(LooseChunkIndex); } } } - AllBlockChunkIndexNeeded.emplace_back(std::move(BlockChunkIndexNeeded)); } - BlobsExistsResult ExistsResult; - if (m_Storage.BuildCacheStorage) { ZEN_TRACE_CPU("BlobCacheExistCheck"); Stopwatch Timer; - tsl::robin_set BlobHashesSet; + std::vector BlobHashes; + BlobHashes.reserve(NeededLooseChunkIndexes.size() + FetchBlockIndexes.size()); - BlobHashesSet.reserve(LooseChunkHashWorks.size() + FetchBlockIndexes.size()); - for (LooseChunkHashWorkData& LooseChunkHashWork : LooseChunkHashWorks) + for (const uint32_t LooseChunkIndex : NeededLooseChunkIndexes) { - BlobHashesSet.insert(m_RemoteContent.ChunkedContent.ChunkHashes[LooseChunkHashWork.RemoteChunkIndex]); + BlobHashes.push_back(m_LooseChunkHashes[LooseChunkIndex]); } + for (uint32_t BlockIndex : FetchBlockIndexes) { - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - BlobHashesSet.insert(BlockDescription.BlockHash); + BlobHashes.push_back(m_BlockDescriptions[BlockIndex].BlockHash); } - if (!BlobHashesSet.empty()) - { - const std::vector BlobHashes(BlobHashesSet.begin(), BlobHashesSet.end()); - const std::vector CacheExistsResult = - m_Storage.BuildCacheStorage->BlobsExists(m_BuildId, BlobHashes); + const std::vector CacheExistsResult = + m_Storage.BuildCacheStorage->BlobsExists(m_BuildId, BlobHashes); - if (CacheExistsResult.size() == BlobHashes.size()) + if (CacheExistsResult.size() == BlobHashes.size()) + { + ExistsResult.ExistingBlobs.reserve(CacheExistsResult.size()); + for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++) { - ExistsResult.ExistingBlobs.reserve(CacheExistsResult.size()); - for (size_t BlobIndex = 0; BlobIndex < BlobHashes.size(); BlobIndex++) + if (CacheExistsResult[BlobIndex].HasBody) { - if (CacheExistsResult[BlobIndex].HasBody) - { - ExistsResult.ExistingBlobs.insert(BlobHashes[BlobIndex]); - } + ExistsResult.ExistingBlobs.insert(BlobHashes[BlobIndex]); } } - ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs(); - if (!ExistsResult.ExistingBlobs.empty() && !m_Options.IsQuiet) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Remote cache : Found {} out of {} needed blobs in {}", - ExistsResult.ExistingBlobs.size(), - BlobHashes.size(), - NiceTimeSpanMs(ExistsResult.ElapsedTimeMs)); - } + } + ExistsResult.ElapsedTimeMs = Timer.GetElapsedTimeMs(); + if (!ExistsResult.ExistingBlobs.empty() && !m_Options.IsQuiet) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Remote cache : Found {} out of {} needed blobs in {}", + ExistsResult.ExistingBlobs.size(), + BlobHashes.size(), + NiceTimeSpanMs(ExistsResult.ElapsedTimeMs)); } } - std::vector BlockRangeWorks; - std::vector FullBlockWorks; + std::vector BlockPartialDownloadModes; + if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off) { - Stopwatch Timer; - - std::vector PartialBlockIndexes; - - for (uint32_t BlockIndex : FetchBlockIndexes) + BlockPartialDownloadModes.resize(m_BlockDescriptions.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); + } + else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::All) + { + BlockPartialDownloadModes.resize(m_BlockDescriptions.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::On); + } + else + { + BlockPartialDownloadModes.reserve(m_BlockDescriptions.size()); + for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++) { - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - - const std::vector BlockChunkIndexNeeded = std::move(AllBlockChunkIndexNeeded[BlockIndex]); - if (!BlockChunkIndexNeeded.empty()) + const bool BlockExistInCache = ExistsResult.ExistingBlobs.contains(m_BlockDescriptions[BlockIndex].BlockHash); + if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::ZenCacheOnly) { - bool WantsToDoPartialBlockDownload = BlockChunkIndexNeeded.size() < BlockDescription.ChunkRawHashes.size(); - bool CanDoPartialBlockDownload = - (BlockDescription.HeaderSize > 0) && - (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); - - bool AllowedToDoPartialRequest = false; - bool BlockExistInCache = ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); - switch (m_Options.PartialBlockRequestMode) - { - case EPartialBlockRequestMode::Off: - break; - case EPartialBlockRequestMode::ZenCacheOnly: - AllowedToDoPartialRequest = BlockExistInCache; - break; - case EPartialBlockRequestMode::Mixed: - case EPartialBlockRequestMode::All: - AllowedToDoPartialRequest = true; - break; - default: - ZEN_ASSERT(false); - break; - } + BlockPartialDownloadModes.push_back(BlockExistInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::On + : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); + } + else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Mixed) + { + BlockPartialDownloadModes.push_back(BlockExistInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::On + : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange); + } + } + } + ZEN_ASSERT(BlockPartialDownloadModes.size() == m_BlockDescriptions.size()); - const uint32_t ChunkStartOffsetInBlock = - gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + ChunkBlockAnalyser::BlockResult PartialBlocks = + BlockAnalyser.CalculatePartialBlockDownloads(NeededBlocks, BlockPartialDownloadModes); - const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), - BlockDescription.ChunkCompressedLengths.end(), - std::uint64_t(ChunkStartOffsetInBlock)); + struct LooseChunkHashWorkData + { + std::vector ChunkTargetPtrs; + uint32_t RemoteChunkIndex = (uint32_t)-1; + }; - if (AllowedToDoPartialRequest && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) - { - ZEN_TRACE_CPU("PartialBlockAnalysis"); - - bool LimitToSingleRange = - BlockExistInCache ? false : m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Mixed; - uint64_t TotalWantedChunksSize = 0; - std::optional> MaybeBlockRanges = - CalculateBlockRanges(BlockIndex, - BlockDescription, - BlockChunkIndexNeeded, - LimitToSingleRange, - ChunkStartOffsetInBlock, - TotalBlockSize, - TotalWantedChunksSize); - ZEN_ASSERT(TotalWantedChunksSize <= TotalBlockSize); - - if (MaybeBlockRanges.has_value()) - { - const std::vector& BlockRanges = MaybeBlockRanges.value(); - ZEN_ASSERT(!BlockRanges.empty()); - BlockRangeWorks.insert(BlockRangeWorks.end(), BlockRanges.begin(), BlockRanges.end()); - TotalRequestCount += BlockRanges.size(); - TotalPartWriteCount += BlockRanges.size(); - - uint64_t RequestedSize = std::accumulate( - BlockRanges.begin(), - BlockRanges.end(), - uint64_t(0), - [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); - PartialBlockIndexes.push_back(BlockIndex); - - if (RequestedSize > TotalWantedChunksSize) - { - if (m_Options.IsVerbose) - { - ZEN_OPERATION_LOG_INFO( - m_LogOutput, - "Requesting {} chunks ({}) from block {} ({}) using {} requests (extra bytes {})", - BlockChunkIndexNeeded.size(), - NiceBytes(RequestedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - BlockRanges.size(), - NiceBytes(RequestedSize - TotalWantedChunksSize)); - } - } - } - else - { - FullBlockWorks.push_back(BlockIndex); - TotalRequestCount++; - TotalPartWriteCount++; - } - } - else - { - FullBlockWorks.push_back(BlockIndex); - TotalRequestCount++; - TotalPartWriteCount++; - } - } - } + TotalRequestCount += NeededLooseChunkIndexes.size(); + TotalPartWriteCount += NeededLooseChunkIndexes.size(); + TotalRequestCount += PartialBlocks.BlockRanges.size(); + TotalPartWriteCount += PartialBlocks.BlockRanges.size(); + TotalRequestCount += PartialBlocks.FullBlockIndexes.size(); + TotalPartWriteCount += PartialBlocks.FullBlockIndexes.size(); - if (!PartialBlockIndexes.empty()) - { - uint64_t TotalFullBlockRequestBytes = 0; - for (uint32_t BlockIndex : FullBlockWorks) - { - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - uint32_t CurrentOffset = - gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + std::vector LooseChunkHashWorks; + for (uint32_t LooseChunkIndex : NeededLooseChunkIndexes) + { + const IoHash& ChunkHash = m_LooseChunkHashes[LooseChunkIndex]; + auto RemoteChunkIndexIt = m_RemoteLookup.ChunkHashToChunkIndex.find(ChunkHash); + ZEN_ASSERT(RemoteChunkIndexIt != m_RemoteLookup.ChunkHashToChunkIndex.end()); + const uint32_t RemoteChunkIndex = RemoteChunkIndexIt->second; - TotalFullBlockRequestBytes += std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), - BlockDescription.ChunkCompressedLengths.end(), - std::uint64_t(CurrentOffset)); - } + std::vector ChunkTargetPtrs = + GetRemainingChunkTargets(SequenceIndexChunksLeftToWriteCounters, RemoteChunkIndex); - uint64_t TotalPartialBlockBytes = 0; - for (uint32_t BlockIndex : PartialBlockIndexes) - { - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - uint32_t CurrentOffset = - gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + ZEN_ASSERT(!ChunkTargetPtrs.empty()); + LooseChunkHashWorks.push_back( + LooseChunkHashWorkData{.ChunkTargetPtrs = ChunkTargetPtrs, .RemoteChunkIndex = RemoteChunkIndex}); + } - TotalPartialBlockBytes += std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), - BlockDescription.ChunkCompressedLengths.end(), - std::uint64_t(CurrentOffset)); - } + ZEN_TRACE_CPU("WriteChunks"); - uint64_t NonPartialTotalBlockBytes = TotalFullBlockRequestBytes + TotalPartialBlockBytes; + m_LogOutput.SetLogOperationProgress((uint32_t)TaskSteps::WriteChunks, (uint32_t)TaskSteps::StepCount); - const uint64_t TotalPartialBlockRequestBytes = - std::accumulate(BlockRangeWorks.begin(), - BlockRangeWorks.end(), - uint64_t(0), - [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); - uint64_t TotalExtraPartialBlocksRequests = BlockRangeWorks.size() - PartialBlockIndexes.size(); + Stopwatch WriteTimer; - uint64_t TotalSavedBlocksSize = TotalPartialBlockBytes - TotalPartialBlockRequestBytes; - double SavedSizePercent = (TotalSavedBlocksSize * 100.0) / NonPartialTotalBlockBytes; + FilteredRate FilteredDownloadedBytesPerSecond; + FilteredRate FilteredWrittenBytesPerSecond; - if (!m_Options.IsQuiet) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Analysis of partial block requests saves download of {} out of {} ({:.1f}%) using {} extra " - "requests. Completed in {}", - NiceBytes(TotalSavedBlocksSize), - NiceBytes(NonPartialTotalBlockBytes), - SavedSizePercent, - TotalExtraPartialBlocksRequests, - NiceTimeSpanMs(ExistsResult.ElapsedTimeMs)); - } - } - } + std::unique_ptr WriteProgressBarPtr( + m_LogOutput.CreateProgressBar(m_Options.PrimeCacheOnly ? "Downloading" : "Writing")); + OperationLogOutput::ProgressBar& WriteProgressBar(*WriteProgressBarPtr); + ParallelWork Work(m_AbortFlag, m_PauseFlag, WorkerThreadPool::EMode::EnableBacklog); + + TotalPartWriteCount += CopyChunkDatas.size(); + TotalPartWriteCount += ScavengedSequenceCopyOperations.size(); BufferedWriteFileCache WriteCache; @@ -1465,13 +1335,23 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) }); } - for (size_t BlockRangeIndex = 0; BlockRangeIndex < BlockRangeWorks.size(); BlockRangeIndex++) + for (size_t BlockRangeIndex = 0; BlockRangeIndex < PartialBlocks.BlockRanges.size();) { ZEN_ASSERT(!m_Options.PrimeCacheOnly); if (m_AbortFlag) { break; } + + size_t RangeCount = 1; + size_t RangesLeft = PartialBlocks.BlockRanges.size() - BlockRangeIndex; + const ChunkBlockAnalyser::BlockRangeDescriptor& CurrentBlockRange = PartialBlocks.BlockRanges[BlockRangeIndex]; + while (RangeCount < RangesLeft && + CurrentBlockRange.BlockIndex == PartialBlocks.BlockRanges[BlockRangeIndex + RangeCount].BlockIndex) + { + RangeCount++; + } + Work.ScheduleWork( m_NetworkPool, [this, @@ -1485,119 +1365,127 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) TotalPartWriteCount, &FilteredWrittenBytesPerSecond, &Work, - &BlockRangeWorks, - BlockRangeIndex](std::atomic&) { + &PartialBlocks, + BlockRangeStartIndex = BlockRangeIndex, + RangeCount](std::atomic&) { if (!m_AbortFlag) { - ZEN_TRACE_CPU("Async_GetPartialBlock"); - - const BlockRangeDescriptor& BlockRange = BlockRangeWorks[BlockRangeIndex]; + ZEN_TRACE_CPU("Async_GetPartialBlockRanges"); FilteredDownloadedBytesPerSecond.Start(); - DownloadPartialBlock( - BlockRange, - ExistsResult, - [this, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - &WritePartsComplete, - &WriteCache, - &Work, - TotalRequestCount, - TotalPartWriteCount, - &FilteredDownloadedBytesPerSecond, - &FilteredWrittenBytesPerSecond, - &BlockRange](IoBuffer&& InMemoryBuffer, const std::filesystem::path& OnDiskPath) { - if (m_DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - - if (!m_AbortFlag) - { - Work.ScheduleWork( - m_IOWorkerPool, - [this, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - &WritePartsComplete, - &WriteCache, - &Work, - TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - &BlockRange, - BlockChunkPath = std::filesystem::path(OnDiskPath), - BlockPartialBuffer = std::move(InMemoryBuffer)](std::atomic&) mutable { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_WritePartialBlock"); + for (size_t BlockRangeIndex = BlockRangeStartIndex; BlockRangeIndex < BlockRangeStartIndex + RangeCount; + BlockRangeIndex++) + { + ZEN_TRACE_CPU("GetPartialBlock"); - const uint32_t BlockIndex = BlockRange.BlockIndex; + const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = PartialBlocks.BlockRanges[BlockRangeIndex]; - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + DownloadPartialBlock( + BlockRange, + ExistsResult, + [this, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + &WritePartsComplete, + &WriteCache, + &Work, + TotalRequestCount, + TotalPartWriteCount, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond, + &BlockRange](IoBuffer&& InMemoryBuffer, const std::filesystem::path& OnDiskPath) { + if (m_DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockPartialBuffer); - } - else + if (!m_AbortFlag) + { + Work.ScheduleWork( + m_IOWorkerPool, + [this, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + &WritePartsComplete, + &WriteCache, + &Work, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + &BlockRange, + BlockChunkPath = std::filesystem::path(OnDiskPath), + BlockPartialBuffer = std::move(InMemoryBuffer)](std::atomic&) mutable { + if (!m_AbortFlag) { - ZEN_ASSERT(!BlockPartialBuffer); - BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockPartialBuffer) + ZEN_TRACE_CPU("Async_WritePartialBlock"); + + const uint32_t BlockIndex = BlockRange.BlockIndex; + + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + + if (BlockChunkPath.empty()) { - throw std::runtime_error( - fmt::format("Could not open downloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); + ZEN_ASSERT(BlockPartialBuffer); + } + else + { + ZEN_ASSERT(!BlockPartialBuffer); + BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockPartialBuffer) + { + throw std::runtime_error( + fmt::format("Could not open downloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); + } } - } - - FilteredWrittenBytesPerSecond.Start(); - if (!WritePartialBlockChunksToCache( - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - CompositeBuffer(std::move(BlockPartialBuffer)), - BlockRange.ChunkBlockIndexStart, - BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, - RemoteChunkIndexNeedsCopyFromSourceFlags, - WriteCache)) - { - std::error_code DummyEc; - RemoveFile(BlockChunkPath, DummyEc); - throw std::runtime_error( - fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); - } + FilteredWrittenBytesPerSecond.Start(); + + if (!WritePartialBlockChunksToCache( + BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + CompositeBuffer(std::move(BlockPartialBuffer)), + BlockRange.ChunkBlockIndexStart, + BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, + RemoteChunkIndexNeedsCopyFromSourceFlags, + WriteCache)) + { + std::error_code DummyEc; + RemoveFile(BlockChunkPath, DummyEc); + throw std::runtime_error( + fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); + } - std::error_code Ec = TryRemoveFile(BlockChunkPath); - if (Ec) - { - ZEN_OPERATION_LOG_DEBUG(m_LogOutput, - "Failed removing file '{}', reason: ({}) {}", - BlockChunkPath, - Ec.value(), - Ec.message()); - } + std::error_code Ec = TryRemoveFile(BlockChunkPath); + if (Ec) + { + ZEN_OPERATION_LOG_DEBUG(m_LogOutput, + "Failed removing file '{}', reason: ({}) {}", + BlockChunkPath, + Ec.value(), + Ec.message()); + } - WritePartsComplete++; - if (WritePartsComplete == TotalPartWriteCount) - { - FilteredWrittenBytesPerSecond.Stop(); + WritePartsComplete++; + if (WritePartsComplete == TotalPartWriteCount) + { + FilteredWrittenBytesPerSecond.Stop(); + } } - } - }, - OnDiskPath.empty() ? WorkerThreadPool::EMode::DisableBacklog - : WorkerThreadPool::EMode::EnableBacklog); - } - }); + }, + OnDiskPath.empty() ? WorkerThreadPool::EMode::DisableBacklog + : WorkerThreadPool::EMode::EnableBacklog); + } + }); + } } }); + BlockRangeIndex += RangeCount; } - for (uint32_t BlockIndex : FullBlockWorks) + for (uint32_t BlockIndex : PartialBlocks.FullBlockIndexes) { if (m_AbortFlag) { @@ -3282,271 +3170,9 @@ BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t RemoteChunkInde } } -BuildsOperationUpdateFolder::BlockRangeDescriptor -BuildsOperationUpdateFolder::MergeBlockRanges(std::span Ranges) -{ - ZEN_ASSERT(Ranges.size() > 1); - const BlockRangeDescriptor& First = Ranges.front(); - const BlockRangeDescriptor& Last = Ranges.back(); - - return BlockRangeDescriptor{.BlockIndex = First.BlockIndex, - .RangeStart = First.RangeStart, - .RangeLength = Last.RangeStart + Last.RangeLength - First.RangeStart, - .ChunkBlockIndexStart = First.ChunkBlockIndexStart, - .ChunkBlockIndexCount = Last.ChunkBlockIndexStart + Last.ChunkBlockIndexCount - First.ChunkBlockIndexStart}; -} - -std::optional> -BuildsOperationUpdateFolder::MakeOptionalBlockRangeVector(uint64_t TotalBlockSize, const BlockRangeDescriptor& Range) -{ - if (Range.RangeLength == TotalBlockSize) - { - return {}; - } - else - { - return std::vector{Range}; - } -}; - -const BuildsOperationUpdateFolder::BlockRangeLimit* -BuildsOperationUpdateFolder::GetBlockRangeLimitForRange(std::span Limits, - uint64_t TotalBlockSize, - std::span Ranges) -{ - if (Ranges.size() > 1) - { - const std::uint64_t WantedSize = - std::accumulate(Ranges.begin(), Ranges.end(), uint64_t(0), [](uint64_t Current, const BlockRangeDescriptor& Range) { - return Current + Range.RangeLength; - }); - - const double RangeRequestedPercent = (WantedSize * 100.0) / TotalBlockSize; - - for (const BlockRangeLimit& Limit : Limits) - { - if (RangeRequestedPercent >= Limit.SizePercent && Ranges.size() > Limit.MaxRangeCount) - { - return &Limit; - } - } - } - return nullptr; -}; - -std::vector -BuildsOperationUpdateFolder::CollapseBlockRanges(const uint64_t AlwaysAcceptableGap, std::span BlockRanges) -{ - ZEN_ASSERT(BlockRanges.size() > 1); - std::vector CollapsedBlockRanges; - - auto BlockRangesIt = BlockRanges.begin(); - CollapsedBlockRanges.push_back(*BlockRangesIt++); - for (; BlockRangesIt != BlockRanges.end(); BlockRangesIt++) - { - BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); - - const uint64_t BothRangeSize = BlockRangesIt->RangeLength + LastRange.RangeLength; - - const uint64_t Gap = BlockRangesIt->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); - if (Gap <= Max(BothRangeSize / 16, AlwaysAcceptableGap)) - { - LastRange.ChunkBlockIndexCount = - (BlockRangesIt->ChunkBlockIndexStart + BlockRangesIt->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; - LastRange.RangeLength = (BlockRangesIt->RangeStart + BlockRangesIt->RangeLength) - LastRange.RangeStart; - } - else - { - CollapsedBlockRanges.push_back(*BlockRangesIt); - } - } - - return CollapsedBlockRanges; -}; - -uint64_t -BuildsOperationUpdateFolder::CalculateNextGap(std::span BlockRanges) -{ - ZEN_ASSERT(BlockRanges.size() > 1); - uint64_t AcceptableGap = (uint64_t)-1; - for (size_t RangeIndex = 0; RangeIndex < BlockRanges.size() - 1; RangeIndex++) - { - const BlockRangeDescriptor& Range = BlockRanges[RangeIndex]; - const BlockRangeDescriptor& NextRange = BlockRanges[RangeIndex + 1]; - - const uint64_t Gap = NextRange.RangeStart - (Range.RangeStart + Range.RangeLength); - AcceptableGap = Min(Gap, AcceptableGap); - } - AcceptableGap = RoundUp(AcceptableGap, 16u * 1024u); - return AcceptableGap; -}; - -std::optional> -BuildsOperationUpdateFolder::CalculateBlockRanges(uint32_t BlockIndex, - const ChunkBlockDescription& BlockDescription, - std::span BlockChunkIndexNeeded, - bool LimitToSingleRange, - const uint64_t ChunkStartOffsetInBlock, - const uint64_t TotalBlockSize, - uint64_t& OutTotalWantedChunksSize) -{ - ZEN_TRACE_CPU("CalculateBlockRanges"); - - std::vector BlockRanges; - { - uint64_t CurrentOffset = ChunkStartOffsetInBlock; - uint32_t ChunkBlockIndex = 0; - uint32_t NeedBlockChunkIndexOffset = 0; - BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; - while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) - { - const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; - if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) - { - if (NextRange.RangeLength > 0) - { - BlockRanges.push_back(NextRange); - NextRange = {.BlockIndex = BlockIndex}; - } - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; - } - else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) - { - if (NextRange.RangeLength == 0) - { - NextRange.RangeStart = CurrentOffset; - NextRange.ChunkBlockIndexStart = ChunkBlockIndex; - } - NextRange.RangeLength += ChunkCompressedLength; - NextRange.ChunkBlockIndexCount++; - ChunkBlockIndex++; - CurrentOffset += ChunkCompressedLength; - NeedBlockChunkIndexOffset++; - } - else - { - ZEN_ASSERT(false); - } - } - if (NextRange.RangeLength > 0) - { - BlockRanges.push_back(NextRange); - } - } - ZEN_ASSERT(!BlockRanges.empty()); - - OutTotalWantedChunksSize = - std::accumulate(BlockRanges.begin(), BlockRanges.end(), uint64_t(0), [](uint64_t Current, const BlockRangeDescriptor& Range) { - return Current + Range.RangeLength; - }); - - double RangeWantedPercent = (OutTotalWantedChunksSize * 100.0) / TotalBlockSize; - - if (BlockRanges.size() == 1) - { - if (m_Options.IsVerbose) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Range request of {} ({:.2f}%) using single range from block {} ({}) as is", - NiceBytes(OutTotalWantedChunksSize), - RangeWantedPercent, - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize)); - } - return BlockRanges; - } - - if (LimitToSingleRange) - { - const BlockRangeDescriptor MergedRange = MergeBlockRanges(BlockRanges); - if (m_Options.IsVerbose) - { - const double RangeRequestedPercent = (MergedRange.RangeLength * 100.0) / TotalBlockSize; - const double WastedPercent = ((MergedRange.RangeLength - OutTotalWantedChunksSize) * 100.0) / MergedRange.RangeLength; - - ZEN_OPERATION_LOG_INFO( - m_LogOutput, - "Range request of {} ({:.2f}%) using {} ranges from block {} ({}) limited to single block range {} ({:.2f}%) wasting " - "{:.2f}% ({})", - NiceBytes(OutTotalWantedChunksSize), - RangeWantedPercent, - BlockRanges.size(), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - NiceBytes(MergedRange.RangeLength), - RangeRequestedPercent, - WastedPercent, - NiceBytes(MergedRange.RangeLength - OutTotalWantedChunksSize)); - } - return MakeOptionalBlockRangeVector(TotalBlockSize, MergedRange); - } - - if (RangeWantedPercent > FullBlockRangePercentLimit) - { - const BlockRangeDescriptor MergedRange = MergeBlockRanges(BlockRanges); - if (m_Options.IsVerbose) - { - const double RangeRequestedPercent = (MergedRange.RangeLength * 100.0) / TotalBlockSize; - const double WastedPercent = ((MergedRange.RangeLength - OutTotalWantedChunksSize) * 100.0) / MergedRange.RangeLength; - - ZEN_OPERATION_LOG_INFO( - m_LogOutput, - "Range request of {} ({:.2f}%) using {} ranges from block {} ({}) exceeds {}%. Merged to single block range {} " - "({:.2f}%) wasting {:.2f}% ({})", - NiceBytes(OutTotalWantedChunksSize), - RangeWantedPercent, - BlockRanges.size(), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - FullBlockRangePercentLimit, - NiceBytes(MergedRange.RangeLength), - RangeRequestedPercent, - WastedPercent, - NiceBytes(MergedRange.RangeLength - OutTotalWantedChunksSize)); - } - return MakeOptionalBlockRangeVector(TotalBlockSize, MergedRange); - } - - std::vector CollapsedBlockRanges = CollapseBlockRanges(16u * 1024u, BlockRanges); - while (GetBlockRangeLimitForRange(ForceMergeLimits, TotalBlockSize, CollapsedBlockRanges)) - { - CollapsedBlockRanges = CollapseBlockRanges(CalculateNextGap(CollapsedBlockRanges), CollapsedBlockRanges); - } - - const std::uint64_t WantedCollapsedSize = - std::accumulate(CollapsedBlockRanges.begin(), - CollapsedBlockRanges.end(), - uint64_t(0), - [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); - - const double CollapsedRangeRequestedPercent = (WantedCollapsedSize * 100.0) / TotalBlockSize; - - if (m_Options.IsVerbose) - { - const double WastedPercent = ((WantedCollapsedSize - OutTotalWantedChunksSize) * 100.0) / WantedCollapsedSize; - - ZEN_OPERATION_LOG_INFO( - m_LogOutput, - "Range request of {} ({:.2f}%) using {} ranges from block {} ({}) collapsed to {} {:.2f}% using {} ranges wasting {:.2f}% " - "({})", - NiceBytes(OutTotalWantedChunksSize), - RangeWantedPercent, - BlockRanges.size(), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - NiceBytes(WantedCollapsedSize), - CollapsedRangeRequestedPercent, - CollapsedBlockRanges.size(), - WastedPercent, - NiceBytes(WantedCollapsedSize - OutTotalWantedChunksSize)); - } - return CollapsedBlockRanges; -} - void BuildsOperationUpdateFolder::DownloadPartialBlock( - const BlockRangeDescriptor BlockRange, + const ChunkBlockAnalyser::BlockRangeDescriptor BlockRange, const BlobsExistsResult& ExistsResult, std::function&& OnDownloaded) { diff --git a/src/zenremotestore/chunking/chunkblock.cpp b/src/zenremotestore/chunking/chunkblock.cpp index c4d8653f4..06cedae3f 100644 --- a/src/zenremotestore/chunking/chunkblock.cpp +++ b/src/zenremotestore/chunking/chunkblock.cpp @@ -10,18 +10,17 @@ #include +#include #include ZEN_THIRD_PARTY_INCLUDES_START #include +#include ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_WITH_TESTS # include # include - -# include -# include #endif // ZEN_WITH_TESTS namespace zen { @@ -455,6 +454,537 @@ FindReuseBlocks(OperationLogOutput& Output, return FilteredReuseBlockIndexes; } +ChunkBlockAnalyser::ChunkBlockAnalyser(OperationLogOutput& LogOutput, + std::span BlockDescriptions, + const Options& Options) +: m_LogOutput(LogOutput) +, m_BlockDescriptions(BlockDescriptions) +, m_Options(Options) +{ +} + +std::vector +ChunkBlockAnalyser::GetNeeded(const tsl::robin_map& ChunkHashToChunkIndex, + std::function&& NeedsBlockChunk) +{ + ZEN_TRACE_CPU("ChunkBlockAnalyser::GetNeeded"); + + std::vector Result; + + std::vector ChunkIsNeeded(ChunkHashToChunkIndex.size()); + for (uint32_t ChunkIndex = 0; ChunkIndex < ChunkHashToChunkIndex.size(); ChunkIndex++) + { + ChunkIsNeeded[ChunkIndex] = NeedsBlockChunk(ChunkIndex); + } + + std::vector BlockSlack(m_BlockDescriptions.size(), 0u); + for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++) + { + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + + uint64_t BlockUsedSize = 0; + uint64_t BlockSize = 0; + + for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++) + { + const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; + if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = It->second; + if (ChunkIsNeeded[RemoteChunkIndex]) + { + BlockUsedSize += BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + } + } + BlockSize += BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + } + BlockSlack[BlockIndex] = BlockSize - BlockUsedSize; + } + + std::vector BlockOrder(m_BlockDescriptions.size()); + std::iota(BlockOrder.begin(), BlockOrder.end(), 0); + + std::sort(BlockOrder.begin(), BlockOrder.end(), [&BlockSlack](uint32_t Lhs, uint32_t Rhs) { + return BlockSlack[Lhs] < BlockSlack[Rhs]; + }); + + std::vector ChunkIsPickedUp(ChunkHashToChunkIndex.size(), false); + + for (uint32_t BlockIndex : BlockOrder) + { + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + + std::vector BlockChunkIndexNeeded; + + for (uint32_t ChunkBlockIndex = 0; ChunkBlockIndex < BlockDescription.ChunkRawHashes.size(); ChunkBlockIndex++) + { + const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; + if (auto It = ChunkHashToChunkIndex.find(ChunkHash); It != ChunkHashToChunkIndex.end()) + { + const uint32_t RemoteChunkIndex = It->second; + if (ChunkIsNeeded[RemoteChunkIndex]) + { + if (!ChunkIsPickedUp[RemoteChunkIndex]) + { + ChunkIsPickedUp[RemoteChunkIndex] = true; + BlockChunkIndexNeeded.push_back(ChunkBlockIndex); + } + } + } + else + { + ZEN_DEBUG("Chunk {} not found in block {}", ChunkHash, BlockDescription.BlockHash); + } + } + + if (!BlockChunkIndexNeeded.empty()) + { + Result.push_back(NeededBlock{.BlockIndex = BlockIndex, .ChunkIndexes = std::move(BlockChunkIndexNeeded)}); + } + } + return Result; +} + +ChunkBlockAnalyser::BlockResult +ChunkBlockAnalyser::CalculatePartialBlockDownloads(std::span NeededBlocks, + std::span BlockPartialDownloadModes) +{ + ZEN_TRACE_CPU("ChunkBlockAnalyser::CalculatePartialBlockDownloads"); + + Stopwatch PartialAnalisysTimer; + + ChunkBlockAnalyser::BlockResult Result; + + uint64_t IdealDownloadTotalSize = 0; + uint64_t AllBlocksTotalBlocksSize = 0; + + for (const NeededBlock& NeededBlock : NeededBlocks) + { + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[NeededBlock.BlockIndex]; + + std::span BlockChunkIndexNeeded(NeededBlock.ChunkIndexes); + if (!NeededBlock.ChunkIndexes.empty()) + { + bool WantsToDoPartialBlockDownload = NeededBlock.ChunkIndexes.size() < BlockDescription.ChunkRawHashes.size(); + bool CanDoPartialBlockDownload = (BlockDescription.HeaderSize > 0) && + (BlockDescription.ChunkCompressedLengths.size() == BlockDescription.ChunkRawHashes.size()); + + EPartialBlockDownloadMode PartialBlockDownloadMode = BlockPartialDownloadModes[NeededBlock.BlockIndex]; + + const uint32_t ChunkStartOffsetInBlock = + gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + + const uint64_t TotalBlockSize = std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), + BlockDescription.ChunkCompressedLengths.end(), + std::uint64_t(ChunkStartOffsetInBlock)); + + AllBlocksTotalBlocksSize += TotalBlockSize; + + if ((PartialBlockDownloadMode != EPartialBlockDownloadMode::Off) && WantsToDoPartialBlockDownload && CanDoPartialBlockDownload) + { + ZEN_TRACE_CPU("PartialBlockAnalysis"); + + uint64_t TotalWantedChunksSize = 0; + std::optional> MaybeBlockRanges = CalculateBlockRanges(NeededBlock.BlockIndex, + BlockDescription, + NeededBlock.ChunkIndexes, + PartialBlockDownloadMode, + ChunkStartOffsetInBlock, + TotalBlockSize, + TotalWantedChunksSize); + ZEN_ASSERT(TotalWantedChunksSize <= TotalBlockSize); + IdealDownloadTotalSize += TotalWantedChunksSize; + + if (MaybeBlockRanges.has_value()) + { + const std::vector& BlockRanges = MaybeBlockRanges.value(); + ZEN_ASSERT(!BlockRanges.empty()); + + uint64_t RequestedSize = + std::accumulate(BlockRanges.begin(), + BlockRanges.end(), + uint64_t(0), + [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); + + if ((PartialBlockDownloadMode != EPartialBlockDownloadMode::Exact) && ((RequestedSize * 100) / TotalBlockSize) >= 200) + { + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Requesting {} chunks ({}) from block {} ({}) using full block request (extra bytes {})", + NeededBlock.ChunkIndexes.size(), + NiceBytes(RequestedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + NiceBytes(TotalBlockSize - TotalWantedChunksSize)); + } + Result.FullBlockIndexes.push_back(NeededBlock.BlockIndex); + } + else + { + Result.BlockRanges.insert(Result.BlockRanges.end(), BlockRanges.begin(), BlockRanges.end()); + + if (RequestedSize > TotalWantedChunksSize) + { + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Requesting {} chunks ({}) from block {} ({}) using {} requests (extra bytes {})", + NeededBlock.ChunkIndexes.size(), + NiceBytes(RequestedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + BlockRanges.size(), + NiceBytes(RequestedSize - TotalWantedChunksSize)); + } + } + } + } + else + { + Result.FullBlockIndexes.push_back(NeededBlock.BlockIndex); + } + } + else + { + Result.FullBlockIndexes.push_back(NeededBlock.BlockIndex); + IdealDownloadTotalSize += TotalBlockSize; + } + } + } + + if (!Result.BlockRanges.empty() && !m_Options.IsQuiet) + { + tsl::robin_set PartialBlockIndexes; + uint64_t PartialBlocksTotalSize = std::accumulate(Result.BlockRanges.begin(), + Result.BlockRanges.end(), + uint64_t(0u), + [&](uint64_t Current, const BlockRangeDescriptor& Range) { + PartialBlockIndexes.insert(Range.BlockIndex); + return Current + Range.RangeLength; + }); + + uint64_t FullBlocksTotalSize = + std::accumulate(Result.FullBlockIndexes.begin(), + Result.FullBlockIndexes.end(), + uint64_t(0u), + [&](uint64_t Current, uint32_t BlockIndex) { + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + uint32_t CurrentOffset = + gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + + return Current + std::accumulate(BlockDescription.ChunkCompressedLengths.begin(), + BlockDescription.ChunkCompressedLengths.end(), + std::uint64_t(CurrentOffset)); + }); + + uint64_t PartialBlockRequestCount = Result.BlockRanges.size(); + uint64_t PartialBlockCount = PartialBlockIndexes.size(); + + uint64_t TotalExtraPartialBlocksRequestCount = PartialBlockRequestCount - PartialBlockCount; + uint64_t ActualPartialDownloadTotalSize = FullBlocksTotalSize + PartialBlocksTotalSize; + + uint64_t IdealSkippedSize = AllBlocksTotalBlocksSize - IdealDownloadTotalSize; + uint64_t ActualSkippedSize = AllBlocksTotalBlocksSize - ActualPartialDownloadTotalSize; + + double PercentOfIdealPartialSkippedSize = (ActualSkippedSize * 100.0) / IdealSkippedSize; + + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Analysis of partial block requests saves download of {} out of {}, {:.1f}% of possible {} using {} extra " + "requests. Completed in {}", + NiceBytes(ActualSkippedSize), + NiceBytes(AllBlocksTotalBlocksSize), + PercentOfIdealPartialSkippedSize, + NiceBytes(IdealSkippedSize), + TotalExtraPartialBlocksRequestCount, + NiceTimeSpanMs(PartialAnalisysTimer.GetElapsedTimeMs())); + } + + return Result; +} + +ChunkBlockAnalyser::BlockRangeDescriptor +ChunkBlockAnalyser::MergeBlockRanges(std::span Ranges) +{ + ZEN_ASSERT(Ranges.size() > 1); + const BlockRangeDescriptor& First = Ranges.front(); + const BlockRangeDescriptor& Last = Ranges.back(); + + return BlockRangeDescriptor{.BlockIndex = First.BlockIndex, + .RangeStart = First.RangeStart, + .RangeLength = Last.RangeStart + Last.RangeLength - First.RangeStart, + .ChunkBlockIndexStart = First.ChunkBlockIndexStart, + .ChunkBlockIndexCount = Last.ChunkBlockIndexStart + Last.ChunkBlockIndexCount - First.ChunkBlockIndexStart}; +} + +std::optional> +ChunkBlockAnalyser::MakeOptionalBlockRangeVector(uint64_t TotalBlockSize, const BlockRangeDescriptor& Range) +{ + if (Range.RangeLength == TotalBlockSize) + { + return {}; + } + else + { + return std::vector{Range}; + } +}; + +const ChunkBlockAnalyser::BlockRangeLimit* +ChunkBlockAnalyser::GetBlockRangeLimitForRange(std::span Limits, + uint64_t TotalBlockSize, + std::span Ranges) +{ + if (Ranges.size() > 1) + { + const std::uint64_t WantedSize = + std::accumulate(Ranges.begin(), Ranges.end(), uint64_t(0), [](uint64_t Current, const BlockRangeDescriptor& Range) { + return Current + Range.RangeLength; + }); + + const double RangeRequestedPercent = (WantedSize * 100.0) / TotalBlockSize; + + for (const BlockRangeLimit& Limit : Limits) + { + if (RangeRequestedPercent >= Limit.SizePercent && Ranges.size() > Limit.MaxRangeCount) + { + return &Limit; + } + } + } + return nullptr; +}; + +std::vector +ChunkBlockAnalyser::CollapseBlockRanges(const uint64_t AlwaysAcceptableGap, std::span BlockRanges) +{ + ZEN_ASSERT(BlockRanges.size() > 1); + std::vector CollapsedBlockRanges; + + auto BlockRangesIt = BlockRanges.begin(); + CollapsedBlockRanges.push_back(*BlockRangesIt++); + for (; BlockRangesIt != BlockRanges.end(); BlockRangesIt++) + { + BlockRangeDescriptor& LastRange = CollapsedBlockRanges.back(); + + const uint64_t BothRangeSize = BlockRangesIt->RangeLength + LastRange.RangeLength; + + const uint64_t Gap = BlockRangesIt->RangeStart - (LastRange.RangeStart + LastRange.RangeLength); + if (Gap <= Max(BothRangeSize / 16, AlwaysAcceptableGap)) + { + LastRange.ChunkBlockIndexCount = + (BlockRangesIt->ChunkBlockIndexStart + BlockRangesIt->ChunkBlockIndexCount) - LastRange.ChunkBlockIndexStart; + LastRange.RangeLength = (BlockRangesIt->RangeStart + BlockRangesIt->RangeLength) - LastRange.RangeStart; + } + else + { + CollapsedBlockRanges.push_back(*BlockRangesIt); + } + } + + return CollapsedBlockRanges; +}; + +uint64_t +ChunkBlockAnalyser::CalculateNextGap(std::span BlockRanges) +{ + ZEN_ASSERT(BlockRanges.size() > 1); + uint64_t AcceptableGap = (uint64_t)-1; + for (size_t RangeIndex = 0; RangeIndex < BlockRanges.size() - 1; RangeIndex++) + { + const BlockRangeDescriptor& Range = BlockRanges[RangeIndex]; + const BlockRangeDescriptor& NextRange = BlockRanges[RangeIndex + 1]; + + const uint64_t Gap = NextRange.RangeStart - (Range.RangeStart + Range.RangeLength); + AcceptableGap = Min(Gap, AcceptableGap); + } + AcceptableGap = RoundUp(AcceptableGap, 16u * 1024u); + return AcceptableGap; +}; + +std::optional> +ChunkBlockAnalyser::CalculateBlockRanges(uint32_t BlockIndex, + const ChunkBlockDescription& BlockDescription, + std::span BlockChunkIndexNeeded, + EPartialBlockDownloadMode PartialBlockDownloadMode, + const uint64_t ChunkStartOffsetInBlock, + const uint64_t TotalBlockSize, + uint64_t& OutTotalWantedChunksSize) +{ + ZEN_TRACE_CPU("CalculateBlockRanges"); + + if (PartialBlockDownloadMode == EPartialBlockDownloadMode::Off) + { + return {}; + } + + std::vector BlockRanges; + { + uint64_t CurrentOffset = ChunkStartOffsetInBlock; + uint32_t ChunkBlockIndex = 0; + uint32_t NeedBlockChunkIndexOffset = 0; + BlockRangeDescriptor NextRange{.BlockIndex = BlockIndex}; + while (NeedBlockChunkIndexOffset < BlockChunkIndexNeeded.size() && ChunkBlockIndex < BlockDescription.ChunkRawHashes.size()) + { + const uint32_t ChunkCompressedLength = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + if (ChunkBlockIndex < BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + { + if (NextRange.RangeLength > 0) + { + BlockRanges.push_back(NextRange); + NextRange = {.BlockIndex = BlockIndex}; + } + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + } + else if (ChunkBlockIndex == BlockChunkIndexNeeded[NeedBlockChunkIndexOffset]) + { + if (NextRange.RangeLength == 0) + { + NextRange.RangeStart = CurrentOffset; + NextRange.ChunkBlockIndexStart = ChunkBlockIndex; + } + NextRange.RangeLength += ChunkCompressedLength; + NextRange.ChunkBlockIndexCount++; + ChunkBlockIndex++; + CurrentOffset += ChunkCompressedLength; + NeedBlockChunkIndexOffset++; + } + else + { + ZEN_ASSERT(false); + } + } + if (NextRange.RangeLength > 0) + { + BlockRanges.push_back(NextRange); + } + } + ZEN_ASSERT(!BlockRanges.empty()); + + OutTotalWantedChunksSize = + std::accumulate(BlockRanges.begin(), BlockRanges.end(), uint64_t(0), [](uint64_t Current, const BlockRangeDescriptor& Range) { + return Current + Range.RangeLength; + }); + + double RangeWantedPercent = (OutTotalWantedChunksSize * 100.0) / TotalBlockSize; + + if (BlockRanges.size() == 1) + { + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Range request of {} ({:.2f}%) using single range from block {} ({}) as is", + NiceBytes(OutTotalWantedChunksSize), + RangeWantedPercent, + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize)); + } + return BlockRanges; + } + + if (PartialBlockDownloadMode == EPartialBlockDownloadMode::Exact) + { + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Range request of {} ({:.2f}%) using {} ranges from block {} ({})", + NiceBytes(OutTotalWantedChunksSize), + RangeWantedPercent, + BlockRanges.size(), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize)); + } + return BlockRanges; + } + + if (PartialBlockDownloadMode == EPartialBlockDownloadMode::SingleRange) + { + const BlockRangeDescriptor MergedRange = MergeBlockRanges(BlockRanges); + if (m_Options.IsVerbose) + { + const double RangeRequestedPercent = (MergedRange.RangeLength * 100.0) / TotalBlockSize; + const double WastedPercent = ((MergedRange.RangeLength - OutTotalWantedChunksSize) * 100.0) / MergedRange.RangeLength; + + ZEN_OPERATION_LOG_INFO( + m_LogOutput, + "Range request of {} ({:.2f}%) using {} ranges from block {} ({}) limited to single block range {} ({:.2f}%) wasting " + "{:.2f}% ({})", + NiceBytes(OutTotalWantedChunksSize), + RangeWantedPercent, + BlockRanges.size(), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + NiceBytes(MergedRange.RangeLength), + RangeRequestedPercent, + WastedPercent, + NiceBytes(MergedRange.RangeLength - OutTotalWantedChunksSize)); + } + return MakeOptionalBlockRangeVector(TotalBlockSize, MergedRange); + } + + if (RangeWantedPercent > FullBlockRangePercentLimit) + { + const BlockRangeDescriptor MergedRange = MergeBlockRanges(BlockRanges); + if (m_Options.IsVerbose) + { + const double RangeRequestedPercent = (MergedRange.RangeLength * 100.0) / TotalBlockSize; + const double WastedPercent = ((MergedRange.RangeLength - OutTotalWantedChunksSize) * 100.0) / MergedRange.RangeLength; + + ZEN_OPERATION_LOG_INFO( + m_LogOutput, + "Range request of {} ({:.2f}%) using {} ranges from block {} ({}) exceeds {}%. Merged to single block range {} " + "({:.2f}%) wasting {:.2f}% ({})", + NiceBytes(OutTotalWantedChunksSize), + RangeWantedPercent, + BlockRanges.size(), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + FullBlockRangePercentLimit, + NiceBytes(MergedRange.RangeLength), + RangeRequestedPercent, + WastedPercent, + NiceBytes(MergedRange.RangeLength - OutTotalWantedChunksSize)); + } + return MakeOptionalBlockRangeVector(TotalBlockSize, MergedRange); + } + + std::vector CollapsedBlockRanges = CollapseBlockRanges(16u * 1024u, BlockRanges); + while (GetBlockRangeLimitForRange(ForceMergeLimits, TotalBlockSize, CollapsedBlockRanges)) + { + CollapsedBlockRanges = CollapseBlockRanges(CalculateNextGap(CollapsedBlockRanges), CollapsedBlockRanges); + } + + const std::uint64_t WantedCollapsedSize = + std::accumulate(CollapsedBlockRanges.begin(), + CollapsedBlockRanges.end(), + uint64_t(0), + [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); + + const double CollapsedRangeRequestedPercent = (WantedCollapsedSize * 100.0) / TotalBlockSize; + + if (m_Options.IsVerbose) + { + const double WastedPercent = ((WantedCollapsedSize - OutTotalWantedChunksSize) * 100.0) / WantedCollapsedSize; + + ZEN_OPERATION_LOG_INFO( + m_LogOutput, + "Range request of {} ({:.2f}%) using {} ranges from block {} ({}) collapsed to {} {:.2f}% using {} ranges wasting {:.2f}% " + "({})", + NiceBytes(OutTotalWantedChunksSize), + RangeWantedPercent, + BlockRanges.size(), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + NiceBytes(WantedCollapsedSize), + CollapsedRangeRequestedPercent, + CollapsedBlockRanges.size(), + WastedPercent, + NiceBytes(WantedCollapsedSize - OutTotalWantedChunksSize)); + } + return CollapsedBlockRanges; +} + #if ZEN_WITH_TESTS namespace testutils { @@ -476,7 +1006,7 @@ namespace testutils { } // namespace testutils -TEST_CASE("project.store.block") +TEST_CASE("chunkblock.block") { using namespace std::literals; using namespace testutils; @@ -504,7 +1034,7 @@ TEST_CASE("project.store.block") HeaderSize)); } -TEST_CASE("project.store.reuseblocks") +TEST_CASE("chunkblock.reuseblocks") { using namespace std::literals; using namespace testutils; diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h index 9e5bf8d91..6800444e0 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -218,33 +219,6 @@ private: uint64_t ElapsedTimeMs = 0; }; - struct BlockRangeDescriptor - { - uint32_t BlockIndex = (uint32_t)-1; - uint64_t RangeStart = 0; - uint64_t RangeLength = 0; - uint32_t ChunkBlockIndexStart = 0; - uint32_t ChunkBlockIndexCount = 0; - }; - - struct BlockRangeLimit - { - uint16_t SizePercent; - uint16_t MaxRangeCount; - }; - - static constexpr uint16_t FullBlockRangePercentLimit = 95; - - static constexpr BuildsOperationUpdateFolder::BlockRangeLimit ForceMergeLimits[] = { - {.SizePercent = FullBlockRangePercentLimit, .MaxRangeCount = 1}, - {.SizePercent = 90, .MaxRangeCount = 2}, - {.SizePercent = 85, .MaxRangeCount = 8}, - {.SizePercent = 80, .MaxRangeCount = 16}, - {.SizePercent = 70, .MaxRangeCount = 32}, - {.SizePercent = 60, .MaxRangeCount = 48}, - {.SizePercent = 2, .MaxRangeCount = 56}, - {.SizePercent = 0, .MaxRangeCount = 64}}; - void ScanCacheFolder(tsl::robin_map& OutCachedChunkHashesFound, tsl::robin_map& OutCachedSequenceHashesFound); void ScanTempBlocksFolder(tsl::robin_map& OutCachedBlocksFound); @@ -299,25 +273,9 @@ private: ParallelWork& Work, std::function&& OnDownloaded); - BlockRangeDescriptor MergeBlockRanges(std::span Ranges); - std::optional> MakeOptionalBlockRangeVector(uint64_t TotalBlockSize, - const BlockRangeDescriptor& Range); - const BlockRangeLimit* GetBlockRangeLimitForRange(std::span Limits, - uint64_t TotalBlockSize, - std::span Ranges); - std::vector CollapseBlockRanges(const uint64_t AlwaysAcceptableGap, - std::span BlockRanges); - uint64_t CalculateNextGap(std::span BlockRanges); - std::optional> CalculateBlockRanges(uint32_t BlockIndex, - const ChunkBlockDescription& BlockDescription, - std::span BlockChunkIndexNeeded, - bool LimitToSingleRange, - const uint64_t ChunkStartOffsetInBlock, - const uint64_t TotalBlockSize, - uint64_t& OutTotalWantedChunksSize); - void DownloadPartialBlock(const BlockRangeDescriptor BlockRange, - const BlobsExistsResult& ExistsResult, - std::function&& OnDownloaded); + void DownloadPartialBlock(const ChunkBlockAnalyser::BlockRangeDescriptor BlockRange, + const BlobsExistsResult& ExistsResult, + std::function&& OnDownloaded); std::vector WriteLocalChunkToCache(CloneQueryInterface* CloneQuery, const CopyChunkData& CopyData, diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h index d339b0f94..57710fcf5 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h @@ -7,6 +7,10 @@ #include #include +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + #include #include @@ -73,6 +77,96 @@ std::vector FindReuseBlocks(OperationLogOutput& Output, std::span ChunkIndexes, std::vector& OutUnusedChunkIndexes); +class ChunkBlockAnalyser +{ +public: + struct Options + { + bool IsQuiet = false; + bool IsVerbose = false; + }; + + ChunkBlockAnalyser(OperationLogOutput& LogOutput, std::span BlockDescriptions, const Options& Options); + + struct BlockRangeDescriptor + { + uint32_t BlockIndex = (uint32_t)-1; + uint64_t RangeStart = 0; + uint64_t RangeLength = 0; + uint32_t ChunkBlockIndexStart = 0; + uint32_t ChunkBlockIndexCount = 0; + }; + + struct NeededBlock + { + uint32_t BlockIndex; + std::vector ChunkIndexes; + }; + + std::vector GetNeeded(const tsl::robin_map& ChunkHashToChunkIndex, + std::function&& NeedsBlockChunk); + + enum EPartialBlockDownloadMode + { + Off, + SingleRange, + On, + Exact + }; + + struct BlockResult + { + std::vector BlockRanges; + std::vector FullBlockIndexes; + }; + + BlockResult CalculatePartialBlockDownloads(std::span NeededBlocks, + std::span BlockPartialDownloadModes); + +private: + struct BlockRangeLimit + { + uint16_t SizePercent; + uint16_t MaxRangeCount; + }; + + static constexpr uint16_t FullBlockRangePercentLimit = 95; + + static constexpr BlockRangeLimit ForceMergeLimits[] = {{.SizePercent = FullBlockRangePercentLimit, .MaxRangeCount = 1}, + {.SizePercent = 90, .MaxRangeCount = 2}, + {.SizePercent = 85, .MaxRangeCount = 8}, + {.SizePercent = 80, .MaxRangeCount = 16}, + {.SizePercent = 75, .MaxRangeCount = 32}, + {.SizePercent = 70, .MaxRangeCount = 48}, + {.SizePercent = 4, .MaxRangeCount = 82}, + {.SizePercent = 0, .MaxRangeCount = 96}}; + + BlockRangeDescriptor MergeBlockRanges(std::span Ranges); + std::optional> MakeOptionalBlockRangeVector(uint64_t TotalBlockSize, + const BlockRangeDescriptor& Range); + const BlockRangeLimit* GetBlockRangeLimitForRange(std::span Limits, + uint64_t TotalBlockSize, + std::span Ranges); + std::vector CollapseBlockRanges(const uint64_t AlwaysAcceptableGap, + std::span BlockRanges); + uint64_t CalculateNextGap(std::span BlockRanges); + std::optional> CalculateBlockRanges(uint32_t BlockIndex, + const ChunkBlockDescription& BlockDescription, + std::span BlockChunkIndexNeeded, + EPartialBlockDownloadMode PartialBlockDownloadMode, + const uint64_t ChunkStartOffsetInBlock, + const uint64_t TotalBlockSize, + uint64_t& OutTotalWantedChunksSize); + + OperationLogOutput& m_LogOutput; + const std::span m_BlockDescriptions; + const Options m_Options; +}; + +#if ZEN_WITH_TESTS + +class CbWriter; void chunkblock_forcelink(); +#endif // ZEN_WITH_TESTS } // namespace zen -- cgit v1.2.3 From 606eb90fa55185a033923d310dc92a78a4a2d0f8 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 20 Feb 2026 13:50:27 +0100 Subject: update .gitignore to exclude .claude/ --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3d98ecff8..eaf1656a6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore .DS_Store +.claude/ # User-specific files *.suo -- cgit v1.2.3 From b927d006e8d33fcf8d2f5c1bce6b3c052839d32a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 22 Feb 2026 21:03:33 +0100 Subject: disable msys logic in blake3 to fix Git Bash build issues --- thirdparty/xmake.lua | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua index 07605a016..6fead7b50 100644 --- a/thirdparty/xmake.lua +++ b/thirdparty/xmake.lua @@ -92,9 +92,11 @@ target("blake3") end if is_arch("x86_64", "x64") then - if is_subhost("msys", "cygwin") then - add_files("blake3/c/*x86-64_windows_gnu.S") - elseif is_plat("windows") then + -- sbo: this breaks when using MINGW on windows (which is what you get in Git Bash), so just disable it + -- if is_subhost("msys", "cygwin") then + -- add_files("blake3/c/*x86-64_windows_gnu.S") + -- elseif is_plat("windows") then + if is_plat("windows") then add_files("blake3/c/*x86-64_windows_msvc.asm") else add_files("blake3/c/*x86-64_unix.S") -- cgit v1.2.3 From 73f3eb4feedf3bca0ddc832a89a05f09813c6858 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 23 Feb 2026 11:08:24 +0100 Subject: implemented base64 decoding (#777) Co-authored-by: Stefan Boberg --- src/zencore/base64.cpp | 192 ++++++++++++++++++++++++++++++++++- src/zencore/include/zencore/base64.h | 4 + 2 files changed, 194 insertions(+), 2 deletions(-) diff --git a/src/zencore/base64.cpp b/src/zencore/base64.cpp index 1f56ee6c3..fdf5f2d66 100644 --- a/src/zencore/base64.cpp +++ b/src/zencore/base64.cpp @@ -1,6 +1,10 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include +#include +#include + +#include namespace zen { @@ -11,7 +15,6 @@ static const uint8_t EncodingAlphabet[64] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'}; /** The table used to convert an ascii character into a 6 bit value */ -#if 0 static const uint8_t DecodingAlphabet[256] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x00-0x0f 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0x10-0x1f @@ -30,7 +33,6 @@ static const uint8_t DecodingAlphabet[256] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, // 0xe0-0xef 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF // 0xf0-0xff }; -#endif // 0 template uint32_t @@ -104,4 +106,190 @@ Base64::Encode(const uint8_t* Source, uint32_t Length, CharType* Dest) template uint32_t Base64::Encode(const uint8_t* Source, uint32_t Length, char* Dest); template uint32_t Base64::Encode(const uint8_t* Source, uint32_t Length, wchar_t* Dest); +template +bool +Base64::Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength) +{ + // Length must be a multiple of 4 + if (Length % 4 != 0) + { + OutLength = 0; + return false; + } + + uint8_t* DecodedBytes = Dest; + + // Process 4 encoded characters at a time, producing 3 decoded bytes + while (Length > 0) + { + // Count padding characters at the end + uint32_t PadCount = 0; + if (Source[3] == '=') + { + PadCount++; + if (Source[2] == '=') + { + PadCount++; + } + } + + // Look up each character in the decoding table + uint8_t A = DecodingAlphabet[static_cast(Source[0])]; + uint8_t B = DecodingAlphabet[static_cast(Source[1])]; + uint8_t C = (PadCount >= 2) ? 0 : DecodingAlphabet[static_cast(Source[2])]; + uint8_t D = (PadCount >= 1) ? 0 : DecodingAlphabet[static_cast(Source[3])]; + + // Check for invalid characters (0xFF means not in the base64 alphabet) + if (A == 0xFF || B == 0xFF || C == 0xFF || D == 0xFF) + { + OutLength = 0; + return false; + } + + // Reconstruct the 24-bit value from 4 6-bit chunks + uint32_t ByteTriplet = (A << 18) | (B << 12) | (C << 6) | D; + + // Extract the 3 bytes + *DecodedBytes++ = static_cast(ByteTriplet >> 16); + if (PadCount < 2) + { + *DecodedBytes++ = static_cast((ByteTriplet >> 8) & 0xFF); + } + if (PadCount < 1) + { + *DecodedBytes++ = static_cast(ByteTriplet & 0xFF); + } + + Source += 4; + Length -= 4; + } + + OutLength = uint32_t(DecodedBytes - Dest); + return true; +} + +template bool Base64::Decode(const char* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); +template bool Base64::Decode(const wchar_t* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); + +////////////////////////////////////////////////////////////////////////// +// +// Testing related code follows... +// + +#if ZEN_WITH_TESTS + +using namespace std::string_literals; + +TEST_CASE("Base64") +{ + auto EncodeString = [](std::string_view Input) -> std::string { + std::string Result; + Result.resize(Base64::GetEncodedDataSize(uint32_t(Input.size()))); + Base64::Encode(reinterpret_cast(Input.data()), uint32_t(Input.size()), Result.data()); + return Result; + }; + + auto DecodeString = [](std::string_view Input) -> std::string { + std::string Result; + Result.resize(Base64::GetMaxDecodedDataSize(uint32_t(Input.size()))); + uint32_t DecodedLength = 0; + bool Success = Base64::Decode(Input.data(), uint32_t(Input.size()), reinterpret_cast(Result.data()), DecodedLength); + CHECK(Success); + Result.resize(DecodedLength); + return Result; + }; + + SUBCASE("Encode") + { + CHECK(EncodeString("") == ""s); + CHECK(EncodeString("f") == "Zg=="s); + CHECK(EncodeString("fo") == "Zm8="s); + CHECK(EncodeString("foo") == "Zm9v"s); + CHECK(EncodeString("foob") == "Zm9vYg=="s); + CHECK(EncodeString("fooba") == "Zm9vYmE="s); + CHECK(EncodeString("foobar") == "Zm9vYmFy"s); + } + + SUBCASE("Decode") + { + CHECK(DecodeString("") == ""s); + CHECK(DecodeString("Zg==") == "f"s); + CHECK(DecodeString("Zm8=") == "fo"s); + CHECK(DecodeString("Zm9v") == "foo"s); + CHECK(DecodeString("Zm9vYg==") == "foob"s); + CHECK(DecodeString("Zm9vYmE=") == "fooba"s); + CHECK(DecodeString("Zm9vYmFy") == "foobar"s); + } + + SUBCASE("RoundTrip") + { + auto RoundTrip = [&](const std::string& Input) { + std::string Encoded = EncodeString(Input); + std::string Decoded = DecodeString(Encoded); + CHECK(Decoded == Input); + }; + + RoundTrip("Hello, World!"); + RoundTrip("Base64 encoding test with various lengths"); + RoundTrip("A"); + RoundTrip("AB"); + RoundTrip("ABC"); + RoundTrip("ABCD"); + RoundTrip("\x00\x01\x02\xff\xfe\xfd"s); + } + + SUBCASE("BinaryRoundTrip") + { + // Test with all byte values 0-255 + uint8_t AllBytes[256]; + for (int i = 0; i < 256; ++i) + { + AllBytes[i] = static_cast(i); + } + + char Encoded[Base64::GetEncodedDataSize(256) + 1]; + Base64::Encode(AllBytes, 256, Encoded); + + uint8_t Decoded[256]; + uint32_t DecodedLength = 0; + bool Success = Base64::Decode(Encoded, uint32_t(strlen(Encoded)), Decoded, DecodedLength); + CHECK(Success); + CHECK(DecodedLength == 256); + CHECK(memcmp(AllBytes, Decoded, 256) == 0); + } + + SUBCASE("DecodeInvalidInput") + { + uint8_t Dest[64]; + uint32_t OutLength = 0; + + // Length not a multiple of 4 + CHECK_FALSE(Base64::Decode("abc", 3u, Dest, OutLength)); + + // Invalid character + CHECK_FALSE(Base64::Decode("ab!d", 4u, Dest, OutLength)); + } + + SUBCASE("EncodedDataSize") + { + CHECK(Base64::GetEncodedDataSize(0) == 0); + CHECK(Base64::GetEncodedDataSize(1) == 4); + CHECK(Base64::GetEncodedDataSize(2) == 4); + CHECK(Base64::GetEncodedDataSize(3) == 4); + CHECK(Base64::GetEncodedDataSize(4) == 8); + CHECK(Base64::GetEncodedDataSize(5) == 8); + CHECK(Base64::GetEncodedDataSize(6) == 8); + } + + SUBCASE("MaxDecodedDataSize") + { + CHECK(Base64::GetMaxDecodedDataSize(0) == 0); + CHECK(Base64::GetMaxDecodedDataSize(4) == 3); + CHECK(Base64::GetMaxDecodedDataSize(8) == 6); + CHECK(Base64::GetMaxDecodedDataSize(12) == 9); + } +} + +#endif + } // namespace zen diff --git a/src/zencore/include/zencore/base64.h b/src/zencore/include/zencore/base64.h index 4d78b085f..08d9f3043 100644 --- a/src/zencore/include/zencore/base64.h +++ b/src/zencore/include/zencore/base64.h @@ -11,7 +11,11 @@ struct Base64 template static uint32_t Encode(const uint8_t* Source, uint32_t Length, CharType* Dest); + template + static bool Decode(const CharType* Source, uint32_t Length, uint8_t* Dest, uint32_t& OutLength); + static inline constexpr int32_t GetEncodedDataSize(uint32_t Size) { return ((Size + 2) / 3) * 4; } + static inline constexpr int32_t GetMaxDecodedDataSize(uint32_t Length) { return (Length / 4) * 3; } }; } // namespace zen -- cgit v1.2.3 From 01445315564ab527566ec200e0182d8968a80d6f Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 23 Feb 2026 11:09:11 +0100 Subject: changed command names and descriptions to use class members instead of string literals in zen.cpp (#776) --- src/zen/cmds/admin_cmd.h | 40 +++++++++++++++---- src/zen/cmds/bench_cmd.h | 5 ++- src/zen/cmds/cache_cmd.h | 20 ++++++++-- src/zen/cmds/copy_cmd.h | 5 ++- src/zen/cmds/dedup_cmd.h | 5 ++- src/zen/cmds/info_cmd.h | 5 ++- src/zen/cmds/print_cmd.h | 10 ++++- src/zen/cmds/projectstore_cmd.h | 58 ++++++++++++++++++++------- src/zen/cmds/rpcreplay_cmd.h | 15 +++++-- src/zen/cmds/run_cmd.h | 5 ++- src/zen/cmds/serve_cmd.h | 5 ++- src/zen/cmds/status_cmd.h | 5 ++- src/zen/cmds/top_cmd.h | 10 ++++- src/zen/cmds/trace_cmd.h | 7 ++-- src/zen/cmds/up_cmd.h | 15 +++++-- src/zen/cmds/vfs_cmd.h | 5 ++- src/zen/zen.cpp | 86 ++++++++++++++++++++--------------------- 17 files changed, 211 insertions(+), 90 deletions(-) diff --git a/src/zen/cmds/admin_cmd.h b/src/zen/cmds/admin_cmd.h index 87ef8091b..83bcf8893 100644 --- a/src/zen/cmds/admin_cmd.h +++ b/src/zen/cmds/admin_cmd.h @@ -13,6 +13,9 @@ namespace zen { class ScrubCommand : public StorageCommand { public: + static constexpr char Name[] = "scrub"; + static constexpr char Description[] = "Scrub zen storage (verify data integrity)"; + ScrubCommand(); ~ScrubCommand(); @@ -20,7 +23,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"scrub", "Scrub zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_DryRun = false; bool m_NoGc = false; @@ -33,6 +36,9 @@ private: class GcCommand : public StorageCommand { public: + static constexpr char Name[] = "gc"; + static constexpr char Description[] = "Garbage collect zen storage"; + GcCommand(); ~GcCommand(); @@ -40,7 +46,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"gc", "Garbage collect zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_SmallObjects{false}; bool m_SkipCid{false}; @@ -62,6 +68,9 @@ private: class GcStatusCommand : public StorageCommand { public: + static constexpr char Name[] = "gc-status"; + static constexpr char Description[] = "Garbage collect zen storage status check"; + GcStatusCommand(); ~GcStatusCommand(); @@ -69,7 +78,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"gc-status", "Garbage collect zen storage status check"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_Details = false; }; @@ -77,6 +86,9 @@ private: class GcStopCommand : public StorageCommand { public: + static constexpr char Name[] = "gc-stop"; + static constexpr char Description[] = "Request cancel of running garbage collection in zen storage"; + GcStopCommand(); ~GcStopCommand(); @@ -84,7 +96,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"gc-stop", "Request cancel of running garbage collection in zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; @@ -93,6 +105,9 @@ private: class JobCommand : public ZenCmdBase { public: + static constexpr char Name[] = "jobs"; + static constexpr char Description[] = "Show/cancel zen background jobs"; + JobCommand(); ~JobCommand(); @@ -100,7 +115,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"jobs", "Show/cancel zen background jobs"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::uint64_t m_JobId = 0; bool m_Cancel = 0; @@ -111,6 +126,9 @@ private: class LoggingCommand : public ZenCmdBase { public: + static constexpr char Name[] = "logs"; + static constexpr char Description[] = "Show/control zen logging"; + LoggingCommand(); ~LoggingCommand(); @@ -118,7 +136,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"logs", "Show/control zen logging"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_CacheWriteLog; std::string m_CacheAccessLog; @@ -133,6 +151,9 @@ private: class FlushCommand : public StorageCommand { public: + static constexpr char Name[] = "flush"; + static constexpr char Description[] = "Flush storage"; + FlushCommand(); ~FlushCommand(); @@ -140,7 +161,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"flush", "Flush zen storage"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; @@ -149,6 +170,9 @@ private: class CopyStateCommand : public StorageCommand { public: + static constexpr char Name[] = "copy-state"; + static constexpr char Description[] = "Copy zen server disk state"; + CopyStateCommand(); ~CopyStateCommand(); @@ -156,7 +180,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"copy-state", "Copy zen server disk state"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_DataPath; std::filesystem::path m_TargetPath; bool m_SkipLogs = false; diff --git a/src/zen/cmds/bench_cmd.h b/src/zen/cmds/bench_cmd.h index ed123be75..7fbf85340 100644 --- a/src/zen/cmds/bench_cmd.h +++ b/src/zen/cmds/bench_cmd.h @@ -9,6 +9,9 @@ namespace zen { class BenchCommand : public ZenCmdBase { public: + static constexpr char Name[] = "bench"; + static constexpr char Description[] = "Utility command for benchmarking"; + BenchCommand(); ~BenchCommand(); @@ -17,7 +20,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"bench", "Benchmarking utility command"}; + cxxopts::Options m_Options{Name, Description}; bool m_PurgeStandbyLists = false; bool m_SingleProcess = false; }; diff --git a/src/zen/cmds/cache_cmd.h b/src/zen/cmds/cache_cmd.h index 4dc05bbdc..4f5b90f4d 100644 --- a/src/zen/cmds/cache_cmd.h +++ b/src/zen/cmds/cache_cmd.h @@ -9,6 +9,9 @@ namespace zen { class DropCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "drop"; + static constexpr char Description[] = "Drop cache namespace or bucket"; + DropCommand(); ~DropCommand(); @@ -16,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"drop", "Drop cache namespace or bucket"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_NamespaceName; std::string m_BucketName; @@ -25,13 +28,16 @@ private: class CacheInfoCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "cache-info"; + static constexpr char Description[] = "Info on cache, namespace or bucket"; + CacheInfoCommand(); ~CacheInfoCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"cache-info", "Info on cache, namespace or bucket"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_NamespaceName; std::string m_SizeInfoBucketNames; @@ -42,26 +48,32 @@ private: class CacheStatsCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "cache-stats"; + static constexpr char Description[] = "Stats on cache"; + CacheStatsCommand(); ~CacheStatsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"cache-stats", "Stats info on cache"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; class CacheDetailsCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "cache-details"; + static constexpr char Description[] = "Details on cache"; + CacheDetailsCommand(); ~CacheDetailsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"cache-details", "Detailed info on cache"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_CSV = false; bool m_Details = false; diff --git a/src/zen/cmds/copy_cmd.h b/src/zen/cmds/copy_cmd.h index e1a5dcb82..757a8e691 100644 --- a/src/zen/cmds/copy_cmd.h +++ b/src/zen/cmds/copy_cmd.h @@ -11,6 +11,9 @@ namespace zen { class CopyCommand : public ZenCmdBase { public: + static constexpr char Name[] = "copy"; + static constexpr char Description[] = "Copy file(s)"; + CopyCommand(); ~CopyCommand(); @@ -19,7 +22,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"copy", "Copy files efficiently"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_CopySource; std::filesystem::path m_CopyTarget; bool m_NoClone = false; diff --git a/src/zen/cmds/dedup_cmd.h b/src/zen/cmds/dedup_cmd.h index 5b8387dd2..835b35e92 100644 --- a/src/zen/cmds/dedup_cmd.h +++ b/src/zen/cmds/dedup_cmd.h @@ -11,6 +11,9 @@ namespace zen { class DedupCommand : public ZenCmdBase { public: + static constexpr char Name[] = "dedup"; + static constexpr char Description[] = "Dedup files"; + DedupCommand(); ~DedupCommand(); @@ -19,7 +22,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"dedup", "Deduplicate files"}; + cxxopts::Options m_Options{Name, Description}; std::vector m_Positional; std::filesystem::path m_DedupSource; std::filesystem::path m_DedupTarget; diff --git a/src/zen/cmds/info_cmd.h b/src/zen/cmds/info_cmd.h index 231565bfd..dc108b8a2 100644 --- a/src/zen/cmds/info_cmd.h +++ b/src/zen/cmds/info_cmd.h @@ -9,6 +9,9 @@ namespace zen { class InfoCommand : public ZenCmdBase { public: + static constexpr char Name[] = "info"; + static constexpr char Description[] = "Show high level Zen server information"; + InfoCommand(); ~InfoCommand(); @@ -17,7 +20,7 @@ public: // virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"info", "Show high level zen store information"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; diff --git a/src/zen/cmds/print_cmd.h b/src/zen/cmds/print_cmd.h index 6c1529b7c..f4a97e218 100644 --- a/src/zen/cmds/print_cmd.h +++ b/src/zen/cmds/print_cmd.h @@ -11,6 +11,9 @@ namespace zen { class PrintCommand : public ZenCmdBase { public: + static constexpr char Name[] = "print"; + static constexpr char Description[] = "Print compact binary object"; + PrintCommand(); ~PrintCommand(); @@ -19,7 +22,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"print", "Print compact binary object"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_Filename; bool m_ShowCbObjectTypeInfo = false; }; @@ -29,6 +32,9 @@ private: class PrintPackageCommand : public ZenCmdBase { public: + static constexpr char Name[] = "printpackage"; + static constexpr char Description[] = "Print compact binary package"; + PrintPackageCommand(); ~PrintPackageCommand(); @@ -37,7 +43,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"printpkg", "Print compact binary package"}; + cxxopts::Options m_Options{Name, Description}; std::filesystem::path m_Filename; bool m_ShowCbObjectTypeInfo = false; }; diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index 56ef858f5..e415b41b7 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -16,6 +16,9 @@ class ProjectStoreCommand : public ZenCmdBase class DropProjectCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-drop"; + static constexpr char Description[] = "Drop project or project oplog"; + DropProjectCommand(); ~DropProjectCommand(); @@ -23,7 +26,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-drop", "Drop project or project oplog"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -33,13 +36,16 @@ private: class ProjectInfoCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-info"; + static constexpr char Description[] = "Info on project or project oplog"; + ProjectInfoCommand(); ~ProjectInfoCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-info", "Info on project or project oplog"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -48,6 +54,9 @@ private: class CreateProjectCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-create"; + static constexpr char Description[] = "Create a project"; + CreateProjectCommand(); ~CreateProjectCommand(); @@ -55,7 +64,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-create", "Create project, the project must not already exist."}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectId; std::string m_RootDir; @@ -68,6 +77,9 @@ private: class CreateOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-create"; + static constexpr char Description[] = "Create a project oplog"; + CreateOplogCommand(); ~CreateOplogCommand(); @@ -75,7 +87,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-create", "Create oplog in an existing project, the oplog must not already exist."}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectId; std::string m_OplogId; @@ -86,6 +98,9 @@ private: class ExportOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-export"; + static constexpr char Description[] = "Export project store oplog"; + ExportOplogCommand(); ~ExportOplogCommand(); @@ -93,8 +108,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-export", - "Export project store oplog to cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -145,6 +159,9 @@ private: class ImportOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-import"; + static constexpr char Description[] = "Import project store oplog"; + ImportOplogCommand(); ~ImportOplogCommand(); @@ -152,8 +169,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-import", - "Import project store oplog from cloud (--cloud), file system (--file) or other Zen instance (--zen)"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -198,14 +214,16 @@ private: class SnapshotOplogCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-snapshot"; + static constexpr char Description[] = "Snapshot project store oplog"; + SnapshotOplogCommand(); ~SnapshotOplogCommand(); - virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-snapshot", "Snapshot external file references in project store oplog into zen"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -214,26 +232,32 @@ private: class ProjectStatsCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-stats"; + static constexpr char Description[] = "Stats on project store"; + ProjectStatsCommand(); ~ProjectStatsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-stats", "Stats info on project store"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; class ProjectOpDetailsCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "project-op-details"; + static constexpr char Description[] = "Detail info on ops inside a project store oplog"; + ProjectOpDetailsCommand(); ~ProjectOpDetailsCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"project-op-details", "Detail info on ops inside a project store oplog"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_Details = false; bool m_OpDetails = false; @@ -247,13 +271,16 @@ private: class OplogMirrorCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-mirror"; + static constexpr char Description[] = "Mirror project store oplog to file system"; + OplogMirrorCommand(); ~OplogMirrorCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-mirror", "Mirror oplog to file system"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; @@ -268,13 +295,16 @@ private: class OplogValidateCommand : public ProjectStoreCommand { public: + static constexpr char Name[] = "oplog-validate"; + static constexpr char Description[] = "Validate oplog for missing references"; + OplogValidateCommand(); ~OplogValidateCommand(); virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"oplog-validate", "Validate oplog for missing references"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; diff --git a/src/zen/cmds/rpcreplay_cmd.h b/src/zen/cmds/rpcreplay_cmd.h index a6363b614..332a3126c 100644 --- a/src/zen/cmds/rpcreplay_cmd.h +++ b/src/zen/cmds/rpcreplay_cmd.h @@ -9,6 +9,9 @@ namespace zen { class RpcStartRecordingCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "rpc-record-start"; + static constexpr char Description[] = "Starts recording of cache rpc requests on a host"; + RpcStartRecordingCommand(); ~RpcStartRecordingCommand(); @@ -16,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"rpc-record-start", "Starts recording of cache rpc requests on a host"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_RecordingPath; }; @@ -24,6 +27,9 @@ private: class RpcStopRecordingCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "rpc-record-stop"; + static constexpr char Description[] = "Stops recording of cache rpc requests on a host"; + RpcStopRecordingCommand(); ~RpcStopRecordingCommand(); @@ -31,13 +37,16 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"rpc-record-stop", "Stops recording of cache rpc requests on a host"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; }; class RpcReplayCommand : public CacheStoreCommand { public: + static constexpr char Name[] = "rpc-record-replay"; + static constexpr char Description[] = "Replays a previously recorded session of rpc requests"; + RpcReplayCommand(); ~RpcReplayCommand(); @@ -45,7 +54,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"rpc-record-replay", "Replays a previously recorded session of cache rpc requests to a target host"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_RecordingPath; bool m_OnHost = false; diff --git a/src/zen/cmds/run_cmd.h b/src/zen/cmds/run_cmd.h index 570a2e63a..300c08c5b 100644 --- a/src/zen/cmds/run_cmd.h +++ b/src/zen/cmds/run_cmd.h @@ -9,6 +9,9 @@ namespace zen { class RunCommand : public ZenCmdBase { public: + static constexpr char Name[] = "run"; + static constexpr char Description[] = "Run command with special options"; + RunCommand(); ~RunCommand(); @@ -17,7 +20,7 @@ public: virtual ZenCmdCategory& CommandCategory() const override { return g_UtilitiesCategory; } private: - cxxopts::Options m_Options{"run", "Run executable"}; + cxxopts::Options m_Options{Name, Description}; int m_RunCount = 0; int m_RunTime = -1; std::string m_BaseDirectory; diff --git a/src/zen/cmds/serve_cmd.h b/src/zen/cmds/serve_cmd.h index ac74981f2..22f430948 100644 --- a/src/zen/cmds/serve_cmd.h +++ b/src/zen/cmds/serve_cmd.h @@ -11,6 +11,9 @@ namespace zen { class ServeCommand : public ZenCmdBase { public: + static constexpr char Name[] = "serve"; + static constexpr char Description[] = "Serve files from a directory"; + ServeCommand(); ~ServeCommand(); @@ -18,7 +21,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"serve", "Serve files from a tree"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; std::string m_ProjectName; std::string m_OplogName; diff --git a/src/zen/cmds/status_cmd.h b/src/zen/cmds/status_cmd.h index dc103a196..df5df3066 100644 --- a/src/zen/cmds/status_cmd.h +++ b/src/zen/cmds/status_cmd.h @@ -11,6 +11,9 @@ namespace zen { class StatusCommand : public ZenCmdBase { public: + static constexpr char Name[] = "status"; + static constexpr char Description[] = "Show zen status"; + StatusCommand(); ~StatusCommand(); @@ -20,7 +23,7 @@ public: private: int GetLockFileEffectivePort() const; - cxxopts::Options m_Options{"status", "Show zen status"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; std::filesystem::path m_DataDir; }; diff --git a/src/zen/cmds/top_cmd.h b/src/zen/cmds/top_cmd.h index 74167ecfd..aeb196558 100644 --- a/src/zen/cmds/top_cmd.h +++ b/src/zen/cmds/top_cmd.h @@ -9,6 +9,9 @@ namespace zen { class TopCommand : public ZenCmdBase { public: + static constexpr char Name[] = "top"; + static constexpr char Description[] = "Monitor zen server activity"; + TopCommand(); ~TopCommand(); @@ -16,12 +19,15 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"top", "Show dev UI"}; + cxxopts::Options m_Options{Name, Description}; }; class PsCommand : public ZenCmdBase { public: + static constexpr char Name[] = "ps"; + static constexpr char Description[] = "Enumerate running zen server instances"; + PsCommand(); ~PsCommand(); @@ -29,7 +35,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"ps", "Enumerate running Zen server instances"}; + cxxopts::Options m_Options{Name, Description}; }; } // namespace zen diff --git a/src/zen/cmds/trace_cmd.h b/src/zen/cmds/trace_cmd.h index a6c9742b7..6eb0ba22b 100644 --- a/src/zen/cmds/trace_cmd.h +++ b/src/zen/cmds/trace_cmd.h @@ -6,11 +6,12 @@ namespace zen { -/** Scrub storage - */ class TraceCommand : public ZenCmdBase { public: + static constexpr char Name[] = "trace"; + static constexpr char Description[] = "Control zen realtime tracing"; + TraceCommand(); ~TraceCommand(); @@ -18,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"trace", "Control zen realtime tracing"}; + cxxopts::Options m_Options{Name, Description}; std::string m_HostName; bool m_Stop = false; std::string m_TraceHost; diff --git a/src/zen/cmds/up_cmd.h b/src/zen/cmds/up_cmd.h index 2e822d5fc..270db7f88 100644 --- a/src/zen/cmds/up_cmd.h +++ b/src/zen/cmds/up_cmd.h @@ -11,6 +11,9 @@ namespace zen { class UpCommand : public ZenCmdBase { public: + static constexpr char Name[] = "up"; + static constexpr char Description[] = "Bring zen server up"; + UpCommand(); ~UpCommand(); @@ -18,7 +21,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"up", "Bring up zen service"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; bool m_ShowConsole = false; bool m_ShowLog = false; @@ -28,6 +31,9 @@ private: class AttachCommand : public ZenCmdBase { public: + static constexpr char Name[] = "attach"; + static constexpr char Description[] = "Add a sponsor process to a running zen service"; + AttachCommand(); ~AttachCommand(); @@ -35,7 +41,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"attach", "Add a sponsor process to a running zen service"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; int m_OwnerPid = 0; std::filesystem::path m_DataDir; @@ -44,6 +50,9 @@ private: class DownCommand : public ZenCmdBase { public: + static constexpr char Name[] = "down"; + static constexpr char Description[] = "Bring zen server down"; + DownCommand(); ~DownCommand(); @@ -51,7 +60,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"down", "Bring down zen service"}; + cxxopts::Options m_Options{Name, Description}; uint16_t m_Port = 0; bool m_ForceTerminate = false; std::filesystem::path m_ProgramBaseDir; diff --git a/src/zen/cmds/vfs_cmd.h b/src/zen/cmds/vfs_cmd.h index 5deaa02fa..9009c774b 100644 --- a/src/zen/cmds/vfs_cmd.h +++ b/src/zen/cmds/vfs_cmd.h @@ -9,6 +9,9 @@ namespace zen { class VfsCommand : public StorageCommand { public: + static constexpr char Name[] = "vfs"; + static constexpr char Description[] = "Manage virtual file system"; + VfsCommand(); ~VfsCommand(); @@ -16,7 +19,7 @@ public: virtual cxxopts::Options& Options() override { return m_Options; } private: - cxxopts::Options m_Options{"vfs", "Manage virtual file system"}; + cxxopts::Options m_Options{Name, Description}; std::string m_Verb; std::string m_HostName; diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 018f77738..bdc2b4003 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -379,56 +379,56 @@ main(int argc, char** argv) const char* CmdSummary; } Commands[] = { // clang-format off - {"attach", &AttachCmd, "Add a sponsor process to a running zen service"}, - {"bench", &BenchCmd, "Utility command for benchmarking"}, - {BuildsCommand::Name, &BuildsCmd, BuildsCommand::Description}, - {"cache-details", &CacheDetailsCmd, "Details on cache"}, - {"cache-info", &CacheInfoCmd, "Info on cache, namespace or bucket"}, + {AttachCommand::Name, &AttachCmd, AttachCommand::Description}, + {BenchCommand::Name, &BenchCmd, BenchCommand::Description}, + {BuildsCommand::Name, &BuildsCmd, BuildsCommand::Description}, + {CacheDetailsCommand::Name, &CacheDetailsCmd, CacheDetailsCommand::Description}, + {CacheInfoCommand::Name, &CacheInfoCmd, CacheInfoCommand::Description}, {CacheGetCommand::Name, &CacheGetCmd, CacheGetCommand::Description}, {CacheGenerateCommand::Name, &CacheGenerateCmd, CacheGenerateCommand::Description}, - {"cache-stats", &CacheStatsCmd, "Stats on cache"}, - {"copy", &CopyCmd, "Copy file(s)"}, - {"copy-state", &CopyStateCmd, "Copy zen server disk state"}, - {"dedup", &DedupCmd, "Dedup files"}, - {"down", &DownCmd, "Bring zen server down"}, - {"drop", &DropCmd, "Drop cache namespace or bucket"}, + {CacheStatsCommand::Name, &CacheStatsCmd, CacheStatsCommand::Description}, + {CopyCommand::Name, &CopyCmd, CopyCommand::Description}, + {CopyStateCommand::Name, &CopyStateCmd, CopyStateCommand::Description}, + {DedupCommand::Name, &DedupCmd, DedupCommand::Description}, + {DownCommand::Name, &DownCmd, DownCommand::Description}, + {DropCommand::Name, &DropCmd, DropCommand::Description}, #if ZEN_WITH_COMPUTE_SERVICES {ExecCommand::Name, &ExecCmd, ExecCommand::Description}, #endif - {"gc-status", &GcStatusCmd, "Garbage collect zen storage status check"}, - {"gc-stop", &GcStopCmd, "Request cancel of running garbage collection in zen storage"}, - {"gc", &GcCmd, "Garbage collect zen storage"}, - {"info", &InfoCmd, "Show high level Zen server information"}, - {"jobs", &JobCmd, "Show/cancel zen background jobs"}, - {"logs", &LoggingCmd, "Show/control zen logging"}, - {"oplog-create", &CreateOplogCmd, "Create a project oplog"}, - {"oplog-export", &ExportOplogCmd, "Export project store oplog"}, - {"oplog-import", &ImportOplogCmd, "Import project store oplog"}, - {"oplog-mirror", &OplogMirrorCmd, "Mirror project store oplog to file system"}, - {"oplog-snapshot", &SnapshotOplogCmd, "Snapshot project store oplog"}, + {GcStatusCommand::Name, &GcStatusCmd, GcStatusCommand::Description}, + {GcStopCommand::Name, &GcStopCmd, GcStopCommand::Description}, + {GcCommand::Name, &GcCmd, GcCommand::Description}, + {InfoCommand::Name, &InfoCmd, InfoCommand::Description}, + {JobCommand::Name, &JobCmd, JobCommand::Description}, + {LoggingCommand::Name, &LoggingCmd, LoggingCommand::Description}, + {CreateOplogCommand::Name, &CreateOplogCmd, CreateOplogCommand::Description}, + {ExportOplogCommand::Name, &ExportOplogCmd, ExportOplogCommand::Description}, + {ImportOplogCommand::Name, &ImportOplogCmd, ImportOplogCommand::Description}, + {OplogMirrorCommand::Name, &OplogMirrorCmd, OplogMirrorCommand::Description}, + {SnapshotOplogCommand::Name, &SnapshotOplogCmd, SnapshotOplogCommand::Description}, {OplogDownloadCommand::Name, &OplogDownload, OplogDownloadCommand::Description}, - {"oplog-validate", &OplogValidateCmd, "Validate oplog for missing references"}, - {"print", &PrintCmd, "Print compact binary object"}, - {"printpackage", &PrintPkgCmd, "Print compact binary package"}, - {"project-create", &CreateProjectCmd, "Create a project"}, - {"project-op-details", &ProjectOpDetailsCmd, "Detail info on ops inside a project store oplog"}, - {"project-drop", &ProjectDropCmd, "Drop project or project oplog"}, - {"project-info", &ProjectInfoCmd, "Info on project or project oplog"}, - {"project-stats", &ProjectStatsCmd, "Stats on project store"}, - {"ps", &PsCmd, "Enumerate running zen server instances"}, - {"rpc-record-replay", &RpcReplayCmd, "Replays a previously recorded session of rpc requests"}, - {"rpc-record-start", &RpcStartRecordingCmd, "Starts recording of cache rpc requests on a host"}, - {"rpc-record-stop", &RpcStopRecordingCmd, "Stops recording of cache rpc requests on a host"}, - {"run", &RunCmd, "Run command with special options"}, - {"scrub", &ScrubCmd, "Scrub zen storage (verify data integrity)"}, - {"serve", &ServeCmd, "Serve files from a directory"}, - {"status", &StatusCmd, "Show zen status"}, - {"top", &TopCmd, "Monitor zen server activity"}, - {"trace", &TraceCmd, "Control zen realtime tracing"}, - {"up", &UpCmd, "Bring zen server up"}, + {OplogValidateCommand::Name, &OplogValidateCmd, OplogValidateCommand::Description}, + {PrintCommand::Name, &PrintCmd, PrintCommand::Description}, + {PrintPackageCommand::Name, &PrintPkgCmd, PrintPackageCommand::Description}, + {CreateProjectCommand::Name, &CreateProjectCmd, CreateProjectCommand::Description}, + {ProjectOpDetailsCommand::Name, &ProjectOpDetailsCmd, ProjectOpDetailsCommand::Description}, + {DropProjectCommand::Name, &ProjectDropCmd, DropProjectCommand::Description}, + {ProjectInfoCommand::Name, &ProjectInfoCmd, ProjectInfoCommand::Description}, + {ProjectStatsCommand::Name, &ProjectStatsCmd, ProjectStatsCommand::Description}, + {PsCommand::Name, &PsCmd, PsCommand::Description}, + {RpcReplayCommand::Name, &RpcReplayCmd, RpcReplayCommand::Description}, + {RpcStartRecordingCommand::Name, &RpcStartRecordingCmd, RpcStartRecordingCommand::Description}, + {RpcStopRecordingCommand::Name, &RpcStopRecordingCmd, RpcStopRecordingCommand::Description}, + {RunCommand::Name, &RunCmd, RunCommand::Description}, + {ScrubCommand::Name, &ScrubCmd, ScrubCommand::Description}, + {ServeCommand::Name, &ServeCmd, ServeCommand::Description}, + {StatusCommand::Name, &StatusCmd, StatusCommand::Description}, + {TopCommand::Name, &TopCmd, TopCommand::Description}, + {TraceCommand::Name, &TraceCmd, TraceCommand::Description}, + {UpCommand::Name, &UpCmd, UpCommand::Description}, {VersionCommand::Name, &VersionCmd, VersionCommand::Description}, - {"vfs", &VfsCmd, "Manage virtual file system"}, - {"flush", &FlushCmd, "Flush storage"}, + {VfsCommand::Name, &VfsCmd, VfsCommand::Description}, + {FlushCommand::Name, &FlushCmd, FlushCommand::Description}, {WipeCommand::Name, &WipeCmd, WipeCommand::Description}, {WorkspaceCommand::Name, &WorkspaceCmd, WorkspaceCommand::Description}, {WorkspaceShareCommand::Name, &WorkspaceShareCmd, WorkspaceShareCommand::Description}, -- cgit v1.2.3 From 9aac0fd369b87e965fb34b5168646387de7ea1cd Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 23 Feb 2026 11:19:52 +0100 Subject: implement yaml generation (#774) this implements a yaml generation strategy similar to the JSON generation where we just build a string instead of building a ryml tree. This also removes the dependency on ryml for reduced binary/build times. --- src/zencore/compactbinaryyaml.cpp | 427 +- src/zencore/xmake.lua | 2 - thirdparty/ryml/.github/codeql.yml | 3 - thirdparty/ryml/.github/release.sh | 127 - thirdparty/ryml/.github/reqs.sh | 315 - thirdparty/ryml/.github/setenv.sh | 434 -- thirdparty/ryml/.github/workflows/benchmarks.yml | 147 - thirdparty/ryml/.github/workflows/clang.yml | 262 - thirdparty/ryml/.github/workflows/clang_tidy.yml | 72 - thirdparty/ryml/.github/workflows/codeql.yml | 44 - thirdparty/ryml/.github/workflows/coverage.yml | 94 - thirdparty/ryml/.github/workflows/emscripten.yml | 71 - thirdparty/ryml/.github/workflows/gcc.yml | 168 - thirdparty/ryml/.github/workflows/install.yml | 105 - thirdparty/ryml/.github/workflows/macosx.yml | 103 - thirdparty/ryml/.github/workflows/rarearchs.yml | 123 - thirdparty/ryml/.github/workflows/release.yml | 366 -- thirdparty/ryml/.github/workflows/samples.yml | 60 - thirdparty/ryml/.github/workflows/windows.yml | 130 - thirdparty/ryml/.gitignore | 51 - thirdparty/ryml/.gitmodules | 3 - thirdparty/ryml/.lgtm.yml | 2 - thirdparty/ryml/CONTRIBUTING.md | 18 - thirdparty/ryml/LICENSE.txt | 20 - thirdparty/ryml/MANIFEST.in | 3 - thirdparty/ryml/README.md | 1136 ---- thirdparty/ryml/ROADMAP.md | 18 - thirdparty/ryml/api/python/.gitignore | 141 - thirdparty/ryml/api/python/Makefile | 94 - thirdparty/ryml/api/python/bm/bm_parse.py | 237 - thirdparty/ryml/api/python/requirements.txt | 5 - thirdparty/ryml/api/python/ryml/__init__.py | 2 - thirdparty/ryml/api/python/tests/test_parse.py | 488 -- thirdparty/ryml/api/ryml.i | 662 --- thirdparty/ryml/changelog/0.1.0.md | 44 - thirdparty/ryml/changelog/0.2.0.md | 29 - thirdparty/ryml/changelog/0.2.1.md | 235 - thirdparty/ryml/changelog/0.2.2.md | 1 - thirdparty/ryml/changelog/0.2.3.md | 285 - thirdparty/ryml/changelog/0.3.0.md | 104 - thirdparty/ryml/changelog/0.4.0.md | 229 - thirdparty/ryml/changelog/0.4.1.md | 3 - thirdparty/ryml/changelog/0.5.0.md | 174 - thirdparty/ryml/cmake/uninstall.cmake | 24 - thirdparty/ryml/compat.cmake | 11 - thirdparty/ryml/ext/c4core/.github/release.sh | 129 - thirdparty/ryml/ext/c4core/.github/reqs.sh | 314 - thirdparty/ryml/ext/c4core/.github/setenv.sh | 443 -- .../ryml/ext/c4core/.github/vagrant/Vagrantfile | 80 - .../ext/c4core/.github/vagrant/macos/Vagrantfile | 71 - .../c4core/.github/vagrant/vagrant-provision.sh | 71 - .../ryml/ext/c4core/.github/workflows/arch.yml | 116 - .../ext/c4core/.github/workflows/benchmarks.yml | 250 - .../ryml/ext/c4core/.github/workflows/clang.yml | 230 - .../ext/c4core/.github/workflows/clang_tidy.yml | 97 - .../ryml/ext/c4core/.github/workflows/codeql.yml | 43 - .../ryml/ext/c4core/.github/workflows/coverage.yml | 150 - .../ext/c4core/.github/workflows/emscripten.yml | 99 - .../ryml/ext/c4core/.github/workflows/gcc.yml | 181 - .../ryml/ext/c4core/.github/workflows/libcxx.yml | 111 - .../ryml/ext/c4core/.github/workflows/macosx.yml | 103 - .../ryml/ext/c4core/.github/workflows/release.yml | 197 - .../ext/c4core/.github/workflows/test_install.yml | 104 - .../ryml/ext/c4core/.github/workflows/windows.yml | 157 - thirdparty/ryml/ext/c4core/.gitignore | 34 - thirdparty/ryml/ext/c4core/.gitmodules | 9 - thirdparty/ryml/ext/c4core/LICENSE-BOOST.txt | 26 - thirdparty/ryml/ext/c4core/LICENSE.txt | 20 - thirdparty/ryml/ext/c4core/README.md | 381 -- thirdparty/ryml/ext/c4core/ROADMAP.md | 23 - thirdparty/ryml/ext/c4core/bm/bm.yml | 32 - thirdparty/ryml/ext/c4core/bm/bm_atox.cpp | 478 -- thirdparty/ryml/ext/c4core/bm/bm_charconv.cpp | 1617 ----- thirdparty/ryml/ext/c4core/bm/bm_charconv.hpp | 454 -- thirdparty/ryml/ext/c4core/bm/bm_format.cpp | 900 --- thirdparty/ryml/ext/c4core/bm/bm_itoa_threads.cpp | 356 -- thirdparty/ryml/ext/c4core/bm/bm_plot_c4core.py | 266 - thirdparty/ryml/ext/c4core/bm/bm_xtoa.cpp | 1538 ----- thirdparty/ryml/ext/c4core/bm/float/measure.py | 65 - thirdparty/ryml/ext/c4core/bm/float/read.cpp | 170 - thirdparty/ryml/ext/c4core/bm/ryu.cmake | 37 - thirdparty/ryml/ext/c4core/changelog/0.1.0.md | 3 - thirdparty/ryml/ext/c4core/changelog/0.1.1.md | 5 - thirdparty/ryml/ext/c4core/changelog/0.1.10.md | 106 - thirdparty/ryml/ext/c4core/changelog/0.1.11.md | 59 - thirdparty/ryml/ext/c4core/changelog/0.1.2.md | 4 - thirdparty/ryml/ext/c4core/changelog/0.1.3.md | 1 - thirdparty/ryml/ext/c4core/changelog/0.1.4.md | 6 - thirdparty/ryml/ext/c4core/changelog/0.1.5.md | 2 - thirdparty/ryml/ext/c4core/changelog/0.1.6.md | 2 - thirdparty/ryml/ext/c4core/changelog/0.1.7.md | 5 - thirdparty/ryml/ext/c4core/changelog/0.1.8.md | 45 - thirdparty/ryml/ext/c4core/changelog/0.1.9.md | 31 - thirdparty/ryml/ext/c4core/changelog/current.md | 0 thirdparty/ryml/ext/c4core/cmake/.gitignore | 1 - .../ryml/ext/c4core/cmake/ConfigurationTypes.cmake | 120 - .../ryml/ext/c4core/cmake/CreateSourceGroup.cmake | 31 - thirdparty/ryml/ext/c4core/cmake/Doxyfile.full.in | 2566 -------- thirdparty/ryml/ext/c4core/cmake/Doxyfile.in | 2566 -------- .../ext/c4core/cmake/ExternalProjectUtils.cmake | 215 - thirdparty/ryml/ext/c4core/cmake/FindD3D12.cmake | 75 - thirdparty/ryml/ext/c4core/cmake/FindDX12.cmake | 76 - thirdparty/ryml/ext/c4core/cmake/GetFlags.cmake | 53 - thirdparty/ryml/ext/c4core/cmake/GetNames.cmake | 51 - thirdparty/ryml/ext/c4core/cmake/LICENSE.txt | 20 - thirdparty/ryml/ext/c4core/cmake/PVS-Studio.cmake | 275 - thirdparty/ryml/ext/c4core/cmake/PatchUtils.cmake | 25 - thirdparty/ryml/ext/c4core/cmake/PrintVar.cmake | 27 - thirdparty/ryml/ext/c4core/cmake/README.md | 25 - .../ryml/ext/c4core/cmake/TargetArchitecture.cmake | 180 - .../ext/c4core/cmake/Toolchain-Arm-ubuntu.cmake | 29 - .../ryml/ext/c4core/cmake/Toolchain-Armv7.cmake | 84 - .../ryml/ext/c4core/cmake/Toolchain-PS4.cmake | 73 - .../ryml/ext/c4core/cmake/Toolchain-XBoxOne.cmake | 93 - .../ryml/ext/c4core/cmake/amalgamate_utils.py | 219 - thirdparty/ryml/ext/c4core/cmake/bm-xp/.gitignore | 1 - thirdparty/ryml/ext/c4core/cmake/bm-xp/README.md | 7 - thirdparty/ryml/ext/c4core/cmake/bm-xp/bm.js | 475 -- thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_plot.py | 746 --- thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_run.py | 248 - thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_serve.py | 502 -- thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_util.py | 147 - .../ryml/ext/c4core/cmake/bm-xp/example_c4core.py | 1061 ---- .../ryml/ext/c4core/cmake/bm-xp/example_mintm.py | 1061 ---- .../ryml/ext/c4core/cmake/bm-xp/requirements.txt | 14 - .../ext/c4core/cmake/bm-xp/template/index.html | 45 - .../ryml/ext/c4core/cmake/c4CatSources.cmake | 105 - thirdparty/ryml/ext/c4core/cmake/c4Doxygen.cmake | 121 - .../ryml/ext/c4core/cmake/c4DoxygenConfig.cmake | 24 - .../cmake/c4GetTargetPropertyRecursive.cmake | 186 - thirdparty/ryml/ext/c4core/cmake/c4Project.cmake | 3691 ------------ .../ryml/ext/c4core/cmake/c4SanitizeTarget.cmake | 292 - .../ryml/ext/c4core/cmake/c4StaticAnalysis.cmake | 154 - .../ryml/ext/c4core/cmake/c4stlAddTarget.cmake | 5 - .../ryml/ext/c4core/cmake/compat/c4/gcc-4.8.hpp | 69 - .../ext/c4core/cmake/compat/gtest_gcc-4.8.patch | 97 - .../ryml/ext/c4core/cmake/requirements_doc.txt | 3 - thirdparty/ryml/ext/c4core/compat.cmake | 16 - thirdparty/ryml/ext/c4core/src/c4/allocator.hpp | 405 -- thirdparty/ryml/ext/c4core/src/c4/base64.cpp | 220 - thirdparty/ryml/ext/c4core/src/c4/base64.hpp | 98 - thirdparty/ryml/ext/c4core/src/c4/bitmask.hpp | 330 - thirdparty/ryml/ext/c4core/src/c4/blob.hpp | 50 - thirdparty/ryml/ext/c4core/src/c4/c4_pop.hpp | 19 - thirdparty/ryml/ext/c4core/src/c4/c4_push.hpp | 37 - thirdparty/ryml/ext/c4core/src/c4/c4core.natvis | 168 - thirdparty/ryml/ext/c4core/src/c4/char_traits.cpp | 10 - thirdparty/ryml/ext/c4core/src/c4/char_traits.hpp | 98 - thirdparty/ryml/ext/c4core/src/c4/charconv.hpp | 2407 -------- thirdparty/ryml/ext/c4core/src/c4/common.hpp | 15 - thirdparty/ryml/ext/c4core/src/c4/compiler.hpp | 117 - thirdparty/ryml/ext/c4core/src/c4/config.hpp | 39 - thirdparty/ryml/ext/c4core/src/c4/cpu.hpp | 139 - thirdparty/ryml/ext/c4core/src/c4/ctor_dtor.hpp | 462 -- thirdparty/ryml/ext/c4core/src/c4/dump.hpp | 579 -- thirdparty/ryml/ext/c4core/src/c4/enum.hpp | 276 - thirdparty/ryml/ext/c4core/src/c4/error.cpp | 227 - thirdparty/ryml/ext/c4core/src/c4/error.hpp | 432 -- thirdparty/ryml/ext/c4core/src/c4/export.hpp | 18 - .../ext/c4core/src/c4/ext/debugbreak/.gitignore | 10 - .../ryml/ext/c4core/src/c4/ext/debugbreak/COPYING | 23 - .../ext/c4core/src/c4/ext/debugbreak/GNUmakefile | 28 - .../ext/debugbreak/HOW-TO-USE-DEBUGBREAK-GDB-PY.md | 33 - .../ext/c4core/src/c4/ext/debugbreak/README.md | 127 - .../c4core/src/c4/ext/debugbreak/debugbreak-gdb.py | 183 - .../ext/c4core/src/c4/ext/debugbreak/debugbreak.h | 174 - .../c4core/src/c4/ext/debugbreak/test/break-c++.cc | 9 - .../ext/c4core/src/c4/ext/debugbreak/test/break.c | 9 - .../c4core/src/c4/ext/debugbreak/test/break.gdb | 2 - .../ext/c4core/src/c4/ext/debugbreak/test/fib.c | 21 - .../ext/c4core/src/c4/ext/debugbreak/test/fib.gdb | 2 - .../src/c4/ext/debugbreak/test/test-debugbreak.gdb | 6 - .../ext/c4core/src/c4/ext/debugbreak/test/trap.c | 8 - .../ext/c4core/src/c4/ext/debugbreak/test/trap.gdb | 3 - .../ryml/ext/c4core/src/c4/ext/fast_float.hpp | 28 - .../ext/c4core/src/c4/ext/fast_float/.cirrus.yml | 22 - .../c4/ext/fast_float/.github/workflows/alpine.yml | 27 - .../.github/workflows/amalgamate-ubuntu20.yml | 25 - .../fast_float/.github/workflows/msys2-clang.yml | 39 - .../c4/ext/fast_float/.github/workflows/msys2.yml | 45 - .../ext/fast_float/.github/workflows/ubuntu18.yml | 35 - .../.github/workflows/ubuntu20-cxx20.yml | 19 - .../ext/fast_float/.github/workflows/ubuntu20.yml | 29 - .../ext/fast_float/.github/workflows/vs15-ci.yml | 29 - .../fast_float/.github/workflows/vs16-arm-ci.yml | 21 - .../ext/fast_float/.github/workflows/vs16-ci.yml | 29 - .../fast_float/.github/workflows/vs16-clang-ci.yml | 27 - .../fast_float/.github/workflows/vs16-cxx20.yml | 24 - .../ext/c4core/src/c4/ext/fast_float/.gitignore | 4 - .../ext/c4core/src/c4/ext/fast_float/.travis.yml | 242 - .../ryml/ext/c4core/src/c4/ext/fast_float/AUTHORS | 2 - .../ext/c4core/src/c4/ext/fast_float/CONTRIBUTORS | 6 - .../c4core/src/c4/ext/fast_float/LICENSE-APACHE | 201 - .../ext/c4core/src/c4/ext/fast_float/LICENSE-MIT | 23 - .../ext/c4core/src/c4/ext/fast_float/README.md | 216 - .../ext/c4core/src/c4/ext/fast_float/ci/script.sh | 18 - .../src/c4/ext/fast_float/cmake/config.cmake.in | 4 - .../fast_float/include/fast_float/ascii_number.h | 231 - .../c4/ext/fast_float/include/fast_float/bigint.h | 590 -- .../include/fast_float/decimal_to_binary.h | 194 - .../include/fast_float/digit_comparison.h | 423 -- .../ext/fast_float/include/fast_float/fast_float.h | 63 - .../ext/fast_float/include/fast_float/fast_table.h | 699 --- .../fast_float/include/fast_float/float_common.h | 362 -- .../fast_float/include/fast_float/parse_number.h | 113 - .../include/fast_float/simple_decimal_conversion.h | 360 -- .../src/c4/ext/fast_float/script/amalgamate.py | 56 - .../src/c4/ext/fast_float/script/analysis.py | 36 - .../c4/ext/fast_float/script/table_generation.py | 31 - .../src/c4/ext/fast_float/tests/basictest.cpp | 704 --- .../fast_float/tests/build_tests/issue72/foo.cpp | 2 - .../fast_float/tests/build_tests/issue72/main.cpp | 2 - .../fast_float/tests/build_tests/issue72/test.h | 2 - .../c4/ext/fast_float/tests/example_comma_test.cpp | 15 - .../src/c4/ext/fast_float/tests/example_test.cpp | 14 - .../src/c4/ext/fast_float/tests/exhaustive32.cpp | 63 - .../c4/ext/fast_float/tests/exhaustive32_64.cpp | 78 - .../ext/fast_float/tests/exhaustive32_midpoint.cpp | 148 - .../c4/ext/fast_float/tests/long_exhaustive32.cpp | 63 - .../ext/fast_float/tests/long_exhaustive32_64.cpp | 66 - .../src/c4/ext/fast_float/tests/long_random64.cpp | 106 - .../src/c4/ext/fast_float/tests/long_test.cpp | 58 - .../fast_float/tests/powersoffive_hardround.cpp | 134 - .../src/c4/ext/fast_float/tests/random64.cpp | 109 - .../src/c4/ext/fast_float/tests/random_string.cpp | 240 - .../ext/fast_float/tests/short_random_string.cpp | 234 - .../src/c4/ext/fast_float/tests/string_test.cpp | 279 - .../ryml/ext/c4core/src/c4/ext/fast_float_all.h | 2947 --------- thirdparty/ryml/ext/c4core/src/c4/ext/rng/rng.hpp | 192 - .../ryml/ext/c4core/src/c4/ext/sg14/README.md | 1 - .../ext/c4core/src/c4/ext/sg14/inplace_function.h | 347 -- thirdparty/ryml/ext/c4core/src/c4/format.cpp | 56 - thirdparty/ryml/ext/c4core/src/c4/format.hpp | 900 --- thirdparty/ryml/ext/c4core/src/c4/hash.hpp | 95 - thirdparty/ryml/ext/c4core/src/c4/language.cpp | 16 - thirdparty/ryml/ext/c4core/src/c4/language.hpp | 275 - .../ryml/ext/c4core/src/c4/memory_resource.cpp | 338 -- .../ryml/ext/c4core/src/c4/memory_resource.hpp | 568 -- thirdparty/ryml/ext/c4core/src/c4/memory_util.cpp | 30 - thirdparty/ryml/ext/c4core/src/c4/memory_util.hpp | 774 --- thirdparty/ryml/ext/c4core/src/c4/platform.hpp | 44 - thirdparty/ryml/ext/c4core/src/c4/preprocessor.hpp | 123 - thirdparty/ryml/ext/c4core/src/c4/restrict.hpp | 51 - thirdparty/ryml/ext/c4core/src/c4/span.hpp | 517 -- thirdparty/ryml/ext/c4core/src/c4/std/std.hpp | 10 - thirdparty/ryml/ext/c4core/src/c4/std/std_fwd.hpp | 10 - thirdparty/ryml/ext/c4core/src/c4/std/string.hpp | 97 - .../ryml/ext/c4core/src/c4/std/string_fwd.hpp | 56 - thirdparty/ryml/ext/c4core/src/c4/std/tuple.hpp | 184 - thirdparty/ryml/ext/c4core/src/c4/std/vector.hpp | 88 - .../ryml/ext/c4core/src/c4/std/vector_fwd.hpp | 60 - thirdparty/ryml/ext/c4core/src/c4/substr.hpp | 2246 ------- thirdparty/ryml/ext/c4core/src/c4/substr_fwd.hpp | 16 - thirdparty/ryml/ext/c4core/src/c4/szconv.hpp | 64 - thirdparty/ryml/ext/c4core/src/c4/type_name.hpp | 125 - thirdparty/ryml/ext/c4core/src/c4/types.hpp | 492 -- thirdparty/ryml/ext/c4core/src/c4/unrestrict.hpp | 17 - thirdparty/ryml/ext/c4core/src/c4/utf.cpp | 56 - thirdparty/ryml/ext/c4core/src/c4/utf.hpp | 16 - thirdparty/ryml/ext/c4core/src/c4/windows.hpp | 10 - thirdparty/ryml/ext/c4core/src/c4/windows_pop.hpp | 41 - thirdparty/ryml/ext/c4core/src/c4/windows_push.hpp | 102 - thirdparty/ryml/ext/c4core/tbump.toml | 48 - .../ryml/ext/c4core/test/c4/libtest/archetypes.cpp | 9 - .../ryml/ext/c4core/test/c4/libtest/archetypes.hpp | 551 -- .../ext/c4core/test/c4/libtest/supprwarn_pop.hpp | 12 - .../ext/c4core/test/c4/libtest/supprwarn_push.hpp | 48 - .../ryml/ext/c4core/test/c4/libtest/test.cpp | 7 - thirdparty/ryml/ext/c4core/test/c4/main.cpp | 3 - thirdparty/ryml/ext/c4core/test/c4/test.hpp | 324 - thirdparty/ryml/ext/c4core/test/printintegers.py | 107 - thirdparty/ryml/ext/c4core/test/test_allocator.cpp | 246 - thirdparty/ryml/ext/c4core/test/test_base64.cpp | 265 - thirdparty/ryml/ext/c4core/test/test_bitmask.cpp | 385 -- thirdparty/ryml/ext/c4core/test/test_blob.cpp | 43 - .../ryml/ext/c4core/test/test_char_traits.cpp | 67 - thirdparty/ryml/ext/c4core/test/test_charconv.cpp | 2743 --------- thirdparty/ryml/ext/c4core/test/test_ctor_dtor.cpp | 306 - thirdparty/ryml/ext/c4core/test/test_dump.cpp | 1220 ---- thirdparty/ryml/ext/c4core/test/test_enum.cpp | 158 - .../ryml/ext/c4core/test/test_enum_common.hpp | 213 - thirdparty/ryml/ext/c4core/test/test_error.cpp | 635 -- .../ryml/ext/c4core/test/test_error_exception.cpp | 108 - thirdparty/ryml/ext/c4core/test/test_format.cpp | 1054 ---- thirdparty/ryml/ext/c4core/test/test_log.cpp | 70 - .../ryml/ext/c4core/test/test_memory_resource.cpp | 255 - .../ryml/ext/c4core/test/test_memory_util.cpp | 415 -- thirdparty/ryml/ext/c4core/test/test_numbers.hpp | 1863 ------ .../ryml/ext/c4core/test/test_preprocessor.cpp | 55 - .../test_singleheader/libc4core_singleheader.cpp | 2 - thirdparty/ryml/ext/c4core/test/test_span.cpp | 944 --- .../ryml/ext/c4core/test/test_std_string.cpp | 125 - .../ryml/ext/c4core/test/test_std_vector.cpp | 133 - thirdparty/ryml/ext/c4core/test/test_substr.cpp | 4507 -------------- thirdparty/ryml/ext/c4core/test/test_szconv.cpp | 166 - thirdparty/ryml/ext/c4core/test/test_type_name.cpp | 49 - thirdparty/ryml/ext/c4core/test/test_types.cpp | 81 - thirdparty/ryml/ext/c4core/test/test_utf.cpp | 48 - thirdparty/ryml/ext/c4core/test/utfchars.inc | 6274 -------------------- thirdparty/ryml/ext/c4core/tools/amalgamate.py | 143 - thirdparty/ryml/ext/testbm.cmake | 4 - thirdparty/ryml/img/first_comparison_yaml_cpp.png | Bin 25969 -> 0 bytes thirdparty/ryml/pyproject.toml | 8 - thirdparty/ryml/requirements.txt | 8 - thirdparty/ryml/samples/add_subdirectory/run.sh | 10 - thirdparty/ryml/samples/custom_c4core/run.sh | 10 - thirdparty/ryml/samples/fetch_content/run.sh | 12 - thirdparty/ryml/samples/find_package/run.sh | 26 - thirdparty/ryml/samples/quickstart.cpp | 4161 ------------- .../ryml/samples/singleheader/amalgamate.cmake | 18 - thirdparty/ryml/samples/singleheader/run.sh | 10 - thirdparty/ryml/samples/singleheaderlib/lib.cpp | 2 - .../ryml/samples/singleheaderlib/run_shared.sh | 10 - .../ryml/samples/singleheaderlib/run_static.sh | 10 - thirdparty/ryml/setup.py | 125 - thirdparty/ryml/src/c4/yml/common.cpp | 117 - thirdparty/ryml/src/c4/yml/common.hpp | 278 - thirdparty/ryml/src/c4/yml/detail/checks.hpp | 200 - thirdparty/ryml/src/c4/yml/detail/parser_dbg.hpp | 137 - thirdparty/ryml/src/c4/yml/detail/print.hpp | 128 - thirdparty/ryml/src/c4/yml/detail/stack.hpp | 274 - thirdparty/ryml/src/c4/yml/emit.def.hpp | 960 --- thirdparty/ryml/src/c4/yml/emit.hpp | 490 -- thirdparty/ryml/src/c4/yml/export.hpp | 18 - thirdparty/ryml/src/c4/yml/node.cpp | 30 - thirdparty/ryml/src/c4/yml/node.hpp | 1276 ---- thirdparty/ryml/src/c4/yml/parse.cpp | 5724 ------------------ thirdparty/ryml/src/c4/yml/parse.hpp | 706 --- thirdparty/ryml/src/c4/yml/preprocess.cpp | 110 - thirdparty/ryml/src/c4/yml/preprocess.hpp | 99 - thirdparty/ryml/src/c4/yml/std/map.hpp | 45 - thirdparty/ryml/src/c4/yml/std/std.hpp | 8 - thirdparty/ryml/src/c4/yml/std/string.hpp | 9 - thirdparty/ryml/src/c4/yml/std/vector.hpp | 53 - thirdparty/ryml/src/c4/yml/tree.cpp | 2183 ------- thirdparty/ryml/src/c4/yml/tree.hpp | 1495 ----- thirdparty/ryml/src/c4/yml/writer.hpp | 229 - thirdparty/ryml/src/c4/yml/yml.hpp | 10 - thirdparty/ryml/src/ryml-gdbtypes.py | 391 -- thirdparty/ryml/src/ryml.hpp | 11 - thirdparty/ryml/src/ryml.natvis | 194 - thirdparty/ryml/src/ryml_std.hpp | 6 - thirdparty/ryml/tbump.toml | 56 - thirdparty/ryml/test/callbacks_tester.hpp | 77 - thirdparty/ryml/test/test_basic.cpp | 304 - thirdparty/ryml/test/test_block_folded.cpp | 1574 ----- thirdparty/ryml/test/test_block_literal.cpp | 1261 ---- thirdparty/ryml/test/test_callbacks.cpp | 356 -- thirdparty/ryml/test/test_case.cpp | 898 --- thirdparty/ryml/test/test_case.hpp | 533 -- thirdparty/ryml/test/test_double_quoted.cpp | 610 -- thirdparty/ryml/test/test_emit.cpp | 491 -- thirdparty/ryml/test/test_empty_file.cpp | 79 - thirdparty/ryml/test/test_empty_map.cpp | 43 - thirdparty/ryml/test/test_empty_scalar.cpp | 353 -- thirdparty/ryml/test/test_empty_seq.cpp | 40 - thirdparty/ryml/test/test_explicit_key.cpp | 419 -- thirdparty/ryml/test/test_generic_map.cpp | 89 - thirdparty/ryml/test/test_generic_seq.cpp | 47 - thirdparty/ryml/test/test_github_issues.cpp | 590 -- thirdparty/ryml/test/test_group.cpp | 732 --- thirdparty/ryml/test/test_group.hpp | 210 - thirdparty/ryml/test/test_indentation.cpp | 340 -- thirdparty/ryml/test/test_json.cpp | 516 -- thirdparty/ryml/test/test_location.cpp | 720 --- thirdparty/ryml/test/test_map_of_seq.cpp | 201 - thirdparty/ryml/test/test_merge.cpp | 225 - thirdparty/ryml/test/test_nested_mapx2.cpp | 73 - thirdparty/ryml/test/test_nested_mapx3.cpp | 103 - thirdparty/ryml/test/test_nested_mapx4.cpp | 190 - thirdparty/ryml/test/test_nested_seqx2.cpp | 133 - thirdparty/ryml/test/test_nested_seqx3.cpp | 187 - thirdparty/ryml/test/test_nested_seqx4.cpp | 124 - thirdparty/ryml/test/test_noderef.cpp | 813 --- thirdparty/ryml/test/test_null_val.cpp | 519 -- thirdparty/ryml/test/test_number.cpp | 217 - thirdparty/ryml/test/test_parser.cpp | 566 -- thirdparty/ryml/test/test_plain_scalar.cpp | 800 --- thirdparty/ryml/test/test_preprocess.cpp | 53 - thirdparty/ryml/test/test_scalar_names.cpp | 94 - thirdparty/ryml/test/test_seq_of_map.cpp | 348 -- thirdparty/ryml/test/test_serialize.cpp | 499 -- thirdparty/ryml/test/test_simple_anchor.cpp | 1405 ----- thirdparty/ryml/test/test_simple_doc.cpp | 526 -- thirdparty/ryml/test/test_simple_map.cpp | 1050 ---- thirdparty/ryml/test/test_simple_seq.cpp | 695 --- thirdparty/ryml/test/test_simple_set.cpp | 144 - thirdparty/ryml/test/test_single_quoted.cpp | 356 -- .../test_singleheader/libryml_singleheader.cpp | 3 - thirdparty/ryml/test/test_stack.cpp | 857 --- thirdparty/ryml/test/test_style.cpp | 616 -- thirdparty/ryml/test/test_suite.cpp | 612 -- .../ryml/test/test_suite/test_suite_common.hpp | 44 - .../ryml/test/test_suite/test_suite_events.cpp | 607 -- .../ryml/test/test_suite/test_suite_events.hpp | 45 - .../test/test_suite/test_suite_events_emitter.cpp | 289 - .../ryml/test/test_suite/test_suite_parts.cpp | 220 - .../ryml/test/test_suite/test_suite_parts.hpp | 28 - thirdparty/ryml/test/test_tag_property.cpp | 1149 ---- thirdparty/ryml/test/test_tree.cpp | 3924 ------------ thirdparty/ryml/test/test_yaml_events.cpp | 467 -- thirdparty/ryml/tools/amalgamate.py | 130 - thirdparty/ryml/tools/parse_emit.cpp | 116 - thirdparty/ryml/tools/test_suite/Dockerfile | 33 - thirdparty/ryml/tools/test_suite/run_test_suite.sh | 18 - thirdparty/ryml/tools/yaml_events.cpp | 92 - thirdparty/xmake.lua | 15 - 407 files changed, 302 insertions(+), 128804 deletions(-) delete mode 100644 thirdparty/ryml/.github/codeql.yml delete mode 100644 thirdparty/ryml/.github/release.sh delete mode 100644 thirdparty/ryml/.github/reqs.sh delete mode 100644 thirdparty/ryml/.github/setenv.sh delete mode 100644 thirdparty/ryml/.github/workflows/benchmarks.yml delete mode 100644 thirdparty/ryml/.github/workflows/clang.yml delete mode 100644 thirdparty/ryml/.github/workflows/clang_tidy.yml delete mode 100644 thirdparty/ryml/.github/workflows/codeql.yml delete mode 100644 thirdparty/ryml/.github/workflows/coverage.yml delete mode 100644 thirdparty/ryml/.github/workflows/emscripten.yml delete mode 100644 thirdparty/ryml/.github/workflows/gcc.yml delete mode 100644 thirdparty/ryml/.github/workflows/install.yml delete mode 100644 thirdparty/ryml/.github/workflows/macosx.yml delete mode 100644 thirdparty/ryml/.github/workflows/rarearchs.yml delete mode 100644 thirdparty/ryml/.github/workflows/release.yml delete mode 100644 thirdparty/ryml/.github/workflows/samples.yml delete mode 100644 thirdparty/ryml/.github/workflows/windows.yml delete mode 100644 thirdparty/ryml/.gitignore delete mode 100644 thirdparty/ryml/.gitmodules delete mode 100644 thirdparty/ryml/.lgtm.yml delete mode 100644 thirdparty/ryml/CONTRIBUTING.md delete mode 100644 thirdparty/ryml/LICENSE.txt delete mode 100644 thirdparty/ryml/MANIFEST.in delete mode 100644 thirdparty/ryml/README.md delete mode 100644 thirdparty/ryml/ROADMAP.md delete mode 100644 thirdparty/ryml/api/python/.gitignore delete mode 100644 thirdparty/ryml/api/python/Makefile delete mode 100644 thirdparty/ryml/api/python/bm/bm_parse.py delete mode 100644 thirdparty/ryml/api/python/requirements.txt delete mode 100644 thirdparty/ryml/api/python/ryml/__init__.py delete mode 100644 thirdparty/ryml/api/python/tests/test_parse.py delete mode 100644 thirdparty/ryml/api/ryml.i delete mode 100644 thirdparty/ryml/changelog/0.1.0.md delete mode 100644 thirdparty/ryml/changelog/0.2.0.md delete mode 100644 thirdparty/ryml/changelog/0.2.1.md delete mode 100644 thirdparty/ryml/changelog/0.2.2.md delete mode 100644 thirdparty/ryml/changelog/0.2.3.md delete mode 100644 thirdparty/ryml/changelog/0.3.0.md delete mode 100644 thirdparty/ryml/changelog/0.4.0.md delete mode 100644 thirdparty/ryml/changelog/0.4.1.md delete mode 100644 thirdparty/ryml/changelog/0.5.0.md delete mode 100644 thirdparty/ryml/cmake/uninstall.cmake delete mode 100644 thirdparty/ryml/compat.cmake delete mode 100644 thirdparty/ryml/ext/c4core/.github/release.sh delete mode 100644 thirdparty/ryml/ext/c4core/.github/reqs.sh delete mode 100644 thirdparty/ryml/ext/c4core/.github/setenv.sh delete mode 100644 thirdparty/ryml/ext/c4core/.github/vagrant/Vagrantfile delete mode 100644 thirdparty/ryml/ext/c4core/.github/vagrant/macos/Vagrantfile delete mode 100755 thirdparty/ryml/ext/c4core/.github/vagrant/vagrant-provision.sh delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/arch.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/benchmarks.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/clang.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/clang_tidy.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/codeql.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/coverage.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/emscripten.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/gcc.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/libcxx.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/macosx.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/release.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/test_install.yml delete mode 100644 thirdparty/ryml/ext/c4core/.github/workflows/windows.yml delete mode 100644 thirdparty/ryml/ext/c4core/.gitignore delete mode 100644 thirdparty/ryml/ext/c4core/.gitmodules delete mode 100644 thirdparty/ryml/ext/c4core/LICENSE-BOOST.txt delete mode 100644 thirdparty/ryml/ext/c4core/LICENSE.txt delete mode 100644 thirdparty/ryml/ext/c4core/README.md delete mode 100644 thirdparty/ryml/ext/c4core/ROADMAP.md delete mode 100644 thirdparty/ryml/ext/c4core/bm/bm.yml delete mode 100644 thirdparty/ryml/ext/c4core/bm/bm_atox.cpp delete mode 100644 thirdparty/ryml/ext/c4core/bm/bm_charconv.cpp delete mode 100644 thirdparty/ryml/ext/c4core/bm/bm_charconv.hpp delete mode 100644 thirdparty/ryml/ext/c4core/bm/bm_format.cpp delete mode 100644 thirdparty/ryml/ext/c4core/bm/bm_itoa_threads.cpp delete mode 100644 thirdparty/ryml/ext/c4core/bm/bm_plot_c4core.py delete mode 100644 thirdparty/ryml/ext/c4core/bm/bm_xtoa.cpp delete mode 100644 thirdparty/ryml/ext/c4core/bm/float/measure.py delete mode 100644 thirdparty/ryml/ext/c4core/bm/float/read.cpp delete mode 100644 thirdparty/ryml/ext/c4core/bm/ryu.cmake delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.0.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.1.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.10.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.11.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.2.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.3.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.4.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.5.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.6.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.7.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.8.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/0.1.9.md delete mode 100644 thirdparty/ryml/ext/c4core/changelog/current.md delete mode 100644 thirdparty/ryml/ext/c4core/cmake/.gitignore delete mode 100644 thirdparty/ryml/ext/c4core/cmake/ConfigurationTypes.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/CreateSourceGroup.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/Doxyfile.full.in delete mode 100644 thirdparty/ryml/ext/c4core/cmake/Doxyfile.in delete mode 100644 thirdparty/ryml/ext/c4core/cmake/ExternalProjectUtils.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/FindD3D12.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/FindDX12.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/GetFlags.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/GetNames.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/LICENSE.txt delete mode 100644 thirdparty/ryml/ext/c4core/cmake/PVS-Studio.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/PatchUtils.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/PrintVar.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/README.md delete mode 100644 thirdparty/ryml/ext/c4core/cmake/TargetArchitecture.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/Toolchain-Arm-ubuntu.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/Toolchain-Armv7.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/Toolchain-PS4.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/Toolchain-XBoxOne.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/amalgamate_utils.py delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/.gitignore delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/README.md delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/bm.js delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_plot.py delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_run.py delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_serve.py delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_util.py delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/example_c4core.py delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/example_mintm.py delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/requirements.txt delete mode 100644 thirdparty/ryml/ext/c4core/cmake/bm-xp/template/index.html delete mode 100644 thirdparty/ryml/ext/c4core/cmake/c4CatSources.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/c4Doxygen.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/c4DoxygenConfig.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/c4GetTargetPropertyRecursive.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/c4Project.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/c4SanitizeTarget.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/c4StaticAnalysis.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/c4stlAddTarget.cmake delete mode 100644 thirdparty/ryml/ext/c4core/cmake/compat/c4/gcc-4.8.hpp delete mode 100644 thirdparty/ryml/ext/c4core/cmake/compat/gtest_gcc-4.8.patch delete mode 100644 thirdparty/ryml/ext/c4core/cmake/requirements_doc.txt delete mode 100644 thirdparty/ryml/ext/c4core/compat.cmake delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/allocator.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/base64.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/base64.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/bitmask.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/blob.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/c4_pop.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/c4_push.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/c4core.natvis delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/char_traits.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/char_traits.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/charconv.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/common.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/compiler.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/config.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/cpu.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ctor_dtor.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/dump.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/enum.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/error.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/error.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/export.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/.gitignore delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/COPYING delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/GNUmakefile delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/HOW-TO-USE-DEBUGBREAK-GDB-PY.md delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/README.md delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/debugbreak-gdb.py delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/debugbreak.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break-c++.cc delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break.c delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break.gdb delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/fib.c delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/fib.gdb delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/test-debugbreak.gdb delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/trap.c delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/trap.gdb delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.cirrus.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/alpine.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/amalgamate-ubuntu20.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/msys2-clang.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/msys2.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu18.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu20-cxx20.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu20.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs15-ci.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-arm-ci.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-ci.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-clang-ci.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-cxx20.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.gitignore delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.travis.yml delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/AUTHORS delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/CONTRIBUTORS delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/LICENSE-APACHE delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/LICENSE-MIT delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/README.md delete mode 100755 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/ci/script.sh delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/cmake/config.cmake.in delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/ascii_number.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/bigint.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/decimal_to_binary.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/digit_comparison.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/fast_float.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/fast_table.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/float_common.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/parse_number.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/simple_decimal_conversion.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/amalgamate.py delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/analysis.py delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/table_generation.py delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/basictest.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/foo.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/main.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/test.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/example_comma_test.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/example_test.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32_64.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32_midpoint.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_exhaustive32.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_exhaustive32_64.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_random64.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_test.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/powersoffive_hardround.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/random64.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/random_string.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/short_random_string.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/string_test.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/fast_float_all.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/rng/rng.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/sg14/README.md delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/ext/sg14/inplace_function.h delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/format.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/format.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/hash.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/language.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/language.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/memory_resource.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/memory_resource.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/memory_util.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/memory_util.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/platform.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/preprocessor.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/restrict.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/span.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/std/std.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/std/std_fwd.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/std/string.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/std/string_fwd.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/std/tuple.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/std/vector.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/std/vector_fwd.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/substr.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/substr_fwd.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/szconv.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/type_name.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/types.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/unrestrict.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/utf.cpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/utf.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/windows.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/windows_pop.hpp delete mode 100644 thirdparty/ryml/ext/c4core/src/c4/windows_push.hpp delete mode 100644 thirdparty/ryml/ext/c4core/tbump.toml delete mode 100644 thirdparty/ryml/ext/c4core/test/c4/libtest/archetypes.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/c4/libtest/archetypes.hpp delete mode 100644 thirdparty/ryml/ext/c4core/test/c4/libtest/supprwarn_pop.hpp delete mode 100644 thirdparty/ryml/ext/c4core/test/c4/libtest/supprwarn_push.hpp delete mode 100644 thirdparty/ryml/ext/c4core/test/c4/libtest/test.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/c4/main.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/c4/test.hpp delete mode 100644 thirdparty/ryml/ext/c4core/test/printintegers.py delete mode 100644 thirdparty/ryml/ext/c4core/test/test_allocator.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_base64.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_bitmask.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_blob.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_char_traits.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_charconv.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_ctor_dtor.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_dump.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_enum.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_enum_common.hpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_error.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_error_exception.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_format.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_log.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_memory_resource.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_memory_util.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_numbers.hpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_preprocessor.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_singleheader/libc4core_singleheader.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_span.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_std_string.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_std_vector.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_substr.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_szconv.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_type_name.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_types.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/test_utf.cpp delete mode 100644 thirdparty/ryml/ext/c4core/test/utfchars.inc delete mode 100644 thirdparty/ryml/ext/c4core/tools/amalgamate.py delete mode 100644 thirdparty/ryml/ext/testbm.cmake delete mode 100644 thirdparty/ryml/img/first_comparison_yaml_cpp.png delete mode 100644 thirdparty/ryml/pyproject.toml delete mode 100644 thirdparty/ryml/requirements.txt delete mode 100755 thirdparty/ryml/samples/add_subdirectory/run.sh delete mode 100755 thirdparty/ryml/samples/custom_c4core/run.sh delete mode 100755 thirdparty/ryml/samples/fetch_content/run.sh delete mode 100755 thirdparty/ryml/samples/find_package/run.sh delete mode 100644 thirdparty/ryml/samples/quickstart.cpp delete mode 100644 thirdparty/ryml/samples/singleheader/amalgamate.cmake delete mode 100755 thirdparty/ryml/samples/singleheader/run.sh delete mode 100644 thirdparty/ryml/samples/singleheaderlib/lib.cpp delete mode 100755 thirdparty/ryml/samples/singleheaderlib/run_shared.sh delete mode 100755 thirdparty/ryml/samples/singleheaderlib/run_static.sh delete mode 100644 thirdparty/ryml/setup.py delete mode 100644 thirdparty/ryml/src/c4/yml/common.cpp delete mode 100644 thirdparty/ryml/src/c4/yml/common.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/detail/checks.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/detail/parser_dbg.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/detail/print.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/detail/stack.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/emit.def.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/emit.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/export.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/node.cpp delete mode 100644 thirdparty/ryml/src/c4/yml/node.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/parse.cpp delete mode 100644 thirdparty/ryml/src/c4/yml/parse.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/preprocess.cpp delete mode 100644 thirdparty/ryml/src/c4/yml/preprocess.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/std/map.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/std/std.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/std/string.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/std/vector.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/tree.cpp delete mode 100644 thirdparty/ryml/src/c4/yml/tree.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/writer.hpp delete mode 100644 thirdparty/ryml/src/c4/yml/yml.hpp delete mode 100644 thirdparty/ryml/src/ryml-gdbtypes.py delete mode 100644 thirdparty/ryml/src/ryml.hpp delete mode 100644 thirdparty/ryml/src/ryml.natvis delete mode 100644 thirdparty/ryml/src/ryml_std.hpp delete mode 100644 thirdparty/ryml/tbump.toml delete mode 100644 thirdparty/ryml/test/callbacks_tester.hpp delete mode 100644 thirdparty/ryml/test/test_basic.cpp delete mode 100644 thirdparty/ryml/test/test_block_folded.cpp delete mode 100644 thirdparty/ryml/test/test_block_literal.cpp delete mode 100644 thirdparty/ryml/test/test_callbacks.cpp delete mode 100644 thirdparty/ryml/test/test_case.cpp delete mode 100644 thirdparty/ryml/test/test_case.hpp delete mode 100644 thirdparty/ryml/test/test_double_quoted.cpp delete mode 100644 thirdparty/ryml/test/test_emit.cpp delete mode 100644 thirdparty/ryml/test/test_empty_file.cpp delete mode 100644 thirdparty/ryml/test/test_empty_map.cpp delete mode 100644 thirdparty/ryml/test/test_empty_scalar.cpp delete mode 100644 thirdparty/ryml/test/test_empty_seq.cpp delete mode 100644 thirdparty/ryml/test/test_explicit_key.cpp delete mode 100644 thirdparty/ryml/test/test_generic_map.cpp delete mode 100644 thirdparty/ryml/test/test_generic_seq.cpp delete mode 100644 thirdparty/ryml/test/test_github_issues.cpp delete mode 100644 thirdparty/ryml/test/test_group.cpp delete mode 100644 thirdparty/ryml/test/test_group.hpp delete mode 100644 thirdparty/ryml/test/test_indentation.cpp delete mode 100644 thirdparty/ryml/test/test_json.cpp delete mode 100644 thirdparty/ryml/test/test_location.cpp delete mode 100644 thirdparty/ryml/test/test_map_of_seq.cpp delete mode 100644 thirdparty/ryml/test/test_merge.cpp delete mode 100644 thirdparty/ryml/test/test_nested_mapx2.cpp delete mode 100644 thirdparty/ryml/test/test_nested_mapx3.cpp delete mode 100644 thirdparty/ryml/test/test_nested_mapx4.cpp delete mode 100644 thirdparty/ryml/test/test_nested_seqx2.cpp delete mode 100644 thirdparty/ryml/test/test_nested_seqx3.cpp delete mode 100644 thirdparty/ryml/test/test_nested_seqx4.cpp delete mode 100644 thirdparty/ryml/test/test_noderef.cpp delete mode 100644 thirdparty/ryml/test/test_null_val.cpp delete mode 100644 thirdparty/ryml/test/test_number.cpp delete mode 100644 thirdparty/ryml/test/test_parser.cpp delete mode 100644 thirdparty/ryml/test/test_plain_scalar.cpp delete mode 100644 thirdparty/ryml/test/test_preprocess.cpp delete mode 100644 thirdparty/ryml/test/test_scalar_names.cpp delete mode 100644 thirdparty/ryml/test/test_seq_of_map.cpp delete mode 100644 thirdparty/ryml/test/test_serialize.cpp delete mode 100644 thirdparty/ryml/test/test_simple_anchor.cpp delete mode 100644 thirdparty/ryml/test/test_simple_doc.cpp delete mode 100644 thirdparty/ryml/test/test_simple_map.cpp delete mode 100644 thirdparty/ryml/test/test_simple_seq.cpp delete mode 100644 thirdparty/ryml/test/test_simple_set.cpp delete mode 100644 thirdparty/ryml/test/test_single_quoted.cpp delete mode 100644 thirdparty/ryml/test/test_singleheader/libryml_singleheader.cpp delete mode 100644 thirdparty/ryml/test/test_stack.cpp delete mode 100644 thirdparty/ryml/test/test_style.cpp delete mode 100644 thirdparty/ryml/test/test_suite.cpp delete mode 100644 thirdparty/ryml/test/test_suite/test_suite_common.hpp delete mode 100644 thirdparty/ryml/test/test_suite/test_suite_events.cpp delete mode 100644 thirdparty/ryml/test/test_suite/test_suite_events.hpp delete mode 100644 thirdparty/ryml/test/test_suite/test_suite_events_emitter.cpp delete mode 100644 thirdparty/ryml/test/test_suite/test_suite_parts.cpp delete mode 100644 thirdparty/ryml/test/test_suite/test_suite_parts.hpp delete mode 100644 thirdparty/ryml/test/test_tag_property.cpp delete mode 100644 thirdparty/ryml/test/test_tree.cpp delete mode 100644 thirdparty/ryml/test/test_yaml_events.cpp delete mode 100644 thirdparty/ryml/tools/amalgamate.py delete mode 100644 thirdparty/ryml/tools/parse_emit.cpp delete mode 100644 thirdparty/ryml/tools/test_suite/Dockerfile delete mode 100755 thirdparty/ryml/tools/test_suite/run_test_suite.sh delete mode 100644 thirdparty/ryml/tools/yaml_events.cpp diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp index 5122e952a..b308af418 100644 --- a/src/zencore/compactbinaryyaml.cpp +++ b/src/zencore/compactbinaryyaml.cpp @@ -14,11 +14,6 @@ #include #include -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - namespace zen { ////////////////////////////////////////////////////////////////////////// @@ -26,193 +21,349 @@ namespace zen { class CbYamlWriter { public: - explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_StrBuilder(InBuilder) { m_NodeStack.push_back(m_Tree.rootref()); } + explicit CbYamlWriter(StringBuilderBase& InBuilder) : m_Builder(InBuilder) {} void WriteField(CbFieldView Field) { - ryml::NodeRef Node; + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); - if (m_IsFirst) + switch (Type) { - Node = Top(); + case CbFieldType::Object: + case CbFieldType::UniformObject: + WriteMapEntries(Field, 0); + break; + case CbFieldType::Array: + case CbFieldType::UniformArray: + WriteSeqEntries(Field, 0); + break; + default: + WriteScalarValue(Field); + m_Builder << '\n'; + break; + } + } + + void WriteMapEntry(CbFieldView Field, int32_t Indent) + { + WriteIndent(Indent); + WriteMapEntryContent(Field, Indent); + } + + void WriteSeqEntry(CbFieldView Field, int32_t Indent) + { + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); - m_IsFirst = false; + if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject) + { + bool First = true; + for (CbFieldView MapChild : Field) + { + if (First) + { + WriteIndent(Indent); + m_Builder << "- "; + First = false; + } + else + { + WriteIndent(Indent + 1); + } + WriteMapEntryContent(MapChild, Indent + 1); + } + } + else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray) + { + WriteIndent(Indent); + m_Builder << "-\n"; + WriteSeqEntries(Field, Indent + 1); } else { - Node = Top().append_child(); + WriteIndent(Indent); + m_Builder << "- "; + WriteScalarValue(Field); + m_Builder << '\n'; } + } - if (std::u8string_view Name = Field.GetU8Name(); !Name.empty()) +private: + void WriteMapEntries(CbFieldView MapField, int32_t Indent) + { + for (CbFieldView Child : MapField) { - Node.set_key_serialized(ryml::csubstr((const char*)Name.data(), Name.size())); + WriteIndent(Indent); + WriteMapEntryContent(Child, Indent); } + } + + void WriteMapEntryContent(CbFieldView Field, int32_t Indent) + { + std::u8string_view Name = Field.GetU8Name(); + m_Builder << std::string_view(reinterpret_cast(Name.data()), Name.size()); - switch (CbValue Accessor = Field.GetValue(); Accessor.GetType()) + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); + + if (IsContainer(Type)) { - case CbFieldType::Null: - Node.set_val("null"); - break; - case CbFieldType::Object: - case CbFieldType::UniformObject: - Node |= ryml::MAP; - m_NodeStack.push_back(Node); - for (CbFieldView It : Field) + m_Builder << ":\n"; + WriteFieldValue(Field, Indent + 1); + } + else + { + m_Builder << ": "; + WriteScalarValue(Field); + m_Builder << '\n'; + } + } + + void WriteSeqEntries(CbFieldView SeqField, int32_t Indent) + { + for (CbFieldView Child : SeqField) + { + CbValue Accessor = Child.GetValue(); + CbFieldType Type = Accessor.GetType(); + + if (Type == CbFieldType::Object || Type == CbFieldType::UniformObject) + { + bool First = true; + for (CbFieldView MapChild : Child) { - WriteField(It); + if (First) + { + WriteIndent(Indent); + m_Builder << "- "; + First = false; + } + else + { + WriteIndent(Indent + 1); + } + WriteMapEntryContent(MapChild, Indent + 1); } - m_NodeStack.pop_back(); + } + else if (Type == CbFieldType::Array || Type == CbFieldType::UniformArray) + { + WriteIndent(Indent); + m_Builder << "-\n"; + WriteSeqEntries(Child, Indent + 1); + } + else + { + WriteIndent(Indent); + m_Builder << "- "; + WriteScalarValue(Child); + m_Builder << '\n'; + } + } + } + + void WriteFieldValue(CbFieldView Field, int32_t Indent) + { + CbValue Accessor = Field.GetValue(); + CbFieldType Type = Accessor.GetType(); + + switch (Type) + { + case CbFieldType::Object: + case CbFieldType::UniformObject: + WriteMapEntries(Field, Indent); break; case CbFieldType::Array: case CbFieldType::UniformArray: - Node |= ryml::SEQ; - m_NodeStack.push_back(Node); - for (CbFieldView It : Field) - { - WriteField(It); - } - m_NodeStack.pop_back(); + WriteSeqEntries(Field, Indent); break; - case CbFieldType::Binary: - { - ExtendableStringBuilder<256> Builder; - const MemoryView Value = Accessor.AsBinary(); - ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); - - Node.set_key_serialized(Builder.c_str()); - } + case CbFieldType::CustomById: + WriteCustomById(Field.GetValue().AsCustomById(), Indent); break; - case CbFieldType::String: - { - const std::u8string_view U8String = Accessor.AsU8String(); - Node.set_val(ryml::csubstr((const char*)U8String.data(), U8String.size())); - } + case CbFieldType::CustomByName: + WriteCustomByName(Field.GetValue().AsCustomByName(), Indent); + break; + default: + WriteScalarValue(Field); + m_Builder << '\n'; + break; + } + } + + void WriteScalarValue(CbFieldView Field) + { + CbValue Accessor = Field.GetValue(); + switch (Accessor.GetType()) + { + case CbFieldType::Null: + m_Builder << "null"; + break; + case CbFieldType::BoolFalse: + m_Builder << "false"; + break; + case CbFieldType::BoolTrue: + m_Builder << "true"; break; case CbFieldType::IntegerPositive: - Node << Accessor.AsIntegerPositive(); + m_Builder << Accessor.AsIntegerPositive(); break; case CbFieldType::IntegerNegative: - Node << Accessor.AsIntegerNegative(); + m_Builder << Accessor.AsIntegerNegative(); break; case CbFieldType::Float32: if (const float Value = Accessor.AsFloat32(); std::isfinite(Value)) - { - Node << Value; - } + m_Builder.Append(fmt::format("{}", Value)); else - { - Node << "null"; - } + m_Builder << "null"; break; case CbFieldType::Float64: if (const double Value = Accessor.AsFloat64(); std::isfinite(Value)) - { - Node << Value; - } + m_Builder.Append(fmt::format("{}", Value)); else + m_Builder << "null"; + break; + case CbFieldType::String: { - Node << "null"; + const std::u8string_view U8String = Accessor.AsU8String(); + WriteString(std::string_view(reinterpret_cast(U8String.data()), U8String.size())); } break; - case CbFieldType::BoolFalse: - Node << "false"; - break; - case CbFieldType::BoolTrue: - Node << "true"; + case CbFieldType::Hash: + WriteString(Accessor.AsHash().ToHexString()); break; case CbFieldType::ObjectAttachment: case CbFieldType::BinaryAttachment: - Node << Accessor.AsAttachment().ToHexString(); - break; - case CbFieldType::Hash: - Node << Accessor.AsHash().ToHexString(); + WriteString(Accessor.AsAttachment().ToHexString()); break; case CbFieldType::Uuid: - Node << fmt::format("{}", Accessor.AsUuid()); + WriteString(fmt::format("{}", Accessor.AsUuid())); break; case CbFieldType::DateTime: - Node << DateTime(Accessor.AsDateTimeTicks()).ToIso8601(); + WriteString(DateTime(Accessor.AsDateTimeTicks()).ToIso8601()); break; case CbFieldType::TimeSpan: if (const TimeSpan Span(Accessor.AsTimeSpanTicks()); Span.GetDays() == 0) - { - Node << Span.ToString("%h:%m:%s.%n"); - } + WriteString(Span.ToString("%h:%m:%s.%n")); else - { - Node << Span.ToString("%d.%h:%m:%s.%n"); - } + WriteString(Span.ToString("%d.%h:%m:%s.%n")); break; case CbFieldType::ObjectId: - Node << fmt::format("{}", Accessor.AsObjectId()); + WriteString(fmt::format("{}", Accessor.AsObjectId())); break; - case CbFieldType::CustomById: - { - CbCustomById Custom = Accessor.AsCustomById(); + case CbFieldType::Binary: + WriteBase64(Accessor.AsBinary()); + break; + default: + ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); + break; + } + } - Node |= ryml::MAP; + void WriteCustomById(CbCustomById Custom, int32_t Indent) + { + WriteIndent(Indent); + m_Builder << "Id: "; + m_Builder.Append(fmt::format("{}", Custom.Id)); + m_Builder << '\n'; + + WriteIndent(Indent); + m_Builder << "Data: "; + WriteBase64(Custom.Data); + m_Builder << '\n'; + } - ryml::NodeRef IdNode = Node.append_child(); - IdNode.set_key("Id"); - IdNode.set_val_serialized(fmt::format("{}", Custom.Id)); + void WriteCustomByName(CbCustomByName Custom, int32_t Indent) + { + WriteIndent(Indent); + m_Builder << "Name: "; + WriteString(std::string_view(reinterpret_cast(Custom.Name.data()), Custom.Name.size())); + m_Builder << '\n'; + + WriteIndent(Indent); + m_Builder << "Data: "; + WriteBase64(Custom.Data); + m_Builder << '\n'; + } - ryml::NodeRef DataNode = Node.append_child(); - DataNode.set_key("Data"); + void WriteBase64(MemoryView Value) + { + ZEN_ASSERT(Value.GetSize() <= 512 * 1024 * 1024); + ExtendableStringBuilder<256> Buf; + const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); + const size_t EncodedIndex = Buf.AddUninitialized(size_t(EncodedSize)); + Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Buf.Data() + EncodedIndex); + WriteString(Buf.ToView()); + } - ExtendableStringBuilder<256> Builder; - const MemoryView& Value = Custom.Data; - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + void WriteString(std::string_view Str) + { + if (NeedsQuoting(Str)) + { + m_Builder << '\''; + for (char C : Str) + { + if (C == '\'') + m_Builder << "''"; + else + m_Builder << C; + } + m_Builder << '\''; + } + else + { + m_Builder << Str; + } + } - DataNode.set_val_serialized(Builder.c_str()); - } - break; - case CbFieldType::CustomByName: - { - CbCustomByName Custom = Accessor.AsCustomByName(); + void WriteIndent(int32_t Indent) + { + for (int32_t I = 0; I < Indent; ++I) + m_Builder << " "; + } - Node |= ryml::MAP; + static bool NeedsQuoting(std::string_view Str) + { + if (Str.empty()) + return false; - ryml::NodeRef NameNode = Node.append_child(); - NameNode.set_key("Name"); - std::string_view Name = std::string_view((const char*)Custom.Name.data(), Custom.Name.size()); - NameNode.set_val_serialized(std::string(Name)); + char First = Str[0]; + if (First == ' ' || First == '\n' || First == '\t' || First == '\r' || First == '*' || First == '&' || First == '%' || + First == '@' || First == '`') + return true; - ryml::NodeRef DataNode = Node.append_child(); - DataNode.set_key("Data"); + if (Str.size() >= 2 && Str[0] == '<' && Str[1] == '<') + return true; - ExtendableStringBuilder<256> Builder; - const MemoryView& Value = Custom.Data; - const uint32_t EncodedSize = Base64::GetEncodedDataSize(uint32_t(Value.GetSize())); - const size_t EncodedIndex = Builder.AddUninitialized(size_t(EncodedSize)); - Base64::Encode(static_cast(Value.GetData()), uint32_t(Value.GetSize()), Builder.Data() + EncodedIndex); + char Last = Str.back(); + if (Last == ' ' || Last == '\n' || Last == '\t' || Last == '\r') + return true; - DataNode.set_val_serialized(Builder.c_str()); - } - break; - default: - ZEN_ASSERT_FORMAT(false, "invalid field type: {}", uint8_t(Accessor.GetType())); - break; + for (char C : Str) + { + if (C == '#' || C == ':' || C == '-' || C == '?' || C == ',' || C == '\n' || C == '{' || C == '}' || C == '[' || C == ']' || + C == '\'' || C == '"') + return true; } - if (m_NodeStack.size() == 1) + return false; + } + + static bool IsContainer(CbFieldType Type) + { + switch (Type) { - std::string Yaml = ryml::emitrs_yaml(m_Tree); - m_StrBuilder << Yaml; + case CbFieldType::Object: + case CbFieldType::UniformObject: + case CbFieldType::Array: + case CbFieldType::UniformArray: + case CbFieldType::CustomById: + case CbFieldType::CustomByName: + return true; + default: + return false; } } -private: - StringBuilderBase& m_StrBuilder; - bool m_IsFirst = true; - - ryml::Tree m_Tree; - std::vector m_NodeStack; - ryml::NodeRef& Top() { return m_NodeStack.back(); } + StringBuilderBase& m_Builder; }; void @@ -229,6 +380,32 @@ CompactBinaryToYaml(const CbArrayView& Array, StringBuilderBase& Builder) Writer.WriteField(Array.AsFieldView()); } +void +CompactBinaryToYaml(MemoryView Data, StringBuilderBase& InBuilder) +{ + std::vector Fields = ReadCompactBinaryStream(Data); + if (Fields.empty()) + return; + + CbYamlWriter Writer(InBuilder); + if (Fields.size() == 1) + { + Writer.WriteField(Fields[0]); + return; + } + + if (Fields[0].HasName()) + { + for (const CbFieldView& Field : Fields) + Writer.WriteMapEntry(Field, 0); + } + else + { + for (const CbFieldView& Field : Fields) + Writer.WriteSeqEntry(Field, 0); + } +} + #if ZEN_WITH_TESTS void cbyaml_forcelink() diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index a3fd4dacb..9a67175a0 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -33,8 +33,6 @@ target('zencore') add_deps("timesinceprocessstart") add_deps("doctest") add_deps("fmt") - add_deps("ryml") - add_packages("json11") if is_plat("linux", "macosx") then diff --git a/thirdparty/ryml/.github/codeql.yml b/thirdparty/ryml/.github/codeql.yml deleted file mode 100644 index fa58c1d8b..000000000 --- a/thirdparty/ryml/.github/codeql.yml +++ /dev/null @@ -1,3 +0,0 @@ -query-filters: - - exclude: - id: cpp/unsigned-comparison-zero diff --git a/thirdparty/ryml/.github/release.sh b/thirdparty/ryml/.github/release.sh deleted file mode 100644 index 43eb6e71b..000000000 --- a/thirdparty/ryml/.github/release.sh +++ /dev/null @@ -1,127 +0,0 @@ -#!/bin/bash - - -# useful to iterate when fixing the release: -# ver=0.2.1 ; ( set -x ; git tag -d v$ver ; git push origin :v$ver ) ; (set -x ; set -e ; tbump --only-patch --non-interactive $ver ; git add -u ; git commit --amend --no-edit ; git tag --annotate --message "v$ver" "v$ver" ; git push -f --tags origin ) - - -function c4_release_create() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - c4_release_bump $ver ; \ - c4_release_commit $ver $branch \ - ) -} - -function c4_release_redo() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - c4_release_delete $ver ; \ - c4_release_bump $ver ; \ - c4_release_amend $ver $branch \ - ) -} - -function c4_release_bump() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - tbump --non-interactive --only-patch $ver \ - ) -} - -function c4_release_commit() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - tag=v$ver ; \ - git add -u ; \ - git commit -m $tag ; \ - git tag --annotate --message $tag $tag ; \ - ) -} - -function c4_release_amend() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - tag=v$ver ; \ - git add -u ; \ - git commit --amend -m $tag ; \ - git tag --annotate --message $tag $tag ; \ - ) -} - -function c4_release_delete() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - git tag -d v$ver ; \ - git push origin :v$ver \ - ) -} - -function c4_release_push() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - tag=v$ver ; \ - git push origin $branch ; \ - git push --tags origin $tag \ - ) -} - -function c4_release_force_push() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - tag=v$ver ; \ - git push -f origin $branch ; \ - git push -f --tags origin $tag \ - ) -} - -function _c4_validate_ver() -{ - ver=$1 - if [ -z "$ver" ] ; then \ - exit 1 - fi - ver=$(echo $ver | sed "s:v\(.*\):\1:") - #sver=$(echo $ver | sed "s:\([0-9]*\.[0-9]*\..[0-9]*\).*:\1:") - if [ ! -f changelog/$ver.md ] ; then \ - if [ -f changelog/current.md ] ; then - git mv changelog/current.md changelog/$ver.md - else - echo "ERROR: could not find changelog/$ver.md or changelog/current.md" - exit 1 - fi - fi - echo $ver -} - -function _c4_validate_branch() -{ - branch=$(git rev-parse --abbrev-ref HEAD) - if [ "$branch" != "master" ] ; then - echo "ERROR: release branch must be master" - exit 1 - fi - echo $branch -} diff --git a/thirdparty/ryml/.github/reqs.sh b/thirdparty/ryml/.github/reqs.sh deleted file mode 100644 index b1747747c..000000000 --- a/thirdparty/ryml/.github/reqs.sh +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env bash - -set -x - -# input environment variables: -# OS: the operating system -# CXX_: the compiler version. eg, g++-9 or clang++-6.0 -# BT: the build type -# VG: whether to install valgrind -# ARM: whether to arm cross-compiler and emulator -# GITHUB_WORKFLOW: when run from github -# API: whether to install swig -# CMANY: whether to install cmany - - - -#------------------------------------------------------------------------------- - -function c4_install_test_requirements() -{ - os=$1 - case "$os" in - ubuntu*) - c4_install_test_requirements_ubuntu - return 0 - ;; - macos*) - c4_install_test_requirements_macos - return 0 - ;; - win*) - c4_install_test_requirements_windows - return 0 - ;; - *) - return 0 - ;; - esac -} - -function c4_install_test_requirements_windows() -{ - if [ "$CMANY" == "ON" ] ; then - pip install cmany - fi - if [ "$API" == "ON" ] ; then - choco install swig - which swig - fi - # ensure chocolatey does not override cmake's cpack - which cpack - choco_cpack="/c/ProgramData/Chocolatey/bin/cpack.exe" - if [ -f $choco_cpack ] ; then - newname=$(echo $choco_cpack | sed 's:cpack:choco-cpack:') - mv -vf $choco_cpack $newname - fi - which cpack -} - -function c4_install_test_requirements_macos() -{ - if [ "$CMANY" == "ON" ] ; then - sudo pip3 install cmany - fi -} - -function c4_install_test_requirements_ubuntu() -{ - UBUNTU_RELEASE=$(lsb_release -rs) - UBUNTU_RELEASE_NAME=$(lsb_release -cs) - APT_PKG="" # all - PIP_PKG="" - c4_gather_test_requirements_ubuntu - echo "apt packages: $APT_PKG" - echo "pip packages: $PIP_PKG" - c4_install_test_requirements_ubuntu_impl - echo 'INSTALL COMPLETE!' -} - - -function c4_install_all_possible_requirements_ubuntu() -{ - export CXX_=all - export BT=Coverage - APT_PKG="" # all - PIP_PKG="" - sudo dpkg --add-architecture i386 - c4_gather_test_requirements_ubuntu - _c4_add_arm_compilers - echo "apt packages: $APT_PKG" - echo "pip packages: $PIP_PKG" - c4_install_test_requirements_ubuntu_impl - echo 'INSTALL COMPLETE!' -} - - -function c4_gather_test_requirements_ubuntu() -{ - if [ "$GITHUB_WORKFLOW" != "" ] ; then - sudo dpkg --add-architecture i386 - else - _add_apt build-essential - _add_apt cmake - fi - - _add_apt linux-libc-dev:i386 - _add_apt libc6:i386 - _add_apt libc6-dev:i386 - _add_apt libc6-dbg:i386 - - _c4_gather_compilers "$CXX_" - - _add_apt python3-setuptools - _add_apt python3-pip - - #_add_apt iwyu - #_add_apt cppcheck - #_add_pip cpplint - # oclint? - if [ "$VG" == "ON" ] ; then - _add_apt valgrind - fi - - if [ "$BT" == "Coverage" ]; then - _add_apt lcov - _add_apt libffi-dev - _add_apt libssl-dev - _add_pip requests[security] - _add_pip pyopenssl - _add_pip ndg-httpsclient - _add_pip pyasn1 - _add_pip cpp-coveralls - fi - - if [ "$CMANY" != "" ] ; then - _add_pip cmany - fi - - case "$CXX_" in - arm*) - _c4_add_arm_compilers - ;; - esac -} - - -function c4_install_test_requirements_ubuntu_impl() -{ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key 2>/dev/null | sudo apt-key add - - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - - sudo -E apt-add-repository --yes "deb https://apt.kitware.com/ubuntu/ $UBUNTU_RELEASE_NAME main" - sudo -E add-apt-repository --yes ppa:ubuntu-toolchain-r/test - - if [ "$APT_PKG" != "" ] ; then - #sudo -E apt-get clean - sudo -E apt-get update - sudo -E apt-get install -y --force-yes $APT_PKG - fi - - if [ "$PIP_PKG" != "" ]; then - sudo pip3 install $PIP_PKG - fi -} - - -#------------------------------------------------------------------------------- - -function _c4_add_arm_compilers() -{ - # this is going to be deprecated: - # https://askubuntu.com/questions/1243252/how-to-install-arm-none-eabi-gdb-on-ubuntu-20-04-lts-focal-fossa - sudo -E add-apt-repository --yes ppa:team-gcc-arm-embedded/ppa - - _add_apt gcc-arm-embedded - _add_apt g++-arm-linux-gnueabihf - _add_apt g++-multilib-arm-linux-gnueabihf - _add_apt qemu -} - - -function _c4_gather_compilers() -{ - cxx=$1 - case $cxx in - g++-12 ) _c4_addgcc 12 ;; - g++-11 ) _c4_addgcc 11 ;; - g++-10 ) _c4_addgcc 10 ;; - g++-9 ) _c4_addgcc 9 ;; - g++-8 ) _c4_addgcc 8 ;; - g++-7 ) _c4_addgcc 7 ;; - g++-6 ) _c4_addgcc 6 ;; - g++-5 ) _c4_addgcc 5 ;; - #g++-4.9 ) _c4_addgcc 4.9 ;; # https://askubuntu.com/questions/1036108/install-gcc-4-9-at-ubuntu-18-04 - g++-4.8 ) _c4_addgcc 4.8 ;; - clang++-14 ) _c4_addclang 14 ;; - clang++-13 ) _c4_addclang 13 ;; - clang++-12 ) _c4_addclang 12 ;; - clang++-11 ) _c4_addclang 11 ;; - clang++-10 ) _c4_addclang 10 ;; - clang++-9 ) _c4_addclang 9 ;; - clang++-8 ) _c4_addclang 8 ;; - clang++-7 ) _c4_addclang 7 ;; - clang++-6.0) _c4_addclang 6.0 ;; - clang++-5.0) _c4_addclang 5.0 ;; - clang++-4.0) _c4_addclang 4.0 ;; - clang++-3.9) _c4_addclang 3.9 ;; - all) - all="g++-12 g++-11 g++-10 g++-9 g++-8 g++-7 g++-6 g++-5 clang++-14 clang++-13 clang++-12 clang++-11 clang++-10 clang++-9 clang++-8 clang++-7 clang++-6.0 clang++-5.0 clang++-4.0 clang++-3.9" - echo "installing all compilers: $all" - for cxx in $all ; do - _c4_gather_compilers $cxx - done - ;; - "") - # use default compiler - ;; - arm*) - ;; - *) - echo "unknown compiler: $cxx" - exit 1 - ;; - esac -} - -# add a gcc compiler -function _c4_addgcc() -{ - gccversion=$1 - case $gccversion in - 5 ) - _add_apt gcc-5 "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" - _add_apt gcc-5 "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" - ;; - *) - ;; - esac - _add_apt g++-$gccversion - _add_apt g++-$gccversion-multilib - _add_apt libstdc++-$gccversion-dev - _add_apt lib32stdc++-$gccversion-dev -} - -# add a clang compiler -function _c4_addclang() -{ - clversion=$1 - case $clversion in - # in 18.04, clang9 and later require PPAs - 9 | 10 | 11 | 12 | 13 | 14) - _add_apt clang-$clversion "deb http://apt.llvm.org/$UBUNTU_RELEASE_NAME/ llvm-toolchain-$UBUNTU_RELEASE_NAME-$clversion main" - # libstdc++ is required - #_c4_addgcc 12 - _c4_addgcc 11 - _c4_addgcc 10 - _c4_addgcc 9 - ;; - "") - _add_apt clang - ;; - *) - _add_apt clang-$clversion - ;; - esac - _add_apt g++-multilib # this is required for 32 bit https://askubuntu.com/questions/1057341/unable-to-find-stl-headers-in-ubuntu-18-04 - _add_apt clang-tidy-$clversion -} - -# add libc++ -function _c4_addlibcxx() -{ - _add_apt clang - _add_apt libc++1 - _add_apt libc++abi-dev - _add_apt libc++-dev - #_add_apt libc++1:i386 - #_add_apt libc++abi-dev:i386 - #_add_apt libc++-dev:i386 -} - - -#------------------------------------------------------------------------------- - -# add a pip package to the list -function _add_pip() -{ - pkgs=$* - PIP_PKG="$PIP_PKG $pkgs" - echo "adding to pip packages: $pkgs" -} - -# add a debian package to the list -function _add_apt() -{ - pkgs=$1 - sourceslist=$2 - APT_PKG="$APT_PKG $pkgs" - echo "adding to apt packages: $pkgs" - _add_src "$sourceslist" "# for packages: $pkgs" -} - -# add an apt source -function _add_src() -{ - sourceslist=$1 - comment=$2 - if [ ! -z "$sourceslist" ] ; then - echo "adding apt source: $sourceslist" - sudo bash -c "cat >> /etc/apt/sources.list < $coverage_service" - cmake --build $build_dir --config $BT --target ${PROJ_PFX_TARGET}coverage-submit-$coverage_service -} - -# WIP -function c4_run_static_analysis() -{ - if _c4skipbitlink "$1" ; then return 0 ; fi - id=$1 - linktype=$(_c4linktype $id) - build_dir=`pwd`/build/$id - # https://blog.kitware.com/static-checks-with-cmake-cdash-iwyu-clang-tidy-lwyu-cpplint-and-cppcheck/ - pushd $PROJ_DIR -} - -function c4_cfg_test() -{ - if _c4skipbitlink "$1" ; then return 0 ; fi - id=$1 - # - build_dir=`pwd`/build/$id - install_dir=`pwd`/install/$id - mkdir -p $build_dir - mkdir -p $install_dir - # - if [ "$TOOLCHAIN" != "" ] ; then - toolchain_file=`pwd`/$TOOLCHAIN - if [ ! -f "$toolchain_file" ] ; then - echo "ERROR: toolchain not found: $toolchain_file" - exit 1 - fi - _addcmkflags -DCMAKE_TOOLCHAIN_FILE=$toolchain_file - else - bits=$(_c4bits $id) - linktype=$(_c4linktype $id) - case "$linktype" in - static) _addcmkflags -DBUILD_SHARED_LIBS=OFF ;; - shared) _addcmkflags -DBUILD_SHARED_LIBS=ON ;; - *) - echo "ERROR: unknown linktype: $linktype" - exit 1 - ;; - esac - fi - if [ "$STD" != "" ] ; then - _addcmkflags -DC4_CXX_STANDARD=$STD - _addprojflags CXX_STANDARD=$STD - fi - if [ "$LIBCXX" != "" ] ; then - _addprojflags USE_LIBCXX=$LIBCXX - fi - # - if [ "$DEV" != "OFF" ] ; then - _addprojflags DEV=ON - fi - case "$LINT" in - all ) _addprojflags LINT=ON LINT_TESTS=ON LINT_CLANG_TIDY=ON LINT_PVS_STUDIO=ON ;; - clang-tidy) _addprojflags LINT=ON LINT_TESTS=ON LINT_CLANG_TIDY=ON LINT_PVS_STUDIO=OFF ;; - pvs-studio) _addprojflags LINT=ON LINT_TESTS=ON LINT_CLANG_TIDY=OFF LINT_PVS_STUDIO=ON ;; - * ) _addprojflags LINT=OFF ;; - esac - case "$SAN" in - ALL) _addprojflags SANITIZE=ON ;; - A ) _addprojflags SANITIZE=ON ASAN=ON TSAN=OFF MSAN=OFF UBSAN=OFF ;; - T ) _addprojflags SANITIZE=ON ASAN=OFF TSAN=ON MSAN=OFF UBSAN=OFF ;; - M ) _addprojflags SANITIZE=ON ASAN=OFF TSAN=OFF MSAN=ON UBSAN=OFF ;; - UB ) _addprojflags SANITIZE=ON ASAN=OFF TSAN=OFF MSAN=OFF UBSAN=ON ;; - * ) _addprojflags SANITIZE=OFF ;; - esac - case "$SAN_ONLY" in - ON) _addprojflags SANITIZE_ONLY=ON ;; - * ) _addprojflags SANITIZE_ONLY=OFF ;; - esac - case "$VG" in - ON) _addprojflags VALGRIND=ON VALGRIND_SGCHECK=OFF ;; # FIXME SGCHECK should be ON - * ) _addprojflags VALGRIND=OFF VALGRIND_SGCHECK=OFF ;; - esac - case "$BM" in - ON) _addprojflags BUILD_BENCHMARKS=ON ;; - * ) _addprojflags BUILD_BENCHMARKS=OFF ;; - esac - if [ "$BT" == "Coverage" ] ; then - # the coverage repo tokens can be set in the travis environment: - # export CODECOV_TOKEN=....... - # export COVERALLS_REPO_TOKEN=....... - _addprojflags COVERAGE_CODECOV=ON COVERAGE_CODECOV_SILENT=ON - _addprojflags COVERAGE_COVERALLS=ON COVERAGE_COVERALLS_SILENT=OFF - fi - if [ ! -z "$VERBOSE_MAKEFILES" ] ; then - _addcmkflags -DCMAKE_VERBOSE_MAKEFILES=$VERBOSE_MAKEFILES - fi - _addcmkflags -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - if [ ! -z "$CMAKE_FLAGS" ] ; then - _addcmkflags $CMAKE_FLAGS - fi - - echo "building with additional cmake flags: $CMFLAGS" - - export C4_EXTERN_DIR=`pwd`/build/extern - mkdir -p $C4_EXTERN_DIR - - cmake --version - pwd - - # - # bash quote handling is a fiasco, and I could not find a way of storing - # quoted strings in variables and then expand the variables with correct quotes - # so we have to do this precious jewell of chicanery: - case "$CXX_" in - vs2022) - g='Visual Studio 17 2022' - case "$bits" in - 64) a=x64 ;; - 32) a=Win32 ;; - esac - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT -G "$g" -A $a $CMFLAGS - ;; - vs2019) - g='Visual Studio 16 2019' - case "$bits" in - 64) a=x64 ;; - 32) a=Win32 ;; - esac - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT -G "$g" -A $a $CMFLAGS - ;; - vs2017) - case "$bits" in - 64) g="Visual Studio 15 2017 Win64" ;; - 32) g="Visual Studio 15 2017" ;; - esac - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT -G "$g" $CMFLAGS - ;; - xcode) - g=Xcode - case "$bits" in - 64) a="x86_64" ;; - 32) a="i386" - exit 1 # i386 is deprecated in xcode - ;; - esac - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT -G "$g" -DCMAKE_OSX_ARCHITECTURES=$a $CMFLAGS - ;; - arm*|"") # make sure arm* comes before *g++ or *gcc* - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT $CMFLAGS - ;; - *mingw*) - export CC_=$(echo "$CXX_" | sed 's:clang++:clang:g' | sed 's:g++:gcc:g') - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -G "MinGW Makefiles" \ - -DCMAKE_BUILD_TYPE=$BT $CMFLAGS \ - -DCMAKE_C_COMPILER=$CC_ -DCMAKE_CXX_COMPILER=$CXX_ \ - -DCMAKE_C_FLAGS="-m$bits" -DCMAKE_CXX_FLAGS="-m$bits" - cmake --build $build_dir --target help | sed 1d | sort - ;; - *g++*|*gcc*|*clang*) - export CC_=$(echo "$CXX_" | sed 's:clang++:clang:g' | sed 's:g++:gcc:g') - _c4_choose_clang_tidy $CXX_ - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT $CMFLAGS \ - -DCMAKE_C_COMPILER=$CC_ -DCMAKE_CXX_COMPILER=$CXX_ \ - -DCMAKE_C_FLAGS="-m$bits" -DCMAKE_CXX_FLAGS="-m$bits" - cmake --build $build_dir --target help | sed 1d | sort - ;; - em++) - emcmake cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT $CMFLAGS \ - -DCMAKE_CXX_FLAGS="-s DISABLE_EXCEPTION_CATCHING=0" \ - -DRYML_TEST_TOOLS=OFF - ;; - *) - echo "unknown compiler" - exit 1 - ;; - esac -} - -function _c4_choose_clang_tidy() -{ - cxx=$1 - # only for clang compilers. - case $cxx in - clang*) - # try with version first - clang_tidy_ver=$(echo $cxx | sed "s:++:-tidy:") - clang_tidy=$(echo $cxx | sed "s:++.*:-tidy:") - for n in $clang_tidy_ver $clang_tidy ; do - exe=$(which $n) - echo "searching for $n: $exe" - if [ -z "$exe" ] ; then - echo "could not find $clang_tidy" - else - _addcmkflags "-DCLANG_TIDY=$exe" - return 0 - fi - done - echo "error: could not find clang-tidy for $cxx" - exit 1 - ;; - esac -} - -# add cmake flags without project prefix -function _addcmkflags() -{ - for f in $* ; do - CMFLAGS="$CMFLAGS ${f}" - done -} - -# add cmake flags with project prefix -function _addprojflags() -{ - for f in $* ; do - CMFLAGS="$CMFLAGS -D${PROJ_PFX_CMAKE}${f}" - done -} - -function _c4_parallel_build_flags() -{ - case "$CXX_" in - vs2022|vs2019|vs2017|vs2015) - # https://docs.microsoft.com/en-us/visualstudio/msbuild/msbuild-command-line-reference?view=vs-2019 - # https://stackoverflow.com/questions/2619198/how-to-get-number-of-cores-in-win32 - if [ -z "$NUM_JOBS_BUILD" ] ; then - echo "/maxcpucount:$NUMBER_OF_PROCESSORS" - else - echo "/maxcpucount:$NUM_JOBS_BUILD" - fi - ;; - xcode) - # https://stackoverflow.com/questions/5417835/how-to-modify-the-number-of-parallel-compilation-with-xcode - # https://gist.github.com/nlutsenko/ee245fbd239087d22137 - if [ -z "$NUM_JOBS_BUILD" ] ; then - echo "-IDEBuildOperationMaxNumberOfConcurrentCompileTasks=$(sysctl -n hw.ncpu)" - else - echo "-IDEBuildOperationMaxNumberOfConcurrentCompileTasks=$NUM_JOBS_BUILD" - fi - ;; - *g++*|*gcc*|*clang*|em++) - if [ -z "$NUM_JOBS_BUILD" ] ; then - echo "-j $(nproc)" - else - echo "-j $NUM_JOBS_BUILD" - fi - ;; - "") # allow empty compiler - ;; - *) - echo "unknown compiler" - exit 1 - ;; - esac -} - -function _c4_generator_build_flags() -{ - case "$CXX_" in - vs2022|vs2019|vs2017|vs2015) - ;; - xcode) - # WTF??? - # https://github.com/biojppm/rapidyaml/pull/97/checks?check_run_id=1504677928#step:7:964 - # https://stackoverflow.com/questions/51153525/xcode-10-unable-to-attach-db-error - echo "-UseModernBuildSystem=NO" - ;; - *g++*|*gcc*|*clang*|em++) - ;; - "") # allow empty compiler - ;; - *) - echo "unknown compiler" - exit 1 - ;; - esac -} diff --git a/thirdparty/ryml/.github/workflows/benchmarks.yml b/thirdparty/ryml/.github/workflows/benchmarks.yml deleted file mode 100644 index 61654643f..000000000 --- a/thirdparty/ryml/.github/workflows/benchmarks.yml +++ /dev/null @@ -1,147 +0,0 @@ -name: benchmarks - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: -DRYML_TEST_SUITE=ON - NUM_JOBS_BUILD: # 4 - - -jobs: - - gettag: - runs-on: ubuntu-latest - steps: - # use fetch-depth to ensure all tags are fetched - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}} - - name: Variables (from tag) - if: contains(github.ref, 'tags/v') - run: | - # https://github.community/t/how-to-get-just-the-tag-name/16241/11 - SRC_TAG=${GITHUB_REF#refs/tags/} - SRC_VERSION=${GITHUB_REF#refs/tags/v} - cat < vars.sh - export SRC_TAG=$SRC_TAG - export SRC_VERSION=$SRC_VERSION - EOF - - name: Variables (from commit, no tag) - if: ${{ !contains(github.ref, 'tags/v') }} - run: | - set -x - branch_name=${GITHUB_REF#refs/heads/} - # builds triggered from PRs have the branch_name like this: refs/pull/150/merge - # so filter to eg pr0150_merge - branch_name=`echo $branch_name | sed "s:refs/pull/\([0-9]*\)/\(.*\):pr0\1_\2:"` - # sanitize the branch name; eg merge/foo-bar -> merge_foo_bar - branch_name=`echo $branch_name | sed 's:[/.-]:_:g'` - git config --global --add safe.directory $(pwd) - SRC_TAG=$(git describe || git rev-parse --short HEAD) # eg v0.2.0-110-gda837e0 - SRC_VERSION="${branch_name}-${SRC_TAG}" - cat < vars.sh - export SRC_TAG=$SRC_TAG - export SRC_VERSION=$SRC_VERSION - EOF - - name: Verify vars.sh - run: cat vars.sh ; source vars.sh ; echo $SRC_TAG ; echo $SRC_VERSION - - name: Save vars.sh - uses: actions/upload-artifact@v3 - with: {name: vars.sh, path: ./vars.sh} - - benchmarks: - name: bm/c++${{matrix.std}}/${{matrix.cxx}}/${{matrix.bt}} - needs: gettag - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip benchmarks')) || - contains(github.event.head_commit.message, 'only benchmarks') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 11, cxx: g++-10, bt: Release, os: ubuntu-20.04, bitlinks: static64 static32} - - {std: 17, cxx: g++-10, bt: Release, os: ubuntu-20.04, bitlinks: static64 static32} - - {std: 20, cxx: g++-10, bt: Release, os: ubuntu-20.04, bitlinks: static64 static32} - # - - {std: 17, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: static64 static32} - - {std: 20, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: static64 static32} - - {std: 17, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: static64 static32} - - {std: 20, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: static64 static32} - # - - {std: 17, cxx: xcode, xcver: 13, bt: Release, os: macos-11, bitlinks: static64} - env: {BM: ON, STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - # use fetch-depth to ensure all tags are fetched - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - name: Download vars.sh - uses: actions/download-artifact@v3 - with: {name: vars.sh, path: ./} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: Install python 3.10 for plotting - uses: actions/setup-python@v4 - with: { python-version: '3.10' } - - name: install benchmark plotting dependencies - run: | - which python - which pip - python --version - pip --version - pip install -v -r ext/c4core/cmake/bm-xp/requirements.txt - python -c 'import munch ; print("ok!") ; exit(0)' - echo $? - - name: shared64-configure--------------------------------------------------- - run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_target shared64 ryml-bm-build} - - {name: shared64-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared64 ryml-bm-run} - - {name: shared64-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared64 ryml-bm-plot} - - name: static64-configure--------------------------------------------------- - run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_target static64 ryml-bm-build} - - {name: static64-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static64 ryml-bm-run} - - {name: static64-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static64 ryml-bm-plot} - - name: static32-configure--------------------------------------------------- - run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_target static32 ryml-bm-build} - - {name: static32-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static32 ryml-bm-run} - - {name: static32-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static32 ryml-bm-plot} - - name: shared32-configure--------------------------------------------------- - run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_target shared32 ryml-bm-build} - - {name: shared32-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared32 ryml-bm-run} - - {name: shared32-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared32 ryml-bm-plot} - - name: gather benchmark results - run: | - set -x - source vars.sh - echo SRC_TAG=$SRC_TAG - echo SRC_VERSION=$SRC_VERSION - desc=$SRC_TAG - for bl in ${{matrix.bitlinks}} ; do - dst=$(echo ryml-benchmark_results/$desc/x86_64/${{matrix.cxx}}-${{matrix.bt}}-c++${{matrix.std}}-$bl | sed 's:++-:xx:g' | sed 's:+:x:g') - mkdir -p $dst - find build -name bm-results - mv -vf build/$bl/bm/bm-results/* $dst/. - done - - name: upload benchmark result artifacts - uses: actions/upload-artifact@v3 - with: - name: ryml-benchmark_results - path: ryml-benchmark_results/ diff --git a/thirdparty/ryml/.github/workflows/clang.yml b/thirdparty/ryml/.github/workflows/clang.yml deleted file mode 100644 index f0b7ad009..000000000 --- a/thirdparty/ryml/.github/workflows/clang.yml +++ /dev/null @@ -1,262 +0,0 @@ -name: clang - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: -DRYML_TEST_SUITE=ON - NUM_JOBS_BUILD: # 4 - - -jobs: - clang_canary: - name: ${{matrix.cxx}}/canary/c++${{matrix.std}}/${{matrix.bt}} - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip clang')) || - contains(github.event.head_commit.message, 'only clang') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 17, cxx: clang++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 17, cxx: clang++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 20, cxx: clang++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 20, cxx: clang++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: clang++-6.0, bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: clang++-6.0, bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", - CMAKE_FLAGS: "${{matrix.cmkflags}}", - VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - clang_canary_tabtokens: - name: tabtokens/${{matrix.cxx}}/canary/c++${{matrix.std}}/${{matrix.bt}} - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip clang')) || - contains(github.event.head_commit.message, 'only clang') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 17, cxx: clang++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: static64, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 17, cxx: clang++-10 , bt: Release, os: ubuntu-20.04, bitlinks: static64, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 20, cxx: clang++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: static64, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 20, cxx: clang++-10 , bt: Release, os: ubuntu-20.04, bitlinks: static64, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 11, cxx: clang++-6.0, bt: Debug , os: ubuntu-20.04, bitlinks: static64, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 11, cxx: clang++-6.0, bt: Release, os: ubuntu-20.04, bitlinks: static64, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", - CMAKE_FLAGS: "${{matrix.cmkflags}}", - VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - #---------------------------------------------------------------------------- - clang_extended: - name: ${{matrix.cxx}}/extended/c++${{matrix.std}}/${{matrix.bt}} - needs: clang_canary - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip clang')) || - contains(github.event.head_commit.message, 'only clang') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 11, cxx: clang++-10 , bt: Debug , vg: on, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10 , bt: Release, vg: on, os: ubuntu-20.04} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", - CMAKE_FLAGS: "${{matrix.cmkflags}}", - VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - #---------------------------------------------------------------------------- - clang_sanitize: - name: ${{matrix.cxx}}/san/c++${{matrix.std}}/${{matrix.bt}} - needs: clang_canary - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip clang')) || - contains(github.event.head_commit.message, 'only clang') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - # these jobs take much longer, so run only one bitlink pair per job to profit from parallelism - - {std: 11, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: shared64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: static64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: shared32, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: static32, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: shared64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: static64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: shared32, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: static32, os: ubuntu-20.04} - - {std: 17, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: shared64, os: ubuntu-20.04} - - {std: 17, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: static64, os: ubuntu-20.04} - - {std: 17, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: shared32, os: ubuntu-20.04} - - {std: 17, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: static32, os: ubuntu-20.04} - - {std: 17, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: shared64, os: ubuntu-20.04} - - {std: 17, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: static64, os: ubuntu-20.04} - - {std: 17, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: shared32, os: ubuntu-20.04} - - {std: 17, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: static32, os: ubuntu-20.04} - - {std: 20, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: shared64, os: ubuntu-20.04} - - {std: 20, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: static64, os: ubuntu-20.04} - - {std: 20, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: shared32, os: ubuntu-20.04} - - {std: 20, cxx: clang++-10, bt: Debug , vg: ON, san: ALL, bitlinks: static32, os: ubuntu-20.04} - - {std: 20, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: shared64, os: ubuntu-20.04} - - {std: 20, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: static64, os: ubuntu-20.04} - - {std: 20, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: shared32, os: ubuntu-20.04} - - {std: 20, cxx: clang++-10, bt: Release, vg: ON, san: ALL, bitlinks: static32, os: ubuntu-20.04} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - -# #---------------------------------------------------------------------------- -# # https://blog.kitware.com/static-checks-with-cmake-cdash-iwyu-clang-tidy-lwyu-cpplint-and-cppcheck/ -# static_analysis: -# if: | -# (!contains(github.event.head_commit.message, 'skip all')) || -# (!contains(github.event.head_commit.message, 'skip clang')) || -# contains(github.event.head_commit.message, 'only clang') -# continue-on-error: true -# runs-on: ${{matrix.os}} -# strategy: -# fail-fast: false -# matrix: -# include: -# # these jobs take much longer, so run only one bitlink pair per job to profit from parallelism -# - {std: 11, cxx: clang++-10, bt: Debug , bitlinks: shared64, os: ubuntu-20.04} -# - {std: 11, cxx: clang++-10, bt: Release, bitlinks: shared64, os: ubuntu-20.04} -# - {std: 14, cxx: clang++-10, bt: Debug , bitlinks: shared64, os: ubuntu-20.04} -# - {std: 14, cxx: clang++-10, bt: Release, bitlinks: shared64, os: ubuntu-20.04} -# - {std: 17, cxx: clang++-10, bt: Debug , bitlinks: shared64, os: ubuntu-20.04} -# - {std: 17, cxx: clang++-10, bt: Release, bitlinks: shared64, os: ubuntu-20.04} -# - {std: 20, cxx: clang++-10, bt: Debug , bitlinks: shared64, os: ubuntu-20.04} -# - {std: 20, cxx: clang++-10, bt: Release, bitlinks: shared64, os: ubuntu-20.04} -# env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} -# steps: -# - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} -# - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} -# - {name: show info, run: source .github/setenv.sh && c4_show_info} -# - name: shared64-configure--------------------------------------------------- -# run: source .github/setenv.sh && c4_cfg_test shared64 -# - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} -# - {name: clang-tidy, run: cmake "-DCMAKE_CXX_CLANG_TIDY=/usr/bin/clang-tidy-3.9;-checks=*" ../path/to/source} -# - {name: cppcheck, run: cmake "-DCMAKE_CXX_CPPCHECK=/usr/bin/cppcheck;--std=c++11" ../path/to/source} -# - {name: cpplint, run: cmake "-DCMAKE_CXX_CPPLINT=/usr/local/bin/cpplint;--linelength=179" ..} -# - {name: include-what-you-use, run: cmake "-DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=/usr/bin/iwyu;--transitive_includes_only" ..} -# - {name: link-what-you-use, run: cmake -DCMAKE_LINK_WHAT_YOU_USE=TRUE ..} diff --git a/thirdparty/ryml/.github/workflows/clang_tidy.yml b/thirdparty/ryml/.github/workflows/clang_tidy.yml deleted file mode 100644 index 5c9a7d872..000000000 --- a/thirdparty/ryml/.github/workflows/clang_tidy.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: clang_tidy - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: -DRYML_TEST_SUITE=ON - NUM_JOBS_BUILD: # 4 - - -jobs: - clang_tidy: - name: ${{matrix.cxx}}/tidy/c++${{matrix.std}}/${{matrix.bt}} - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip clang')) || - contains(github.event.head_commit.message, 'only clang') - continue-on-error: false - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - # clang tidy takes a long time, so don't do multiple bits/linktypes - - {std: 11, cxx: clang++-10, bt: Debug , lint: clang-tidy, bitlinks: shared64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Debug , lint: clang-tidy, bitlinks: static64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Debug , lint: clang-tidy, bitlinks: shared32, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: Debug , lint: clang-tidy, bitlinks: static32, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: ReleaseWithDebInfo, lint: clang-tidy, bitlinks: shared64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: ReleaseWithDebInfo, lint: clang-tidy, bitlinks: static64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: ReleaseWithDebInfo, lint: clang-tidy, bitlinks: shared32, os: ubuntu-20.04} - - {std: 11, cxx: clang++-10, bt: ReleaseWithDebInfo, lint: clang-tidy, bitlinks: static32, os: ubuntu-20.04} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} diff --git a/thirdparty/ryml/.github/workflows/codeql.yml b/thirdparty/ryml/.github/workflows/codeql.yml deleted file mode 100644 index 3ad8a5819..000000000 --- a/thirdparty/ryml/.github/workflows/codeql.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: "CodeQL" - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - schedule: - - cron: "41 14 * * 5" - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ cpp ] - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - config-file: ./.github/codeql.yml - queries: +security-and-quality - - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:${{ matrix.language }}" diff --git a/thirdparty/ryml/.github/workflows/coverage.yml b/thirdparty/ryml/.github/workflows/coverage.yml deleted file mode 100644 index 52da02c56..000000000 --- a/thirdparty/ryml/.github/workflows/coverage.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: coverage - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: -DRYML_TEST_SUITE=ON - NUM_JOBS_BUILD: # 4 - - -jobs: - #---------------------------------------------------------------------------- - coverage: - name: coverage/c++${{matrix.std}} - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip coverage')) || - contains(github.event.head_commit.message, 'only coverage') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 11, cxx: g++-9, bt: Coverage, os: ubuntu-20.04} - - {std: 17, cxx: g++-9, bt: Coverage, os: ubuntu-20.04} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}", - CODECOV_TOKEN: "${{secrets.CODECOV_TOKEN}}", - COVERALLS_REPO_TOKEN: "${{secrets.COVERALLS_REPO_TOKEN}}", - # coveralls disabled: https://github.com/lemurheavy/coveralls-public/issues/1665 - # https://docs.coveralls.io/parallel-build-webhook - #COVERALLS_PARALLEL: true - } - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-coverage, run: source .github/setenv.sh && c4_run_target static64 ryml-coverage} - - name: static64-coverage-artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-static64-cxx${{matrix.std}} - path: | - build/static64/lcov - build/static64/coverage3-final_filtered.lcov - - {name: static64-submit-codecov, run: source .github/setenv.sh && c4_submit_coverage static64 codecov} - # coveralls disabled: https://github.com/lemurheavy/coveralls-public/issues/1665 - #- {name: static64-submit-coveralls, run: "if [ \"${{matrix.std}}\" == \"17\" ] ; then ( source .github/setenv.sh && c4_submit_coverage static64 coveralls ) ; fi" } - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-coverage, run: source .github/setenv.sh && c4_run_target static32 ryml-coverage} - - name: static32-coverage-artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-static32-cxx${{matrix.std}} - path: | - build/static32/lcov - build/static32/coverage3-final_filtered.lcov - - {name: static32-submit-codecov, run: source .github/setenv.sh && c4_submit_coverage static32 codecov} - # coveralls disabled: https://github.com/lemurheavy/coveralls-public/issues/1665 - #- {name: static32-submit-coveralls, run: source .github/setenv.sh && c4_submit_coverage static32 coveralls} - - # coveralls disabled: https://github.com/lemurheavy/coveralls-public/issues/1665 - # https://github.com/lemurheavy/coveralls-public/issues/1665 - #coveralls_finish: - # needs: [coverage] - # runs-on: ubuntu-latest - # steps: - # - name: coveralls-notify - # #continue-on-error: true - # uses: coverallsapp/github-action@master - # with: - # github-token: ${{ secrets.github_token }} - # parallel-finished: true - diff --git a/thirdparty/ryml/.github/workflows/emscripten.yml b/thirdparty/ryml/.github/workflows/emscripten.yml deleted file mode 100644 index bc920d7ca..000000000 --- a/thirdparty/ryml/.github/workflows/emscripten.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: emscripten - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: '-DRYML_TEST_SUITE=OFF' - NUM_JOBS_BUILD: # 4 - EMSCRIPTEN_CACHE_FOLDER: 'emsdk-cache' - -jobs: - - #---------------------------------------------------------------------------- - emscripten: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip emscripten')) || - contains(github.event.head_commit.message, 'only emscripten') - name: emscripten/${{matrix.emver}}/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - #- {std: 11, cxx: em++, emver: 2.0.34, bt: Debug , os: ubuntu-latest, bitlinks: static32} - - {std: 11, cxx: em++, emver: 2.0.34, bt: Release, os: ubuntu-latest, bitlinks: static32} - #- {std: 20, cxx: em++, emver: 2.0.34, bt: Debug , os: ubuntu-latest, bitlinks: static32} - - {std: 20, cxx: em++, emver: 2.0.34, bt: Release, os: ubuntu-latest, bitlinks: static32} - #- {std: 11, cxx: em++, emver: 3.0.0 , bt: Debug , os: ubuntu-latest, bitlinks: static32} - - {std: 11, cxx: em++, emver: 3.0.0 , bt: Release, os: ubuntu-latest, bitlinks: static32} - #- {std: 20, cxx: em++, emver: 3.0.0 , bt: Debug , os: ubuntu-latest, bitlinks: static32} - - {std: 20, cxx: em++, emver: 3.0.0 , bt: Release, os: ubuntu-latest, bitlinks: static32} - env: - STD: "${{matrix.std}}" - CXX_: "${{matrix.cxx}}" - BT: "${{matrix.bt}}" - BITLINKS: "${{matrix.bitlinks}}" - VG: "${{matrix.vg}}" - SAN: "${{matrix.san}}" - LINT: "${{matrix.lint}}" - OS: "${{matrix.os}}" - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - name: setup emscripten cache - id: cache-system-libraries - uses: actions/cache@v3 - with: {path: "${{env.EMSCRIPTEN_CACHE_FOLDER}}", key: "${{matrix.emver}}-${{runner.os}}"} - - name: setup emscripten - uses: mymindstorm/setup-emsdk@v11 - with: {version: "${{matrix.emver}}", actions-cache-folder: "${{env.EMSCRIPTEN_CACHE_FOLDER}}"} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} diff --git a/thirdparty/ryml/.github/workflows/gcc.yml b/thirdparty/ryml/.github/workflows/gcc.yml deleted file mode 100644 index fd0337ed7..000000000 --- a/thirdparty/ryml/.github/workflows/gcc.yml +++ /dev/null @@ -1,168 +0,0 @@ -name: gcc - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: -DRYML_TEST_SUITE=ON - NUM_JOBS_BUILD: # 4 - - -jobs: - gcc_canary: - name: ${{matrix.cxx}}/canary/${{matrix.bt}} - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip gcc')) || - contains(github.event.head_commit.message, 'only gcc') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 11, cxx: g++-7 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-7 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 20, cxx: g++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 20, cxx: g++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-5 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-5 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-4.8 , bt: Debug, os: ubuntu-18.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-4.8 , bt: Release, os: ubuntu-18.04, bitlinks: shared64 static32} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - gcc_tabtokens: - name: tabtokens/${{matrix.cxx}}/canary/${{matrix.bt}} - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip gcc')) || - contains(github.event.head_commit.message, 'only gcc') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 11, cxx: g++-7 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 11, cxx: g++-7 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 20, cxx: g++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 20, cxx: g++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 11, cxx: g++-5 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - - {std: 11, cxx: g++-5 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32, cmkflags: "-DRYML_WITH_TAB_TOKENS=ON"} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", - CMAKE_FLAGS: "${{matrix.cmkflags}}", - VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - #---------------------------------------------------------------------------- - gcc_extended: - name: ${{matrix.cxx}}/extended/${{matrix.bt}} - needs: gcc_canary - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip gcc')) || - contains(github.event.head_commit.message, 'only gcc') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - # VALGRIND - - {std: 11, cxx: g++-10, bt: Debug , vg: ON, os: ubuntu-20.04} - - {std: 11, cxx: g++-10, bt: Release, vg: ON, os: ubuntu-20.04} - - {std: 14, cxx: g++-10, bt: Debug , vg: ON, os: ubuntu-20.04} - - {std: 14, cxx: g++-10, bt: Release, vg: ON, os: ubuntu-20.04} - - {std: 17, cxx: g++-10, bt: Debug , vg: ON, os: ubuntu-20.04} - - {std: 17, cxx: g++-10, bt: Release, vg: ON, os: ubuntu-20.04} - - {std: 20, cxx: g++-10, bt: Debug , vg: ON, os: ubuntu-20.04} - - {std: 20, cxx: g++-10, bt: Release, vg: ON, os: ubuntu-20.04} - - {std: 11, cxx: g++-4.8, bt: Debug, vg: ON, os: ubuntu-18.04} - - {std: 11, cxx: g++-4.8, bt: Release, vg: ON, os: ubuntu-18.04} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} diff --git a/thirdparty/ryml/.github/workflows/install.yml b/thirdparty/ryml/.github/workflows/install.yml deleted file mode 100644 index b8b74c682..000000000 --- a/thirdparty/ryml/.github/workflows/install.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: test_install - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - -jobs: - - #---------------------------------------------------------------------------- - install_tests: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip test_install')) || - contains(github.event.head_commit.message, 'only test_install') - name: ${{matrix.name}}/${{matrix.bt}} - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {name: find_package/linux , sdir: test/test_install , os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Release, vars: "-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/linux , sdir: test/test_install , os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Debug , vars: "-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/linux/libcxx, sdir: test/test_install , os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Release, vars: "-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: "-DRYML_USE_LIBCXX=ON"} - - {name: find_package/linux/libcxx, sdir: test/test_install , os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Debug , vars: "-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: "-DRYML_USE_LIBCXX=ON"} - - {name: find_package/macos , sdir: test/test_install , os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Release, vars: "-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/macos , sdir: test/test_install , os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Debug , vars: "-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/ryml -DRYML_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/win , sdir: test/test_install , os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Release, vars: "-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/cmake -DRYML_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/win , sdir: test/test_install , os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Debug , vars: "-Dryml_DIR=$GITHUB_WORKSPACE/$PDIR/cmake -DRYML_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - # - - {name: find_library/linux , sdir: test/test_install , os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Release, vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/linux , sdir: test/test_install , os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Debug , vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/linux/libcxx, sdir: test/test_install , os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Release, vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: "-DRYML_USE_LIBCXX=ON"} - - {name: find_library/linux/libcxx, sdir: test/test_install , os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Debug , vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: "-DRYML_USE_LIBCXX=ON"} - - {name: find_library/macos , sdir: test/test_install , os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Release, vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/macos , sdir: test/test_install , os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Debug , vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/win , sdir: test/test_install , os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Release, vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/win , sdir: test/test_install , os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Debug , vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DRYML_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - # - - {name: singleheader/linux , sdir: test/test_singleheader, os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Release, vars: , commonvars: } - - {name: singleheader/linux , sdir: test/test_singleheader, os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Debug , vars: , commonvars: } - - {name: singleheader/linux/libcxx, sdir: test/test_singleheader, os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Release, vars: , commonvars: "-DRYML_USE_LIBCXX=ON"} - - {name: singleheader/linux/libcxx, sdir: test/test_singleheader, os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Debug , vars: , commonvars: "-DRYML_USE_LIBCXX=ON"} - - {name: singleheader/macos , sdir: test/test_singleheader, os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Release, vars: , commonvars: } - - {name: singleheader/macos , sdir: test/test_singleheader, os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Debug , vars: , commonvars: } - - {name: singleheader/win , sdir: test/test_singleheader, os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Release, vars: , commonvars: } - - {name: singleheader/win , sdir: test/test_singleheader, os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Debug , vars: , commonvars: } - env: - CXX_: "${{matrix.cxx}}" - BT: "${{matrix.bt}}" - OS: "${{matrix.os}}" - BDIR: "build/${{matrix.name}}-${{matrix.bt}}" - IDIR: "install/${{matrix.name}}-${{matrix.bt}}" - PDIR: "prefix/${{matrix.name}}-${{matrix.bt}}" - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: Install python 3.9 - uses: actions/setup-python@v4 - with: { python-version: 3.9 } - - name: preinstall - run: | - if [ "${{matrix.sdir}}" == "test/test_install" ] ; then - mkdir -p $BDIR-staging - cmake -S . -B $BDIR-staging -DCMAKE_INSTALL_PREFIX=$PDIR -DCMAKE_BUILD_TYPE=${{matrix.bt}} ${{matrix.gen}} ${{matrix.commonvars}} - cmake --build $BDIR-staging --config ${{matrix.bt}} --target ${{matrix.tgt}} -j - cmake --build $BDIR-staging --config ${{matrix.bt}} --target install - fi - - name: configure - run: | - mkdir -p $BDIR - mkdir -p $IDIR - cmake -S ${{matrix.sdir}} -B $BDIR \ - -DRYML_BUILD_TESTS=ON \ - -DRYML_VALGRIND=OFF \ - -DCMAKE_BUILD_TYPE=${{matrix.bt}} \ - -DCMAKE_INSTALL_PREFIX=$IDIR \ - ${{matrix.gen}} \ - ${{matrix.vars}} \ - ${{matrix.commonvars}} - - name: build - run: | - cmake --build $BDIR --config ${{matrix.bt}} --target ryml-test-build -j - - name: run - run: | - cmake --build $BDIR --config ${{matrix.bt}} --target ryml-test-run diff --git a/thirdparty/ryml/.github/workflows/macosx.yml b/thirdparty/ryml/.github/workflows/macosx.yml deleted file mode 100644 index a9fb97ffb..000000000 --- a/thirdparty/ryml/.github/workflows/macosx.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: macosx - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: -DRYML_TEST_SUITE=ON - NUM_JOBS_BUILD: # 4 - - -# ubuntu-20.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.2.0 -# clang: 8.0.1, 9.0.1, 10.0.0 -# ubuntu-18.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.1.0 -# clang: 6.0.0, 8.0.0, 9.0.0 -# macos-11.0: macOS Big Sur 11.0 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 10.0.1 -# gcc-8 gcc-9 -# macos-10.15: macOS Catalina 10.15 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 11.0.0 -# gcc-8 gcc-9 -# windows-2019: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md -# vs2019 -# windows-2016: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md -# vs2017 -jobs: - - #---------------------------------------------------------------------------- - macosx: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip macosx')) || - contains(github.event.head_commit.message, 'only macosx') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 11, cxx: xcode, xcver: 13, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 11, cxx: xcode, xcver: 13, bt: Release, os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 13, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 13, bt: Release, os: macos-11, bitlinks: shared64 static64} - # - - {std: 11, cxx: xcode, xcver: 12, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 11, cxx: xcode, xcver: 12, bt: Release, os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 12, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 12, bt: Release, os: macos-11, bitlinks: shared64 static64} - # - - {std: 11, cxx: xcode, xcver: 11, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 11, cxx: xcode, xcver: 11, bt: Release, os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 11, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 11, bt: Release, os: macos-11, bitlinks: shared64 static64} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: xcode, uses: maxim-lobanov/setup-xcode@v1, with: {xcode-version: "${{matrix.xcver}}" }} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} diff --git a/thirdparty/ryml/.github/workflows/rarearchs.yml b/thirdparty/ryml/.github/workflows/rarearchs.yml deleted file mode 100644 index 8306b54ae..000000000 --- a/thirdparty/ryml/.github/workflows/rarearchs.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: rarearchs - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - rarearchs: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip rarearchs')) || - contains(github.event.head_commit.message, 'only rarearchs') - name: ${{matrix.arch}}/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - include: - - {std: 11, bt: Debug , arch: aarch64, distro: ubuntu20.04} - - {std: 11, bt: Release, arch: aarch64, distro: ubuntu20.04} - - {std: 14, bt: Debug , arch: aarch64, distro: ubuntu20.04} - - {std: 14, bt: Release, arch: aarch64, distro: ubuntu20.04} - - {std: 17, bt: Debug , arch: aarch64, distro: ubuntu20.04} - - {std: 17, bt: Release, arch: aarch64, distro: ubuntu20.04} - # - - {std: 11, bt: Debug , arch: ppc64le, distro: ubuntu20.04} - - {std: 11, bt: Release, arch: ppc64le, distro: ubuntu20.04} - - {std: 14, bt: Debug , arch: ppc64le, distro: ubuntu20.04} - - {std: 14, bt: Release, arch: ppc64le, distro: ubuntu20.04} - - {std: 17, bt: Debug , arch: ppc64le, distro: ubuntu20.04} - - {std: 17, bt: Release, arch: ppc64le, distro: ubuntu20.04} - # - - {std: 11, bt: Debug , arch: s390x , distro: ubuntu20.04} - - {std: 11, bt: Release, arch: s390x , distro: ubuntu20.04} - - {std: 14, bt: Debug , arch: s390x , distro: ubuntu20.04} - - {std: 14, bt: Release, arch: s390x , distro: ubuntu20.04} - - {std: 17, bt: Debug , arch: s390x , distro: ubuntu20.04} - - {std: 17, bt: Release, arch: s390x , distro: ubuntu20.04} - # - #- {std: 11, bt: Debug , arch: armv6 , distro: bullseye} - #- {std: 11, bt: Release, arch: armv6 , distro: bullseye} - #- {std: 14, bt: Debug , arch: armv6 , distro: bullseye} - #- {std: 14, bt: Release, arch: armv6 , distro: bullseye} - #- {std: 17, bt: Debug , arch: armv6 , distro: bullseye} - #- {std: 17, bt: Release, arch: armv6 , distro: bullseye} - # - #- {std: 11, bt: Debug , arch: armv7 , distro: ubuntu20.04} - #- {std: 11, bt: Release, arch: armv7 , distro: ubuntu20.04} - #- {std: 14, bt: Debug , arch: armv7 , distro: ubuntu20.04} - #- {std: 14, bt: Release, arch: armv7 , distro: ubuntu20.04} - #- {std: 17, bt: Debug , arch: armv7 , distro: ubuntu20.04} - #- {std: 17, bt: Release, arch: armv7 , distro: ubuntu20.04} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - name: test - uses: uraimo/run-on-arch-action@v2.3.0 - with: - arch: ${{matrix.arch}} - distro: ${{matrix.distro}} - install: | - set -x - start_time=$SECONDS - time apt-get update -y - time apt-get install -y \ - git \ - build-essential - # arm platforms need an up-to-date cmake: - # https://gitlab.kitware.com/cmake/cmake/-/issues/20568 - if [ "${{matrix.arch}}" == "armv6" ] || [ "${{matrix.arch}}" == "armv7" ] ; then - time apt-get install -y \ - gpg \ - wget \ - apt-transport-https - wget --no-check-certificate -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null - echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null - time apt-get update -y - rm /usr/share/keyrings/kitware-archive-keyring.gpg - time apt-get install kitware-archive-keyring - time apt-get update -y - fi - time apt-get install -y cmake cmake-data - cmake --version - echo "install took $((SECONDS - start_time))" - run: | - set -x - start_time=$SECONDS - uname -a - pwd - ls -lFhp . - # - bdir=build_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}} - idir=install_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}} - mkdir -p $bdir - # - time cmake -S . -B $bdir \ - -DCMAKE_INSTALL_PREFIX=$idir \ - -DCMAKE_BUILD_TYPE=${{matrix.bt}} \ - -DC4_CXX_STANDARD=${{matrix.std}} \ - -DCXX_STANDARD=${{matrix.std}} \ - -DRYML_DEV=ON \ - -DRYML_TEST_SUITE=ON \ - -DRYML_BUILD_BENCHMARKS=OFF \ - -DRYML_SANITIZE=OFF \ - -DRYML_LINT=OFF \ - -DRYML_VALGRIND=OFF - # - time cmake --build $bdir -j 3 --target ryml-test-build - # - time cmake --build $bdir -j 3 --target ryml-test-run - echo "run took $((SECONDS - start_time))" diff --git a/thirdparty/ryml/.github/workflows/release.yml b/thirdparty/ryml/.github/workflows/release.yml deleted file mode 100644 index 3797359b8..000000000 --- a/thirdparty/ryml/.github/workflows/release.yml +++ /dev/null @@ -1,366 +0,0 @@ -name: release - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - tags: - - v0.* - - v1.* - - v2.* - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PKG_NAME: rapidyaml- - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: -DRYML_TEST_SUITE=OFF - NUM_JOBS_BUILD: # 4 - - -# useful to iterate when fixing the release: -# ver=0.2.1 ; ( set -x ; git tag -d v$ver ; git push origin :v$ver ) ; (set -x ; set -e ; tbump --only-patch --non-interactive $ver ; git add -u ; git commit --amend --no-edit ; git tag --annotate --message "v$ver" "v$ver" ; git push -f --tags origin ) - -jobs: - - gettag: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip release')) || - contains(github.event.head_commit.message, 'only release') - runs-on: ubuntu-latest - steps: - # use fetch-depth to ensure all tags are fetched - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}} - - name: Variables (from tag) - if: contains(github.ref, 'tags/v') - run: | - # https://github.community/t/how-to-get-just-the-tag-name/16241/11 - SRC_TAG=${GITHUB_REF#refs/tags/} - SRC_VERSION=${GITHUB_REF#refs/tags/v} - cat < vars.sh - export SRC_TAG=$SRC_TAG - export SRC_VERSION=$SRC_VERSION - EOF - - name: Variables (from commit, no tag) - if: ${{ !contains(github.ref, 'tags/v') }} - run: | - set -x - branch_name=${GITHUB_REF#refs/heads/} - # builds triggered from PRs have the branch_name like this: refs/pull/150/merge - # so filter to eg pr0150_merge - branch_name=`echo $branch_name | sed "s:refs/pull/\([0-9]*\)/\(.*\):pr0\1_\2:"` - # sanitize the branch name; eg merge/foo-bar -> merge_foo_bar - branch_name=`echo $branch_name | sed 's:[/.-]:_:g'` - SRC_TAG=$(git describe || git rev-parse --short HEAD) # eg v0.2.0-110-gda837e0 - SRC_VERSION="${branch_name}-${SRC_TAG}" - cat < vars.sh - export SRC_TAG=$SRC_TAG - export SRC_VERSION=$SRC_VERSION - EOF - - name: Verify vars.sh - run: cat vars.sh ; source vars.sh ; echo $SRC_TAG ; echo $SRC_VERSION - - name: Save vars.sh - uses: actions/upload-artifact@v3 - with: {name: vars.sh, path: ./vars.sh} - - #---------------------------------------------------------------------------- - # create source packages - src: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip release')) || - contains(github.event.head_commit.message, 'only release') - needs: gettag - runs-on: ubuntu-latest - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - name: Download vars.sh - uses: actions/download-artifact@v3 - with: {name: vars.sh, path: ./} - - name: Install python 3.9 - uses: actions/setup-python@v4 - with: { python-version: 3.9 } - - name: Install requirements - run: | - sudo -E pip install git-archive-all - - name: Create source packages - run: | - pwd - ls -lFhp - source vars.sh - echo SRC_TAG=$SRC_TAG - echo SRC_VERSION=$SRC_VERSION - id=${PROJ_PKG_NAME}${SRC_VERSION} - name=${id}-src - mkdir -p assets - git-archive-all --prefix $name assets/$name.tgz - git-archive-all --prefix $name assets/$name.zip - python --version - python tools/amalgamate.py assets/$id.hpp - - name: Save source artifacts - uses: actions/upload-artifact@v3 - with: {name: assets, path: assets} - - #---------------------------------------------------------------------------- - # create c++ packages - cpp: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip release')) || - contains(github.event.head_commit.message, 'only release') - name: cpp/${{matrix.config.os}}/${{matrix.config.gen}} - needs: gettag - runs-on: ${{matrix.config.os}} - env: {DEV: OFF, BT: Release, OS: "${{matrix.config.os}}", CXX_: "${{matrix.config.cxx}}", GEN: "${{matrix.config.gen}}"} - strategy: - fail-fast: false - matrix: - config: - # name of the artifact | suffix (gen) | suffix (package) | cpack gen | mime type | os | cxx - - {name: Ubuntu 20.04 deb , sfxg: unix64.deb, sfxp: ubuntu-20.04.deb , gen: DEB , mime: vnd.debian.binary-package, os: ubuntu-20.04 } - #- {name: Ubuntu 18.04 deb , sfxg: unix64.deb, sfxp: ubuntu-18.04.deb , gen: DEB , mime: vnd.debian.binary-package, os: ubuntu-18.04 } - - {name: Windows VS2019 zip, sfxg: win64.zip , sfxp: windows-vs2019.zip , gen: ZIP , mime: zip , os: windows-2019, cxx: vs2019} - - {name: MacOSX sh , sfxg: apple64.sh, sfxp: macosx-xcode.sh , gen: STGZ , mime: x-sh , os: macos-11.0 , cxx: xcode } - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - name: Download vars.sh - uses: actions/download-artifact@v3 - with: {name: vars.sh, path: ./} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info } - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_target shared64} - - name: shared64-pack - run: source .github/setenv.sh && c4_package shared64 $GEN - - name: shared64-normalize - run: | - set -x - source vars.sh - mkdir -p assets - asset_src=`ls -1 ./build/shared64/*-${{matrix.config.sfxg}}` - asset_dst=./assets/${PROJ_PKG_NAME}${SRC_VERSION}-${{matrix.config.sfxp}} - [ ! -f $asset_src ] && exit 1 - cp -fav $asset_src $asset_dst - - name: Save artifacts - uses: actions/upload-artifact@v3 - with: {name: assets, path: assets} - - #---------------------------------------------------------------------------- - # create python packages - # adapted from https://github.com/pikepdf/pikepdf/blob/master/.github/workflows/build_wheels.yml - - python_src: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip release')) || - contains(github.event.head_commit.message, 'only release') - name: python/src - runs-on: ubuntu-latest - steps: - # use fetch-depth to ensure all tags are fetched - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}} - - name: install python 3.9 - uses: actions/setup-python@v4 - with: { python-version: 3.9 } - - name: package python src packages - run: | - python --version - pip install -v -r requirements.txt - python setup.py sdist --formats=zip - - name: normalize src package names - run: | - sdist_orig=`find dist -type f -name 'rapidyaml-*.zip'` - [ ! -f $sdist_orig ] && exit 1 - sdist=`echo $sdist_orig | sed 's:\.zip:-python_src.zip:'` - mv -fv $sdist_orig $sdist - - name: Save artifacts - uses: actions/upload-artifact@v3 - with: {name: dist, path: dist} - - python_wheels: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip release')) || - contains(github.event.head_commit.message, 'only release') - name: python/${{matrix.config.cibw_pyv}}/${{matrix.config.osname}}/${{matrix.config.cibw_arch}} - runs-on: ${{matrix.config.os}} - env: - CMAKE_FLAGS: "${{matrix.config.cmakeflags}} -DRYML_DEV=OFF -DRYML_BUILD_API=ON -DRYML_API_TESTS=OFF -DRYML_API_BENCHMARKS=OFF" - CIBW_BUILD: "cp${{matrix.config.cibw_pyv}}-${{matrix.config.cibw_platform}}" - CIBW_ARCHS: "${{matrix.config.cibw_arch}}" - strategy: - fail-fast: false - matrix: - config: - # the 3-digit versions NEED to be quoted to prevent the version being read as float. (!) - - {pythonv: '3.11', cibw_pyv: 311, cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04} - - {pythonv: '3.11', cibw_pyv: 311, cibw_arch: i686 , cibw_platform: manylinux_i686 , osname: linux, os: ubuntu-20.04} - - {pythonv: '3.10', cibw_pyv: 310, cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04} - - {pythonv: '3.10', cibw_pyv: 310, cibw_arch: i686 , cibw_platform: manylinux_i686 , osname: linux, os: ubuntu-20.04} - - {pythonv: 3.9 , cibw_pyv: 39 , cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04} - - {pythonv: 3.9 , cibw_pyv: 39 , cibw_arch: i686 , cibw_platform: manylinux_i686 , osname: linux, os: ubuntu-20.04} - - {pythonv: 3.8 , cibw_pyv: 38 , cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04} - - {pythonv: 3.8 , cibw_pyv: 38 , cibw_arch: i686 , cibw_platform: manylinux_i686 , osname: linux, os: ubuntu-20.04} - - {pythonv: 3.7 , cibw_pyv: 37 , cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04} - - {pythonv: 3.7 , cibw_pyv: 37 , cibw_arch: i686 , cibw_platform: manylinux_i686 , osname: linux, os: ubuntu-20.04} - - {pythonv: 3.6 , cibw_pyv: 36 , cibw_arch: x86_64, cibw_platform: manylinux_x86_64, osname: linux, os: ubuntu-20.04} - - {pythonv: 3.6 , cibw_pyv: 36 , cibw_arch: i686 , cibw_platform: manylinux_i686 , osname: linux, os: ubuntu-20.04} - # the windows builds are disabled because they are causing problems and preventing the release. - # the problems are related to CMakeExtension forcing the use of Ninja - # which does not play well with the -G 'Visual Studio...' option used below. - # fixing this looks like it will be time-intensive. - #- {pythonv: '3.11', cibw_pyv: 311, cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'} - #- {pythonv: '3.11', cibw_pyv: 311, cibw_arch: x86 , cibw_platform: win32 , osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'} - #- {pythonv: '3.10', cibw_pyv: 310, cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'} - #- {pythonv: '3.10', cibw_pyv: 310, cibw_arch: x86 , cibw_platform: win32 , osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'} - #- {pythonv: 3.9 , cibw_pyv: 39 , cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'} - #- {pythonv: 3.9 , cibw_pyv: 39 , cibw_arch: x86 , cibw_platform: win32 , osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'} - #- {pythonv: 3.8 , cibw_pyv: 38 , cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'} - #- {pythonv: 3.8 , cibw_pyv: 38 , cibw_arch: x86 , cibw_platform: win32 , osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'} - - {pythonv: 3.7 , cibw_pyv: 37 , cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'} - #- {pythonv: 3.7 , cibw_pyv: 37 , cibw_arch: x86 , cibw_platform: win32 , osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'} - - {pythonv: 3.6 , cibw_pyv: 36 , cibw_arch: AMD64 , cibw_platform: win_amd64, osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A x64'} - #- {pythonv: 3.6 , cibw_pyv: 36 , cibw_arch: x86 , cibw_platform: win32 , osname: win , os: windows-2019, cxx: vs2019} #, cmakeflags: '-G "Visual Studio 16 2019" -A Win32'} - ## macosx builds are generating a SIGSEGV when importing. (!) - ## https://github.com/biojppm/rapidyaml/actions/runs/3062528713/jobs/4943611397#step:7:269 - #- {pythonv: '3.11', cibw_pyv: 311, cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15} - #- {pythonv: '3.10', cibw_pyv: 310, cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15} - #- {pythonv: 3.9 , cibw_pyv: 39 , cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15} - #- {pythonv: 3.8 , cibw_pyv: 38 , cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15} - #- {pythonv: 3.7 , cibw_pyv: 37 , cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15} - #- {pythonv: 3.6 , cibw_pyv: 36 , cibw_arch: x86_64, cibw_platform: macosx_x86_64, osname: macos, os: macos-10.15} - steps: - # use fetch-depth to ensure all tags are fetched - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}} - - name: create wheel - uses: joerick/cibuildwheel@v2.9.0 - - name: rename wheelhouse -> dist - run: | - mv -fv wheelhouse dist - ls -lFhp dist/ - - name: Save artifacts for publishing to PyPI - uses: actions/upload-artifact@v3 - with: {name: dist, path: dist} - # run the tests - - name: install python ${{matrix.config.pythonv}} - uses: actions/setup-python@v4 - with: - python-version: '${{matrix.config.pythonv}}' - - name: test with python ${{matrix.config.pythonv}} - run: | - set -x - echo "python ${{matrix.config.pythonv}} ${{matrix.config.py_arch}} ${{matrix.config.cibw_arch}}" - # skip 32 bit tests, as Python 32 bits are not available in ubuntu - arch="${{matrix.config.cibw_arch}}" - if [ "$arch" == "x86" ] || [ "$arch" == "i686" ] ; then - exit 0 - fi - python --version - python -c 'import sys ; import struct ; print("python:", sys.version, struct.calcsize("P") * 8, "bits")' - pip --version - pip install -v -r requirements.txt - pip install -v -r api/python/requirements.txt - for whl in dist/* ; do - pip install -v $whl - pip show -f rapidyaml - python -c 'import ryml ; print("ryml", ryml.version, ryml.version_tuple)' - python -c 'import ryml ; tree = ryml.parse_in_arena(b"{foo: bar}") ; assert tree.key(1) == b"foo" ; assert tree.val(1) == b"bar" ; print(str(tree.key(1), "utf8")) ; print(str(tree.val(1), "utf8"))' - python -m pytest -vvv api/python/tests - pip uninstall -y -v rapidyaml - done - - - #---------------------------------------------------------------------------- - release: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip release')) || - contains(github.event.head_commit.message, 'only release') - runs-on: ubuntu-latest - needs: - - src - - cpp - - python_src - - python_wheels - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - name: Gather artifacts - ./assets - uses: actions/download-artifact@v3 - with: {name: assets, path: assets} - - name: Gather artifacts - ./dist - uses: actions/download-artifact@v3 - with: {name: dist, path: dist} - - name: Verify existing artifacts - run: | - ls -lFhp assets/ - ls -lFhp dist/ - # - # Github - - name: Restore vars.sh - if: contains(github.ref, 'tags/v') - uses: actions/download-artifact@v3 - with: {name: vars.sh, path: ./} - - name: Save vars for following steps - if: contains(github.ref, 'tags/v') - id: vars - run: | - source vars.sh - version_body=${{github.workspace}}/changelog/$SRC_VERSION.md - if [ ! -f $version_body ] ; then - echo "version body file was not found: $version_body" - exit 1 - fi - echo "VERSION=$SRC_VERSION >> $GITHUB_OUTPUT" - echo "VERSION_BODY=$version_body >> $GITHUB_OUTPUT" - - name: Move Python packages to assets folder - if: contains(github.ref, 'tags/v') - run: mv -fv dist/*src.zip assets/. - - name: Create Github Release - if: contains(github.ref, 'tags/v') - id: create_release - uses: actions/create-release@v1 - env: { GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" } - with: - tag_name: ${{github.ref}} - release_name: Release ${{steps.vars.outputs.VERSION}} - body_path: ${{steps.vars.outputs.VERSION_BODY}} - draft: true - prerelease: ${{contains(github.ref, 'rc')}} - - name: Upload assets to Github Release - if: contains(github.ref, 'tags/v') - uses: dwenegar/upload-release-assets@v1 - env: { GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" } - with: - release_id: ${{steps.create_release.outputs.id}} - assets_path: ./assets/ - # - # PyPI (test) - - name: Publish python packages to test PyPI - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - repository_url: https://test.pypi.org/legacy/ - user: __token__ - password: ${{secrets.PYPI_TOKEN_TEST}} - verbose: true - skip_existing: true - # - # PyPI (production) - - name: Publish python packages to production PyPI - if: contains(github.ref, 'tags/v') - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{secrets.PYPI_TOKEN}} - verbose: true diff --git a/thirdparty/ryml/.github/workflows/samples.yml b/thirdparty/ryml/.github/workflows/samples.yml deleted file mode 100644 index d4ad65494..000000000 --- a/thirdparty/ryml/.github/workflows/samples.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: samples - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: -DRYML_TEST_SUITE=ON - NUM_JOBS_BUILD: # 4 - -jobs: - - #---------------------------------------------------------------------------- - samples: - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip samples')) || - contains(github.event.head_commit.message, 'only samples') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {bt: Debug , os: ubuntu-20.04} - - {bt: Release, os: ubuntu-20.04} - - {bt: Debug , os: windows-2019} - - {bt: Release, os: windows-2019} - - {bt: Debug , os: macos-latest} - - {bt: Release, os: macos-latest} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", - VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}", - CMANY: ON, RYMLSHA: "${{github.event.pull_request.head.sha}}" } - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0 } } # use fetch-depth to ensure all tags are fetched - - {name: python3, uses: actions/setup-python@v4, with: {python-version: 3.7}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info } - # - - {name: singleheader, run: cd samples/singleheader && ./run.sh $BT } - - {name: singleheaderlib-static, run: cd samples/singleheaderlib && ./run_static.sh $BT } - - {name: singleheaderlib-shared, run: cd samples/singleheaderlib && ./run_shared.sh $BT } - - {name: add_subdirectory, run: cd samples/add_subdirectory && ./run.sh $BT } - - {name: find_package, run: cd samples/find_package && ./run.sh $BT } - - {name: custom_c4core, run: cd samples/custom_c4core && ./run.sh $BT } - - {name: fetch_content, run: cd samples/fetch_content && ./run.sh $BT $RYMLSHA } diff --git a/thirdparty/ryml/.github/workflows/windows.yml b/thirdparty/ryml/.github/workflows/windows.yml deleted file mode 100644 index f39c6e19d..000000000 --- a/thirdparty/ryml/.github/workflows/windows.yml +++ /dev/null @@ -1,130 +0,0 @@ -name: windows - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: ryml- - PROJ_PFX_CMAKE: RYML_ - CMAKE_FLAGS: -DRYML_TEST_SUITE=ON - NUM_JOBS_BUILD: # 4 - - -jobs: - vs: - name: ${{matrix.cxx}}/c++${{matrix.std}}/${{matrix.bt}} - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip windows')) || - contains(github.event.head_commit.message, 'only windows') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - # vs2017 is only availble in windows-2016 - #- {std: 11, cxx: vs2017, bt: Debug , os: windows-2016, bitlinks: shared64 static32} - #- {std: 11, cxx: vs2017, bt: Release, os: windows-2016, bitlinks: shared64 static32} - #- {std: 14, cxx: vs2017, bt: Debug , os: windows-2016, bitlinks: shared64 static32} - #- {std: 14, cxx: vs2017, bt: Release, os: windows-2016, bitlinks: shared64 static32} - # - - {std: 11, cxx: vs2019, bt: Debug , os: windows-2019, bitlinks: shared64 static32} - - {std: 11, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: shared64 static32} - - {std: 17, cxx: vs2019, bt: Debug , os: windows-2019, bitlinks: shared64 static32} - - {std: 17, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: shared64 static32} - # - - {std: 11, cxx: vs2022, bt: Debug , os: windows-2022, bitlinks: shared64 static32} - - {std: 11, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: shared64 static32} - - {std: 17, cxx: vs2022, bt: Debug , os: windows-2022, bitlinks: shared64 static32} - - {std: 17, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: shared64 static32} - - {std: 20, cxx: vs2022, bt: Debug , os: windows-2022, bitlinks: shared64 static32} - - {std: 20, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: shared64 static32} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - -# TODO: -# mingw: -# name: mingw/${{matrix.platform}}/c++${{matrix.std}}/${{matrix.bt}} -# if: | -# (!contains(github.event.head_commit.message, 'skip all')) || -# (!contains(github.event.head_commit.message, 'skip windows')) || -# contains(github.event.head_commit.message, 'only windows') -# continue-on-error: true -# runs-on: ${{matrix.os}} -# strategy: -# fail-fast: false -# matrix: -# include: -# - {std: 11, platform: x86, cxx: i686-w64-mingw32-g++, bt: Debug , os: windows-latest, bitlinks: shared32 static32} -# - {std: 11, platform: x64, cxx: x86_64-w64-mingw32-g++, bt: Debug , os: windows-latest, bitlinks: shared64 static64} -# - {std: 11, platform: x86, cxx: i686-w64-mingw32-g++, bt: Release, os: windows-latest, bitlinks: shared32 static32} -# - {std: 11, platform: x64, cxx: x86_64-w64-mingw32-g++, bt: Release, os: windows-latest, bitlinks: shared64 static64} -# - {std: 17, platform: x86, cxx: i686-w64-mingw32-g++, bt: Debug , os: windows-latest, bitlinks: shared32 static32} -# - {std: 17, platform: x64, cxx: x86_64-w64-mingw32-g++, bt: Debug , os: windows-latest, bitlinks: shared64 static64} -# - {std: 17, platform: x86, cxx: i686-w64-mingw32-g++, bt: Release, os: windows-latest, bitlinks: shared32 static32} -# - {std: 17, platform: x64, cxx: x86_64-w64-mingw32-g++, bt: Release, os: windows-latest, bitlinks: shared64 static64} -# env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} -# steps: -# - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} -# - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} -# - name: install mingw -# uses: egor-tensin/setup-mingw@v2 -# with: -# platform: "${{matrix.platform}}" -# - name: shared64-configure--------------------------------------------------- -# run: source .github/setenv.sh && c4_cfg_test shared64 -# - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} -# - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} -# - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} -# - name: static64-configure--------------------------------------------------- -# run: source .github/setenv.sh && c4_cfg_test static64 -# - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} -# - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} -# - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} -# - name: shared32-configure--------------------------------------------------- -# run: source .github/setenv.sh && c4_cfg_test shared32 -# - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} -# - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} -# - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} -# - name: static32-configure--------------------------------------------------- -# run: source .github/setenv.sh && c4_cfg_test static32 -# - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} -# - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} -# - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} diff --git a/thirdparty/ryml/.gitignore b/thirdparty/ryml/.gitignore deleted file mode 100644 index 641b684ab..000000000 --- a/thirdparty/ryml/.gitignore +++ /dev/null @@ -1,51 +0,0 @@ -# text editor files -*.bck -\#* -*~ -.cquery_cached_index/ -.clangd/ -.ccls-cache/ -.cache/ -__pycache__/ - -# gdb files -.gdbinit -setup.gdb - -# valgrind files -callgrind* -vgcore* - -# Visual Studio files -.vs/ -.vscode/ -# QtCreator files -CMakeLists.txt.user* -# Eclipse -.project -.cproject -/.settings/ -# KDevelop files -*.kdev4 - -# build files -build/ -install/ -.python-version -compile_commands.json - -# test files -/Testing/ - -# python packaging -.eggs/ -dist/ -rapidyaml.egg-info/ -wheelhouse/ - -# continuous integration files -.ci/.vagrant - -# amalgamation files -src/c4/c4core_all.hpp -src_singleheader/ diff --git a/thirdparty/ryml/.gitmodules b/thirdparty/ryml/.gitmodules deleted file mode 100644 index b8f0b0ef4..000000000 --- a/thirdparty/ryml/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "extern/c4core"] - path = ext/c4core - url = https://github.com/biojppm/c4core diff --git a/thirdparty/ryml/.lgtm.yml b/thirdparty/ryml/.lgtm.yml deleted file mode 100644 index c1376d236..000000000 --- a/thirdparty/ryml/.lgtm.yml +++ /dev/null @@ -1,2 +0,0 @@ -queries: -- exclude: cpp/unsigned-comparison-zero diff --git a/thirdparty/ryml/CONTRIBUTING.md b/thirdparty/ryml/CONTRIBUTING.md deleted file mode 100644 index c8e847bfa..000000000 --- a/thirdparty/ryml/CONTRIBUTING.md +++ /dev/null @@ -1,18 +0,0 @@ -# Contributing - -Thanks for your contribution! - -* Make sure to clone the project with `git clone --recursive` so that - the submodules are initialized correctly. -* To enable both tests and benchmarks, configure ryml with `-DRYML_DEV=ON` - when calling cmake. To enable only tests, use `-DRYML_BUILD_TESTS=ON`; to - enable only benchmarks use `-DRYML_BUILD_BENCHMARKS=ON`. All these flags - are disabled by default. -* Code style for pull requests should respect the existing code style: - ```c++ - if(foo) // no space before parens - { // curly brackets on next line - // no tabs; indent with 4 spaces - bar(); - } - ``` diff --git a/thirdparty/ryml/LICENSE.txt b/thirdparty/ryml/LICENSE.txt deleted file mode 100644 index 47b6b4394..000000000 --- a/thirdparty/ryml/LICENSE.txt +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2018, Joao Paulo Magalhaes - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - diff --git a/thirdparty/ryml/MANIFEST.in b/thirdparty/ryml/MANIFEST.in deleted file mode 100644 index cdeb27468..000000000 --- a/thirdparty/ryml/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -# MANIFEST.in must be in root directory. -# See https://github.com/pypa/setuptools/issues/2615 -graft ext diff --git a/thirdparty/ryml/README.md b/thirdparty/ryml/README.md deleted file mode 100644 index ddfa35454..000000000 --- a/thirdparty/ryml/README.md +++ /dev/null @@ -1,1136 +0,0 @@ -# Rapid YAML -[![MIT Licensed](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/biojppm/rapidyaml/blob/master/LICENSE.txt) -[![release](https://img.shields.io/github/v/release/biojppm/rapidyaml?color=g&include_prereleases&label=release%20&sort=semver)](https://github.com/biojppm/rapidyaml/releases) -[![PyPI](https://img.shields.io/pypi/v/rapidyaml?color=g)](https://pypi.org/project/rapidyaml/) -[![Docs](https://img.shields.io/badge/docs-docsforge-blue)](https://rapidyaml.docsforge.com/) -[![Gitter](https://badges.gitter.im/rapidyaml/community.svg)](https://gitter.im/rapidyaml/community) - -[![test](https://github.com/biojppm/rapidyaml/workflows/test/badge.svg?branch=master)](https://github.com/biojppm/rapidyaml/actions) - -[![Codecov](https://codecov.io/gh/biojppm/rapidyaml/branch/master/graph/badge.svg?branch=master)](https://codecov.io/gh/biojppm/rapidyaml) - - -Or ryml, for short. ryml is a C++ library to parse and emit YAML, -and do it fast, on everything from x64 to bare-metal chips without -operating system. (If you are looking to use your programs with a YAML tree -as a configuration tree with override facilities, take a look at -[c4conf](https://github.com/biojppm/c4conf)). - -ryml parses both read-only and in-situ source buffers; the resulting -data nodes hold only views to sub-ranges of the source buffer. No -string copies or duplications are done, and no virtual functions are -used. The data tree is a flat index-based structure stored in a single -array. Serialization happens only at your direct request, after -parsing / before emitting. Internally, the data tree representation -stores only string views and has no knowledge of types, but of course, -every node can have a YAML type tag. ryml makes it easy and fast to -read and modify the data tree. - -ryml is available as a single header file, or it can be used as a -simple library with cmake -- both separately (ie -build->install->`find_package()`) or together with your project (ie with -`add_subdirectory()`). (See below for examples). - -ryml can use custom global and per-tree memory allocators and error -handler callbacks, and is exception-agnostic. ryml provides a default -implementation for the allocator (using `std::malloc()`) and error -handlers (using using `std::abort()` is provided, but you can opt out -and provide your own memory allocation and eg, exception-throwing -callbacks. - -ryml does not depend on the STL, ie, it does not use any std container -as part of its data structures), but it can serialize and deserialize -these containers into the data tree, with the use of optional -headers. ryml ships with [c4core](https://github.com/biojppm/c4core), a -small C++ utilities multiplatform library. - -ryml is written in C++11, and compiles cleanly with: -* Visual Studio 2015 and later -* clang++ 3.9 and later -* g++ 4.8 and later -* Intel Compiler - -ryml is [extensively unit-tested in Linux, Windows and -MacOS](https://github.com/biojppm/rapidyaml/actions). The tests cover -x64, x86, wasm (emscripten), arm, aarch64, ppc64le and s390x -architectures, and include analysing ryml with: - * valgrind - * clang-tidy - * clang sanitizers: - * memory - * address - * undefined behavior - * thread - * [LGTM.com](https://lgtm.com/projects/g/biojppm/rapidyaml) - -ryml also [runs in -bare-metal](https://github.com/biojppm/rapidyaml/issues/193), and -[RISC-V -architectures](https://github.com/biojppm/c4core/pull/69). Both of -these are pending implementation of CI actions for continuous -validation, but ryml has been proven to work there. - -ryml is [available in Python](https://pypi.org/project/rapidyaml/), -and can very easily be compiled to JavaScript through emscripten (see -below). - -See also [the changelog](https://github.com/biojppm/rapidyaml/tree/master/changelog) -and [the roadmap](https://github.com/biojppm/rapidyaml/tree/master/ROADMAP.md). - - - - ------- - -## Table of contents -* [Is it rapid?](#is-it-rapid) - * [Comparison with yaml-cpp](#comparison-with-yaml-cpp) - * [Performance reading JSON](#performance-reading-json) - * [Performance emitting](#performance-emitting) -* [Quick start](#quick-start) -* [Using ryml in your project](#using-ryml-in-your-project) - * [Package managers](#package-managers) - * [Single header file](#single-header-file) - * [As a library](#as-a-library) - * [Quickstart samples](#quickstart-samples) - * [CMake build settings for ryml](#cmake-build-settings-for-ryml) - * [Forcing ryml to use a different c4core version](#forcing-ryml-to-use-a-different-c4core-version) -* [Other languages](#other-languages) - * [JavaScript](#javascript) - * [Python](#python) -* [YAML standard conformance](#yaml-standard-conformance) - * [Test suite status](#test-suite-status) -* [Known limitations](#known-limitations) -* [Alternative libraries](#alternative-libraries) -* [License](#license) - - ------- - -## Is it rapid? - -You bet! On a i7-6800K CPU @3.40GHz: - * ryml parses YAML at about ~150MB/s on Linux and ~100MB/s on Windows (vs2017). - * **ryml parses JSON at about ~450MB/s on Linux**, faster than sajson (didn't - try yet on Windows). - * compared against the other existing YAML libraries for C/C++: - * ryml is in general between 2 and 3 times faster than [libyaml](https://github.com/yaml/libyaml) - * ryml is in general between 10 and 70 times faster than - [yaml-cpp](https://github.com/jbeder/yaml-cpp), and in some cases as - much as 100x and [even - 200x](https://github.com/biojppm/c4core/pull/16#issuecomment-700972614) faster. - -[Here's the benchmark](./bm/bm_parse.cpp). Using different -approaches within ryml (in-situ/read-only vs. with/without reuse), a YAML / -JSON buffer is repeatedly parsed, and compared against other libraries. - -### Comparison with yaml-cpp - -The first result set is for Windows, and is using a [appveyor.yml config -file](./bm/cases/appveyor.yml). A comparison of these results is -summarized on the table below: - -| Read rates (MB/s) | ryml | yamlcpp | compared | -|------------------------------|--------|---------|--------------| -| appveyor / vs2017 / Release | 101.5 | 5.3 | 20x / 5.2% | -| appveyor / vs2017 / Debug | 6.4 | 0.0844 | 76x / 1.3% | - - -The next set of results is taken in Linux, comparing g++ 8.2 and clang++ 7.0.1 in -parsing a YAML buffer from a [travis.yml config -file](./bm/cases/travis.yml) or a JSON buffer from a [compile_commands.json -file](./bm/cases/compile_commands.json). You -can [see the full results here](./bm/results/parse.linux.i7_6800K.md). -Summarizing: - -| Read rates (MB/s) | ryml | yamlcpp | compared | -|-----------------------------|--------|---------|------------| -| json / clang++ / Release | 453.5 | 15.1 | 30x / 3% | -| json / g++ / Release | 430.5 | 16.3 | 26x / 4% | -| json / clang++ / Debug | 61.9 | 1.63 | 38x / 3% | -| json / g++ / Debug | 72.6 | 1.53 | 47x / 2% | -| travis / clang++ / Release | 131.6 | 8.08 | 16x / 6% | -| travis / g++ / Release | 176.4 | 8.23 | 21x / 5% | -| travis / clang++ / Debug | 10.2 | 1.08 | 9x / 1% | -| travis / g++ / Debug | 12.5 | 1.01 | 12x / 8% | - -The 450MB/s read rate for JSON puts ryml squarely in the same ballpark -as [RapidJSON](https://github.com/Tencent/rapidjson) and other fast json -readers -([data from here](https://lemire.me/blog/2018/05/03/how-fast-can-you-parse-json/)). -Even parsing full YAML is at ~150MB/s, which is still in that performance -ballpark, albeit at its lower end. This is something to be proud of, as the -YAML specification is much more complex than JSON: [23449 vs 1969 words](https://www.arp242.net/yaml-config.html#its-pretty-complex). - - -### Performance reading JSON - -So how does ryml compare against other JSON readers? Well, it's one of the -fastest! - -The benchmark is the [same as above](./bm/parse.cpp), and it is reading -the [compile_commands.json](./bm/cases/compile_commands.json), The `_arena` -suffix notes parsing a read-only buffer (so buffer copies are performed), -while the `_inplace` suffix means that the source buffer can be parsed in -place. The `_reuse` means the data tree and/or parser are reused on each -benchmark repeat. - -Here's what we get with g++ 8.2: - -| Benchmark | Release,MB/s | Debug,MB/s | -|:----------------------|-------------:|------------:| -| rapidjson_arena | 509.9 | 43.4 | -| rapidjson_inplace | 1329.4 | 68.2 | -| sajson_inplace | 434.2 | 176.5 | -| sajson_arena | 430.7 | 175.6 | -| jsoncpp_arena | 183.6 | ? 187.9 | -| nlohmann_json_arena | 115.8 | 21.5 | -| yamlcpp_arena | 16.6 | 1.6 | -| libyaml_arena | 113.9 | 35.7 | -| libyaml_arena_reuse | 114.6 | 35.9 | -| ryml_arena | 388.6 | 36.9 | -| ryml_inplace | 393.7 | 36.9 | -| ryml_arena_reuse | 446.2 | 74.6 | -| ryml_inplace_reuse | 457.1 | 74.9 | - -You can verify that (at least for this test) ryml beats most json -parsers at their own game, with the only exception of -[rapidjson](https://github.com/Tencent/rapidjson). And actually, in -Debug, [rapidjson](https://github.com/Tencent/rapidjson) is slower -than ryml, and [sajson](https://github.com/chadaustin/sajson) -manages to be faster (but not sure about jsoncpp; need to scrutinize there -the suspicious fact that the Debug result is faster than the Release result). - - -### Performance emitting - -[Emitting benchmarks](bm/bm_emit.cpp) also show similar speedups from -the existing libraries, also anecdotally reported by some users [(eg, -here's a user reporting 25x speedup from -yaml-cpp)](https://github.com/biojppm/rapidyaml/issues/28#issue-553855608). Also, in -some cases (eg, block folded multiline scalars), the speedup is as -high as 200x (eg, 7.3MB/s -> 1.416MG/s). - - -### CI results and request for files - -While a more effective way of showing the benchmark results is not -available yet, you can browse through the [runs of the benchmark -workflow in the -CI](https://github.com/biojppm/rapidyaml/actions/workflows/benchmarks.yml) -to scroll through the results for yourself. - -Also, if you have a case where ryml behaves very nicely or not as nicely as -claimed above, we would definitely like to see it! Please submit a pull request -adding the file to [bm/cases](bm/cases), or just send us the files. - - ------- - -## Quick start - -If you're wondering whether ryml's speed comes at a usage cost, you -need not: with ryml, you can have your cake and eat it too. Being -rapid is definitely NOT the same as being unpractical, so ryml was -written with easy AND efficient usage in mind, and comes with a two -level API for accessing and traversing the data tree. - -The following snippet is a quick overview taken from [the quickstart -sample](samples/quickstart.cpp). After cloning ryml (don't forget the -`--recursive` flag for git), you can very -easily build and run this executable using any of the build samples, -eg the [`add_subdirectory()` sample](samples/add_subdirectory/). - -```c++ -// Parse YAML code in place, potentially mutating the buffer. -// It is also possible to: -// - parse a read-only buffer using parse_in_arena() -// - reuse an existing tree (advised) -// - reuse an existing parser (advised) -char yml_buf[] = "{foo: 1, bar: [2, 3], john: doe}"; -ryml::Tree tree = ryml::parse_in_place(yml_buf); - -// Note: it will always be significantly faster to use mutable -// buffers and reuse tree+parser. -// -// Below you will find samples that show how to achieve reuse; but -// please note that for brevity and clarity, many of the examples -// here are parsing immutable buffers, and not reusing tree or -// parser. - - -//------------------------------------------------------------------ -// API overview - -// ryml has a two-level API: -// -// The lower level index API is based on the indices of nodes, -// where the node's id is the node's position in the tree's data -// array. This API is very efficient, but somewhat difficult to use: -size_t root_id = tree.root_id(); -size_t bar_id = tree.find_child(root_id, "bar"); // need to get the index right -CHECK(tree.is_map(root_id)); // all of the index methods are in the tree -CHECK(tree.is_seq(bar_id)); // ... and receive the subject index - -// The node API is a lightweight abstraction sitting on top of the -// index API, but offering a much more convenient interaction: -ryml::ConstNodeRef root = tree.rootref(); -ryml::ConstNodeRef bar = tree["bar"]; -CHECK(root.is_map()); -CHECK(bar.is_seq()); -// A node ref is a lightweight handle to the tree and associated id: -CHECK(root.tree() == &tree); // a node ref points at its tree, WITHOUT refcount -CHECK(root.id() == root_id); // a node ref's id is the index of the node -CHECK(bar.id() == bar_id); // a node ref's id is the index of the node - -// The node API translates very cleanly to the index API, so most -// of the code examples below are using the node API. - -// One significant point of the node API is that it holds a raw -// pointer to the tree. Care must be taken to ensure the lifetimes -// match, so that a node will never access the tree after the tree -// went out of scope. - - -//------------------------------------------------------------------ -// To read the parsed tree - -// ConstNodeRef::operator[] does a lookup, is O(num_children[node]). -CHECK(tree["foo"].is_keyval()); -CHECK(tree["foo"].key() == "foo"); -CHECK(tree["foo"].val() == "1"); -CHECK(tree["bar"].is_seq()); -CHECK(tree["bar"].has_key()); -CHECK(tree["bar"].key() == "bar"); -// maps use string keys, seqs use integral keys: -CHECK(tree["bar"][0].val() == "2"); -CHECK(tree["bar"][1].val() == "3"); -CHECK(tree["john"].val() == "doe"); -// An integral key is the position of the child within its parent, -// so even maps can also use int keys, if the key position is -// known. -CHECK(tree[0].id() == tree["foo"].id()); -CHECK(tree[1].id() == tree["bar"].id()); -CHECK(tree[2].id() == tree["john"].id()); -// Tree::operator[](int) searches a ***root*** child by its position. -CHECK(tree[0].id() == tree["foo"].id()); // 0: first child of root -CHECK(tree[1].id() == tree["bar"].id()); // 1: first child of root -CHECK(tree[2].id() == tree["john"].id()); // 2: first child of root -// NodeRef::operator[](int) searches a ***node*** child by its position: -CHECK(bar[0].val() == "2"); // 0 means first child of bar -CHECK(bar[1].val() == "3"); // 1 means second child of bar -// NodeRef::operator[](string): -// A string key is the key of the node: lookup is by name. So it -// is only available for maps, and it is NOT available for seqs, -// since seq members do not have keys. -CHECK(tree["foo"].key() == "foo"); -CHECK(tree["bar"].key() == "bar"); -CHECK(tree["john"].key() == "john"); -CHECK(bar.is_seq()); -// CHECK(bar["BOOM!"].is_seed()); // error, seqs do not have key lookup - -// Note that maps can also use index keys as well as string keys: -CHECK(root["foo"].id() == root[0].id()); -CHECK(root["bar"].id() == root[1].id()); -CHECK(root["john"].id() == root[2].id()); - -// IMPORTANT. The ryml tree uses indexed linked lists for storing -// children, so the complexity of `Tree::operator[csubstr]` and -// `Tree::operator[size_t]` is linear on the number of root -// children. If you use `Tree::operator[]` with a large tree where -// the root has many children, you will see a performance hit. -// -// To avoid this hit, you can create your own accelerator -// structure. For example, before doing a lookup, do a single -// traverse at the root level to fill an `map` -// mapping key names to node indices; with a node index, a lookup -// (via `Tree::get()`) is O(1), so this way you can get O(log n) -// lookup from a key. (But please do not use `std::map` if you -// care about performance; use something else like a flat map or -// sorted vector). -// -// As for node refs, the difference from `NodeRef::operator[]` and -// `ConstNodeRef::operator[]` to `Tree::operator[]` is that the -// latter refers to the root node, whereas the former are invoked -// on their target node. But the lookup process works the same for -// both and their algorithmic complexity is the same: they are -// both linear in the number of direct children. But of course, -// depending on the data, that number may be very different from -// one to another. - -//------------------------------------------------------------------ -// Hierarchy: - -{ - ryml::ConstNodeRef foo = root.first_child(); - ryml::ConstNodeRef john = root.last_child(); - CHECK(tree.size() == 6); // O(1) number of nodes in the tree - CHECK(root.num_children() == 3); // O(num_children[root]) - CHECK(foo.num_siblings() == 3); // O(num_children[parent(foo)]) - CHECK(foo.parent().id() == root.id()); // parent() is O(1) - CHECK(root.first_child().id() == root["foo"].id()); // first_child() is O(1) - CHECK(root.last_child().id() == root["john"].id()); // last_child() is O(1) - CHECK(john.first_sibling().id() == foo.id()); - CHECK(foo.last_sibling().id() == john.id()); - // prev_sibling(), next_sibling(): (both are O(1)) - CHECK(foo.num_siblings() == root.num_children()); - CHECK(foo.prev_sibling().id() == ryml::NONE); // foo is the first_child() - CHECK(foo.next_sibling().key() == "bar"); - CHECK(foo.next_sibling().next_sibling().key() == "john"); - CHECK(foo.next_sibling().next_sibling().next_sibling().id() == ryml::NONE); // john is the last_child() -} - - -//------------------------------------------------------------------ -// Iterating: -{ - ryml::csubstr expected_keys[] = {"foo", "bar", "john"}; - // iterate children using the high-level node API: - { - size_t count = 0; - for(ryml::ConstNodeRef const& child : root.children()) - CHECK(child.key() == expected_keys[count++]); - } - // iterate siblings using the high-level node API: - { - size_t count = 0; - for(ryml::ConstNodeRef const& child : root["foo"].siblings()) - CHECK(child.key() == expected_keys[count++]); - } - // iterate children using the lower-level tree index API: - { - size_t count = 0; - for(size_t child_id = tree.first_child(root_id); child_id != ryml::NONE; child_id = tree.next_sibling(child_id)) - CHECK(tree.key(child_id) == expected_keys[count++]); - } - // iterate siblings using the lower-level tree index API: - // (notice the only difference from above is in the loop - // preamble, which calls tree.first_sibling(bar_id) instead of - // tree.first_child(root_id)) - { - size_t count = 0; - for(size_t child_id = tree.first_sibling(bar_id); child_id != ryml::NONE; child_id = tree.next_sibling(child_id)) - CHECK(tree.key(child_id) == expected_keys[count++]); - } -} - - -//------------------------------------------------------------------ -// Gotchas: -CHECK(!tree["bar"].has_val()); // seq is a container, so no val -CHECK(!tree["bar"][0].has_key()); // belongs to a seq, so no key -CHECK(!tree["bar"][1].has_key()); // belongs to a seq, so no key -//CHECK(tree["bar"].val() == BOOM!); // ... so attempting to get a val is undefined behavior -//CHECK(tree["bar"][0].key() == BOOM!); // ... so attempting to get a key is undefined behavior -//CHECK(tree["bar"][1].key() == BOOM!); // ... so attempting to get a key is undefined behavior - - -//------------------------------------------------------------------ -// Deserializing: use operator>> -{ - int foo = 0, bar0 = 0, bar1 = 0; - std::string john; - root["foo"] >> foo; - root["bar"][0] >> bar0; - root["bar"][1] >> bar1; - root["john"] >> john; // requires from_chars(std::string). see serialization samples below. - CHECK(foo == 1); - CHECK(bar0 == 2); - CHECK(bar1 == 3); - CHECK(john == "doe"); -} - - -//------------------------------------------------------------------ -// Modifying existing nodes: operator<< vs operator= - -// As implied by its name, ConstNodeRef is a reference to a const -// node. It can be used to read from the node, but not write to it -// or modify the hierarchy of the node. If any modification is -// desired then a NodeRef must be used instead: -ryml::NodeRef wroot = tree.rootref(); - -// operator= assigns an existing string to the receiving node. -// This pointer will be in effect until the tree goes out of scope -// so beware to only assign from strings outliving the tree. -wroot["foo"] = "says you"; -wroot["bar"][0] = "-2"; -wroot["bar"][1] = "-3"; -wroot["john"] = "ron"; -// Now the tree is _pointing_ at the memory of the strings above. -// That is OK because those are static strings and will outlive -// the tree. -CHECK(root["foo"].val() == "says you"); -CHECK(root["bar"][0].val() == "-2"); -CHECK(root["bar"][1].val() == "-3"); -CHECK(root["john"].val() == "ron"); -// WATCHOUT: do not assign from temporary objects: -// { -// std::string crash("will dangle"); -// root["john"] = ryml::to_csubstr(crash); -// } -// CHECK(root["john"] == "dangling"); // CRASH! the string was deallocated - -// operator<< first serializes the input to the tree's arena, then -// assigns the serialized string to the receiving node. This avoids -// constraints with the lifetime, since the arena lives with the tree. -CHECK(tree.arena().empty()); -wroot["foo"] << "says who"; // requires to_chars(). see serialization samples below. -wroot["bar"][0] << 20; -wroot["bar"][1] << 30; -wroot["john"] << "deere"; -CHECK(root["foo"].val() == "says who"); -CHECK(root["bar"][0].val() == "20"); -CHECK(root["bar"][1].val() == "30"); -CHECK(root["john"].val() == "deere"); -CHECK(tree.arena() == "says who2030deere"); // the result of serializations to the tree arena -// using operator<< instead of operator=, the crash above is avoided: -{ - std::string ok("in_scope"); - // root["john"] = ryml::to_csubstr(ok); // don't, will dangle - wroot["john"] << ryml::to_csubstr(ok); // OK, copy to the tree's arena -} -CHECK(root["john"] == "in_scope"); // OK! -CHECK(tree.arena() == "says who2030deerein_scope"); // the result of serializations to the tree arena - - -//------------------------------------------------------------------ -// Adding new nodes: - -// adding a keyval node to a map: -CHECK(root.num_children() == 3); -wroot["newkeyval"] = "shiny and new"; // using these strings -wroot.append_child() << ryml::key("newkeyval (serialized)") << "shiny and new (serialized)"; // serializes and assigns the serialization -CHECK(root.num_children() == 5); -CHECK(root["newkeyval"].key() == "newkeyval"); -CHECK(root["newkeyval"].val() == "shiny and new"); -CHECK(root["newkeyval (serialized)"].key() == "newkeyval (serialized)"); -CHECK(root["newkeyval (serialized)"].val() == "shiny and new (serialized)"); -CHECK( ! tree.in_arena(root["newkeyval"].key())); // it's using directly the static string above -CHECK( ! tree.in_arena(root["newkeyval"].val())); // it's using directly the static string above -CHECK( tree.in_arena(root["newkeyval (serialized)"].key())); // it's using a serialization of the string above -CHECK( tree.in_arena(root["newkeyval (serialized)"].val())); // it's using a serialization of the string above -// adding a val node to a seq: -CHECK(root["bar"].num_children() == 2); -wroot["bar"][2] = "oh so nice"; -wroot["bar"][3] << "oh so nice (serialized)"; -CHECK(root["bar"].num_children() == 4); -CHECK(root["bar"][2].val() == "oh so nice"); -CHECK(root["bar"][3].val() == "oh so nice (serialized)"); -// adding a seq node: -CHECK(root.num_children() == 5); -wroot["newseq"] |= ryml::SEQ; -wroot.append_child() << ryml::key("newseq (serialized)") |= ryml::SEQ; -CHECK(root.num_children() == 7); -CHECK(root["newseq"].num_children() == 0); -CHECK(root["newseq (serialized)"].num_children() == 0); -// adding a map node: -CHECK(root.num_children() == 7); -wroot["newmap"] |= ryml::MAP; -wroot.append_child() << ryml::key("newmap (serialized)") |= ryml::SEQ; -CHECK(root.num_children() == 9); -CHECK(root["newmap"].num_children() == 0); -CHECK(root["newmap (serialized)"].num_children() == 0); -// -// When the tree is mutable, operator[] does not mutate the tree -// until the returned node is written to. -// -// Until such time, the NodeRef object keeps in itself the required -// information to write to the proper place in the tree. This is -// called being in a "seed" state. -// -// This means that passing a key/index which does not exist will -// not mutate the tree, but will instead store (in the node) the -// proper place of the tree to be able to do so, if and when it is -// required. -// -// This is a significant difference from eg, the behavior of -// std::map, which mutates the map immediately within the call to -// operator[]. -// -// All of the points above apply only if the tree is mutable. If -// the tree is const, then a NodeRef cannot be obtained from it; -// only a ConstNodeRef, which can never be used to mutate the -// tree. -CHECK(!root.has_child("I am not nothing")); -ryml::NodeRef nothing = wroot["I am nothing"]; -CHECK(nothing.valid()); // points at the tree, and a specific place in the tree -CHECK(nothing.is_seed()); // ... but nothing is there yet. -CHECK(!root.has_child("I am nothing")); // same as above -ryml::NodeRef something = wroot["I am something"]; -ryml::ConstNodeRef constsomething = wroot["I am something"]; -CHECK(!root.has_child("I am something")); // same as above -CHECK(something.valid()); -CHECK(something.is_seed()); // same as above -CHECK(!constsomething.valid()); // NOTE: because a ConstNodeRef - // cannot be used to mutate a - // tree, it is only valid() if it - // is pointing at an existing - // node. -something = "indeed"; // this will commit to the tree, mutating at the proper place -CHECK(root.has_child("I am something")); -CHECK(root["I am something"].val() == "indeed"); -CHECK(something.valid()); -CHECK(!something.is_seed()); // now the tree has this node, so the - // ref is no longer a seed -// now the constref is also valid (but it needs to be reassigned): -ryml::ConstNodeRef constsomethingnew = wroot["I am something"]; -CHECK(constsomethingnew.valid()); -// note that the old constref is now stale, because it only keeps -// the state at creation: -CHECK(!constsomething.valid()); - - -//------------------------------------------------------------------ -// Emitting: - -// emit to a FILE* -ryml::emit_yaml(tree, stdout); // there is also emit_json() -// emit to a stream -std::stringstream ss; -ss << tree; -std::string stream_result = ss.str(); -// emit to a buffer: -std::string str_result = ryml::emitrs_yaml(tree); // there is also emitrs_json() -// can emit to any given buffer: -char buf[1024]; -ryml::csubstr buf_result = ryml::emit_yaml(tree, buf); -// now check -ryml::csubstr expected_result = R"(foo: says who -bar: -- 20 -- 30 -- oh so nice -- oh so nice (serialized) -john: in_scope -newkeyval: shiny and new -newkeyval (serialized): shiny and new (serialized) -newseq: [] -newseq (serialized): [] -newmap: {} -newmap (serialized): [] -I am something: indeed -)"; -CHECK(buf_result == expected_result); -CHECK(str_result == expected_result); -CHECK(stream_result == expected_result); -// There are many possibilities to emit to buffer; -// please look at the emit sample functions below. - -//------------------------------------------------------------------ -// ConstNodeRef vs NodeRef - -ryml::NodeRef noderef = tree["bar"][0]; -ryml::ConstNodeRef constnoderef = tree["bar"][0]; - -// ConstNodeRef cannot be used to mutate the tree, but a NodeRef can: -//constnoderef = "21"; // compile error -//constnoderef << "22"; // compile error -noderef = "21"; // ok, can assign because it's not const -CHECK(tree["bar"][0].val() == "21"); -noderef << "22"; // ok, can serialize and assign because it's not const -CHECK(tree["bar"][0].val() == "22"); - -// it is not possible to obtain a NodeRef from a ConstNodeRef: -// noderef = constnoderef; // compile error - -// it is always possible to obtain a ConstNodeRef from a NodeRef: -constnoderef = noderef; // ok can assign const <- nonconst - -// If a tree is const, then only ConstNodeRef's can be -// obtained from that tree: -ryml::Tree const& consttree = tree; -//noderef = consttree["bar"][0]; // compile error -noderef = tree["bar"][0]; // ok -constnoderef = consttree["bar"][0]; // ok - -// ConstNodeRef and NodeRef can be compared for equality. -// Equality means they point at the same node. -CHECK(constnoderef == noderef); -CHECK(!(constnoderef != noderef)); - -//------------------------------------------------------------------ -// Dealing with UTF8 -ryml::Tree langs = ryml::parse_in_arena(R"( -en: Planet (Gas) -fr: Planète (Gazeuse) -ru: Планета (Газ) -ja: 惑星(ガス) -zh: 行星(气体) -# UTF8 decoding only happens in double-quoted strings,\ -# as per the YAML standard -decode this: "\u263A \xE2\x98\xBA" -and this as well: "\u2705 \U0001D11E" -)"); -// in-place UTF8 just works: -CHECK(langs["en"].val() == "Planet (Gas)"); -CHECK(langs["fr"].val() == "Planète (Gazeuse)"); -CHECK(langs["ru"].val() == "Планета (Газ)"); -CHECK(langs["ja"].val() == "惑星(ガス)"); -CHECK(langs["zh"].val() == "行星(气体)"); -// and \x \u \U codepoints are decoded (but only when they appear -// inside double-quoted strings, as dictated by the YAML -// standard): -CHECK(langs["decode this"].val() == "☺ ☺"); -CHECK(langs["and this as well"].val() == "✅ 𝄞"); - -//------------------------------------------------------------------ -// Getting the location of nodes in the source: -ryml::Parser parser; -ryml::Tree tree2 = parser.parse_in_arena("expected.yml", expected_result); -ryml::Location loc = parser.location(tree2["bar"][1]); -CHECK(parser.location_contents(loc).begins_with("30")); -CHECK(loc.line == 3u); -CHECK(loc.col == 4u); -``` - -The [quickstart.cpp sample](./samples/quickstart.cpp) (from which the -above overview was taken) has many more detailed examples, and should -be your first port of call to find out any particular point about -ryml's API. It is tested in the CI, and thus has the correct behavior. -There you can find the following subjects being addressed: - -```c++ -sample_substr(); ///< about ryml's string views (from c4core) -sample_parse_file(); ///< ready-to-go example of parsing a file from disk -sample_parse_in_place(); ///< parse a mutable YAML source buffer -sample_parse_in_arena(); ///< parse a read-only YAML source buffer -sample_parse_reuse_tree(); ///< parse into an existing tree, maybe into a node -sample_parse_reuse_parser(); ///< reuse an existing parser -sample_parse_reuse_tree_and_parser(); ///< how to reuse existing trees and parsers -sample_iterate_trees(); ///< visit individual nodes and iterate through trees -sample_create_trees(); ///< programatically create trees -sample_tree_arena(); ///< interact with the tree's serialization arena -sample_fundamental_types(); ///< serialize/deserialize fundamental types -sample_formatting(); ///< control formatting when serializing/deserializing -sample_base64(); ///< encode/decode base64 -sample_user_scalar_types(); ///< serialize/deserialize scalar (leaf/string) types -sample_user_container_types(); ///< serialize/deserialize container (map or seq) types -sample_std_types(); ///< serialize/deserialize STL containers -sample_emit_to_container(); ///< emit to memory, eg a string or vector-like container -sample_emit_to_stream(); ///< emit to a stream, eg std::ostream -sample_emit_to_file(); ///< emit to a FILE* -sample_emit_nested_node(); ///< pick a nested node as the root when emitting -sample_json(); ///< JSON parsing and emitting -sample_anchors_and_aliases(); ///< deal with YAML anchors and aliases -sample_tags(); ///< deal with YAML type tags -sample_docs(); ///< deal with YAML docs -sample_error_handler(); ///< set a custom error handler -sample_global_allocator(); ///< set a global allocator for ryml -sample_per_tree_allocator(); ///< set per-tree allocators -sample_static_trees(); ///< how to use static trees in ryml -sample_location_tracking(); ///< track node locations in the parsed source tree -``` - - ------- - -## Using ryml in your project - -### Package managers - -If you opt for package managers, here's where ryml is available so far -(thanks to all the contributors!): - * [vcpkg](https://vcpkg.io/en/packages.html): `vcpkg install ryml` - * Arch Linux/Manjaro: - * [rapidyaml-git (AUR)](https://aur.archlinux.org/packages/rapidyaml-git/) - * [python-rapidyaml-git (AUR)](https://aur.archlinux.org/packages/python-rapidyaml-git/) - * [PyPI](https://pypi.org/project/rapidyaml/) - -Although package managers are very useful for quickly getting up to -speed, the advised way is still to bring ryml as a submodule of your -project, building both together. This makes it easy to track any -upstream changes in ryml. Also, ryml is small and quick to build, so -there's not much of a cost for building it with your project. - -### Single header file -ryml is provided chiefly as a cmake library project, but it can also -be used as a single header file, and there is a [tool to -amalgamate](./tools/amalgamate.py) the code into a single header -file. The amalgamated header file is provided with each release, but -you can also generate a customized file suiting your particular needs -(or commit): - -```console -[user@host rapidyaml]$ python3 tools/amalgamate.py -h -usage: amalgamate.py [-h] [--c4core | --no-c4core] [--fastfloat | --no-fastfloat] [--stl | --no-stl] [output] - -positional arguments: - output output file. defaults to stdout - -optional arguments: - -h, --help show this help message and exit - --c4core amalgamate c4core together with ryml. this is the default. - --no-c4core amalgamate c4core together with ryml. the default is --c4core. - --fastfloat enable fastfloat library. this is the default. - --no-fastfloat enable fastfloat library. the default is --fastfloat. - --stl enable stl interop. this is the default. - --no-stl enable stl interop. the default is --stl. -``` - -The amalgamated header file contains all the function declarations and -definitions. To use it in the project, `#include` the header at will -in any header or source file in the project, but in one source file, -and only in that one source file, `#define` the macro -`RYML_SINGLE_HDR_DEFINE_NOW` **before including the header**. This -will enable the function definitions. For example: -```c++ -// foo.h -#include - -// foo.cpp -// ensure that foo.h is not included before this define! -#define RYML_SINGLE_HDR_DEFINE_NOW -#include -``` - -If you wish to package the single header into a shared library, then -you will need to define the preprocessor symbol `RYML_SHARED` during -compilation. - - -### As a library -The single header file is a good approach to quickly try the library, -but if you wish to make good use of CMake and its tooling ecosystem, -(and get better compile times), then ryml has you covered. - -As with any other cmake library, you have the option to integrate ryml into -your project's build setup, thereby building ryml together with your -project, or -- prior to configuring your project -- you can have ryml -installed either manually or through package managers. - -Currently [cmake](https://cmake.org/) is required to build ryml; we -recommend a recent cmake version, at least 3.13. - -Note that ryml uses submodules. Take care to use the `--recursive` flag -when cloning the repo, to ensure ryml's submodules are checked out as well: -```bash -git clone --recursive https://github.com/biojppm/rapidyaml -``` -If you omit `--recursive`, after cloning you -will have to do `git submodule update --init --recursive` -to ensure ryml's submodules are checked out. - -### Quickstart samples - -These samples show different ways of getting ryml into your application. All the -samples use [the same quickstart executable -source](./samples/quickstart.cpp), but are built in different ways, -showing several alternatives to integrate ryml into your project. We -also encourage you to refer to the [quickstart source](./samples/quickstart.cpp) itself, which -extensively covers most of the functionality that you may want out of -ryml. - -Each sample brings a `run.sh` script with the sequence of commands -required to successfully build and run the application (this is a bash -script and runs in Linux and MacOS, but it is also possible to run in -Windows via Git Bash or the WSL). Click on the links below to find out -more about each sample: - -| Sample name | ryml is part of build? | cmake file | commands | -|:-------------------|--------------------------|:-------------|:-------------| -| [`singleheader`](./samples/singleheader) | **yes**
ryml brought as a single header file,
not as a library | [`CMakeLists.txt`](./samples/singleheader/CMakeLists.txt) | [`run.sh`](./samples/singleheader/run.sh) | -| [`singleheaderlib`](./samples/singleheaderlib) | **yes**
ryml brought as a library
but from the single header file | [`CMakeLists.txt`](./samples/singleheaderlib/CMakeLists.txt) | [`run_shared.sh` (shared library)](./samples/singleheaderlib/run_shared.sh)
[`run_static.sh` (static library)](./samples/singleheaderlib/run_static.sh) | -| [`add_subdirectory`](./samples/add_subdirectory) | **yes** | [`CMakeLists.txt`](./samples/add_subdirectory/CMakeLists.txt) | [`run.sh`](./samples/add_subdirectory/run.sh) | -| [`fetch_content`](./samples/fetch_content) | **yes** | [`CMakeLists.txt`](./samples/fetch_content/CMakeLists.txt) | [`run.sh`](./samples/fetch_content/run.sh) | -| [`find_package`](./samples/find_package) | **no**
needs prior install or package | [`CMakeLists.txt`](./samples/find_package/CMakeLists.txt) | [`run.sh`](./samples/find_package/run.sh) | - -### CMake build settings for ryml -The following cmake variables can be used to control the build behavior of -ryml: - - * `RYML_WITH_TAB_TOKENS=ON/OFF`. Enable/disable support for tabs as - valid container tokens after `:` and `-`. Defaults to `OFF`, - because this may cost up to 10% in processing time. - * `RYML_DEFAULT_CALLBACKS=ON/OFF`. Enable/disable ryml's default - implementation of error and allocation callbacks. Defaults to `ON`. - * `RYML_STANDALONE=ON/OFF`. ryml uses - [c4core](https://github.com/biojppm/c4core), a C++ library with low-level - multi-platform utilities for C++. When `RYML_STANDALONE=ON`, c4core is - incorporated into ryml as if it is the same library. Defaults to `ON`. - -If you're developing ryml or just debugging problems with ryml itself, the -following cmake variables can be helpful: - * `RYML_DEV=ON/OFF`: a bool variable which enables development targets such as - unit tests, benchmarks, etc. Defaults to `OFF`. - * `RYML_DBG=ON/OFF`: a bool variable which enables verbose prints from - parsing code; can be useful to figure out parsing problems. Defaults to - `OFF`. - -#### Forcing ryml to use a different c4core version - -ryml is strongly coupled to c4core, and this is reinforced by the fact -that c4core is a submodule of the current repo. However, it is still -possible to use a c4core version different from the one in the repo -(of course, only if there are no incompatibilities between the -versions). You can find out how to achieve this by looking at the -[`custom_c4core` sample](./samples/custom_c4core/CMakeLists.txt). - - ------- - -## Other languages - -One of the aims of ryml is to provide an efficient YAML API for other -languages. JavaScript is fully available, and there is already a -cursory implementation for Python using only the low-level API. After -ironing out the general approach, other languages are likely to -follow (all of this is possible because we're using -[SWIG](http://www.swig.org/), which makes it easy to do so). - -### JavaScript - -A JavaScript+WebAssembly port is available, compiled through [emscripten](https://emscripten.org/). - - -### Python - -(Note that this is a work in progress. Additions will be made and things will -be changed.) With that said, here's an example of the Python API: - -```python -import ryml - -# ryml cannot accept strings because it does not take ownership of the -# source buffer; only bytes or bytearrays are accepted. -src = b"{HELLO: a, foo: b, bar: c, baz: d, seq: [0, 1, 2, 3]}" - -def check(tree): - # for now, only the index-based low-level API is implemented - assert tree.size() == 10 - assert tree.root_id() == 0 - assert tree.first_child(0) == 1 - assert tree.next_sibling(1) == 2 - assert tree.first_sibling(5) == 2 - assert tree.last_sibling(1) == 5 - # use bytes objects for queries - assert tree.find_child(0, b"foo") == 1 - assert tree.key(1) == b"foo") - assert tree.val(1) == b"b") - assert tree.find_child(0, b"seq") == 5 - assert tree.is_seq(5) - # to loop over children: - for i, ch in enumerate(ryml.children(tree, 5)): - assert tree.val(ch) == [b"0", b"1", b"2", b"3"][i] - # to loop over siblings: - for i, sib in enumerate(ryml.siblings(tree, 5)): - assert tree.key(sib) == [b"HELLO", b"foo", b"bar", b"baz", b"seq"][i] - # to walk over all elements - visited = [False] * tree.size() - for n, indentation_level in ryml.walk(tree): - # just a dumb emitter - left = " " * indentation_level - if tree.is_keyval(n): - print("{}{}: {}".format(left, tree.key(n), tree.val(n)) - elif tree.is_val(n): - print("- {}".format(left, tree.val(n)) - elif tree.is_keyseq(n): - print("{}{}:".format(left, tree.key(n)) - visited[inode] = True - assert False not in visited - # NOTE about encoding! - k = tree.get_key(5) - print(k) # '' - assert k == b"seq" # ok, as expected - assert k != "seq" # not ok - NOTE THIS! - assert str(k) != "seq" # not ok - assert str(k, "utf8") == "seq" # ok again - -# parse immutable buffer -tree = ryml.parse_in_arena(src) -check(tree) # OK - -# parse mutable buffer. -# requires bytearrays or objects offering writeable memory -mutable = bytearray(src) -tree = ryml.parse_in_place(mutable) -check(tree) # OK -``` -As expected, the performance results so far are encouraging. In -a [timeit benchmark](api/python/parse_bm.py) compared -against [PyYaml](https://pyyaml.org/) -and [ruamel.yaml](https://yaml.readthedocs.io/en/latest/), ryml parses -quicker by generally 100x and up to 400x: -``` -+----------------------------------------+-------+----------+----------+-----------+ -| style_seqs_blck_outer1000_inner100.yml | count | time(ms) | avg(ms) | avg(MB/s) | -+----------------------------------------+-------+----------+----------+-----------+ -| parse:RuamelYamlParse | 1 | 4564.812 | 4564.812 | 0.173 | -| parse:PyYamlParse | 1 | 2815.426 | 2815.426 | 0.280 | -| parse:RymlParseInArena | 38 | 588.024 | 15.474 | 50.988 | -| parse:RymlParseInArenaReuse | 38 | 466.997 | 12.289 | 64.202 | -| parse:RymlParseInPlace | 38 | 579.770 | 15.257 | 51.714 | -| parse:RymlParseInPlaceReuse | 38 | 462.932 | 12.182 | 64.765 | -+----------------------------------------+-------+----------+----------+-----------+ -``` -(Note that the parse timings above are somewhat biased towards ryml, because -it does not perform any type conversions in Python-land: return types -are merely `memoryviews` to the source buffer, possibly copied to the tree's -arena). - -As for emitting, the improvement can be as high as 3000x: -``` -+----------------------------------------+-------+-----------+-----------+-----------+ -| style_maps_blck_outer1000_inner100.yml | count | time(ms) | avg(ms) | avg(MB/s) | -+----------------------------------------+-------+-----------+-----------+-----------+ -| emit_yaml:RuamelYamlEmit | 1 | 18149.288 | 18149.288 | 0.054 | -| emit_yaml:PyYamlEmit | 1 | 2683.380 | 2683.380 | 0.365 | -| emit_yaml:RymlEmitToNewBuffer | 88 | 861.726 | 9.792 | 99.976 | -| emit_yaml:RymlEmitReuse | 88 | 437.931 | 4.976 | 196.725 | -+----------------------------------------+-------+-----------+-----------+-----------+ -``` - - ------- - -## YAML standard conformance - -ryml is close to feature complete. Most of the YAML features are well -covered in the unit tests, and expected to work, unless in the -exceptions noted below. - -Of course, there are many dark corners in YAML, and there certainly -can appear cases which ryml fails to parse. Your [bug reports or pull -requests](https://github.com/biojppm/rapidyaml/issues) are very -welcome. - -See also [the roadmap](./ROADMAP.md) for a list of future work. - - -### Known limitations - -ryml deliberately makes no effort to follow the standard in the -following situations: - -* Containers are not accepted as mapping keys: keys must be scalars. -* Tab characters after `:` and `-` are not accepted tokens, unless - ryml is compiled with the macro `RYML_WITH_TAB_TOKENS`. This - requirement exists because checking for tabs introduces branching - into the parser's hot code and in some cases costs as much as 10% - in parsing time. -* Anchor names must not end with a terminating colon: eg `&anchor: key: val`. -* Non-unique map keys are allowed. Enforcing key uniqueness in the - parser or in the tree would cause log-linear parsing complexity (for - root children on a mostly flat tree), and would increase code size - through added structural, logical and cyclomatic complexity. So - enforcing uniqueness in the parser would hurt users who may not care - about it (they may not care either because non-uniqueness is OK for - their use case, or because it is impossible to occur). On the other - hand, any user who requires uniqueness can easily enforce it by - doing a post-parse walk through the tree. So choosing to not enforce - key uniqueness adheres to the spirit of "don't pay for what you - don't use". -* `%YAML` directives have no effect and are ignored. -* `%TAG` directives are limited to a default maximum of 4 instances - per `Tree`. To increase this maximum, define the preprocessor symbol - `RYML_MAX_TAG_DIRECTIVES` to a suitable value. This arbitrary limit - reflects the usual practice of having at most 1 or 2 tag directives; - also, be aware that this feature is under consideration for removal - in YAML 1.3. - -Also, ryml tends to be on the permissive side where the YAML standard -dictates there should be an error; in many of these cases, ryml will -tolerate the input. This may be good or bad, but in any case is being -improved on (meaning ryml will grow progressively less tolerant of -YAML errors in the coming releases). So we strongly suggest to stay -away from those dark corners of YAML which are generally a source of -problems, which is a good practice anyway. - -If you do run into trouble and would like to investigate conformance -of your YAML code, beware of existing online YAML linters, many of -which are not fully conformant; instead, try using -[https://play.yaml.io](https://play.yaml.io), an amazing tool which -lets you dynamically input your YAML and continuously see the results -from all the existing parsers (kudos to @ingydotnet and the people -from the YAML test suite). And of course, if you detect anything wrong -with ryml, please [open an -issue](https://github.com/biojppm/rapidyaml/issues) so that we can -improve. - - -### Test suite status - -As part of its CI testing, ryml uses the [YAML test -suite](https://github.com/yaml/yaml-test-suite). This is an extensive -set of reference cases covering the full YAML spec. Each of these -cases have several subparts: - * `in-yaml`: mildly, plainly or extremely difficult-to-parse YAML - * `in-json`: equivalent JSON (where possible/meaningful) - * `out-yaml`: equivalent standard YAML - * `emit-yaml`: equivalent standard YAML - * `events`: reference results (ie, expected tree) - -When testing, ryml parses each of the 4 yaml/json parts, then emits -the parsed tree, then parses the emitted result and verifies that -emission is idempotent, ie that the emitted result is semantically the -same as its input without any loss of information. To ensure -consistency, this happens over four levels of parse/emission -pairs. And to ensure correctness, each of the stages is compared -against the `events` spec from the test, which constitutes the -reference. The tests also check for equality between the reference -events in the test case and the events emitted by ryml from the data -tree parsed from the test case input. All of this is then carried out -combining several variations: both unix `\n` vs windows `\r\n` line -endings, emitting to string, file or streams, which results in ~250 -tests per case part. With multiple parts per case and ~400 reference -cases in the test suite, this makes over several hundred thousand -individual tests to which ryml is subjected, which are added to the -unit tests in ryml, which also employ the same extensive -combinatorial approach. - -Also, note that in [their own words](http://matrix.yaml.io/), the -tests from the YAML test suite *contain a lot of edge cases that don't -play such an important role in real world examples*. And yet, despite -the extreme focus of the test suite, currently ryml only fails a minor -fraction of the test cases, mostly related with the deliberate -limitations noted above. Other than those limitations, by far the main -issue with ryml is that several standard-mandated parse errors fail to -materialize. For the up-to-date list of ryml failures in the -test-suite, refer to the [list of known -exceptions](test/test_suite/test_suite_parts.cpp) from ryml's test -suite runner, which is used as part of ryml's CI process. - - ------- - -## Alternative libraries - -Why this library? Because none of the existing libraries was quite -what I wanted. When I started this project in 2018, I was aware of these two -alternative C/C++ libraries: - - * [libyaml](https://github.com/yaml/libyaml). This is a bare C - library. It does not create a representation of the data tree, so - I don't see it as practical. My initial idea was to wrap parsing - and emitting around libyaml's convenient event handling, but to my - surprise I found out it makes heavy use of allocations and string - duplications when parsing. I briefly pondered on sending PRs to - reduce these allocation needs, but not having a permanent tree to - store the parsed data was too much of a downside. - * [yaml-cpp](https://github.com/jbeder/yaml-cpp). This library may - be full of functionality, but is heavy on the use of - node-pointer-based structures like `std::map`, allocations, string - copies, polymorphism and slow C++ stream serializations. This is - generally a sure way of making your code slower, and strong - evidence of this can be seen in the benchmark results above. - -Recently [libfyaml](https://github.com/pantoniou/libfyaml) -appeared. This is a newer C library, fully conformant to the YAML -standard with an amazing 100% success in the test suite; it also offers -the tree as a data structure. As a downside, it does not work in -Windows, and it is also multiple times slower parsing and emitting. - -When performance and low latency are important, using contiguous -structures for better cache behavior and to prevent the library from -trampling caches, parsing in place and using non-owning strings is of -central importance. Hence this Rapid YAML library which, with minimal -compromise, bridges the gap from efficiency to usability. This library -takes inspiration from -[RapidJSON](https://github.com/Tencent/rapidjson) and -[RapidXML](http://rapidxml.sourceforge.net/). - ------- -## License - -ryml is permissively licensed under the [MIT license](LICENSE.txt). - diff --git a/thirdparty/ryml/ROADMAP.md b/thirdparty/ryml/ROADMAP.md deleted file mode 100644 index ca43b7c77..000000000 --- a/thirdparty/ryml/ROADMAP.md +++ /dev/null @@ -1,18 +0,0 @@ -# Roadmap - -Roughly in order of priority: - - * Cleanup: - * Review & cleanup API surface. - * Turn calls to `C4_ASSERT()` into calls to `RYML_ASSERT()` - * Add emit formatting controls: - * add single-line flow formatter - * add multi-line flow formatters - * indenting - * non indenting - * keep current block formatter - * add customizable linebreak limits (number of columns) to every formatter - * add per node format flags - * (lesser priority) add auto formatter using reasonable heuristics to - switch between other existing formatters - * Investigate possibility of comment-preserving roundtrips diff --git a/thirdparty/ryml/api/python/.gitignore b/thirdparty/ryml/api/python/.gitignore deleted file mode 100644 index 830e7e4d6..000000000 --- a/thirdparty/ryml/api/python/.gitignore +++ /dev/null @@ -1,141 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Version file generated by setuptools_scm -version.py - -# SWIG produced file -ryml/ryml.py diff --git a/thirdparty/ryml/api/python/Makefile b/thirdparty/ryml/api/python/Makefile deleted file mode 100644 index bdb5a6ffd..000000000 --- a/thirdparty/ryml/api/python/Makefile +++ /dev/null @@ -1,94 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT - -# Use bash even on Windows -SHELL := /bin/bash - -# On Windows the activate script is stored in a different location. -ACTIVATE_SCRIPT := venv/bin/activate -ifeq ($(OS),Windows_NT) -ACTIVATE_SCRIPT := venv/Scripts/activate -endif - -# How to invoke python -PYTHON := python -# How to invoke pytest -PYTEST := $(PYTHON) -m pytest -vvv - -ACTIVATE=[[ -e $(ACTIVATE_SCRIPT) ]] && source $(ACTIVATE_SCRIPT); - -.PHONY: clean -clean: - rm -rf dist *.egg-info - rm -rf ../../build ../../.egg* - rm -rf ryml/*.so ryml/ryml.py ryml/include ryml/lib - -.PHONY: venv-clean -venv-clean: - rm -rf venv - - -$(ACTIVATE_SCRIPT): requirements.txt Makefile - make venv - @touch $(ACTIVATE_SCRIPT) - -.PHONY: venv -venv: - virtualenv --python=python3 --always-copy venv - # Packaging tooling. - ${ACTIVATE} pip install -U pip - # Setup requirements. - ${ACTIVATE} pip install -v -r requirements.txt - ${ACTIVATE} pip install -v -e ../.. - @${ACTIVATE} $(PYTHON) -c "from ryml.version import version as v; print('Installed version:', v)" - -.PHONY: build-sdist -build-sdist: | $(ACTIVATE_SCRIPT) - ${ACTIVATE} (cd ../..; $(PYTHON) -m build --sdist --outdir $(PWD)/dist) - - -.PHONY: build-wheel -build-wheel: | $(ACTIVATE_SCRIPT) - rm -rf dist - $(MAKE) build-sdist - @ls -l dist/*.tar.gz - ${ACTIVATE} pip wheel -v dist/*.tar.gz --wheel-dir $(PWD)/dist - -.PHONY: build -build: - rm -rf build dist - $(MAKE) build-sdist - $(MAKE) build-wheel - -# PYPI_TEST = --repository-url https://test.pypi.org/legacy/ -PYPI_TEST = --repository testpypi - -.PHONY: upload-test -upload-test: | $(ACTIVATE_SCRIPT) - make clean - make build-sdist - ${ACTIVATE} twine upload ${PYPI_TEST} dist/* - -.PHONY: upload -upload: | $(ACTIVATE_SCRIPT) - make clean - make build-sdist - ${ACTIVATE} twine upload --verbose dist/* - -.PHONY: check -check: | $(ACTIVATE_SCRIPT) - make clean - make build-wheel - ${ACTIVATE} twine check dist/*.whl - -.PHONY: install -install: | $(ACTIVATE_SCRIPT) - ${ACTIVATE} $(PYTHON) setup.py install - -.PHONY: test -test: | $(ACTIVATE_SCRIPT) - ${ACTIVATE} $(PYTEST) tests - -.PHONY: version -version: | $(ACTIVATE_SCRIPT) - ${ACTIVATE} $(PYTHON) setup.py --version diff --git a/thirdparty/ryml/api/python/bm/bm_parse.py b/thirdparty/ryml/api/python/bm/bm_parse.py deleted file mode 100644 index 294be679b..000000000 --- a/thirdparty/ryml/api/python/bm/bm_parse.py +++ /dev/null @@ -1,237 +0,0 @@ -import ryml -import ruamel.yaml -import yaml -import timeit -import time -import copy -import prettytable -import os.path -from collections import OrderedDict as odict - - -def _nodbg(*args, **kwargs): - pass - - -def _dbg(*args, **kwargs): - print(*args, **kwargs, file=sys.stderr, flush=True) - - -dbg = _dbg - - -class RunResults: - - __slots__ = ('name', 'time_ms', 'count', 'avg', 'MBps') - - def __init__(self, name, time_ms, count, num_bytes): - self.name = name - self.time_ms = time_ms - self.count = count - self.avg = time_ms / count - num_megabytes = count * num_bytes / 1.0e6 - num_seconds = time_ms / 1000.0 - self.MBps = num_megabytes / num_seconds - - def __str__(self): - fmt = "{}: count={} time={:.3f}ms avg={:.3f}ms MB/s={:.3f}" - fmt = fmt.format(self.name, self.count, self.time_ms, self.avg, self.MBps) - return fmt - - -class BmCase: - - def __init__(self, filename): - with open(filename, "r") as f: - src = f.read() - self.filename = filename - self.src_as_str = src - self.src_as_bytes = bytes(src, "utf8") - self.src_as_bytearray = bytearray(src, "utf8") - self.src_as_bytearray_orig = copy.copy(self.src_as_bytearray) - self.emittree = ryml.parse_in_arena(self.src_as_bytearray) - self.emitbuf = bytearray(4 * len(self.src_as_str)) # should be enough - - def run(self, bm_method_name, cls): - def run_bm(obj, subject): - obj.count = 0 - t = timeit.Timer(subject) - delta = time.time() - result = t.autorange() #lambda number, time_taken: time_taken > 1.0) - delta = 1000. * (time.time() - delta) - return delta, obj.count - obj = cls(self) - if not hasattr(obj, bm_method_name): - return None - name = bm_method_name + ":" + cls.__name__ - dbg(name, "...") - method = getattr(obj, bm_method_name) - reset_name = 'reset_' + bm_method_name - reset_fn = getattr(obj, reset_name, None) - def bm_fn(): - method(self) - obj.count += 1 - if reset_fn is not None: - reset_fn(self) - delta, count = run_bm(obj, bm_fn) - # correct the benchmark to account for the time spent - # resetting - if reset_fn is not None: - # find out how much it takes to reset the bytearray - if not hasattr(obj, 'bm_reset_done'): - def bm_reset(): - reset_fn(self) - obj.count += 1 - rdelta, rcount = run_bm(obj, bm_reset) - obj.bm_reset_time_per_iteration = rdelta / rcount - dbg(name, "reset_time_per_iteration={:.3f}us".format(obj.bm_reset_time_per_iteration * 1000.0)) - obj.bm_reset_done = True - reset_correction = count * obj.bm_reset_time_per_iteration - dbg(name, "delta={:.3f}ms".format(delta), "reset_correction={:.3f}ms({:.2f}%)".format(reset_correction, 100.0 * reset_correction / delta)) - delta -= reset_correction - ret = RunResults(name, delta, count, len(self.src_as_str)) - dbg(name, "ok:", ret) - return ret - - -def run(case, benchmarks, approaches): - for bm in benchmarks: - results = odict() - for cls in approaches: - r = case.run(bm, cls) - if r is None: - continue - results[r.name] = r - table = prettytable.PrettyTable() - name = os.path.basename(case.filename) - table.field_names = [name, "count", "time(ms)", "avg(ms)", "avg(MB/s)"] - table.align[name] = "l" - def i(v): return "{:5d}".format(v) - def f(v): return "{:8.3f}".format(v) - for v in results.values(): - table.add_row([v.name, i(v.count), f(v.time_ms), f(v.avg), f(v.MBps)]) - print(table) - - -class BmCaseRun: - def __init__(self, case): - self.reset_bytearray = False - - -class RymlParseInArena(BmCaseRun): - - def parse(self, case): - _ = ryml.parse_in_arena(case.src_as_bytearray) - - -class RymlParseInArenaReuse(BmCaseRun): - - def __init__(self, case): - self.tree = ryml.Tree() - - def parse(self, case): - ryml.parse_in_arena(case.src_as_bytearray, tree=self.tree) - - def reset_parse(self, case): - self.tree.clear() - self.tree.clear_arena() - - -class RymlParseInPlace(BmCaseRun): - - def parse(self, case): - _ = ryml.parse_in_place(case.src_as_bytearray) - - def reset_parse(self, case): - case.src_as_bytearray = copy.copy(case.src_as_bytearray_orig) - - -class RymlParseInPlaceReuse(BmCaseRun): - - def __init__(self, case): - self.tree = ryml.Tree() - - def parse(self, case): - ryml.parse_in_place(case.src_as_bytearray, tree=self.tree) - - def reset_parse(self, case): - self.tree.clear() - self.tree.clear_arena() - case.src_as_bytearray = copy.copy(case.src_as_bytearray_orig) - - -class RuamelYamlParse(BmCaseRun): - - def parse(self, case): - _ = ruamel.yaml.load(case.src_as_str, Loader=ruamel.yaml.Loader) - - -class PyYamlParse(BmCaseRun): - - def parse(self, case): - _ = yaml.safe_load(case.src_as_str) - - -class RymlEmitToNewBuffer(BmCaseRun): - - def emit_yaml(self, case): - _ = ryml.emit_yaml(case.emittree) - - def emit_json(self, case): - _ = ryml.emit_json(case.emittree) - - -class RymlEmitReuse(BmCaseRun): - - def emit_yaml(self, case): - _ = ryml.emit_yaml_in_place(case.emittree, case.emitbuf) - - def emit_json(self, case): - _ = ryml.emit_json_in_place(case.emittree, case.emitbuf) - - -class RuamelYamlEmit: - - def __init__(self, case): - case.ruamel_emittree = ruamel.yaml.load(case.src_as_str, Loader=ruamel.yaml.Loader) - - def emit_yaml(self, case): - # https://stackoverflow.com/a/47617341/5875572 - class MyToStr: - def __init__(self, *args, **kwargs): - self.s = b"" - def write(self, s): - self.s += s - dumper = MyToStr() - ruamel.yaml.YAML().dump(case.ruamel_emittree, MyToStr()) - - -class PyYamlEmit: - - def __init__(self, case): - case.pyyaml_emittree = yaml.load(case.src_as_str, Loader=yaml.Loader) - - def emit_yaml(self, case): - _ = yaml.dump(case.pyyaml_emittree) - - -if __name__ == "__main__": - import sys - if len(sys.argv) < 2: - raise Exception("") - filename = sys.argv[1] - if filename.endswith("outer1000_inner1000.yml"): # this one is too heavy for the Python libs - exit(0) - case = BmCase(filename) - run(case, benchmarks=('parse', ), - approaches=(RuamelYamlParse, - PyYamlParse, - RymlParseInArena, - RymlParseInArenaReuse, - RymlParseInPlace, - RymlParseInPlaceReuse)) - run(case, benchmarks=('emit_yaml', 'emit_json', ), - approaches=(RuamelYamlEmit, - PyYamlEmit, - RymlEmitToNewBuffer, - RymlEmitReuse)) diff --git a/thirdparty/ryml/api/python/requirements.txt b/thirdparty/ryml/api/python/requirements.txt deleted file mode 100644 index 86c126420..000000000 --- a/thirdparty/ryml/api/python/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -ruamel.yaml -ninja -pyyaml -prettytable -pytest diff --git a/thirdparty/ryml/api/python/ryml/__init__.py b/thirdparty/ryml/api/python/ryml/__init__.py deleted file mode 100644 index 8bbb62199..000000000 --- a/thirdparty/ryml/api/python/ryml/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from ryml.ryml import * -from .version import * diff --git a/thirdparty/ryml/api/python/tests/test_parse.py b/thirdparty/ryml/api/python/tests/test_parse.py deleted file mode 100644 index bb7973cb3..000000000 --- a/thirdparty/ryml/api/python/tests/test_parse.py +++ /dev/null @@ -1,488 +0,0 @@ -import ryml -from ryml.ryml import _same_ptr, _same_mem -import unittest - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -class TestSubstrInterop(unittest.TestCase): - - # ------------------------------------------------ - # str - - # CAN create c4::csubstr from string object - def test11_str2csubstr(self): - s = "asdasd" - m = ryml.as_csubstr(s) - self.assertTrue(_same_ptr(s, m)) - self.assertTrue(_same_mem(s, m)) - self.assertEqual(s, ryml.u(m)) - # - m = ryml.as_csubstr(m) - self.assertTrue(_same_ptr(s, m)) - self.assertTrue(_same_mem(s, m)) - self.assertEqual(s, ryml.u(m)) - - # CANNOT create c4::substr from string object - def test12_str2substr(self): - s = "" - with self.assertRaises(TypeError) as context: - _ = ryml.as_substr(s) - self.assertTrue(type(context.exception), TypeError) - - # ------------------------------------------------ - # bytes - - # CAN create c4::csubstr from string object - def test21_bytes2csubstr(self): - s = b"foo21" - m = ryml.as_csubstr(s) - self.assertTrue(_same_ptr(s, m)) - self.assertTrue(_same_mem(s, m)) - self.assertEqual(s, m) - # - m = ryml.as_csubstr(m) - self.assertTrue(_same_ptr(s, m)) - self.assertTrue(_same_mem(s, m)) - self.assertEqual(s, m) - - # CANNOT create c4::csubstr from string object - def test22_bytes2substr(self): - s = b"foo22" - with self.assertRaises(TypeError) as context: - _ = ryml.as_substr(s) - self.assertTrue(type(context.exception), TypeError) - - # ------------------------------------------------ - # bytearray - - # CAN create c4::csubstr from string object - def test31_bytes2csubstr(self): - s = bytearray("foo31", "utf8") - m = ryml.as_csubstr(s) - self.assertTrue(_same_ptr(s, m)) - self.assertTrue(_same_mem(s, m)) - self.assertEqual(s, m) - # - m = ryml.as_csubstr(m) - self.assertTrue(_same_ptr(s, m)) - self.assertTrue(_same_mem(s, m)) - self.assertEqual(s, m) - - # CANNOT create c4::csubstr from string object - def test32_bytes2substr(self): - s = bytearray("foo31", "utf8") - m = ryml.as_csubstr(s) - self.assertTrue(_same_ptr(s, m)) - self.assertTrue(_same_mem(s, m)) - self.assertEqual(s, m) - # - m = ryml.as_csubstr(m) - self.assertTrue(_same_ptr(s, m)) - self.assertTrue(_same_mem(s, m)) - self.assertEqual(s, m) - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -def _addmap(t, node, k=None): - m = t.append_child(node) - if k is None: - t.to_map(m) - else: - t.to_map(m, k) - return m - - -def _addseq(t, node, k=None): - m = t.append_child(node) - if k is None: - t.to_seq(m) - else: - t.to_seq(m, k) - return m - - -def _addval(t, node, k, v=None): - ch = t.append_child(node) - if v is None: - t.to_val(ch, k) - else: - t.to_keyval(ch, k, v) - return ch - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -def check_tree_mod(ut, t): - # some convenient shorthands - eq = ut.assertEqual - def _addval_and_check(node, k, v=None): - ch = _addval(t, node, k, v) - pos = t.child_pos(node, ch) - eq(t.child(node, pos), ch) - if v is not None: - eq(t.find_child(node, k), ch) - eq(t.child(node, pos), t.find_child(node, k)) - return ch - def _addseq_and_check(node, k): - ch = _addseq(t, node, k) - eq(t.find_child(node, k), ch) - return ch - def _addmap_and_check(node, k): - ch = _addmap(t, node, k) - eq(t.find_child(node, k), ch) - return ch - m = _addmap_and_check(t.root_id(), "check_tree_mod_map") - _addval_and_check(m, "k1", "v1") - _addval_and_check(m, "k2", "v2") - _addval_and_check(m, "k3", "v3") - eq(t.num_children(m), 3) - eq(t.num_siblings(t.first_child(m)), 3) - s = _addseq_and_check(t.root_id(), "check_tree_mod_seq") - _addval_and_check(s, "v1") - _addval_and_check(s, "v2") - _addval_and_check(s, "v3") - eq(t.num_children(s), 3) - eq(t.num_siblings(t.first_child(m)), 3) - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -class SimpleTestCase: - - yaml = "{'HELLO': a, foo: \"b\", bar: c, baz: d, seq: [0, 1, 2, 3]}" - - def check(self, ut, t, is_json=False): - # some convenient shorthands - eq = ut.assertEqual - ne = ut.assertNotEqual - fs = ut.assertFalse - tr = ut.assertTrue - # - eq(t.size(), 10) - tr(t.is_root(0)) - eq(t.num_children(0), 5) - eq(t.find_child(0, b"HELLO"), 1) - eq(t.find_child(0, b"foo"), 2) - eq(t.find_child(0, b"bar"), 3) - eq(t.find_child(0, b"baz"), 4) - eq(t.find_child(0, b"seq"), 5) - eq(t.parent(0), ryml.NONE) - eq(t.parent(1), 0) - eq(t.parent(2), 0) - eq(t.parent(3), 0) - eq(t.parent(4), 0) - eq(t.parent(5), 0) - fs(t.is_root(1)) - fs(t.is_root(2)) - fs(t.is_root(3)) - fs(t.is_root(4)) - fs(t.is_root(5)) - fs(t.has_child(0, b"foozzie")) - fs(t.has_child(0, b"bark")) - fs(t.has_child(0, b"bart")) - fs(t.has_child(0, b"bazk")) - eq(t.next_sibling(0), ryml.NONE) - eq(t.prev_sibling(0), ryml.NONE) - eq(t.prev_sibling(1), ryml.NONE) - eq(t.next_sibling(5), ryml.NONE) - tr(t.has_child(0, b"HELLO")) - tr(t.has_child(0, b"foo")) - tr(t.has_child(0, b"bar")) - tr(t.has_child(0, b"baz")) - eq(t.key(1), b"HELLO") - eq(t.key(2), b"foo") - eq(t.key(3), b"bar") - eq(t.key(4), b"baz") - eq(t.key(5), b"seq") - eq(t.val(1), b"a") - eq(t.val(2), b"b") - eq(t.val(3), b"c") - eq(t.val(4), b"d") - eq(t.val(6), b"0") - eq(t.val(7), b"1") - eq(t.val(8), b"2") - eq(t.val(9), b"3") - if not is_json: - tr(t.is_key_quoted(1)) - fs(t.is_key_quoted(2)) - fs(t.is_key_quoted(3)) - fs(t.is_key_quoted(4)) - fs(t.is_key_quoted(5)) - else: - tr(t.is_key_quoted(1)) - tr(t.is_key_quoted(2)) - tr(t.is_key_quoted(3)) - tr(t.is_key_quoted(4)) - tr(t.is_key_quoted(5)) - if not is_json: - fs(t.is_val_quoted(1)) - tr(t.is_val_quoted(2)) - fs(t.is_val_quoted(3)) - fs(t.is_val_quoted(4)) - fs(t.is_val_quoted(5)) - fs(t.is_val_quoted(6)) - fs(t.is_val_quoted(7)) - fs(t.is_val_quoted(8)) - fs(t.is_val_quoted(9)) - else: - tr(t.is_val_quoted(1)) - tr(t.is_val_quoted(2)) - tr(t.is_val_quoted(3)) - tr(t.is_val_quoted(4)) - fs(t.is_val_quoted(5)) - fs(t.is_val_quoted(6)) - fs(t.is_val_quoted(7)) - fs(t.is_val_quoted(8)) - fs(t.is_val_quoted(9)) - if not is_json: - tr(t.is_quoted(1)) - tr(t.is_quoted(2)) - fs(t.is_quoted(3)) - fs(t.is_quoted(4)) - fs(t.is_quoted(5)) - fs(t.is_quoted(6)) - fs(t.is_quoted(7)) - fs(t.is_quoted(8)) - fs(t.is_quoted(9)) - else: - tr(t.is_quoted(1)) - tr(t.is_quoted(2)) - tr(t.is_quoted(3)) - tr(t.is_quoted(4)) - tr(t.is_quoted(5)) - fs(t.is_quoted(6)) - fs(t.is_quoted(7)) - fs(t.is_quoted(8)) - fs(t.is_quoted(9)) - tr(t.has_sibling(1, b"bar")) - tr(t.has_sibling(1, b"baz")) - tr(t.has_sibling(2, b"foo")) - tr(t.has_sibling(2, b"baz")) - tr(t.has_sibling(3, b"foo")) - tr(t.has_sibling(3, b"bar")) - for i in (1, 2, 3, 4, 5): - eq(t.find_sibling(i, b"HELLO"), 1) - eq(t.find_sibling(i, b"foo"), 2) - eq(t.find_sibling(i, b"bar"), 3) - eq(t.find_sibling(i, b"baz"), 4) - eq(t.find_sibling(i, b"seq"), 5) - # - num = 0 - for id in ryml.children(t): - num += 1 - eq(id, num) - eq(num, t.num_children(t.root_id())) - eq(num, t.num_siblings(t.first_child(t.root_id()))) - # - num = 0 - for id in ryml.children(t, 1): - num += 1 - eq(num, 0) - # - num = 0 - for id in ryml.siblings(t, 1): - num += 1 - eq(id, num) - eq(num, t.num_children(t.root_id())) - eq(num, t.num_siblings(t.first_child(t.root_id()))) - # - num = 0 - for id in ryml.siblings(t, 3): - num += 1 - eq(id, num) - eq(num, 5) - eq(num, t.num_siblings(t.first_child(t.root_id()))) - # - for i, ch in enumerate(ryml.children(t, 5)): - eq(t.val(ch), [b"0", b"1", b"2", b"3"][i]) - sibs = [b"HELLO", b"foo", b"bar", b"baz", b"seq"] - sibs_s = ["HELLO", "foo", "bar", "baz", "seq"] - for i, sib in enumerate(ryml.siblings(t, 5)): - k = t.key(sib) - k_s = str(k, "utf8") - eq(k, sibs[i]) - eq(k_s, sibs_s[i]) - ne(k, sibs_s[i]) - ne(k_s, sibs[i]) - k_s = str(k) - ne(k_s, sibs_s[i]) - ne(k_s, sibs[i]) - num = 0 - for id in ryml.siblings(t, 0): - num += 1 - eq(num, 1) - # - num = 0 - for id, level in ryml.walk(t): - num += 1 - if t.is_root(id): - eq(id, 0) - eq(level, 0) - if t.is_map(id): - eq(id, 0) - eq(level, 0) - if t.is_seq(id): - eq(id, 5) - eq(level, 1) - if t.is_keyval(id): - tr(id > 0 and id < 5) - if t.is_val(id): - tr(id > 5) - eq(level, 2) - eq(num, t.size()) - # - num = 0 - for id in ryml.walk(t, 5): - num += 1 - eq(num, 5) - # - num = 0 - for id in ryml.walk(t, 9): - num += 1 - eq(num, 1) - check_tree_mod(ut, t) - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -class TestRunner(unittest.TestCase): - - def setUp(self): - self._setUp(SimpleTestCase()) - - # allow creating this class with different cases - # if they are added - def _setUp(self, case): - self.case = case - self.src_as_str = str(case.yaml) - self.src_as_bytes = bytes(case.yaml, "utf8") - self.src_as_bytearray = bytearray(case.yaml, "utf8") - - # ---------------------------------------------------------- - def test11_str__arena(self): # cannot read string buffers (or can we?) - tree = ryml.parse_in_arena(self.src_as_str) - self.case.check(self, tree) - - def test12_str__arena__reuse_tree(self): # cannot read string buffers (or can we?) - t = ryml.Tree() - ryml.parse_in_arena(self.src_as_str, tree=t) - self.case.check(self, t) - - def test13_str__inplace(self): # cannot mutate string buffers (or can we?) - with self.assertRaises(TypeError) as context: - ryml.parse_in_place(self.src_as_str) - self.assertTrue(type(context.exception), TypeError) - - # ---------------------------------------------------------- - def test21_bytes__arena(self): - tree = ryml.parse_in_arena(self.src_as_bytes) - self.case.check(self, tree) - - def test22_bytes__arena__reuse_tree(self): - t = ryml.Tree() - r = ryml.parse_in_arena(self.src_as_bytes, tree=t) - self.assertTrue(r is t) - self.case.check(self, t) - - def test23_bytes__inplace(self): # cannot mutate bytes buffers - with self.assertRaises(TypeError) as context: - ryml.parse_in_place(self.src_as_bytes) - self.assertTrue(type(context.exception), TypeError) - - # ---------------------------------------------------------- - def test31_bytearray__arena(self): - tree = ryml.parse_in_arena(self.src_as_bytearray) - self.case.check(self, tree) - - def test32_bytearray__arena__reuse_tree(self): - t = ryml.Tree() - r = ryml.parse_in_arena(self.src_as_bytearray, tree=t) - self.assertTrue(r is t) - self.case.check(self, t) - - def test33_bytearray__inplace(self): # bytearray buffers are mutable - tree = ryml.parse_in_place(self.src_as_bytearray) - self.case.check(self, tree) - - def test34_bytearray__inplace__reuse_tree(self): # bytearray buffers are mutable - t = ryml.Tree() - r = ryml.parse_in_place(self.src_as_bytearray, tree=t) - self.assertTrue(r is t) - self.case.check(self, t) - - # ---------------------------------------------------------- - def test41_emit_yaml(self): - tree = ryml.parse_in_arena(self.src_as_bytearray) - yaml = ryml.emit_yaml(tree) - output_tree = ryml.parse_in_arena(yaml) - self.case.check(self, output_tree) - - def test41_emit_json(self): - tree = ryml.parse_in_arena(self.src_as_bytearray) - json = ryml.emit_json(tree) - output_tree = ryml.parse_in_arena(json) - self.case.check(self, output_tree, is_json=True) - - def test42_compute_emit_yaml_length(self): - tree = ryml.parse_in_arena(self.src_as_bytearray) - yaml = ryml.emit_yaml(tree) - length = ryml.compute_emit_yaml_length(tree) - self.assertEqual(len(yaml), length) - - def test42_compute_emit_json_length(self): - tree = ryml.parse_in_arena(self.src_as_bytearray) - json = ryml.emit_json(tree) - length = ryml.compute_emit_json_length(tree) - self.assertEqual(len(json), length) - - def test43_emit_yaml_inplace(self): - tree = ryml.parse_in_arena(self.src_as_bytearray) - yaml = ryml.emit_yaml(tree) - length = ryml.compute_emit_yaml_length(tree) - self.assertEqual(len(yaml), length) - buf = bytearray(length) - s = ryml.emit_yaml_in_place(tree, buf) - self.assertEqual(len(s), length) - self.assertTrue(s.tobytes().decode('utf-8') == yaml) - self.assertTrue(buf.decode('utf-8') == yaml) - - def test43_emit_json_inplace(self): - tree = ryml.parse_in_arena(self.src_as_bytearray) - json = ryml.emit_json(tree) - length = ryml.compute_emit_json_length(tree) - self.assertEqual(len(json), length) - buf = bytearray(length) - s = ryml.emit_json_in_place(tree, buf) - self.assertEqual(len(s), length) - self.assertTrue(s.tobytes().decode('utf-8') == json) - self.assertTrue(buf.decode('utf-8') == json) - - def test44_emit_yaml_short_buf(self): - tree = ryml.parse_in_arena(self.src_as_bytearray) - length = ryml.compute_emit_yaml_length(tree) - buf = bytearray(length-1) - with self.assertRaises(IndexError): - ryml.emit_yaml_in_place(tree, buf) - - def test44_emit_json_short_buf(self): - tree = ryml.parse_in_arena(self.src_as_bytearray) - length = ryml.compute_emit_json_length(tree) - buf = bytearray(length-1) - with self.assertRaises(IndexError): - ryml.emit_json_in_place(tree, buf) - - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -if __name__ == "__main__": - unittest.main() diff --git a/thirdparty/ryml/api/ryml.i b/thirdparty/ryml/api/ryml.i deleted file mode 100644 index ac226142a..000000000 --- a/thirdparty/ryml/api/ryml.i +++ /dev/null @@ -1,662 +0,0 @@ - -%module ryml - - -//----------------------------------------------------------------------------- -// this block will be pasted verbatim in the generated C++ source file - -%{ -// specifies that the resulting C file should be built as a python -// extension, inserting the module init code -#define SWIG_FILE_WITH_INIT - -#include - -namespace c4 { -namespace yml { - -using substr = c4::substr; -using csubstr = c4::csubstr; - -} /* namespace yml */ -} /* namespace c4 */ - -%} - -//----------------------------------------------------------------------------- - - -%apply (const char *STRING, size_t LENGTH) { (const char *str, size_t len) }; -%apply (char *STRING, size_t LENGTH) { (char *str, size_t len) }; -%newobject emit_malloc; - -%typemap(in) c4::substr { -#if defined(SWIGPYTHON) - Py_buffer view; - int ok = PyObject_CheckBuffer($input); - if(ok) - { - ok = (0 == PyObject_GetBuffer($input, &view, PyBUF_SIMPLE|PyBUF_WRITABLE)); - } - if(ok) - { - $1 = c4::substr((char*)view.buf, view.len); - PyBuffer_Release(&view); - } - else - { - PyErr_SetString(PyExc_TypeError, "could not get mutable memory for c4::csubstr - have you passed a str?"); - SWIG_fail; - } -#else -#error no "in" typemap defined for this export language -#endif -}; - -%typemap(in) c4::csubstr { -#if defined(SWIGPYTHON) - Py_buffer view; - view.buf = nullptr; - int ok = PyObject_CheckBuffer($input); - if(ok) - { - ok = (0 == PyObject_GetBuffer($input, &view, PyBUF_CONTIG_RO)); - } - if(ok) - { - $1 = c4::csubstr((const char*)view.buf, view.len); - PyBuffer_Release(&view); - } - else - { - // https://stackoverflow.com/questions/36098984/python-3-3-c-api-and-utf-8-strings - Py_ssize_t sz = 0; - const char *buf = PyUnicode_AsUTF8AndSize($input, &sz); - if(buf || sz == 0) - { - $1 = c4::csubstr(buf, sz); - } - else - { - PyErr_SetString(PyExc_TypeError, "c4::csubstr: could not get readonly memory from python object"); - SWIG_fail; - } - } -#else -#error no "in" typemap defined for this export language -#endif -}; -// Copy the typecheck code for "char *". -%typemap(typecheck) c4::substr = char *; -%typemap(typecheck) c4::csubstr = const char *; - - -%typemap(out) c4::csubstr { -#if defined(SWIGPYTHON) - if($1.str == nullptr) { - $result = Py_None; - Py_INCREF($result); - } else { - PyObject *obj = PyMemoryView_FromMemory((char*)$1.str, $1.len, PyBUF_READ); - if( ! obj) - { - PyErr_SetString(PyExc_TypeError, "could not get readonly memory from c4::csubstr - have you passed a str?"); - SWIG_fail; - } - $result = obj; - } -#else -#error no "out" typemap defined for this export language -#endif -}; - - -%inline %{ - -void parse_csubstr(c4::csubstr s, c4::yml::Tree *t) -{ - c4::yml::parse_in_arena(s, t); -} - -void parse_substr(c4::substr s, c4::yml::Tree *t) -{ - c4::yml::parse_in_place(s, t); -} - -char * emit_yaml_malloc(c4::yml::Tree const& t, size_t id) -{ - c4::substr buf; - c4::substr ret = c4::yml::emit_yaml(t, id, buf, /*error_on_excess*/false); - if(ret.str == nullptr && ret.len > 0) - { - // Use new[] to parse with delete[] in SWIG. - char * alloc = new char[ret.len + 1]; // we'll return a c-string and not a csubstr - c4::substr alloced_buf(alloc, ret.len); - ret = c4::yml::emit_yaml(t, id, alloced_buf, /*error_on_excess*/true); - ret.str[ret.len] = 0; - } - return ret.str; -} - -char * emit_json_malloc(c4::yml::Tree const& t, size_t id) -{ - c4::substr buf; - c4::substr ret = c4::yml::emit_json(t, id, buf, /*error_on_excess*/false); - if(ret.str == nullptr && ret.len > 0) - { - // Use new[] to parse with delete[] in SWIG. - char * alloc = new char[ret.len + 1]; // we'll return a c-string and not a csubstr - c4::substr alloced_buf(alloc, ret.len); - ret = c4::yml::emit_json(t, id, alloced_buf, /*error_on_excess*/true); - ret.str[ret.len] = 0; - } - return ret.str; -} - -size_t emit_yaml_length(const c4::yml::Tree &t, size_t id) -{ - c4::substr buf; - c4::substr ret = c4::yml::emit_yaml(t, id, buf, /*error_on_excess*/false); - return ret.len; -} - -size_t emit_json_length(const c4::yml::Tree &t, size_t id) -{ - c4::substr buf; - c4::substr ret = c4::yml::emit_json(t, id, buf, /*error_on_excess*/false); - return ret.len; -} - -bool emit_yaml_to_substr(const c4::yml::Tree &t, size_t id, c4::substr s, size_t *OUTPUT) -{ - c4::substr result = c4::yml::emit_yaml(t, id, s, /*error_on_excess*/false); - *OUTPUT = result.len; - return result.str == nullptr; -} - -bool emit_json_to_substr(const c4::yml::Tree &t, size_t id, c4::substr s, size_t *OUTPUT) -{ - c4::substr result = c4::yml::emit_json(t, id, s, /*error_on_excess*/false); - *OUTPUT = result.len; - return result.str == nullptr; -} - - -// force a roundtrip to C++, which triggers a conversion to csubstr and returns it as a memoryview -c4::csubstr _get_as_csubstr(c4::csubstr s) -{ - //printf("_get_as_csubstr: %p[%zu]'%.*s'\n", s.str, s.len, (int)s.len, s.str); - return s; -} - -c4::csubstr _get_as_substr(c4::substr s) -{ - //printf("_get_as_substr: %p[%zu]'%.*s'\n", s.str, s.len, (int)s.len, s.str); - return s; -} - - -// utilities for testing -bool _same_ptr(c4::csubstr l, c4::csubstr r) -{ - return l.str == r.str; -} - -bool _same_mem(c4::csubstr l, c4::csubstr r) -{ - return l.str == r.str && l.len == r.len; -} - - -%} - - -//----------------------------------------------------------------------------- - -%pythoncode %{ - -from deprecation import deprecated - - -def as_csubstr(s): - return _get_as_csubstr(s) - -def as_substr(s): - return _get_as_substr(s) - -def u(memview): - return str(memview, "utf8") - - -def children(tree, node=None): - assert tree is not None - if node is None: - node = tree.root_id() - ch = tree.first_child(node) - while ch != NONE: - yield ch - ch = tree.next_sibling(ch) - - -def siblings(tree, node): - assert tree is not None - if node is None: - return - ch = tree.first_sibling(node) - while ch != NONE: - yield ch - ch = tree.next_sibling(ch) - - -def walk(tree, node=None, indentation_level=0): - assert tree is not None - if node is None: node = tree.root_id() - yield node, indentation_level - ch = tree.first_child(node) - while ch != NONE: - for gc, il in walk(tree, ch, indentation_level + 1): - yield gc, il - ch = tree.next_sibling(ch) - - -@deprecated(deprecated_in="0.5.0", details="Use parse_in_arena() instead") -def parse(buf, **kwargs): - return parse_in_arena(tree, id) -def parse_in_arena(buf, **kwargs): - return _call_parse(parse_csubstr, buf, **kwargs) -def parse_in_place(buf, **kwargs): - _check_valid_for_in_situ(buf) - return _call_parse(parse_substr, buf, **kwargs) - - - -def _call_parse(parse_fn, buf, **kwargs): - tree = kwargs.get("tree", Tree()) - parse_fn(buf, tree) - return tree - - -def _check_valid_for_in_situ(obj): - if type(obj) in (str, bytes): - raise TypeError("cannot parse in situ: " + type(obj).__name__) - - - -@deprecated(deprecated_in="0.5.0", details="Use emit_yaml() instead") -def emit(tree, id=None): - return emit_yaml(tree, id) -def emit_yaml(tree, id=None): - if id is None: - id = tree.root_id() - return emit_yaml_malloc(tree, id) -def emit_json(tree, id=None): - if id is None: - id = tree.root_id() - return emit_json_malloc(tree, id) - - -@deprecated(deprecated_in="0.5.0", details="Use compute_emit_yaml_length() instead") -def compute_emit_length(tree, id=None): - return compute_emit_yaml_length(tree, id) -def compute_emit_yaml_length(tree, id=None): - if id is None: - id = tree.root_id() - return emit_yaml_length(tree, id) -def compute_emit_json_length(tree, id=None): - if id is None: - id = tree.root_id() - return emit_json_length(tree, id) - - -@deprecated(deprecated_in="0.5.0", details="Use emit_yaml_in_place() instead") -def emit_in_place(tree, buf, id=None): - return emit_yaml_in_place(tree, buf, id) -def emit_yaml_in_place(tree, buf, id=None): - return _emit_fn_in_place(tree, buf, id, emit_yaml_to_substr) -def emit_json_in_place(tree, buf, id=None): - return _emit_fn_in_place(tree, buf, id, emit_json_to_substr) -def _emit_fn_in_place(tree, buf, id, fn): - if id is None: - id = tree.root_id() - (failed, expected_size) = fn(tree, id, buf) - if failed: - raise IndexError("Output buffer has {} bytes, but emit requires {} bytes".format( - len(buf), expected_size)) - return memoryview(buf)[:expected_size] - -%} - -//----------------------------------------------------------------------------- - -namespace c4 { -namespace yml { - -constexpr const size_t NONE = (size_t)-1; - -typedef enum { - NOTYPE = 0, ///< no type is set - VAL = (1<<0), ///< a leaf node, has a (possibly empty) value - KEY = (1<<1), ///< is member of a map, must have non-empty key - MAP = (1<<2), ///< a map: a parent of keyvals - SEQ = (1<<3), ///< a seq: a parent of vals - DOC = (1<<4), ///< a document - STREAM = (1<<5)|SEQ, ///< a stream: a seq of docs - KEYREF = (1<<6), ///< a *reference: the key references an &anchor - VALREF = (1<<7), ///< a *reference: the val references an &anchor - KEYANCH = (1<<8), ///< the key has an &anchor - VALANCH = (1<<9), ///< the val has an &anchor - KEYTAG = (1<<10), ///< the key has an explicit tag/type - VALTAG = (1<<11), ///< the val has an explicit tag/type -} NodeType_e; - - -struct NodeType -{ - NodeType_e type; - - NodeType(); - NodeType(NodeType_e t); - ~NodeType(); - - const char *type_str(); - static const char* type_str(NodeType_e t); - - void set(NodeType_e t); - void add(NodeType_e t); - void rem(NodeType_e t); - - bool is_stream() const; - bool is_doc() const; - bool is_container() const; - bool is_map() const; - bool is_seq() const; - bool has_val() const; - bool has_key() const; - bool is_val() const; - bool is_keyval() const; - bool has_key_tag() const; - bool has_val_tag() const; - bool has_key_anchor() const; - bool has_val_anchor() const; - bool has_anchor() const; - bool is_key_ref() const; - bool is_val_ref() const; - bool is_ref() const; - bool is_anchor_or_ref() const; - bool is_key_quoted() const; - bool is_val_quoted() const; - bool is_quoted() const; -}; - - -struct Tree -{ - Tree(); - ~Tree(); - - void reserve(size_t node_capacity); - void reserve_arena(size_t node_capacity); - void clear(); - void clear_arena(); - - size_t size() const; - size_t capacity() const; - size_t slack() const; - - size_t arena_size() const; - size_t arena_capacity() const; - size_t arena_slack() const; - - void resolve(); - -public: - - // getters - - NodeType_e type(size_t node) const; - const char* type_str(size_t node) const; - - c4::csubstr key (size_t node) const; - c4::csubstr key_tag (size_t node) const; - c4::csubstr key_ref (size_t node) const; - c4::csubstr key_anchor(size_t node) const; - c4::yml::NodeScalar keysc(size_t node) const; - - c4::csubstr val (size_t node) const; - c4::csubstr val_tag (size_t node) const; - c4::csubstr val_ref (size_t node) const; - c4::csubstr val_anchor(size_t node) const; - c4::yml::NodeScalar valsc(size_t node) const; - -public: - - // node predicates - - bool is_root(size_t node) const; - bool is_stream(size_t node) const; - bool is_doc(size_t node) const; - bool is_container(size_t node) const; - bool is_map(size_t node) const; - bool is_seq(size_t node) const; - bool has_val(size_t node) const; - bool has_key(size_t node) const; - bool is_val(size_t node) const; - bool is_keyval(size_t node) const; - bool has_key_tag(size_t node) const; - bool has_val_tag(size_t node) const; - bool has_key_anchor(size_t node) const; - bool has_val_anchor(size_t node) const; - bool is_key_ref(size_t node) const; - bool is_val_ref(size_t node) const; - bool is_ref(size_t node) const; - bool is_anchor_or_ref(size_t node) const; - bool is_key_quoted(size_t node) const; - bool is_val_quoted(size_t node) const; - bool is_quoted(size_t node) const; - bool is_anchor(size_t node) const; - bool parent_is_seq(size_t node) const; - bool parent_is_map(size_t node) const; - bool empty(size_t node) const; - bool has_anchor(size_t node, c4::csubstr a) const; - -public: - - // hierarchy predicates - - bool has_parent(size_t node) const; - bool has_child(size_t node, c4::csubstr key) const; - //bool has_child(size_t node, size_t ch) const; - bool has_children(size_t node) const; - bool has_sibling(size_t node, c4::csubstr key) const; - //bool has_sibling(size_t node, size_t sib) const; - bool has_other_siblings(size_t node) const; - -public: - - // hierarchy getters - - size_t root_id() const; - - size_t parent(size_t node) const; - size_t prev_sibling(size_t node) const; - size_t next_sibling(size_t node) const; - size_t num_children(size_t node) const; - size_t child_pos(size_t node, size_t ch) const; - size_t first_child(size_t node) const; - size_t last_child(size_t node) const; - size_t child(size_t node, size_t pos) const; - size_t find_child(size_t node, c4::csubstr key) const; - size_t num_siblings(size_t node) const; - size_t num_other_siblings(size_t node) const; - size_t sibling_pos(size_t node, size_t sib) const; - size_t first_sibling(size_t node) const; - size_t last_sibling(size_t node) const; - size_t sibling(size_t node, size_t pos) const; - size_t find_sibling(size_t node, c4::csubstr key) const; - -public: - - void to_keyval(size_t node, c4::csubstr key, c4::csubstr val, int more_flags=0); - void to_map(size_t node, c4::csubstr key, int more_flags=0); - void to_seq(size_t node, c4::csubstr key, int more_flags=0); - void to_val(size_t node, c4::csubstr val, int more_flags=0); - void to_stream(size_t node, int more_flags=0); - void to_map(size_t node, int more_flags=0); - void to_seq(size_t node, int more_flags=0); - void to_doc(size_t node, int more_flags=0); - - void set_key_tag(size_t node, c4::csubstr tag); - void set_key_anchor(size_t node, c4::csubstr anchor); - void set_val_anchor(size_t node, c4::csubstr anchor); - void set_key_ref (size_t node, c4::csubstr ref ); - void set_val_ref (size_t node, c4::csubstr ref ); - - void _set_key(size_t node, c4::csubstr key, int more_flags=0); - void _set_val(size_t node, c4::csubstr val, int more_flags=0); - - void set_val_tag(size_t node, c4::csubstr tag); - void rem_key_anchor(size_t node); - void rem_val_anchor(size_t node); - void rem_key_ref (size_t node); - void rem_val_ref (size_t node); - void rem_anchor_ref(size_t node); - -public: - - /** create and insert a new child of "parent". insert after the (to-be) - * sibling "after", which must be a child of "parent". To insert as the - * first child, set after to NONE */ - size_t insert_child(size_t parent, size_t after); - size_t prepend_child(size_t parent); - size_t append_child(size_t parent); - -public: - - //! create and insert a new sibling of n. insert after "after" - size_t insert_sibling(size_t node, size_t after); - size_t prepend_sibling(size_t node); - size_t append_sibling(size_t node); - -public: - - //! remove an entire branch at once: ie remove the children and the node itself - void remove(size_t node); - - //! remove all the node's children, but keep the node itself - void remove_children(size_t node); - -public: - - void reorder(); - - /** change the node's position in the parent */ - void move(size_t node, size_t after); - - /** change the node's parent and position */ - void move(size_t node, size_t new_parent, size_t after); - /** change the node's parent and position */ - size_t move(Tree * src, size_t node, size_t new_parent, size_t after); - - /** recursively duplicate the node */ - size_t duplicate(size_t node, size_t new_parent, size_t after); - /** recursively duplicate a node from a different tree */ - size_t duplicate(Tree const* src, size_t node, size_t new_parent, size_t after); - - /** recursively duplicate the node's children (but not the node) */ - void duplicate_children(size_t node, size_t parent, size_t after); - /** recursively duplicate the node's children (but not the node), where the node is from a different tree */ - void duplicate_children(Tree const* src, size_t node, size_t parent, size_t after); - - void duplicate_contents(size_t node, size_t where); - - /** duplicate the node's children (but not the node) in a new parent, but - * omit repetitions where a duplicated node has the same key (in maps) or - * value (in seqs). If one of the duplicated children has the same key - * (in maps) or value (in seqs) as one of the parent's children, the one - * that is placed closest to the end will prevail. */ - void duplicate_children_no_rep(size_t node, size_t parent, size_t after); - -}; - -/* -%extend Tree { - - bool has_anchor(size_t node, const char *str, size_t len) const - { - return $self->has_anchor(node, c4::csubstr(str, len)); - } - - bool has_child(size_t node, const char *str, size_t len) const - { - return $self->has_child(node, c4::csubstr(str, len)); - } - - bool has_sibling(size_t node, const char *str, size_t len) const - { - return $self->has_sibling(node, c4::csubstr(str, len)); - } - - size_t find_child(size_t node, const char *str, size_t len) const - { - return $self->find_child(node, c4::csubstr(str, len)); - } - - size_t find_sibling(size_t node, const char *str, size_t len) const - { - return $self->find_sibling(node, c4::csubstr(str, len)); - } - - void to_keyval(size_t node, const char *keystr, size_t keylen, const char *valstr, size_t vallen, int more_flags=0) - { - return $self->to_keyval(node, c4::csubstr(keystr, keylen), c4::csubstr(valstr, vallen), more_flags); - } - - void to_map(size_t node, const char *keystr, size_t keylen, int more_flags=0) - { - return $self->to_map(node, c4::csubstr(keystr, keylen), more_flags); - } - - void to_seq(size_t node, const char *keystr, size_t keylen, int more_flags=0) - { - return $self->to_seq(node, c4::csubstr(keystr, keylen), more_flags); - } - - void to_val(size_t node, const char *valstr, size_t vallen, int more_flags=0) - { - return $self->to_val(node, c4::csubstr(valstr, vallen), more_flags); - } - - void set_key_tag(size_t node, const char *str, size_t len) - { - return $self->set_key_tag(node, c4::csubstr(str, len)); - } - void set_val_tag(size_t node, const char *str, size_t len) - { - return $self->set_val_tag(node, c4::csubstr(str, len)); - } - - void set_key_anchor(size_t node, const char *str, size_t len) - { - return $self->set_key_anchor(node, c4::csubstr(str, len)); - } - void set_val_anchor(size_t node, const char *str, size_t len) - { - return $self->set_val_anchor(node, c4::csubstr(str, len)); - } - - void set_key_ref(size_t node, const char *str, size_t len) - { - return $self->set_key_ref(node, c4::csubstr(str, len)); - } - void set_val_ref(size_t node, const char *str, size_t len) - { - return $self->set_val_ref(node, c4::csubstr(str, len)); - } - -}; -*/ - -} // namespace yml -} // namespace c4 - -//----------------------------------------------------------------------------- diff --git a/thirdparty/ryml/changelog/0.1.0.md b/thirdparty/ryml/changelog/0.1.0.md deleted file mode 100644 index b2ea04861..000000000 --- a/thirdparty/ryml/changelog/0.1.0.md +++ /dev/null @@ -1,44 +0,0 @@ -This is the first ryml release. Future releases will have a more organized changelog; for now, only recent major changes are listed. - -Please be aware that there are still some anticipated breaking changes in the API before releasing the 1.0 major version. These are highlighted in [the repo ROADMAP](https://github.com/biojppm/rapidyaml/blob/v0.1.0/ROADMAP.md). - -* 2020/October - * [MR#89](https://github.com/biojppm/rapidyaml/pull/89): - * fix python API generation in windows - * use github actions for testing and releasing - * [MR#88](https://github.com/biojppm/rapidyaml/pull/88): [fix MacOS compilation and installs](https://github.com/biojppm/rapidyaml/issues/75). This is a fix from [c4core](https://github.com/biojppm/cmake/issues/1). - * [MR#88](https://github.com/biojppm/rapidyaml/pull/88): [fix boolean handling](https://github.com/biojppm/rapidyaml/issues/74). This is a fix from [c4core](https://github.com/biojppm/c4core/pull/18/). `true` and `false` are now parsed correctly into `bool` variables: - ```c++ - auto tree = parse("{foo: true, bar: false}"); - ``` - Emitting `bool` variables still defaults to `0`/`1`, like the default behaviour in the STL. To explicitly request `true`/`false` use `c4::fmt::boolalpha()`: - ```c++ - node << var; // "1" or "0" - node << c4::fmt::boolalpha(var); // "true" or "false" - ``` -* 2020/September - * [***Breaking change***] [MR#85](https://github.com/biojppm/rapidyaml/pull/85) null values in YAML are now parsed to null strings instead of YAML null token "~": - ```c++ - auto tree = parse("{foo: , bar: ''}"); - // previous: - assert(tree["foo"].val() == "~"); - assert(tree["bar"].val() == ""); - // now: - assert(tree["foo"].val() == nullptr); // notice that this is now null - assert(tree["bar"].val() == ""); - ``` - * [MR#85](https://github.com/biojppm/rapidyaml/pull/85) Commas after tags are now allowed: - ```yaml - {foo: !!str, bar: ''} # now the comma does not cause an error - ``` - * [MR#81](https://github.com/biojppm/rapidyaml/pull/81): Always compile with extra pedantic warnings. -* 2020/May - * [***Breaking change***] the error callback now receives a source location object: - ```c++ - // previous - using pfn_error = void (*)(const char* msg, size_t msg_len, void *user_data); - // now: - using pfn_error = void (*)(const char* msg, size_t msg_len, Location location, void *user_data); - ``` - * Parser fixes to improve test suite success: [MR#73](https://github.com/biojppm/rapidyaml/pull/73), [MR#71](https://github.com/biojppm/rapidyaml/pull/71), [MR#68](https://github.com/biojppm/rapidyaml/pull/68), [MR#67](https://github.com/biojppm/rapidyaml/pull/67), [MR#66](https://github.com/biojppm/rapidyaml/pull/66) - * Fix compilation as DLL on windows [MR#69](https://github.com/biojppm/rapidyaml/pull/69) diff --git a/thirdparty/ryml/changelog/0.2.0.md b/thirdparty/ryml/changelog/0.2.0.md deleted file mode 100644 index 4bddffdd6..000000000 --- a/thirdparty/ryml/changelog/0.2.0.md +++ /dev/null @@ -1,29 +0,0 @@ -### New features & improvements -- Enable parsing into nested nodes ([87f4184](https://github.com/biojppm/rapidyaml/commit/87f4184)) -- `as_json()` can now be called with tree and node id ([4c23041](https://github.com/biojppm/rapidyaml/commit/4c23041)) -- Add `Parser::reserve_stack()` ([f31fb9f](https://github.com/biojppm/rapidyaml/commit/f31fb9f)) -- Add uninstall target ([PR #122](https://github.com/biojppm/rapidyaml/pull/122)) -- Update [c4core](https://github.com/biojppm/c4core) to v0.1.1 -- Add a [quickstart sample](samples/quickstart.cpp) with build examples. -- Update [README.md](README.md) to refer to the quickstart -- Add [gdb visualizers](src/ryml-gdbtypes.py) -- Add `SO_VERSION` to shared builds - -### Fixes -- Fix [#139](https://github.com/biojppm/rapidyaml/issues/139): substr and csubstr not found in ryml namespace -- Fix [#131](https://github.com/biojppm/rapidyaml/issues/131): resolve references to map keys -- Fix [#129](https://github.com/biojppm/rapidyaml/issues/129): quoted strings starting with * parsed as references -- Fix [#128](https://github.com/biojppm/rapidyaml/issues/128): segfault on nonexistent anchor -- Fix [#124](https://github.com/biojppm/rapidyaml/issues/124): parse failure in comments with trailing colon -- Fix [#121](https://github.com/biojppm/rapidyaml/issues/121): preserve quotes when emitting scalars -- Fix [#103](https://github.com/biojppm/rapidyaml/issues/103): ambiguous parsing of null/empty scalars -- Fix [#90](https://github.com/biojppm/rapidyaml/issues/90): CMAKE_CXX_STANDARD ignored -- Fix [#40](https://github.com/biojppm/rapidyaml/issues/40): quadratic complexity from use of `sscanf(%f)` -- Fix emitting json to streams ([dc6af83](https://github.com/biojppm/rapidyaml/commit/dc6af83)) -- Set the global memory resource when setting global callbacks ([511cba0](https://github.com/biojppm/rapidyaml/commit/511cba0)) -- Fix python packaging ([PR #102](https://github.com/biojppm/rapidyaml/pull/102)) - -### Special thanks -- @Gei0r -- @litghost -- @costashatz diff --git a/thirdparty/ryml/changelog/0.2.1.md b/thirdparty/ryml/changelog/0.2.1.md deleted file mode 100644 index 3dc9310d3..000000000 --- a/thirdparty/ryml/changelog/0.2.1.md +++ /dev/null @@ -1,235 +0,0 @@ -This release is focused on bug fixes and compliance with the [YAML test suite](https://github.com/yaml/yaml-test-suite). - -### Breaking changes - -- Fix parsing behavior of root-level scalars: now these are parsed into a DOCVAL, not SEQ->VAL ([5ba0d56](https://github.com/biojppm/rapidyaml/pull/144/commits/5ba0d56904daef1509f0073695145c4835ab1b30), from [PR #144](https://github.com/biojppm/rapidyaml/pull/144)). Eg, - ```yaml - --- - this is a scalar - --- # previously this was parsed as - - this is a scalar - ``` -- Cleanup type predicate API ([PR #155](https://github.com/biojppm/rapidyaml/pull/155))): - - ensure all type predicates from `Tree` and `NodeRef` forward to the corresponding predicate in `NodeType` - - remove all type predicates and methods from `NodeData`; use the equivalent call from `Tree` or `NodeRef`. For example, for `is_map()`: - ```c++ - Tree t = parse("{foo: bar}"); - size_t map_id = t.root_id(); - NodeRef map = t.rootref(); - t.get(map_id)->is_map(); // compile error: no longer exists - assert(t.is_map(map_id)); // OK - assert(map.is_map()); // OK - ``` - - Further cleanup to the type predicate API will be done in the future, especially around the `.has_*()` vs corresponding `.is_*()` naming scheme. - - -### New features & improvements - -- `Tree::lookup_path_or_modify()`: add overload to graft existing branches ([PR #141](https://github.com/biojppm/rapidyaml/pull/141)) -- Callbacks: improve test coverage ([PR #141](https://github.com/biojppm/rapidyaml/pull/141)) -- [YAML test suite](https://github.com/yaml/yaml-test-suite) ([PR #144](https://github.com/biojppm/rapidyaml/pull/144), [PR #145](https://github.com/biojppm/rapidyaml/pull/145)): big progress towards compliance with the suite. There are still a number of existing problems, which are the subject of ongoing work. See the [list of current known failures](../test/test_suite/test_suite_parts.cpp) in the test suite file. -- Python wheels and source package are now [uploaded to PyPI](https://pypi.org/project/rapidyaml/) as part of the release process. - - -### Fixes - -#### Anchors and references -- Fix resolving of nodes with keyref+valref ([PR #144](https://github.com/biojppm/rapidyaml/pull/144)): `{&a a: &b b, *b: *a}` -- Fix parsing of implicit scalars when tags are present ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - - &a # test case PW8X - - a - - &a : a - b: &b - - &c : &a - - ? &d - - ? &e - : &a - ``` -- Fix [#151](https://github.com/biojppm/rapidyaml/issues/151): scalars beginning with `*` or `&` or `<<` are now correctly quoted when emitting ([PR #156](https://github.com/biojppm/rapidyaml/pull/156)). -- Also from [PR #156](https://github.com/biojppm/rapidyaml/pull/156), map inheritance nodes like `<<: *anchor` or `<<: [*anchor1, *anchor2]` now have a `KEYREF` flag in their type (until a call to `Tree::resolve()`): - ```c++ - Tree tree = parse("{map: &anchor {foo: bar}, copy: {<<: *anchor}}"); - assert(tree["copy"]["<<"].is_key_ref()); // previously this did not hold - assert(tree["copy"]["<<"].is_val_ref()); // ... but this did - ``` - -#### Tags -- Fix parsing of tag dense maps and seqs ([PR #144](https://github.com/biojppm/rapidyaml/pull/144)): - ```yaml - --- !!map { - k: !!seq [ a, !!str b], - j: !!seq - [ a, !!str b] - --- !!seq [ - !!map { !!str k: v}, - !!map { !!str ? k: v} - ] - --- !!map - !!str foo: !!map # there was a parse error with the multiple tags - !!int 1: !!float 20.0 - !!int 3: !!float 40.0 - --- !!seq - - !!map - !!str k1: v1 - !!str k2: v2 - !!str k3: v3 - ``` - -#### Whitespace -- Fix parsing of double-quoted scalars with tabs ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - "This has a\ttab" - # is now correctly parsed as "This has atab" - ``` -- Fix filtering of leading and trailing whitespace within double-quoted scalars ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - # test case 4ZYM, 7A4E, TL85 - " - foo - - bar - baz - " - # is now correctly parsed as " foo\nbar\nbaz " - ``` -- Fix parsing of tabs within YAML tokens ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - ---scalar # test case K54U - ---{} # test case Q5MG - --- # test case DC7X - a: b - seq: - - a - c: d#X - ``` -- Fix parsing of flow-style maps with ommitted values without any space ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - # test case 4ABK - - {foo: , bar: , baz: } # this was parsed correctly as {foo: ~, bar: ~, baz: ~} - - {foo:, bar:, baz:} # ... but this was parsed as {'foo:': , 'bar:': ~, 'baz:': ~} - ``` - -#### Scalars -- Unescape forward slashes in double quoted string ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - --- escaped slash: "a\/b" # test case 3UYS - # is now parsed as: - --- escaped slash: "a/b" - ``` -- Fix filtering of indented regions in folded scalars ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - # test case 7T8X - - > - - folded - line - - next - line - * bullet - - * list - * lines - - last - line - ``` - is now correctly parsed as `\nfolded line\nnext line\n * bullet\n\n * list\n * lines\n\nlast line\n`. -- Fix parsing of special characters within plain scalars ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - # test case 3MYT - k:#foo - &a !t s - !t s - # now correctly parsed as "k:#foo &a !t s !t s" - ``` -- Fix parsing of comments after complex keys ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - # test case X8DW - ? key - # comment - : value - # now correctly parsed as {key: value} - ``` -- Fix parsing of consecutive complex keys within maps ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)) - ```yaml - # test case 7W2P, ZWK4 - ? a - ? b - c: - ? d - e: - # now correctly parsed as {a: ~, b: ~, c: ~, d: ~, e: ~} - ``` -- Fix [#152](https://github.com/biojppm/rapidyaml/issues/152): parse error with folded scalars that are the last in a container ([PR #157](https://github.com/biojppm/rapidyaml/pull/157)): - ```yaml - exec: - command: - # before the fix, this folded scalar failed to parse - - | - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - parses: no - ``` -- Fix: documents consisting of a quoted scalar now retain the VALQUO flag ([PR #156](https://github.com/biojppm/rapidyaml/pull/156)) - ```c++ - Tree tree = parse("'this is a quoted scalar'"); - assert(tree.rootref().is_doc()); - assert(tree.rootref().is_val()); - assert(tree.rootref().is_val_quoted()); - ``` - - -#### Document structure -- Empty docs are now parsed as a docval with a null node: - ```yaml - --- # test cases 6XDY, 6ZKB, 9BXL, PUW8 - --- - --- - ``` - is now parsed as - ```yaml - --- ~ - --- ~ - --- ~ - ``` -- Prevent creation of DOC nodes from stream-level comments or tags ([PR #145](https://github.com/biojppm/rapidyaml/pull/145)): - ```yaml - !foo "bar" - ... - # Global - %TAG ! tag:example.com,2000:app/ - --- - !foo "bar" - ``` - was parsed as - ```yaml - --- - !foo "bar" - --- - # notice the empty doc in here - --- - !foo "bar" - ``` - and it is now correctly parsed as - ```yaml - --- - !foo "bar" - --- - !foo "bar" - ``` - (other than the known limitation that ryml does not do tag lookup). - - -#### General - -- Fix [#147](https://github.com/biojppm/rapidyaml/issues/147): serialize/deserialize special float values `.nan`, `.inf`, `-.inf` ([PR #149](https://github.com/biojppm/rapidyaml/pull/149)) -- Fix [#142](https://github.com/biojppm/rapidyaml/issues/142): `preprocess_json()`: ensure quoted ranges are skipped when slurping containers -- Ensure error macros expand to a single statement ([PR #141](https://github.com/biojppm/rapidyaml/pull/141)) -- Update c4core to [0.1.4](https://github.com/biojppm/c4core/releases/tag/v0.1.4) - - -### Special thanks - -- @Gei0r - diff --git a/thirdparty/ryml/changelog/0.2.2.md b/thirdparty/ryml/changelog/0.2.2.md deleted file mode 100644 index 508759d97..000000000 --- a/thirdparty/ryml/changelog/0.2.2.md +++ /dev/null @@ -1 +0,0 @@ -Yank python package 0.2.1, was accidentally created while iterating the PyPI submission from the Github action. This release does not add any change, and is functionally the same as [0.2.1](https://github.com/biojppm/rapidyaml/releases/tag/v0.2.1). diff --git a/thirdparty/ryml/changelog/0.2.3.md b/thirdparty/ryml/changelog/0.2.3.md deleted file mode 100644 index 268ebd3ec..000000000 --- a/thirdparty/ryml/changelog/0.2.3.md +++ /dev/null @@ -1,285 +0,0 @@ -This release is focused on bug fixes and compliance with the [YAML test suite](https://github.com/yaml/yaml-test-suite). - -### New features -- Add support for CPU architectures aarch64, ppc64le, s390x. -- Update c4core to [0.1.7](https://github.com/biojppm/c4core/releases/tag/v0.1.7) -- `Tree` and `NodeRef`: add document getter `doc()` and `docref()` - ```c++ - Tree tree = parse(R"(--- - doc0 - --- - doc1 - )"); - NodeRef stream = t.rootref(); - assert(stream.is_stream()); - // tree.doc(i): get the index of the i-th doc node. - // Equivalent to tree.child(tree.root_id(), i) - assert(tree.doc(0) == 1u); - assert(tree.doc(1) == 2u); - // tree.docref(i), same as above, return NodeRef - assert(tree.docref(0).val() == "doc0"); - assert(tree.docref(1).val() == "doc1"); - // stream.doc(i), same as above, given NodeRef - assert(stream.doc(0).val() == "doc0"); - assert(stream.doc(1).val() == "doc1"); - ``` - -### Fixes - -- Fix compilation with `C4CORE_NO_FAST_FLOAT` ([PR #163](https://github.com/biojppm/rapidyaml/pull/163)) - -#### Flow maps - -- Fix parse of multiline plain scalars inside flow maps ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - # test case UT92 - # all parsed as "matches %": 20 - - { matches - % : 20 } - - { matches - %: 20 } - - { matches - %: - 20 } - ``` - - -#### Tags - -- Fix parsing of tags followed by comments in sequences ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - # test case 735Y - - !!map # Block collection - foo : bar - ``` - -#### Quoted scalars -- Fix filtering of tab characters in quoted scalars ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - --- - # test case 5GBF - "Empty line - - as a line feed" - # now correctly parsed as "Empty line\nas a line feed" - --- - # test case PRH3 - ' 1st non-empty - - 2nd non-empty - 3rd non-empty ' - # now correctly parsed as " 1st non-empty\n2nd non-empty 3rd non-empty " - ``` -- Fix filtering of backslash characters in double-quoted scalars ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - # test cases NP9H, Q8AD - "folded - to a space, - - to a line feed, or \ - \ non-content" - # now correctly parsed as "folded to a space,\nto a line feed, or \t \tnon-content" - ``` -- Ensure filtering of multiline quoted scalars ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - # all scalars now correctly parsed as "quoted string", - # both for double and single quotes - --- - "quoted - string" - --- "quoted - string" - --- - - "quoted - string" - --- - - "quoted - string" - --- - "quoted - string": "quoted - string" - --- - "quoted - string": "quoted - string" - ``` - - -#### Block scalars -- Ensure no newlines are added when emitting block scalars ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)) -- Fix parsing of block spec with both chomping and indentation: chomping may come before or after the indentation ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - # the block scalar specs below now have the same effect. - # test cases: D83L, P2AD - - |2- - explicit indent and chomp - - |-2 - chomp and explicit indent - ``` -- Fix [inference of block indentation](https://yaml.org/spec/1.2.2/#8111-block-indentation-indicator) with leading blank lines ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - # test cases: 4QFQ, 7T8X - - > - - - # child1 - # parsed as "\n\n child1" - --- # test case DWX9 - | - - - literal - - - text - - # Comment - # parsed as "\n\nliteral\n \n\ntext\n" - ``` -- Fix parsing of same-indentation block scalars ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - # test case W4TN - # all docs have the same value: "%!PS-Adobe-2.0" - --- | - %!PS-Adobe-2.0 - ... - --- > - %!PS-Adobe-2.0 - ... - --- | - %!PS-Adobe-2.0 - ... - --- > - %!PS-Adobe-2.0 - ... - --- | - %!PS-Adobe-2.0 - --- > - %!PS-Adobe-2.0 - --- | - %!PS-Adobe-2.0 - --- > - %!PS-Adobe-2.0 - ``` -- Folded block scalars: fix folding of newlines at the border of indented parts ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - # test case 6VJK - # now correctly parsed as "Sammy Sosa completed another fine season with great stats.\n\n 63 Home Runs\n 0.288 Batting Average\n\nWhat a year!\n" - > - Sammy Sosa completed another - fine season with great stats. - - 63 Home Runs - 0.288 Batting Average - - What a year! - --- - # test case MJS9 - # now correctly parsed as "foo \n\n \t bar\n\nbaz\n" - > - foo - - bar - - baz - ``` -- Folded block scalars: fix folding of newlines when the indented part is at the begining of the scalar ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - # test case F6MC - a: >2 - more indented - regular - # parsed as a: " more indented\nregular\n" - b: >2 - - - more indented - regular - # parsed as b: "\n\n more indented\nregular\n" - ``` - -#### Plain scalars -- Fix parsing of whitespace within plain scalars ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)): - ```yaml - --- - # test case NB6Z - key: - value - with - - tabs - tabs - - foo - - bar - baz - - # is now correctly parsed as "value with\ntabs tabs\nfoo\nbar baz" - --- - # test case 9YRD, EX5H (trailing whitespace) - a - b - c - d - - e - # is now correctly parsed as "a b c d\ne" - ``` -- Fix parsing of unindented plain scalars at the root level scope ([PR #161](https://github.com/biojppm/rapidyaml/pull/161)) - ```yaml - --- # this parsed - Bare - scalar - is indented - # was correctly parsed as "Bare scalar is indented" - --- # but this failed to parse successfully: - Bare - scalar - is not indented - # is now correctly parsed as "Bare scalar is not indented" - --- # test case NB6Z - value - with - - tabs - tabs - - foo - - bar - baz - - # now correctly parsed as "value with\ntabs tabs\nfoo\nbar baz" - --- - --- # test cases EXG3, 82AN - ---word1 - word2 - # now correctly parsed as "---word1 word2" - ``` -- Fix parsing of comments within plain scalars - ```yaml - # test case 7TMG - --- # now correctly parsed as "word1" - word1 - # comment - --- # now correctly parsed as [word1, word2] - [ word1 - # comment - , word2] - ``` - -#### Python API -- Add missing node predicates in SWIG API definition ([PR #166](https://github.com/biojppm/rapidyaml/pull/166)): - - `is_anchor_or_ref()` - - `is_key_quoted()` - - `is_val_quoted()` - - `is_quoted()` - - -### Thanks - ---- @mbs-c ---- @simu ---- @QuellaZhang diff --git a/thirdparty/ryml/changelog/0.3.0.md b/thirdparty/ryml/changelog/0.3.0.md deleted file mode 100644 index 3bd5875d2..000000000 --- a/thirdparty/ryml/changelog/0.3.0.md +++ /dev/null @@ -1,104 +0,0 @@ -### Breaking changes - -Despite ryml being still in a non-stable 0.x.y version, considerable effort goes into trying to avoid breaking changes. However, this release has to collect on the [semantic versioning](https://semver.org/) prerogative for breaking changes. This is a needed improvement, so sorry for any nuisance! - -**The allocation and error callback logic was revamped** on the [amalgamation PR](https://github.com/biojppm/rapidyaml/pull/172). Now trees and parsers receive (and store) a full `ryml::Callbacks` object instead of the (now removed) `ryml::Allocator` which had a pointer to a (now removed) `ryml::MemoryResourceCallbacks`, which was a (now removed) `ryml::MemoryResource`. To be clear, the `Callbacks` class is unchanged, other than removing some unneeded helper methods. - -These changes were motivated by unfortunate name clashes between `c4::Allocator/ryml::Allocator` and `c4::MemoryResource/ryml::MemoryResource`, occurring if `` or `` were included before ``. They also significantly simplify this part of the API, making it really easier to understand. - -As a consequence of the above changes, the global memory resource getters and setters for ryml were also removed: `ryml::get_memory_resource()/ryml::set_memory_resource()`. - -Here's an example of the required changes in client code. First the old client code (from the quickstart): - -```c++ -struct PerTreeMemoryExample : public ryml::MemoryResource -{ - void *allocate(size_t len, void * hint) override; - void free(void *mem, size_t len) override; -}; - -PerTreeMemoryExample mrp; -PerTreeMemoryExample mr1; -PerTreeMemoryExample mr2; - -ryml::Parser parser = {ryml::Allocator(&mrp)}; -ryml::Tree tree1 = {ryml::Allocator(&mr1)}; -ryml::Tree tree2 = {ryml::Allocator(&mr2)}; -``` - -Should now be rewritten to: - -```c++ -struct PerTreeMemoryExample -{ - ryml::Callbacks callbacks() const; // helper to create the callbacks -}; - -PerTreeMemoryExample mrp; -PerTreeMemoryExample mr1; -PerTreeMemoryExample mr2; - -ryml::Parser parser = {mrp.callbacks()}; -ryml::Tree tree1 = {mr1.callbacks()}; -ryml::Tree tree2 = {mr2.callbacks()}; -``` - - -### New features -- Add amalgamation into a single header file ([PR #172](https://github.com/biojppm/rapidyaml/pull/172)): - - The amalgamated header will be available together with the deliverables from each release. - - To generate the amalgamated header: - ```console - $ python tools/amalgamate.py ryml_all.hpp - ``` - - To use the amalgamated header: - - Include at will in any header of your project. - - In one - and only one - of your project source files, `#define RYML_SINGLE_HDR_DEFINE_NOW` and then `#include `. This will enable the function and class definitions in the header file. For example, here's a sample program: - ```c++ - #include - #define RYML_SINGLE_HDR_DEFINE_NOW // do this before the include - #include - int main() - { - auto tree = ryml::parse("{foo: bar}"); - std::cout << tree["foo"].val() << "\n"; - } - ``` -- Add `Tree::change_type()` and `NodeRef::change_type()` ([PR #171](https://github.com/biojppm/rapidyaml/pull/171)): - ```c++ - // clears a node and sets its type to a different type (one of `VAL`, `SEQ`, `MAP`): - Tree t = parse("{keyval0: val0, keyval1: val1, keyval2: val2}"); - t[0].change_type(VAL); - t[1].change_type(MAP); - t[2].change_type(SEQ); - Tree expected = parse("{keyval0: val0, keyval1: {}, keyval2: []}"); - assert(emitrs(t) == emitrs(expected)); - ``` -- Add support for compilation with emscripten (WebAssembly+javascript) ([PR #176](https://github.com/biojppm/rapidyaml/pull/176)). - -### Fixes - -- Take block literal indentation as relative to current indentation level, rather than as an absolute indentation level ([PR #178](https://github.com/biojppm/rapidyaml/pull/178)): - ```yaml - foo: - - | - child0 - - |2 - child2 # indentation is 4, not 2 - ``` -- Fix parsing when seq member maps start without a key ([PR #178](https://github.com/biojppm/rapidyaml/pull/178)): - ```yaml - # previously this resulted in a parse error - - - : empty key - - - : another empty key - ``` -- Prefer passing `substr` and `csubstr` by value instead of const reference ([PR #171](https://github.com/biojppm/rapidyaml/pull/171)) -- Fix [#173](https://github.com/biojppm/rapidyaml/issues/173): add alias target `ryml::ryml` ([PR #174](https://github.com/biojppm/rapidyaml/pull/174)) -- Speedup compilation of tests by removing linking with yaml-cpp and libyaml. ([PR #177](https://github.com/biojppm/rapidyaml/pull/177)) -- Fix [c4core#53](https://github.com/biojppm/c4core/issues/53): cmake install targets were missing call to `export()` ([PR #179](https://github.com/biojppm/c4core/pull/179)). -- Add missing export to `Tree` ([PR #181](https://github.com/biojppm/c4core/pull/181)). - - -### Thanks - -- @aviktorov diff --git a/thirdparty/ryml/changelog/0.4.0.md b/thirdparty/ryml/changelog/0.4.0.md deleted file mode 100644 index 1c9c43914..000000000 --- a/thirdparty/ryml/changelog/0.4.0.md +++ /dev/null @@ -1,229 +0,0 @@ -This release improves compliance with the [YAML test suite](https://github.com/yaml/yaml-test-suite/) (thanks @ingydotnet and @perlpunk for extensive and helpful cooperation), and adds node location tracking using the parser. - - -### Breaking changes - -As part of the [new feature to track source locations](https://github.com/biojppm/rapidyaml/pull/168), opportunity was taken to address a number of pre-existing API issues. These changes consisted of: - -- Deprecate `c4::yml::parse()` and `c4::yml::Parser::parse()` overloads; all these functions will be removed in short order. Until removal, any call from client code will trigger a compiler warning. -- Add `parse()` alternatives, either `parse_in_place()` or `parse_in_arena()`: - - `parse_in_place()` receives only `substr` buffers, ie mutable YAML source buffers. Trying to pass a `csubstr` buffer to `parse_in_place()` will cause a compile error: - ```c++ - substr readwrite = /*...*/; - Tree tree = parse_in_place(readwrite); // OK - - csubstr readonly = /*...*/; - Tree tree = parse_in_place(readonly); // compile error - ``` - - `parse_in_arena()` receives only `csubstr` buffers, ie immutable YAML source buffers. Prior to parsing, the buffer is copied to the tree's arena, then the copy is parsed in place. Because `parse_in_arena()` is meant for immutable buffers, overloads receiving a `substr` YAML buffer are now declared but marked deprecated, and intentionally left undefined, such that calling `parse_in_arena()` with a `substr` will cause a linker error as well as a compiler warning. - ```c++ - substr readwrite = /*...*/; - Tree tree = parse_in_arena(readwrite); // compile warning+linker error - ``` - This is to prevent an accidental extra copy of the mutable source buffer to the tree's arena: `substr` is implicitly convertible to `csubstr`. If you really intend to parse an originally mutable buffer in the tree's arena, convert it first explicitly to immutable by assigning the `substr` to a `csubstr` prior to calling `parse_in_arena()`: - ```c++ - substr readwrite = /*...*/; - csubstr readonly = readwrite; // ok - Tree tree = parse_in_arena(readonly); // ok - ``` - This problem does not occur with `parse_in_place()` because `csubstr` is not implicitly convertible to `substr`. -- In the python API, `ryml.parse()` was removed and not just deprecated; the `parse_in_arena()` and `parse_in_place()` now replace this. -- `Callbacks`: changed behavior in `Parser` and `Tree`: - - When a tree is copy-constructed or move-constructed to another, the receiving tree will start with the callbacks of the original. - - When a tree is copy-assigned or move-assigned to another, the receiving tree will now change its callbacks to the original. - - When a parser creates a new tree, the tree will now use a copy of the parser's callbacks object. - - When an existing tree is given directly to the parser, both the tree and the parser now retain their own callback objects; any allocation or error during parsing will go through the respective callback object. - - -### New features - -- Add tracking of source code locations. This is useful for reporting semantic errors after the parsing phase (ie where the YAML is syntatically valid and parsing is successful, but the tree contents are semantically invalid). The locations can be obtained lazily from the parser when the first location is queried: - ```c++ - // To obtain locations, use of the parser is needed: - ryml::Parser parser; - ryml::Tree tree = parser.parse_in_arena("source.yml", R"({ - aa: contents, - foo: [one, [two, three]] - })"); - // After parsing, on the first call to obtain a location, - // the parser will cache a lookup structure to accelerate - // tracking the location of a node, with complexity - // O(numchars(srcbuffer)). Then it will do the lookup, with - // complexity O(log(numlines(srcbuffer))). - ryml::Location loc = parser.location(tree.rootref()); - assert(parser.location_contents(loc).begins_with("{")); - // note the location members are zero-based: - assert(loc.offset == 0u); - assert(loc.line == 0u); - assert(loc.col == 0u); - // On the next call to location(), the accelerator is reused - // and only the lookup is done. - loc = parser.location(tree["aa"]); - assert(parser.location_contents(loc).begins_with("aa")); - assert(loc.offset == 2u); - assert(loc.line == 1u); - assert(loc.col == 0u); - // KEYSEQ in flow style: points at the key - loc = parser.location(tree["foo"]); - assert(parser.location_contents(loc).begins_with("foo")); - assert(loc.offset == 16u); - assert(loc.line == 2u); - assert(loc.col == 0u); - loc = parser.location(tree["foo"][0]); - assert(parser.location_contents(loc).begins_with("one")); - assert(loc.line == 2u); - assert(loc.col == 6u); - // SEQ in flow style: location points at the opening '[' (there's no key) - loc = parser.location(tree["foo"][1]); - assert(parser.location_contents(loc).begins_with("[")); - assert(loc.line == 2u); - assert(loc.col == 11u); - loc = parser.location(tree["foo"][1][0]); - assert(parser.location_contents(loc).begins_with("two")); - assert(loc.line == 2u); - assert(loc.col == 12u); - loc = parser.location(tree["foo"][1][1]); - assert(parser.location_contents(loc).begins_with("three")); - assert(loc.line == 2u); - assert(loc.col == 17u); - // NOTE: reusing the parser with a new YAML source buffer - // will invalidate the accelerator. - ``` - See more details in the [quickstart sample](https://github.com/biojppm/rapidyaml/blob/bfb073265abf8c58bbeeeed7fb43270e9205c71c/samples/quickstart.cpp#L3759). Thanks to @cschreib for submitting a working example proving how simple it could be to achieve this. -- `Parser`: - - add `source()` and `filename()` to get the latest buffer and filename to be parsed - - add `callbacks()` to get the parser's callbacks -- Add `from_tag_long()` and `normalize_tag_long()`: - ```c++ - assert(from_tag_long(TAG_MAP) == ""); - assert(normalize_tag_long("!!map") == ""); - ``` -- Add an experimental API to resolve tags based on the tree's tag directives. This API is still imature and will likely be subject to changes, so we won't document it yet. -- Regarding emit styles (see issue [#37](https://github.com/biojppm/rapidyaml/issues/37)): add an experimental API to force flow/block style on container nodes, as well as block-literal/block-folded/double-quoted/single-quoted/plain styles on scalar nodes. This API is also immature and will likely be subject to changes, so we won't document it yet. But if you are desperate for this functionality, the new facilities will let you go further. -- Add preliminary support for bare-metal ARM architectures, with CI tests pending implementation of QEMU action. ([#193](https://github.com/biojppm/rapidyaml/issues/193), [c4core#63](https://github.com/biojppm/c4core/issues/63)). -- Add preliminary support for RISC-V architectures, with CI tests pending availability of RISC-V based github actions. ([c4core#69](https://github.com/biojppm/c4core/pulls/69)). - - -### Fixes - -- Fix edge cases of parsing of explicit keys (ie keys after `?`) ([PR#212](https://github.com/biojppm/rapidyaml/pulls/212)): - ```yaml - # all these were fixed: - ? : # empty - ? explicit key # this comment was not parsed correctly - ? # trailing empty key was not added to the map - ``` -- Fixed parsing of tabs used as whitespace tokens after `:` or `-`. This feature [is costly (see some benchmark results here)](https://github.com/biojppm/rapidyaml/pull/211#issuecomment-1030688035) and thus it is disabled by default, and requires defining a macro or cmake option `RYML_WITH_TAB_TOKENS` to enable ([PR#211](https://github.com/biojppm/rapidyaml/pulls/211)). -- Allow tab indentation in flow seqs ([PR#215](https://github.com/biojppm/rapidyaml/pulls/215)) (6CA3). -- ryml now parses successfully compact JSON code `{"like":"this"}` without any need for preprocessing. This code was not valid YAML 1.1, but was made valid in YAML 1.2. So the `preprocess_json()` functions, used to insert spaces after `:` are no longer necessary and have been removed. If you were using these functions, remove the calls and just pass the original source directly to ryml's parser ([PR#210](https://github.com/biojppm/rapidyaml/pulls/210)). -- Fix handling of indentation when parsing block scalars ([PR#210](https://github.com/biojppm/rapidyaml/pulls/210)): - ```yaml - --- - | - hello - there - --- - | - ciao - qua - --- - - | - hello - there - - | - ciao - qua - --- - foo: | - hello - there - bar: | - ciao - qua - ``` -- Fix parsing of maps when opening a scope with whitespace before the colon ([PR#210](https://github.com/biojppm/rapidyaml/pulls/210)): - ```yaml - foo0 : bar - --- - foo1 : bar # the " :" was causing an assert - --- - foo2 : bar - --- - foo3 : bar - --- - foo4 : bar - ``` -- Ensure container keys preserve quote flags when the key is quoted ([PR#210](https://github.com/biojppm/rapidyaml/pulls/210)). -- Ensure scalars beginning with `%` are emitted with quotes (([PR#216](https://github.com/biojppm/rapidyaml/pulls/216)). -- Fix [#203](https://github.com/biojppm/rapidyaml/issues/203): when parsing, do not convert `null` or `~` to null scalar strings. Now the scalar strings contain the verbatim contents of the original scalar; to query whether a scalar value is null, use `Tree::key_is_null()/val_is_null()` and `NodeRef::key_is_null()/val_is_null()` which return true if it is empty or any of the unquoted strings `~`, `null`, `Null`, or `NULL`. ([PR#207](https://github.com/biojppm/rapidyaml/pulls/207)): -- Fix [#205](https://github.com/biojppm/rapidyaml/issues/205): fix parsing of escaped characters in double-quoted strings: `"\\\"\n\r\t\\/\\0\b\f\a\v\e\_\N\L\P"` ([PR#207](https://github.com/biojppm/rapidyaml/pulls/207)). -- Fix [#204](https://github.com/biojppm/rapidyaml/issues/204): add decoding of unicode codepoints `\x` `\u` `\U` in double-quoted scalars: - ```c++ - Tree tree = parse_in_arena(R"(["\u263A \xE2\x98\xBA \u2705 \U0001D11E"])"); - assert(tree[0].val() == "☺ ☺ ✅ 𝄞"); - ``` - This is mandated by the YAML standard and was missing from ryml ([PR#207](https://github.com/biojppm/rapidyaml/pulls/207)). -- Fix emission of nested nodes which are sequences: when these are given as the emit root, the `- ` from the parent node was added ([PR#210](https://github.com/biojppm/rapidyaml/pulls/210)): - ```c++ - const ryml::Tree tree = ryml::parse_in_arena(R"( - - - Rochefort 10 - - Busch - - Leffe Rituel - - - and so - - many other - - wonderful beers - )"); - // before (error), YAML valid but not expected - //assert(ryml::emitrs(tree[0][3]) == R"(- - and so - // - many other - // - wonderful beers - //)"); - // now: YAML valid and expected - assert(ryml::emitrs(tree[0][3]) == R"(- and so - - many other - - wonderful beers - )"); - ``` -- Fix parsing of isolated `!`: should be an empty val tagged with `!` (UKK06-02) ([PR#215](https://github.com/biojppm/rapidyaml/pulls/215)). -- Fix [#193](https://github.com/biojppm/rapidyaml/issues/193): amalgamated header missing `#include ` which prevented compilation in bare-metal `arm-none-eabi` ([PR #195](https://github.com/biojppm/rapidyaml/pull/195), requiring also [c4core #64](https://github.com/biojppm/c4core/pull/64)). -- Accept `infinity`,`inf` and `nan` as special float values (but not mixed case: eg `InFiNiTy` or `Inf` or `NaN` are not accepted) ([PR #186](https://github.com/biojppm/rapidyaml/pull/186)). -- Accept special float values with upper or mixed case: `.Inf`, `.INF`, `.NaN`, `.NAN`. Previously, only low-case `.inf` and `.nan` were accepted ([PR #186](https://github.com/biojppm/rapidyaml/pull/186)). -- Accept `null` with upper or mixed case: `Null` or `NULL`. Previously, only low-case `null` was accepted ([PR #186](https://github.com/biojppm/rapidyaml/pull/186)). -- Fix [#182](https://github.com/biojppm/rapidyaml/issues/182): add missing export of DLL symbols, and document requirements for compiling shared library from the amalgamated header. [PR #183](https://github.com/biojppm/rapidyaml/pull/183), also [PR c4core#56](https://github.com/biojppm/c4core/pull/56) and [PR c4core#57](https://github.com/biojppm/c4core/pull/57). -- Fix [#185](https://github.com/biojppm/rapidyaml/issues/185): compilation failures in earlier Xcode versions ([PR #187](https://github.com/biojppm/rapidyaml/pull/187) and [PR c4core#61](https://github.com/biojppm/c4core/pull/61)): - - `c4/substr_fwd.hpp`: (failure in Xcode 12 and earlier) forward declaration for `std::allocator` is inside the `inline namespace __1`, unlike later versions. - - `c4/error.hpp`: (failure in debug mode in Xcode 11 and earlier) `__clang_major__` does not mean the same as in the common clang, and as a result the warning `-Wgnu-inline-cpp-without-extern` does not exist there. -- Ensure error messages do not wrap around the buffer when the YAML source line is too long ([PR#210](https://github.com/biojppm/rapidyaml/pulls/210)). -- Ensure error is emitted on unclosed flow sequence characters eg `[[[` ([PR#210](https://github.com/biojppm/rapidyaml/pulls/210)). Same thing for `[]]`. -- Refactor error message building and parser debug logging to use the new dump facilities in c4core ([PR#212](https://github.com/biojppm/rapidyaml/pulls/212)). -- Parse: fix read-after-free when duplicating a parser state node, when pushing to the stack requires a stack buffer resize ([PR#210](https://github.com/biojppm/rapidyaml/pulls/210)). -- Add support for legacy gcc 4.8 ([PR#217](https://github.com/biojppm/rapidyaml/pulls/217)). - - -### Improvements - -- Rewrite filtering of scalars to improve parsing performance ([PR #188](https://github.com/biojppm/rapidyaml/pull/188)). Previously the scalar strings were filtered in place, which resulted in quadratic complexity in terms of scalar length. This did not matter for small scalars fitting the cache (which is the more frequent case), but grew in cost as the scalars grew larger. To achieve linearity, the code was changed so that the strings are now filtered to a temporary scratch space in the parser, and copied back to the output buffer after filtering, if any change occurred. The improvements were large for the folded scalars; the table below shows the benchmark results of throughput (MB/s) for several files containing large scalars of a single type: - | scalar type | before | after | improvement | - |:------------|-------:|-------:|---------:| - | block folded | 276 | 561 | 103% | - | block literal | 331 | 611 | 85% | - | single quoted | 247 | 267 | 8% | - | double quoted | 212 | 230 | 8% | - | plain (unquoted) | 173 | 186 | 8% | - - The cost for small scalars is negligible, with benchmark improvement in the interval of -2% to 5%, so well within the margin of benchmark variability in a regular OS. In the future, this will be optimized again by copying each character in place, thus completely avoiding the staging arena. -- `Callbacks`: add `operator==()` and `operator!=()` ([PR #168](https://github.com/biojppm/rapidyaml/pull/168)). -- `Tree`: on error or assert prefer the error callback stored into the tree's current `Callbacks`, rather than the global `Callbacks` ([PR #168](https://github.com/biojppm/rapidyaml/pull/168)). -- `detail::stack<>`: improve behavior when assigning from objects `Callbacks`, test all rule-of-5 scenarios ([PR #168](https://github.com/biojppm/rapidyaml/pull/168)). -- Improve formatting of error messages. - - -### Thanks - -- @ingydotnet -- @perlpunk -- @cschreib -- @fargies -- @Xeonacid -- @aviktorov -- @xTVaser diff --git a/thirdparty/ryml/changelog/0.4.1.md b/thirdparty/ryml/changelog/0.4.1.md deleted file mode 100644 index c8735aa6e..000000000 --- a/thirdparty/ryml/changelog/0.4.1.md +++ /dev/null @@ -1,3 +0,0 @@ -### Fixes - -- Fix [#223](https://github.com/biojppm/rapidyaml/issues/223): assertion peeking into the last line when it was whitespaces only. diff --git a/thirdparty/ryml/changelog/0.5.0.md b/thirdparty/ryml/changelog/0.5.0.md deleted file mode 100644 index 467706e7c..000000000 --- a/thirdparty/ryml/changelog/0.5.0.md +++ /dev/null @@ -1,174 +0,0 @@ - -### Breaking changes - -- Make the node API const-correct ([PR#267](https://github.com/biojppm/rapidyaml/pull/267)): added `ConstNodeRef` to hold a constant reference to a node. As the name implies, a `ConstNodeRef` object cannot be used in any tree-mutating operation. It is also smaller than the existing `NodeRef` (and faster because it does not need to check its own validity on every access). As a result of this change, there are now some constraints when obtaining a ref from a tree, and existing code is likely to break in this type of situation: - ```c++ - const Tree const_tree = ...; - NodeRef nr = const_tree.rootref(); // ERROR (was ok): cannot obtain a mutating NodeRef from a const Tree - ConstNodeRef cnr = const_tree.rootref(); // ok - - Tree tree = ...; - NodeRef nr = tree.rootref(); // ok - ConstNodeRef cnr = tree.rootref(); // ok (implicit conversion from NodeRef to ConstNodeRef) - // to obtain a ConstNodeRef from a mutable Tree - // while avoiding implicit conversion, use the `c` - // prefix: - ConstNodeRef cnr = tree.crootref(); - // likewise for tree.ref() and tree.cref(). - - nr = cnr; // ERROR: cannot obtain NodeRef from ConstNodeRef - cnr = nr; // ok - ``` - The use of `ConstNodeRef` also needs to be propagated through client code. One such place is when deserializing types: - ```c++ - // needs to be changed from: - template bool read(ryml::NodeRef const& n, T *var); - // ... to: - template bool read(ryml::ConstNodeRef const& n, T *var); - ``` - - The initial version of `ConstNodeRef/NodeRef` had the problem that const methods in the CRTP base did not participate in overload resolution ([#294](https://github.com/biojppm/rapidyaml/issues/294)), preventing calls from `const NodeRef` objects. This was fixed by moving non-const methods to the CRTP base and disabling them with SFINAE ([PR#295](https://github.com/biojppm/rapidyaml/pull/295)). - - Also added disambiguation iteration methods: `.cbegin()`, `.cend()`, `.cchildren()`, `.csiblings()` ([PR#295](https://github.com/biojppm/rapidyaml/pull/295)). -- Deprecate `emit()` and `emitrs()` ([#120](https://github.com/biojppm/rapidyaml/issues/120), [PR#303](https://github.com/biojppm/rapidyaml/pull/303)): use `emit_yaml()` and `emitrs_yaml()` instead. This was done to improve compatibility with Qt, which leaks a macro named `emit`. For more information, see [#120](https://github.com/biojppm/rapidyaml/issues/120). - - In the Python API: - - Deprecate `emit()`, add `emit_yaml()` and `emit_json()`. - - Deprecate `compute_emit_length()`, add `compute_emit_yaml_length()` and `compute_emit_json_length()`. - - Deprecate `emit_in_place()`, add `emit_yaml_in_place()` and `emit_json_in_place()`. - - Calling the deprecated functions will now trigger a warning. -- Location querying is no longer done lazily ([#260](https://github.com/biojppm/rapidyaml/issues/260), [PR#307](https://github.com/biojppm/rapidyaml/pull/307)). It now requires explicit opt-in when instantiating the parser. With this change, the accelerator structure for location querying is now built when parsing: - ```c++ - Parser parser(ParserOptions().locations(true)); - // now parsing also builds location lookup: - Tree t = parser.parse_in_arena("myfile.yml", "foo: bar"); - assert(parser.location(t["foo"]).line == 0u); - ``` - - Locations are disabled by default: - ```c++ - Parser parser; - assert(parser.options().locations() == false); - ``` -- Deprecate `Tree::arena_pos()`: use `Tree::arena_size()` instead ([PR#290](https://github.com/biojppm/rapidyaml/pull/290)). -- Deprecate pointless `has_siblings()`: use `Tree::has_other_siblings()` instead ([PR#330](https://github.com/biojppm/rapidyaml/pull/330). - - -### Performance improvements - -- Improve performance of integer serialization and deserialization (in [c4core](https://github.com/biojppm/c4core)). Eg, on Linux/g++11.2, with integral types: - - `c4::to_chars()` can be expected to be roughly... - - ~40% to 2x faster than `std::to_chars()` - - ~10x-30x faster than `sprintf()` - - ~50x-100x faster than a naive `stringstream::operator<<()` followed by `stringstream::str()` - - `c4::from_chars()` can be expected to be roughly... - - ~10%-30% faster than `std::from_chars()` - - ~10x faster than `scanf()` - - ~30x-50x faster than a naive `stringstream::str()` followed by `stringstream::operator>>()` - For more details, see [the changelog for c4core 0.1.10](https://github.com/biojppm/c4core/releases/tag/v0.1.10). -- Fix [#289](https://github.com/biojppm/rapidyaml/issues/289) and [#331](https://github.com/biojppm/rapidyaml/issues/331) - parsing of single-line flow-style sequences had quadratic complexity, causing long parse times in ultra long lines [PR#293](https://github.com/biojppm/rapidyaml/pull/293)/[PR#332](https://github.com/biojppm/rapidyaml/pull/332). - - This was due to scanning for the token `: ` before scanning for `,` or `]`, which caused line-length scans on every scalar scan. Changing the order of the checks was enough to address the quadratic complexity, and the parse times for flow-style are now in line with block-style. - - As part of this changeset, a significant number of runtime branches was eliminated by separating `Parser::_scan_scalar()` into several different `{seq,map}x{block,flow}` functions specific for each context. Expect some improvement in parse times. - - Also, on Debug builds (or assertion-enabled builds) there was a paranoid assertion calling `Tree::has_child()` in `Tree::insert_child()` that caused quadratic behavior because the assertion had linear complexity. It was replaced with a somewhat equivalent O(1) assertion. - - Now the byte throughput is independent of line size for styles and containers. This can be seen in the table below, which shows parse troughputs in MB/s of 1000 containers of different styles and sizes (flow containers are in a single line): - - | Container | Style | 10elms | 100elms | 1000elms | - |-----------|-------|-------------|--------------|---------------| - | 1000 Maps | block | 50.8MB/s | 57.8MB/s | 63.9MB/s | - | 1000 Maps | flow | 58.2MB/s | 65.9MB/s | 74.5MB/s | - | 1000 Seqs | block | 55.7MB/s | 59.2MB/s | 60.0MB/s | - | 1000 Seqs | flow | 52.8MB/s | 55.6MB/s | 54.5MB/s | -- Fix [#329](https://github.com/biojppm/rapidyaml/issues/329): complexity of `has_sibling()` and `has_child()` is now O(1), previously was linear ([PR#330](https://github.com/biojppm/rapidyaml/pull/330)). - - -### Fixes - -- Fix [#233](https://github.com/biojppm/rapidyaml/issues/233) - accept leading colon in the first key of a flow map (`UNK` node) [PR#234](https://github.com/biojppm/rapidyaml/pull/234): - ```yaml - :foo: # parse error on the leading colon - :bar: a # parse error on the leading colon - :barbar: b # was ok - :barbarbar: c # was ok - foo: # was ok - bar: a # was ok - :barbar: b # was ok - :barbarbar: c # was ol - ``` -- Fix [#253](https://github.com/biojppm/rapidyaml/issues/253): double-quoted emitter should encode carriage-return `\r` to preserve roundtrip equivalence: - ```yaml - Tree tree; - NodeRef root = tree.rootref(); - root |= MAP; - root["s"] = "t\rt"; - root["s"] |= _WIP_VAL_DQUO; - std::string s = emitrs(tree); - EXPECT_EQ(s, "s: \"t\\rt\"\n"); - Tree tree2 = parse_in_arena(to_csubstr(s)); - EXPECT_EQ(tree2["s"].val(), tree["s"].val()); - ``` -- Fix parsing of empty block folded+literal scalars when they are the last child of a container (part of [PR#264](https://github.com/biojppm/rapidyaml/issues/264)): - ```yaml - seq: - - "" - - '' - - > - - | # error, the resulting val included all the YAML from the next node - seq2: - - "" - - '' - - | - - > # error, the resulting val included all the YAML from the next node - map: - a: "" - b: '' - c: > - d: | # error, the resulting val included all the YAML from the next node - map2: - a: "" - b: '' - c: | - d: > # error, the resulting val included all the YAML from the next node - lastly: the last - ``` -- Fix [#274](https://github.com/biojppm/rapidyaml/issues/274) ([PR#296](https://github.com/biojppm/rapidyaml/pull/296)): Lists with unindented items and trailing empty values parse incorrectly: - ```yaml - foo: - - bar - - - baz: qux - ``` - was wrongly parsed as - ```yaml - foo: - - bar - - baz: qux - ``` -- Fix [#277](https://github.com/biojppm/rapidyaml/issues/277) ([PR#340](https://github.com/biojppm/rapidyaml/pull/340)): merge fails with duplicate keys. -- Fix [#337](https://github.com/biojppm/rapidyaml/issues/337) ([PR#338](https://github.com/biojppm/rapidyaml/pull/338)): empty lines in block scalars shall not have tab characters `\t`. -- Fix [#268](https://github.com/biojppm/rapidyaml/issues/268) ([PR#339](https://github.com/biojppm/rapidyaml/pull/339)): don't override key type_bits when copying val. This was causing problematic resolution of anchors/references. -- Fix [#309](https://github.com/biojppm/rapidyaml/issues/309) ([PR#310](https://github.com/biojppm/rapidyaml/pull/310)): emitted scalars containing `@` or `` ` `` should be quoted. - - The quotes should be added only when they lead the scalar. See [#320](https://github.com/biojppm/rapidyaml/issues/320) and [PR#334](https://github.com/biojppm/rapidyaml/pull/334). -- Fix [#297](https://github.com/biojppm/rapidyaml/issues/297) ([PR#298](https://github.com/biojppm/rapidyaml/pull/298)): JSON emitter should escape control characters. -- Fix [#292](https://github.com/biojppm/rapidyaml/issues/292) ([PR#299](https://github.com/biojppm/rapidyaml/pull/299)): JSON emitter should quote version string scalars like `0.1.2`. -- Fix [#291](https://github.com/biojppm/rapidyaml/issues/291) ([PR#299](https://github.com/biojppm/rapidyaml/pull/299)): JSON emitter should quote scalars with leading zero, eg `048`. -- Fix [#280](https://github.com/biojppm/rapidyaml/issues/280) ([PR#281](https://github.com/biojppm/rapidyaml/pull/281)): deserialization of `std::vector` failed because its `operator[]` returns a `reference` instead of `value_type`. -- Fix [#288](https://github.com/biojppm/rapidyaml/issues/288) ([PR#290](https://github.com/biojppm/rapidyaml/pull/290)): segfault on successive calls to `Tree::_grow_arena()`, caused by using the arena position instead of its length as starting point for the new arena capacity. -- Fix [#324](https://github.com/biojppm/rapidyaml/issues/324) ([PR#328](https://github.com/biojppm/rapidyaml/pull/328)): eager assertion prevented moving nodes to the first position in a parent. -- Fix `Tree::_clear_val()`: was clearing key instead ([PR#335](https://github.com/biojppm/rapidyaml/pull/335)). -- YAML test suite events emitter: fix emission of inheriting nodes. The events for `{<<: *anchor, foo: bar}` are now correctly emitted as: - ```yaml - =VAL :<< # previously was =ALI << - =ALI *anchor - =VAL :foo - =VAL :bar - ``` -- Fix [#246](https://github.com/biojppm/rapidyaml/issues/246): add missing `#define` for the include guard of the amalgamated header. -- Fix [#326](https://github.com/biojppm/rapidyaml/issues/326): honor runtime settings for calling debugbreak, add option to disable any calls to debugbreak. -- Fix [cmake#8](https://github.com/biojppm/cmake/issues/8): `SOVERSION` missing from shared libraries. - - -## Python - -- The Python packages for Windows and MacOSX are causing problems in the CI, and were mostly disabled. The problematic packages are successfully made, but then fail to be imported. This was impossible to reproduce outside of the CI, and they were disabled since they were delaying the release. As a consequence, the Python release will have very limited compiled packages for Windows (only Python 3.6 and 3.7) or MacOSX. Help would be appreciated from those interested in these packages. - - -## Thanks - -- @NaN-git -- @dancingbug diff --git a/thirdparty/ryml/cmake/uninstall.cmake b/thirdparty/ryml/cmake/uninstall.cmake deleted file mode 100644 index 5ac7ceb6d..000000000 --- a/thirdparty/ryml/cmake/uninstall.cmake +++ /dev/null @@ -1,24 +0,0 @@ -set(MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt") - -if(NOT EXISTS ${MANIFEST}) - message(FATAL_ERROR "Cannot find install manifest: '${MANIFEST}'") -endif() - -file(STRINGS ${MANIFEST} files) -foreach(file ${files}) - if(EXISTS ${file}) - message(STATUS "Removing file: '${file}'") - - exec_program( - ${CMAKE_COMMAND} ARGS "-E remove ${file}" - OUTPUT_VARIABLE stdout - RETURN_VALUE result - ) - - if(NOT "${result}" STREQUAL 0) - message(FATAL_ERROR "Failed to remove file: '${file}'.") - endif() - else() - MESSAGE(STATUS "File '${file}' does not exist.") - endif() -endforeach(file) diff --git a/thirdparty/ryml/compat.cmake b/thirdparty/ryml/compat.cmake deleted file mode 100644 index bbdd16c31..000000000 --- a/thirdparty/ryml/compat.cmake +++ /dev/null @@ -1,11 +0,0 @@ - -# old gcc-4.8 support -if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND - (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) AND - (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)) - - # c++17 compiler required - set(C4RYML_BUILD_BENCHMARKS OFF CACHE BOOL "" FORCE) - # LLVM required - set(C4RYML_SANITIZE OFF CACHE BOOL "" FORCE) -endif() diff --git a/thirdparty/ryml/ext/c4core/.github/release.sh b/thirdparty/ryml/ext/c4core/.github/release.sh deleted file mode 100644 index 68d24d3d0..000000000 --- a/thirdparty/ryml/ext/c4core/.github/release.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - - -# useful to iterate when fixing the release: -# ver=0.2.1 ; ( set -x ; git tag -d v$ver ; git push origin :v$ver ) ; (set -x ; set -e ; tbump --only-patch --non-interactive $ver ; git add -u ; git commit --amend --no-edit ; git tag --annotate --message "v$ver" "v$ver" ; git push -f --tags origin ) - - -function c4_release_create() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - c4_release_bump $ver ; \ - c4_release_commit $ver $branch \ - ) -} - -function c4_release_redo() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - c4_release_delete $ver ; \ - c4_release_bump $ver ; \ - c4_release_amend $ver $branch \ - ) -} - -function c4_release_bump() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - tbump --non-interactive --only-patch $ver \ - ) -} - -function c4_release_commit() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - tag=v$ver ; \ - git add -u ; \ - git commit -m $tag ; \ - git tag --annotate --message $tag $tag ; \ - ) -} - -function c4_release_amend() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - tag=v$ver ; \ - git add -u ; \ - git commit --amend -m $tag ; \ - git tag --annotate --message $tag $tag ; \ - ) -} - -function c4_release_delete() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - git tag -d v$ver ; \ - git push origin :v$ver \ - ) -} - -function c4_release_push() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - tag=v$ver ; \ - git push origin $branch ; \ - git push --tags origin $tag \ - ) -} - -function c4_release_force_push() -{ - ( \ - set -euxo pipefail ; \ - ver=$(_c4_validate_ver $1) ; \ - branch=$(_c4_validate_branch) ; \ - tag=v$ver ; \ - git push -f origin $branch ; \ - git push -f --tags origin $tag \ - ) -} - -function _c4_validate_ver() -{ - ver=$1 - if [ -z "$ver" ] ; then \ - exit 1 - fi - ver=$(echo $ver | sed "s:v\(.*\):\1:") - #sver=$(echo $ver | sed "s:\([0-9]*\.[0-9]*\..[0-9]*\).*:\1:") - if [ ! -f changelog/$ver.md ] ; then \ - if [ -f changelog/current.md ] ; then - git mv changelog/current.md changelog/$ver.md - touch changelog/current.md - git add changelog/current.md - else - echo "ERROR: could not find changelog/$ver.md or changelog/current.md" - exit 1 - fi - fi - echo $ver -} - -function _c4_validate_branch() -{ - branch=$(git rev-parse --abbrev-ref HEAD) - if [ "$branch" != "master" ] ; then - echo "ERROR: release branch must be master" - exit 1 - fi - echo $branch -} diff --git a/thirdparty/ryml/ext/c4core/.github/reqs.sh b/thirdparty/ryml/ext/c4core/.github/reqs.sh deleted file mode 100644 index 9937616af..000000000 --- a/thirdparty/ryml/ext/c4core/.github/reqs.sh +++ /dev/null @@ -1,314 +0,0 @@ -#!/usr/bin/env bash - -set -x - -# input environment variables: -# OS: the operating system -# CXX_: the compiler version. eg, g++-9 or clang++-6.0 -# BT: the build type -# VG: whether to install valgrind -# ARM: whether to arm cross-compiler and emulator -# GITHUB_WORKFLOW: when run from github -# API: whether to install swig -# CMANY: whether to install cmany - - - -#------------------------------------------------------------------------------- - -function c4_install_test_requirements() -{ - os=$1 - case "$os" in - ubuntu*) - c4_install_test_requirements_ubuntu - return 0 - ;; - macos*) - c4_install_test_requirements_macos - return 0 - ;; - win*) - c4_install_test_requirements_windows - return 0 - ;; - *) - return 0 - ;; - esac -} - -function c4_install_test_requirements_windows() -{ - if [ "$CMANY" == "ON" ] ; then - pip install cmany - fi - if [ "$API" == "ON" ] ; then - choco install swig - which swig - fi - # ensure chocolatey does not override cmake's cpack - which cpack - choco_cpack="/c/ProgramData/Chocolatey/bin/cpack.exe" - if [ -f $choco_cpack ] ; then - newname=$(echo $choco_cpack | sed 's:cpack:choco-cpack:') - mv -vf $choco_cpack $newname - fi - which cpack -} - -function c4_install_test_requirements_macos() -{ - if [ "$CMANY" == "ON" ] ; then - sudo pip3 install cmany - fi -} - -function c4_install_test_requirements_ubuntu() -{ - UBUNTU_RELEASE=$(lsb_release -rs) - UBUNTU_RELEASE_NAME=$(lsb_release -cs) - APT_PKG="" # all - PIP_PKG="" - c4_gather_test_requirements_ubuntu - echo "apt packages: $APT_PKG" - echo "pip packages: $PIP_PKG" - c4_install_test_requirements_ubuntu_impl - echo 'INSTALL COMPLETE!' -} - - -function c4_install_all_possible_requirements_ubuntu() -{ - export CXX_=all - export BT=Coverage - APT_PKG="" # all - PIP_PKG="" - sudo dpkg --add-architecture i386 - c4_gather_test_requirements_ubuntu - _c4_add_arm_compilers - echo "apt packages: $APT_PKG" - echo "pip packages: $PIP_PKG" - c4_install_test_requirements_ubuntu_impl - echo 'INSTALL COMPLETE!' -} - - -function c4_gather_test_requirements_ubuntu() -{ - if [ "$GITHUB_WORKFLOW" != "" ] ; then - sudo dpkg --add-architecture i386 - else - _add_apt build-essential - _add_apt cmake - fi - - _add_apt linux-libc-dev:i386 - _add_apt libc6:i386 - _add_apt libc6-dev:i386 - _add_apt libc6-dbg:i386 - _c4_addlibcxx - - _c4_gather_compilers "$CXX_" - - _add_apt python3-setuptools - _add_apt python3-pip - - #_add_apt iwyu - #_add_apt cppcheck - #_add_pip cpplint - # oclint? - if [ "$VG" == "ON" ] ; then - _add_apt valgrind - fi - - if [ "$BT" == "Coverage" ]; then - _add_apt lcov - _add_apt libffi-dev - _add_apt libssl-dev - _add_pip requests[security] - _add_pip pyopenssl - _add_pip ndg-httpsclient - _add_pip pyasn1 - _add_pip cpp-coveralls - fi - - if [ "$CMANY" != "" ] ; then - _add_pip cmany - fi - - case "$CXX_" in - arm*) - _c4_add_arm_compilers - ;; - esac -} - - -function c4_install_test_requirements_ubuntu_impl() -{ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key 2>/dev/null | sudo apt-key add - - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - - sudo -E apt-add-repository --yes "deb https://apt.kitware.com/ubuntu/ $UBUNTU_RELEASE_NAME main" - sudo -E add-apt-repository --yes ppa:ubuntu-toolchain-r/test - - if [ "$APT_PKG" != "" ] ; then - #sudo -E apt-get clean - sudo -E apt-get update - sudo -E apt-get install -y --force-yes $APT_PKG - fi - - if [ "$PIP_PKG" != "" ]; then - sudo pip3 install $PIP_PKG - fi -} - - -#------------------------------------------------------------------------------- - -function _c4_add_arm_compilers() -{ - # this is going to be deprecated: - # https://askubuntu.com/questions/1243252/how-to-install-arm-none-eabi-gdb-on-ubuntu-20-04-lts-focal-fossa - sudo -E add-apt-repository --yes ppa:team-gcc-arm-embedded/ppa - - _add_apt gcc-arm-embedded - _add_apt g++-arm-linux-gnueabihf - _add_apt g++-multilib-arm-linux-gnueabihf - _add_apt qemu -} - - -function _c4_gather_compilers() -{ - cxx=$1 - case $cxx in - g++-12 ) _c4_addgcc 12 ;; - g++-11 ) _c4_addgcc 11 ;; - g++-10 ) _c4_addgcc 10 ;; - g++-9 ) _c4_addgcc 9 ;; - g++-8 ) _c4_addgcc 8 ;; - g++-7 ) _c4_addgcc 7 ;; - g++-6 ) _c4_addgcc 6 ;; - g++-5 ) _c4_addgcc 5 ;; - #g++-4.9 ) _c4_addgcc 4.9 ;; # https://askubuntu.com/questions/1036108/install-gcc-4-9-at-ubuntu-18-04 - g++-4.8 ) _c4_addgcc 4.8 ;; - clang++-13 ) _c4_addclang 13 ;; - clang++-12 ) _c4_addclang 12 ;; - clang++-11 ) _c4_addclang 11 ;; - clang++-10 ) _c4_addclang 10 ;; - clang++-9 ) _c4_addclang 9 ;; - clang++-8 ) _c4_addclang 8 ;; - clang++-7 ) _c4_addclang 7 ;; - clang++-6.0) _c4_addclang 6.0 ;; - clang++-5.0) _c4_addclang 5.0 ;; - clang++-4.0) _c4_addclang 4.0 ;; - clang++-3.9) _c4_addclang 3.9 ;; - all) - all="g++-11 g++-10 g++-9 g++-8 g++-7 g++-6 g++-5 clang++-12 clang++-11 clang++-10 clang++-9 clang++-8 clang++-7 clang++-6.0 clang++-5.0 clang++-4.0 clang++-3.9" - echo "installing all compilers: $all" - for cxx in $all ; do - _c4_gather_compilers $cxx - done - ;; - "") - # use default compiler - ;; - arm*) - ;; - *) - echo "unknown compiler: $cxx" - exit 1 - ;; - esac -} - -# add a gcc compiler -function _c4_addgcc() -{ - gccversion=$1 - case $gccversion in - 5 ) - _add_apt gcc-5 "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" - _add_apt gcc-5 "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" - ;; - *) - ;; - esac - _add_apt g++-$gccversion - _add_apt g++-$gccversion-multilib - _add_apt libstdc++-$gccversion-dev - _add_apt lib32stdc++-$gccversion-dev -} - -# add a clang compiler -function _c4_addclang() -{ - clversion=$1 - case $clversion in - # in 18.04, clang9 and later require PPAs - 9 | 10 | 11 | 12 | 13) - _add_apt clang-$clversion "deb http://apt.llvm.org/$UBUNTU_RELEASE_NAME/ llvm-toolchain-$UBUNTU_RELEASE_NAME-$clversion main" - # libstdc++ is required - _c4_addgcc 11 - _c4_addgcc 10 - _c4_addgcc 9 - ;; - "") - _add_apt clang - ;; - *) - _add_apt clang-$clversion - ;; - esac - _add_apt g++-multilib # this is required for 32 bit https://askubuntu.com/questions/1057341/unable-to-find-stl-headers-in-ubuntu-18-04 - _add_apt clang-tidy-$clversion -} - -# add libc++ -function _c4_addlibcxx() -{ - _add_apt clang - _add_apt libc++1 - _add_apt libc++abi-dev - _add_apt libc++-dev - #_add_apt libc++1:i386 - #_add_apt libc++abi-dev:i386 - #_add_apt libc++-dev:i386 -} - - -#------------------------------------------------------------------------------- - -# add a pip package to the list -function _add_pip() -{ - pkgs=$* - PIP_PKG="$PIP_PKG $pkgs" - echo "adding to pip packages: $pkgs" -} - -# add a debian package to the list -function _add_apt() -{ - pkgs=$1 - sourceslist=$2 - APT_PKG="$APT_PKG $pkgs" - echo "adding to apt packages: $pkgs" - _add_src "$sourceslist" "# for packages: $pkgs" -} - -# add an apt source -function _add_src() -{ - sourceslist=$1 - comment=$2 - if [ ! -z "$sourceslist" ] ; then - echo "adding apt source: $sourceslist" - sudo bash -c "cat >> /etc/apt/sources.list < $coverage_service" - cmake --build $build_dir --config $BT --target ${PROJ_PFX_TARGET}coverage-submit-$coverage_service -} - -# WIP -function c4_run_static_analysis() -{ - if _c4skipbitlink "$1" ; then return 0 ; fi - id=$1 - linktype=$(_c4linktype $id) - build_dir=`pwd`/build/$id - # https://blog.kitware.com/static-checks-with-cmake-cdash-iwyu-clang-tidy-lwyu-cpplint-and-cppcheck/ - pushd $PROJ_DIR -} - -function c4_cfg_test() -{ - if _c4skipbitlink "$1" ; then return 0 ; fi - id=$1 - # - build_dir=`pwd`/build/$id - install_dir=`pwd`/install/$id - mkdir -p $build_dir - mkdir -p $install_dir - # - if [ "$TOOLCHAIN" != "" ] ; then - toolchain_file=`pwd`/$TOOLCHAIN - if [ ! -f "$toolchain_file" ] ; then - echo "ERROR: toolchain not found: $toolchain_file" - exit 1 - fi - _addcmkflags -DCMAKE_TOOLCHAIN_FILE=$toolchain_file - else - bits=$(_c4bits $id) - linktype=$(_c4linktype $id) - case "$linktype" in - static) _addcmkflags -DBUILD_SHARED_LIBS=OFF ;; - shared) _addcmkflags -DBUILD_SHARED_LIBS=ON ;; - *) - echo "ERROR: unknown linktype: $linktype" - exit 1 - ;; - esac - fi - if [ "$STD" != "" ] ; then - _addcmkflags -DC4_CXX_STANDARD=$STD - _addprojflags CXX_STANDARD=$STD - fi - if [ "$LIBCXX" != "" ] ; then - _addprojflags USE_LIBCXX=$LIBCXX - fi - # - if [ "$DEV" != "OFF" ] ; then - _addprojflags DEV=ON - fi - case "$LINT" in - all ) _addprojflags LINT=ON LINT_TESTS=ON LINT_CLANG_TIDY=ON LINT_PVS_STUDIO=ON ;; - clang-tidy) _addprojflags LINT=ON LINT_TESTS=ON LINT_CLANG_TIDY=ON LINT_PVS_STUDIO=OFF ;; - pvs-studio) _addprojflags LINT=ON LINT_TESTS=ON LINT_CLANG_TIDY=OFF LINT_PVS_STUDIO=ON ;; - * ) _addprojflags LINT=OFF ;; - esac - case "$SAN" in - ALL) _addprojflags SANITIZE=ON ;; - A ) _addprojflags SANITIZE=ON ASAN=ON TSAN=OFF MSAN=OFF UBSAN=OFF ;; - T ) _addprojflags SANITIZE=ON ASAN=OFF TSAN=ON MSAN=OFF UBSAN=OFF ;; - M ) _addprojflags SANITIZE=ON ASAN=OFF TSAN=OFF MSAN=ON UBSAN=OFF ;; - UB ) _addprojflags SANITIZE=ON ASAN=OFF TSAN=OFF MSAN=OFF UBSAN=ON ;; - * ) _addprojflags SANITIZE=OFF ;; - esac - case "$SAN_ONLY" in - ON) _addprojflags SANITIZE_ONLY=ON ;; - * ) _addprojflags SANITIZE_ONLY=OFF ;; - esac - case "$VG" in - ON) _addprojflags VALGRIND=ON VALGRIND_SGCHECK=OFF ;; # FIXME SGCHECK should be ON - * ) _addprojflags VALGRIND=OFF VALGRIND_SGCHECK=OFF ;; - esac - case "$BM" in - ON) _addprojflags BUILD_BENCHMARKS=ON ;; - * ) _addprojflags BUILD_BENCHMARKS=OFF ;; - esac - if [ "$BT" == "Coverage" ] ; then - # the coverage repo tokens can be set in the travis environment: - # export CODECOV_TOKEN=....... - # export COVERALLS_REPO_TOKEN=....... - _addprojflags COVERAGE_CODECOV=ON COVERAGE_CODECOV_SILENT=OFF - _addprojflags COVERAGE_COVERALLS=ON COVERAGE_COVERALLS_SILENT=OFF - fi - if [ ! -z "$VERBOSE_MAKEFILES" ] ; then - _addcmkflags -DCMAKE_VERBOSE_MAKEFILES=$VERBOSE_MAKEFILES - fi - _addcmkflags -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - if [ ! -z "$CMAKE_FLAGS" ] ; then - _addcmkflags $CMAKE_FLAGS - fi - - echo "building with additional cmake flags: $CMFLAGS" - - export C4_EXTERN_DIR=`pwd`/build/extern - mkdir -p $C4_EXTERN_DIR - - cmake --version - pwd - - # - # bash quote handling is a fiasco, and I could not find a way of storing - # quoted strings in variables and then expand the variables with correct quotes - # so we have to do this precious jewell of chicanery: - case "$CXX_" in - vs2022) - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -G 'Visual Studio 17 2022' -A $(_c4vsarchtype $id) \ - $(_c4_add_ehsc_to_vs_arm32 $id) \ - -DCMAKE_BUILD_TYPE=$BT $CMFLAGS - ;; - vs2019) - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -G 'Visual Studio 16 2019' -A $(_c4vsarchtype $id) \ - $(_c4_add_ehsc_to_vs_arm32 $id) \ - -DCMAKE_BUILD_TYPE=$BT $CMFLAGS - ;; - vs2017) - case "$bits" in - 64) g="Visual Studio 15 2017 Win64" ;; - 32) g="Visual Studio 15 2017" ;; - *) exit 1 ;; - esac - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - $(_c4_add_ehsc_to_vs_arm32 $id) \ - -DCMAKE_BUILD_TYPE=$BT -G "$g" $CMFLAGS - ;; - xcode) - g=Xcode - case "$bits" in - 64) a="x86_64" ;; - 32) a="i386" - echo "xcode does not support i386" - exit 1 # i386 is deprecated in xcode - ;; - esac - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT -G "$g" -DCMAKE_OSX_ARCHITECTURES=$a $CMFLAGS - ;; - arm*|"") # make sure arm* comes before *g++ or *gcc* - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT $CMFLAGS - ;; - *g++*|*gcc*|*clang*) - export CC_=$(echo "$CXX_" | sed 's:clang++:clang:g' | sed 's:g++:gcc:g') - _c4_choose_clang_tidy $CXX_ - cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT $CMFLAGS \ - -DCMAKE_C_COMPILER=$CC_ -DCMAKE_CXX_COMPILER=$CXX_ \ - -DCMAKE_C_FLAGS="-std=c99 -m$bits" -DCMAKE_CXX_FLAGS="-m$bits" - cmake --build $build_dir --target help | sed 1d | sort - ;; - em++) - emcmake cmake -S $PROJ_DIR -B $build_dir -DCMAKE_INSTALL_PREFIX="$install_dir" \ - -DCMAKE_BUILD_TYPE=$BT $CMFLAGS -DCMAKE_CXX_FLAGS="-s DISABLE_EXCEPTION_CATCHING=0" - ;; - *) - echo "unknown compiler" - exit 1 - ;; - esac -} - -function _c4_choose_clang_tidy() -{ - cxx=$1 - # only for clang compilers. - case $cxx in - clang*) - # try with version first - clang_tidy_ver=$(echo $cxx | sed "s:++:-tidy:") - clang_tidy=$(echo $cxx | sed "s:++.*:-tidy:") - for n in $clang_tidy_ver $clang_tidy ; do - exe=$(which $n) - echo "searching for $n: $exe" - if [ -z "$exe" ] ; then - echo "could not find $clang_tidy" - else - _addcmkflags "-DCLANG_TIDY=$exe" - return 0 - fi - done - echo "error: could not find clang-tidy for $cxx" - exit 1 - ;; - esac -} - -# add cmake flags without project prefix -function _addcmkflags() -{ - for f in $* ; do - CMFLAGS="$CMFLAGS ${f}" - done -} - -# add cmake flags with project prefix -function _addprojflags() -{ - for f in $* ; do - CMFLAGS="$CMFLAGS -D${PROJ_PFX_CMAKE}${f}" - done -} - -function _c4_add_ehsc_to_vs_arm32() -{ - id=$1 - case "$CXX_" in - vs*) - case "$id" in - arm32|arm32shared|arm32static|shared32arm|static32arm|arm) - echo '-DCMAKE_CXX_FLAGS="/EHsc"' - ;; - *) - esac - ;; - esac -} - -function _c4_parallel_build_flags() -{ - case "$CXX_" in - vs2022|vs2019|vs2017|vs2015) - # https://docs.microsoft.com/en-us/visualstudio/msbuild/msbuild-command-line-reference?view=vs-2019 - # https://stackoverflow.com/questions/2619198/how-to-get-number-of-cores-in-win32 - if [ -z "$NUM_JOBS_BUILD" ] ; then - echo "/maxcpucount:$NUMBER_OF_PROCESSORS" - else - echo "/maxcpucount:$NUM_JOBS_BUILD" - fi - ;; - xcode) - # https://stackoverflow.com/questions/5417835/how-to-modify-the-number-of-parallel-compilation-with-xcode - # https://gist.github.com/nlutsenko/ee245fbd239087d22137 - if [ -z "$NUM_JOBS_BUILD" ] ; then - echo "-IDEBuildOperationMaxNumberOfConcurrentCompileTasks=$(sysctl -n hw.ncpu)" - else - echo "-IDEBuildOperationMaxNumberOfConcurrentCompileTasks=$NUM_JOBS_BUILD" - fi - ;; - *g++*|*gcc*|*clang*|em++) - if [ -z "$NUM_JOBS_BUILD" ] ; then - echo "-j $(nproc)" - else - echo "-j $NUM_JOBS_BUILD" - fi - ;; - "") # allow empty compiler - ;; - *) - echo "unknown compiler" - exit 1 - ;; - esac -} - -function _c4_generator_build_flags() -{ - case "$CXX_" in - vs2022|vs2019|vs2017|vs2015) - ;; - xcode) - # WTF??? - # https://github.com/biojppm/rapidyaml/pull/97/checks?check_run_id=1504677928#step:7:964 - # https://stackoverflow.com/questions/51153525/xcode-10-unable-to-attach-db-error - echo "-UseModernBuildSystem=NO" - ;; - *g++*|*gcc*|*clang*|em++) - ;; - "") # allow empty compiler - ;; - *) - echo "unknown compiler" - exit 1 - ;; - esac -} diff --git a/thirdparty/ryml/ext/c4core/.github/vagrant/Vagrantfile b/thirdparty/ryml/ext/c4core/.github/vagrant/Vagrantfile deleted file mode 100644 index 6f3d9d237..000000000 --- a/thirdparty/ryml/ext/c4core/.github/vagrant/Vagrantfile +++ /dev/null @@ -1,80 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# 1) download and install vagrant: https://www.vagrantup.com/downloads.html -# (do not install ubuntu's 14.04 16.04 version, see https://stackoverflow.com/questions/22717428/vagrant-error-failed-to-mount-folders-in-linux-guest ): -# 2) vagrant plugin install vagrant-vbguest -# 3) vagrant up --provider virtualbox -# 4) vagrant ssh - -# All Vagrant configuration is done below. The "2" in Vagrant.configure -# configures the configuration version (we support older styles for -# backwards compatibility). Please don't change it unless you know what -# you're doing. -Vagrant.configure(2) do |config| - # The most common configuration options are documented and commented below. - # For a complete reference, please see the online documentation at - # https://docs.vagrantup.com. - - # Every Vagrant development environment requires a box. You can search for - # boxes at https://atlas.hashicorp.com/search. - config.vm.box = "generic/ubuntu2004" - - # Disable automatic box update checking. If you disable this, then - # boxes will only be checked for updates when the user runs - # `vagrant box outdated`. This is not recommended. - # config.vm.box_check_update = false - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - # config.vm.network "forwarded_port", guest: 80, host: 8080 - - #config.ssh.username = 'travis' - #config.ssh.password = 'travis' - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - # config.vm.network "private_network", ip: "192.168.33.10" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network "public_network" - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - config.vm.synced_folder "../../../..", "/vagrant" - - #config.vm.synced_folder '.', '/vagrant', disabled: true - - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. - # Example for VirtualBox: - # - # config.vm.provider "virtualbox" do |vb| - # # Display the VirtualBox GUI when booting the machine - # vb.gui = true - # - # # Customize the amount of memory on the VM: - # vb.memory = "1024" - # end - # - # View the documentation for the provider you are using for more - # information on available options. - - # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies - # such as FTP and Heroku are also available. See the documentation at - # https://docs.vagrantup.com/v2/push/atlas.html for more information. - # config.push.define "atlas" do |push| - # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" - # end - - # Enable provisioning with a shell script. Additional provisioners such as - # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the - # documentation for more information about their specific syntax and use. - #config.vm.provision "shell", path: "travis-install.sh" - -end diff --git a/thirdparty/ryml/ext/c4core/.github/vagrant/macos/Vagrantfile b/thirdparty/ryml/ext/c4core/.github/vagrant/macos/Vagrantfile deleted file mode 100644 index 62806c970..000000000 --- a/thirdparty/ryml/ext/c4core/.github/vagrant/macos/Vagrantfile +++ /dev/null @@ -1,71 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# All Vagrant configuration is done below. The "2" in Vagrant.configure -# configures the configuration version (we support older styles for -# backwards compatibility). Please don't change it unless you know what -# you're doing. -Vagrant.configure("2") do |config| - # The most common configuration options are documented and commented below. - # For a complete reference, please see the online documentation at - # https://docs.vagrantup.com. - - # Every Vagrant development environment requires a box. You can search for - # boxes at https://vagrantcloud.com/search. - config.vm.box = "ramsey/macos-catalina" - config.vm.box_version = "1.0.0" - - # Disable automatic box update checking. If you disable this, then - # boxes will only be checked for updates when the user runs - # `vagrant box outdated`. This is not recommended. - # config.vm.box_check_update = false - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - # NOTE: This will enable public access to the opened port - # config.vm.network "forwarded_port", guest: 80, host: 8080 - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine and only allow access - # via 127.0.0.1 to disable public access - # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - # config.vm.network "private_network", ip: "192.168.33.10" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network "public_network" - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - # config.vm.synced_folder "../data", "/vagrant_data" - - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. - # Example for VirtualBox: - # - # config.vm.provider "virtualbox" do |vb| - # # Display the VirtualBox GUI when booting the machine - # vb.gui = true - # - # # Customize the amount of memory on the VM: - # vb.memory = "1024" - # end - # - # View the documentation for the provider you are using for more - # information on available options. - - # Enable provisioning with a shell script. Additional provisioners such as - # Ansible, Chef, Docker, Puppet and Salt are also available. Please see the - # documentation for more information about their specific syntax and use. - # config.vm.provision "shell", inline: <<-SHELL - # apt-get update - # apt-get install -y apache2 - # SHELL -end diff --git a/thirdparty/ryml/ext/c4core/.github/vagrant/vagrant-provision.sh b/thirdparty/ryml/ext/c4core/.github/vagrant/vagrant-provision.sh deleted file mode 100755 index efa958738..000000000 --- a/thirdparty/ryml/ext/c4core/.github/vagrant/vagrant-provision.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -set -x - -# https://askubuntu.com/questions/735201/installing-clang-3-8-on-ubuntu-14-04-3 -wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | sudo apt-key add - - -done=$(grep C4STL /etc/apt/sources.list) -if [ -z "$done" ] ; then - cat >> /etc/apt/sources.list </dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null - echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null - apt-get update -y - rm /usr/share/keyrings/kitware-archive-keyring.gpg - apt-get install kitware-archive-keyring - apt-get update -y - fi - apt-get install -y cmake cmake-data - cmake --version - run: | - set -x - uname -a - pwd - ls -lFhp . - # - bdir=build_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}} - idir=install_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}} - mkdir -p $bdir - # - cmake -S . -B $bdir \ - -DCMAKE_INSTALL_PREFIX=$idir \ - -DCMAKE_BUILD_TYPE=${{matrix.bt}} \ - -DC4_CXX_STANDARD=${{matrix.std}} \ - -DCXX_STANDARD=${{matrix.std}} \ - -DC4CORE_DEV=ON \ - -DC4CORE_BUILD_BENCHMARKS=OFF \ - -DC4CORE_SANITIZE=OFF \ - -DC4CORE_LINT=OFF \ - -DC4CORE_VALGRIND=OFF - # - cmake --build $bdir -j --target c4core-test-build - # - cmake --build $bdir --target c4core-test-run diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/benchmarks.yml b/thirdparty/ryml/ext/c4core/.github/workflows/benchmarks.yml deleted file mode 100644 index acc7dd630..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/benchmarks.yml +++ /dev/null @@ -1,250 +0,0 @@ -name: benchmarks - -defaults: - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -jobs: - - gettag: - runs-on: ubuntu-latest - steps: - # use fetch-depth to ensure all tags are fetched - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}} - - name: Variables (from tag) - if: contains(github.ref, 'tags/v') - run: | - # https://github.community/t/how-to-get-just-the-tag-name/16241/11 - SRC_TAG=${GITHUB_REF#refs/tags/} - SRC_VERSION=${GITHUB_REF#refs/tags/v} - cat < vars.sh - export SRC_TAG=$SRC_TAG - export SRC_VERSION=$SRC_VERSION - EOF - - name: Variables (from commit, no tag) - if: ${{ !contains(github.ref, 'tags/v') }} - run: | - set -x - branch_name=${GITHUB_REF#refs/heads/} - # builds triggered from PRs have the branch_name like this: refs/pull/150/merge - # so filter to eg pr0150_merge - branch_name=`echo $branch_name | sed "s:refs/pull/\([0-9]*\)/\(.*\):pr0\1_\2:"` - # sanitize the branch name; eg merge/foo-bar -> merge_foo_bar - branch_name=`echo $branch_name | sed 's:[/.-]:_:g'` - git config --global --add safe.directory $(pwd) - SRC_TAG=$(git describe || git rev-parse --short HEAD) # eg v0.2.0-110-gda837e0 - SRC_VERSION="${branch_name}-${SRC_TAG}" - cat < vars.sh - export SRC_TAG=$SRC_TAG - export SRC_VERSION=$SRC_VERSION - EOF - - name: Verify vars.sh - run: cat vars.sh ; source vars.sh ; echo $SRC_TAG ; echo $SRC_VERSION - - name: Save vars.sh - uses: actions/upload-artifact@v3 - with: {name: vars.sh, path: ./vars.sh} - - bm_x86_64: - name: bm/x86_64/c++${{matrix.std}}/${{matrix.cxx}}/${{matrix.bt}} - needs: gettag - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip benchmarks')) || - contains(github.event.head_commit.message, 'only benchmarks') - continue-on-error: true - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 17, cxx: g++-10, bt: Release, os: ubuntu-20.04, bitlinks: static64 static32} - - {std: 20, cxx: g++-10, bt: Release, os: ubuntu-20.04, bitlinks: static64 static32} - # - - {std: 17, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: static64 static32} - - {std: 20, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: static64 static32} - - {std: 17, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: static64 static32} - - {std: 20, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: static64 static32} - # - - {std: 17, cxx: xcode, xcver: 13, bt: Release, os: macos-11, bitlinks: static64} - env: {BM: ON, STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - # use fetch-depth to ensure all tags are fetched - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - name: Download vars.sh - uses: actions/download-artifact@v3 - with: {name: vars.sh, path: ./} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: Install python 3.10 for plotting - uses: actions/setup-python@v4 - with: { python-version: '3.10' } - - name: install benchmark plotting dependencies - run: | - which python - which pip - python --version - pip --version - pip install -v -r cmake/bm-xp/requirements.txt - python -c 'import munch ; print("ok!") ; exit(0)' - echo $? - - name: shared64-configure--------------------------------------------------- - run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_target shared64 c4core-bm-build} - - {name: shared64-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared64 c4core-bm-run} - - {name: shared64-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared64 c4core-bm-plot} - - name: static64-configure--------------------------------------------------- - run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_target static64 c4core-bm-build} - - {name: static64-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static64 c4core-bm-run} - - {name: static64-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static64 c4core-bm-plot} - - name: static32-configure--------------------------------------------------- - run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_target static32 c4core-bm-build} - - {name: static32-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static32 c4core-bm-run} - - {name: static32-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target static32 c4core-bm-plot} - - name: shared32-configure--------------------------------------------------- - run: export CMAKE_FLAGS="-DPython_EXECUTABLE=$(which python)" && source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_target shared32 c4core-bm-build} - - {name: shared32-run, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared32 c4core-bm-run} - - {name: shared32-plot, run: export NUM_JOBS_BUILD=1 && source .github/setenv.sh && c4_run_target shared32 c4core-bm-plot} - - name: gather benchmark results - run: | - set -x - source vars.sh - echo SRC_TAG=$SRC_TAG - echo SRC_VERSION=$SRC_VERSION - desc=$SRC_TAG - for bl in ${{matrix.bitlinks}} ; do - dst=$(echo benchmark_results/$desc/x86_64/${{matrix.cxx}}-${{matrix.bt}}-c++${{matrix.std}}-$bl | sed 's:++-:xx:g' | sed 's:+:x:g') - mkdir -p $dst - find build -name bm-results - mv -vf build/$bl/bm/bm-results/* $dst/. - done - - name: upload benchmark result artifacts - uses: actions/upload-artifact@v3 - with: - name: benchmark_results - path: benchmark_results/ - - #-------------------------------------------------------------------------------------------------- - bm_rarearch: - name: bm/${{matrix.arch}}/c++${{matrix.std}}/${{matrix.bt}} - needs: gettag - if: | - (!contains(github.event.head_commit.message, 'skip all')) || - (!contains(github.event.head_commit.message, 'skip benchmarks')) || - contains(github.event.head_commit.message, 'only benchmarks') - continue-on-error: true - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - include: - - {std: 17, bt: Release, arch: aarch64, distro: ubuntu20.04} - # the python dependencies cannot be installed for this one: - #- {std: 17, bt: Release, arch: ppc64le, distro: ubuntu20.04} - # - # the github runners are failing for the following: - #- {std: 11, bt: Release, arch: s390x , distro: ubuntu20.04} - #- {std: 17, bt: Release, arch: s390x , distro: ubuntu20.04} - ## - #- {std: 11, bt: Release, arch: armv6 , distro: ubuntu18.04} - #- {std: 17, bt: Release, arch: armv6 , distro: ubuntu18.04} - ## - #- {std: 11, bt: Release, arch: armv7 , distro: ubuntu18.04} - #- {std: 17, bt: Release, arch: armv7 , distro: ubuntu18.04} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}} - - name: Download vars.sh - uses: actions/download-artifact@v3 - with: {name: vars.sh, path: ./} - - name: test - uses: uraimo/run-on-arch-action@v2.3.0 - with: - arch: ${{matrix.arch}} - distro: ${{matrix.distro}} - install: | - set -x - apt-get update -y - apt-get install -y \ - git \ - build-essential - # arm platforms need an up-to-date cmake: - # https://gitlab.kitware.com/cmake/cmake/-/issues/20568 - if [ "${{matrix.arch}}" == "armv6" ] || [ "${{matrix.arch}}" == "armv7" ] ; then - apt-get install -y \ - gpg \ - wget \ - apt-transport-https - wget --no-check-certificate -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null - echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null - apt-get update -y - rm /usr/share/keyrings/kitware-archive-keyring.gpg - apt-get install kitware-archive-keyring - apt-get update -y - fi - apt-get install -y cmake cmake-data - cmake --version - apt-get install -y python3 python3-pip - run: | - set -x - uname -a - pwd - ls -lFhp . - # - pip3 install -v -r cmake/bm-xp/requirements.txt - # - bdir=build_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}} - idir=install_${{matrix.arch}}_${{matrix.bt}}_${{matrix.std}} - mkdir -p $bdir - # - cmake -S . -B $bdir \ - -DCMAKE_INSTALL_PREFIX=$idir \ - -DCMAKE_BUILD_TYPE=${{matrix.bt}} \ - -DC4_CXX_STANDARD=${{matrix.std}} \ - -DCXX_STANDARD=${{matrix.std}} \ - -DC4CORE_DEV=ON \ - -DC4CORE_BUILD_TESTS=OFF \ - -DC4CORE_BUILD_BENCHMARKS=ON \ - -DC4CORE_SANITIZE=OFF \ - -DC4CORE_LINT=OFF \ - -DC4CORE_VALGRIND=OFF - # - cmake --build $bdir -j --target c4core-bm-build - # - cmake --build $bdir -j 1 --target c4core-bm-run - # - cmake --build $bdir -j 1 --target c4core-bm-plot - # - source vars.sh - echo SRC_TAG=$SRC_TAG - echo SRC_VERSION=$SRC_VERSION - desc=$SRC_TAG - dst=$(echo benchmark_results/$desc/${{matrix.arch}}/${{matrix.bt}}-c++${{matrix.std}} | sed 's:++-:xx:g' | sed 's:+:x:g') - mkdir -p $dst - find $bdir -name bm-results - mv -vf $bdir/bm/bm-results/* $dst/. - - name: upload benchmark result artifacts - uses: actions/upload-artifact@v3 - with: - name: benchmark_results - path: benchmark_results/ diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/clang.yml b/thirdparty/ryml/ext/c4core/.github/workflows/clang.yml deleted file mode 100644 index df2194885..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/clang.yml +++ /dev/null @@ -1,230 +0,0 @@ -name: clang - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -# ubuntu-20.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.2.0 -# clang: 8.0.1, 9.0.1, 10.0.0 -# ubuntu-18.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.1.0 -# clang: 6.0.0, 8.0.0, 9.0.0 -# macos-11.0: macOS Big Sur 11.0 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 10.0.1 -# gcc-8 gcc-9 -# macos-10.15: macOS Catalina 10.15 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 11.0.0 -# gcc-8 gcc-9 -# windows-2019: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md -# vs2019 -# windows-2016: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md -# vs2017 - -jobs: - - #---------------------------------------------------------------------------- - clang_canary: - name: clang_canary/${{matrix.cxx}}/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 20, cxx: clang++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 20, cxx: clang++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: clang++-6.0, bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: clang++-6.0, bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - #---------------------------------------------------------------------------- - clang_extended: - name: clang_extended/${{matrix.cxx}}/c++${{matrix.std}}/${{matrix.bt}}/vg${{matrix.vg}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 20, cxx: clang++-10 , bt: Debug , vg: on, os: ubuntu-18.04} - - {std: 20, cxx: clang++-10 , bt: Release, vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-9 , bt: Debug , vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-9 , bt: Release, vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-8 , bt: Debug , vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-8 , bt: Release, vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-7 , bt: Debug , vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-7 , bt: Release, vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-6.0, bt: Debug , vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-6.0, bt: Release, vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-5.0, bt: Debug , vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-5.0, bt: Release, vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-4.0, bt: Debug , vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-4.0, bt: Release, vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-3.9, bt: Debug , vg: on, os: ubuntu-18.04} - - {std: 11, cxx: clang++-3.9, bt: Release, vg: on, os: ubuntu-18.04} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - #---------------------------------------------------------------------------- - clang_sanitize: - name: clang_sanitize/c++${{matrix.std}}/${{matrix.bt}}/vg${{matrix.vg}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - # these jobs take much longer, so run only one bitlink pair per job to profit from parallelism - - {std: 11, cxx: clang++-10 , bt: Debug , vg: ON, san: ALL, bitlinks: shared64 static64, os: ubuntu-18.04} - - {std: 11, cxx: clang++-10 , bt: Debug , vg: ON, san: ALL, bitlinks: shared32 static32, os: ubuntu-18.04} - - {std: 11, cxx: clang++-10 , bt: Release, vg: ON, san: ALL, bitlinks: shared64 static64, os: ubuntu-18.04} - - {std: 11, cxx: clang++-10 , bt: Release, vg: ON, san: ALL, bitlinks: shared32 static32, os: ubuntu-18.04} - - {std: 14, cxx: clang++-10 , bt: Debug , vg: ON, san: ALL, bitlinks: shared64 static64, os: ubuntu-18.04} - - {std: 14, cxx: clang++-10 , bt: Debug , vg: ON, san: ALL, bitlinks: shared32 static32, os: ubuntu-18.04} - - {std: 14, cxx: clang++-10 , bt: Release, vg: ON, san: ALL, bitlinks: shared64 static64, os: ubuntu-18.04} - - {std: 14, cxx: clang++-10 , bt: Release, vg: ON, san: ALL, bitlinks: shared32 static32, os: ubuntu-18.04} - - {std: 17, cxx: clang++-10 , bt: Debug , vg: ON, san: ALL, bitlinks: shared64 static64, os: ubuntu-18.04} - - {std: 17, cxx: clang++-10 , bt: Debug , vg: ON, san: ALL, bitlinks: shared32 static32, os: ubuntu-18.04} - - {std: 17, cxx: clang++-10 , bt: Release, vg: ON, san: ALL, bitlinks: shared64 static64, os: ubuntu-18.04} - - {std: 17, cxx: clang++-10 , bt: Release, vg: ON, san: ALL, bitlinks: shared32 static32, os: ubuntu-18.04} - - {std: 20, cxx: clang++-10 , bt: Debug , vg: ON, san: ALL, bitlinks: shared64 static64, os: ubuntu-18.04} - - {std: 20, cxx: clang++-10 , bt: Debug , vg: ON, san: ALL, bitlinks: shared32 static32, os: ubuntu-18.04} - - {std: 20, cxx: clang++-10 , bt: Release, vg: ON, san: ALL, bitlinks: shared64 static64, os: ubuntu-18.04} - - {std: 20, cxx: clang++-10 , bt: Release, vg: ON, san: ALL, bitlinks: shared32 static32, os: ubuntu-18.04} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - #---------------------------------------------------------------------------- -# # https://blog.kitware.com/static-checks-with-cmake-cdash-iwyu-clang-tidy-lwyu-cpplint-and-cppcheck/ -# static_analysis: -# continue-on-error: true -# if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct -# runs-on: ${{matrix.os}} -# strategy: -# fail-fast: false -# matrix: -# include: -# # these jobs take much longer, so run only one bitlink pair per job to profit from parallelism -# - {std: 11, cxx: clang++-10, bt: Debug , bitlinks: shared64, os: ubuntu-18.04} -# - {std: 11, cxx: clang++-10, bt: Release, bitlinks: shared64, os: ubuntu-18.04} -# - {std: 14, cxx: clang++-10, bt: Debug , bitlinks: shared64, os: ubuntu-18.04} -# - {std: 14, cxx: clang++-10, bt: Release, bitlinks: shared64, os: ubuntu-18.04} -# - {std: 17, cxx: clang++-10, bt: Debug , bitlinks: shared64, os: ubuntu-18.04} -# - {std: 17, cxx: clang++-10, bt: Release, bitlinks: shared64, os: ubuntu-18.04} -# - {std: 20, cxx: clang++-10, bt: Debug , bitlinks: shared64, os: ubuntu-18.04} -# - {std: 20, cxx: clang++-10, bt: Release, bitlinks: shared64, os: ubuntu-18.04} -# env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} -# steps: -# - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} -# - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} -# - {name: show info, run: source .github/setenv.sh && c4_show_info} -# - name: shared64-configure--------------------------------------------------- -# run: source .github/setenv.sh && c4_cfg_test shared64 -# - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} -# - {name: clang-tidy, run: cmake "-DCMAKE_CXX_CLANG_TIDY=/usr/bin/clang-tidy-3.9;-checks=*" ../path/to/source} -# - {name: cppcheck, run: cmake "-DCMAKE_CXX_CPPCHECK=/usr/bin/cppcheck;--std=c++11" ../path/to/source} -# - {name: cpplint, run: cmake "-DCMAKE_CXX_CPPLINT=/usr/local/bin/cpplint;--linelength=179" ..} -# - {name: include-what-you-use, run: cmake "-DCMAKE_CXX_INCLUDE_WHAT_YOU_USE=/usr/bin/iwyu;--transitive_includes_only" ..} -# - {name: link-what-you-use, run: cmake -DCMAKE_LINK_WHAT_YOU_USE=TRUE ..} diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/clang_tidy.yml b/thirdparty/ryml/ext/c4core/.github/workflows/clang_tidy.yml deleted file mode 100644 index 9d452ec4c..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/clang_tidy.yml +++ /dev/null @@ -1,97 +0,0 @@ -name: clang_tidy - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -# ubuntu-20.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.2.0 -# clang: 8.0.1, 9.0.1, 10.0.0 -# ubuntu-18.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.1.0 -# clang: 6.0.0, 8.0.0, 9.0.0 -# macos-11.0: macOS Big Sur 11.0 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 10.0.1 -# gcc-8 gcc-9 -# macos-10.15: macOS Catalina 10.15 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 11.0.0 -# gcc-8 gcc-9 -# windows-2019: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md -# vs2019 -# windows-2016: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md -# vs2017 - -jobs: - - #---------------------------------------------------------------------------- - clang_tidy: - name: clang_tidy/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - # clang tidy takes a long time, so don't do multiple bits/linktypes - - {std: 11, cxx: clang++-9, bt: Debug , lint: clang-tidy, bitlinks: shared64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-9, bt: Debug , lint: clang-tidy, bitlinks: shared32, os: ubuntu-20.04} - - {std: 11, cxx: clang++-9, bt: Debug , lint: clang-tidy, bitlinks: static64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-9, bt: Debug , lint: clang-tidy, bitlinks: static32, os: ubuntu-20.04} - - {std: 11, cxx: clang++-9, bt: ReleaseWithDebInfo, lint: clang-tidy, bitlinks: shared64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-9, bt: ReleaseWithDebInfo, lint: clang-tidy, bitlinks: shared32, os: ubuntu-20.04} - - {std: 11, cxx: clang++-9, bt: ReleaseWithDebInfo, lint: clang-tidy, bitlinks: static64, os: ubuntu-20.04} - - {std: 11, cxx: clang++-9, bt: ReleaseWithDebInfo, lint: clang-tidy, bitlinks: static32, os: ubuntu-20.04} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/codeql.yml b/thirdparty/ryml/ext/c4core/.github/workflows/codeql.yml deleted file mode 100644 index ca8d78bb7..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/codeql.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: "CodeQL" - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - schedule: - - cron: "59 18 * * 1" - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ cpp ] - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - queries: +security-and-quality - - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:${{ matrix.language }}" diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/coverage.yml b/thirdparty/ryml/ext/c4core/.github/workflows/coverage.yml deleted file mode 100644 index 6f593de14..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/coverage.yml +++ /dev/null @@ -1,150 +0,0 @@ -name: coverage - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -jobs: - - #---------------------------------------------------------------------------- - coverage: - name: coverage/${{matrix.name}} - # if: github.ref == 'refs/heads/master' - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {name: c++11, std: 11, cxx: g++-9, cc: gcc-9, bt: Coverage, os: ubuntu-20.04} - - {name: c++17, std: 17, cxx: g++-9, cc: gcc-9, bt: Coverage, os: ubuntu-20.04} - #- {name: c++20, std: 20, cxx: g++-9, cc: gcc-9, bt: Coverage, os: ubuntu-20.04} - env: { - STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}", - CODECOV_TOKEN: "${{secrets.CODECOV_TOKEN}}", - COVERALLS_REPO_TOKEN: "${{secrets.COVERALLS_REPO_TOKEN}}", - COVERALLS_PARALLEL: true, - } - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_build_target static64 c4core-coverage} - - name: static64-coverage-artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-static64 - path: | - build/static64/lcov/ - build/static64/coverage3-final_filtered.lcov - - {name: static64-submit-codecov, run: source .github/setenv.sh && c4_submit_coverage static64 codecov} - - {name: static64-submit-coveralls, run: source .github/setenv.sh && c4_submit_coverage static64 coveralls, - env: {COVERALLS_FLAG_NAME: "${{matrix.name}}/static64"}} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_build_target static32 c4core-coverage} - - name: static32-coverage-artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-static32 - path: | - build/static32/lcov/ - build/static32/coverage3-final_filtered.lcov - - {name: static32-submit-codecov, run: source .github/setenv.sh && c4_submit_coverage static32 codecov} - - {name: static32-submit-coveralls, run: source .github/setenv.sh && c4_submit_coverage static32 coveralls, - env: {COVERALLS_FLAG_NAME: "${{matrix.name}}/static32"}} - - #---------------------------------------------------------------------------- - coverage_nofastfloat: - name: coverage/${{matrix.name}} - # if: github.ref == 'refs/heads/master' - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {name: nofastfloat/c++11, std: 11, cxx: g++-9, cc: gcc-9, bt: Coverage, os: ubuntu-20.04} - - {name: nofastfloat/c++17, std: 17, cxx: g++-9, cc: gcc-9, bt: Coverage, os: ubuntu-20.04} - env: { - STD: "${{matrix.std}}", - CXX_: "${{matrix.cxx}}", - BT: "${{matrix.bt}}", - OS: "${{matrix.os}}", - CODECOV_TOKEN: "${{secrets.CODECOV_TOKEN}}", - COVERALLS_REPO_TOKEN: "${{secrets.COVERALLS_REPO_TOKEN}}", - COVERALLS_PARALLEL: true, # https://docs.coveralls.io/parallel-build-webhook - COVERALLS_FLAG_NAME: "${{matrix.name}}", - BDIR: "build/nofastfloat-${{matrix.cxx}}-cxx${{matrix.std}}", - IDIR: "install/nofastfloat-${{matrix.cxx}}-cxx${{matrix.std}}", - } - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: nofastfloat-configure------------------------------------------------ - run: | - set -x - mkdir -p $BDIR - mkdir -p $IDIR - cmake -S . -B $BDIR \ - -DC4CORE_WITH_FASTFLOAT=OFF \ - -DC4_CXX_STANDARD=${{matrix.std}} \ - -DC4CORE_CXX_STANDARD=${{matrix.std}} \ - -DC4CORE_BUILD_TESTS=ON \ - -DC4CORE_VALGRIND=OFF \ - -DC4CORE_COVERAGE_CODECOV=ON \ - -DC4CORE_COVERAGE_COVERALLS=ON \ - -DCMAKE_INSTALL_PREFIX=$IDIR \ - -DCMAKE_BUILD_TYPE=Coverage \ - -DCMAKE_CXX_COMPILER=${{matrix.cxx}} \ - -DCMAKE_C_COMPILER=${{matrix.cc}} - - {name: nofastfloat-build, run: cmake --build $BDIR --config Coverage --target c4core-test-build -j} - - {name: nofastfloat-run, run: cmake --build $BDIR --config Coverage --target c4core-coverage} - - name: nofastfloat-coverage-artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-shared32 - path: | - build/nofastfloat-${{matrix.cxx}}-cxx${{matrix.std}}/lcov/ - build/nofastfloat-${{matrix.cxx}}-cxx${{matrix.std}}/coverage3-final_filtered.lcov - - {name: nofastfloat-submit-codecov, run: cmake --build $BDIR --config Coverage --target c4core-coverage-submit-codecov} - - {name: nofastfloat-submit-coveralls, run: cmake --build $BDIR --config Coverage --target c4core-coverage-submit-coveralls} - - # https://github.com/marketplace/actions/coveralls-github-action - coveralls_finish: - needs: [coverage, coverage_nofastfloat] - runs-on: ubuntu-latest - steps: - - name: coveralls-notify - continue-on-error: true - uses: coverallsapp/github-action@master - with: - github-token: ${{ secrets.github_token }} - parallel-finished: true diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/emscripten.yml b/thirdparty/ryml/ext/c4core/.github/workflows/emscripten.yml deleted file mode 100644 index cabe5d293..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/emscripten.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: emscripten - -# running in a local machine: -# /usr/lib/emscripten/emcmake cmake -DCMAKE_BUILD_TYPE=Debug -S . -B build/emscripten -DC4CORE_DEV=ON -DC4CORE_SANITIZE=OFF -DC4CORE_BUILD_BENCHMARKS=OFF -DCMAKE_CXX_FLAGS="-s DISABLE_EXCEPTION_CATCHING=0" && cmake --build build/emscripten/ -j --target c4core-test-charconv && ( cd build/emscripten/test/ && ctest --output-on-failure -R charconv ) - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -# ubuntu-20.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.2.0 -# clang: 8.0.1, 9.0.1, 10.0.0 -# ubuntu-18.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.1.0 -# clang: 6.0.0, 8.0.0, 9.0.0 -# macos-11.0: macOS Big Sur 11.0 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 10.0.1 -# gcc-8 gcc-9 -# macos-10.15: macOS Catalina 10.15 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 11.0.0 -# gcc-8 gcc-9 -# windows-2019: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md -# vs2019 -# windows-2016: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md -# vs2017 - -jobs: - - #---------------------------------------------------------------------------- - emscripten: - name: emscripten/${{matrix.emver}}/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - #- {std: 11, cxx: em++, emver: 2.0.34, bt: Debug , os: ubuntu-latest, bitlinks: static32} - - {std: 11, cxx: em++, emver: 2.0.34, bt: Release, os: ubuntu-latest, bitlinks: static32} - #- {std: 20, cxx: em++, emver: 2.0.34, bt: Debug , os: ubuntu-latest, bitlinks: static32} - - {std: 20, cxx: em++, emver: 2.0.34, bt: Release, os: ubuntu-latest, bitlinks: static32} - #- {std: 11, cxx: em++, emver: 3.0.0 , bt: Debug , os: ubuntu-latest, bitlinks: static32} - - {std: 11, cxx: em++, emver: 3.0.0 , bt: Release, os: ubuntu-latest, bitlinks: static32} - #- {std: 20, cxx: em++, emver: 3.0.0 , bt: Debug , os: ubuntu-latest, bitlinks: static32} - - {std: 20, cxx: em++, emver: 3.0.0 , bt: Release, os: ubuntu-latest, bitlinks: static32} - env: - STD: "${{matrix.std}}" - CXX_: "${{matrix.cxx}}" - BT: "${{matrix.bt}}" - BITLINKS: "${{matrix.bitlinks}}" - VG: "${{matrix.vg}}" - SAN: "${{matrix.san}}" - LINT: "${{matrix.lint}}" - OS: "${{matrix.os}}" - EM_VERSION: "${{matrix.emver}}" - EM_CACHE_FOLDER: 'emsdk-cache' - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - name: setup emscripten cache - id: cache-system-libraries - uses: actions/cache@v3 - with: {path: "${{env.EM_CACHE_FOLDER}}", key: "${{env.EM_VERSION}}-${{runner.os}}"} - - name: setup emscripten - uses: mymindstorm/setup-emsdk@v11 - with: {version: "${{matrix.emver}}", actions-cache-folder: "${{env.EM_CACHE_FOLDER}}"} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/gcc.yml b/thirdparty/ryml/ext/c4core/.github/workflows/gcc.yml deleted file mode 100644 index 5398d5f22..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/gcc.yml +++ /dev/null @@ -1,181 +0,0 @@ -name: gcc - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -# ubuntu-20.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.2.0 -# clang: 8.0.1, 9.0.1, 10.0.0 -# ubuntu-18.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.1.0 -# clang: 6.0.0, 8.0.0, 9.0.0 -# macos-11.0: macOS Big Sur 11.0 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 10.0.1 -# gcc-8 gcc-9 -# macos-10.15: macOS Catalina 10.15 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 11.0.0 -# gcc-8 gcc-9 -# windows-2019: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md -# vs2019 -# windows-2016: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md -# vs2017 - -jobs: - - #---------------------------------------------------------------------------- - gcc_canary: - name: gcc_canary/${{matrix.cxx}}/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 11, cxx: g++-7 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-7 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 20, cxx: g++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 20, cxx: g++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-5 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-5 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-4.8 , bt: Debug, os: ubuntu-18.04, bitlinks: shared64 static32} - - {std: 11, cxx: g++-4.8 , bt: Release, os: ubuntu-18.04, bitlinks: shared64 static32} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - #---------------------------------------------------------------------------- - gcc_extended: - name: gcc_extended/${{matrix.cxx}}/c++${{matrix.std}}/${{matrix.bt}}/vg${{matrix.vg}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - # VALGRIND - - {std: 11, cxx: g++-10, bt: Debug , vg: ON, os: ubuntu-20.04} - - {std: 11, cxx: g++-10, bt: Release, vg: ON, os: ubuntu-20.04} - - {std: 14, cxx: g++-10, bt: Debug , vg: ON, os: ubuntu-20.04} - - {std: 14, cxx: g++-10, bt: Release, vg: ON, os: ubuntu-20.04} - - {std: 17, cxx: g++-10, bt: Debug , vg: ON, os: ubuntu-20.04} - - {std: 17, cxx: g++-10, bt: Release, vg: ON, os: ubuntu-20.04} - - {std: 20, cxx: g++-10, bt: Debug , vg: ON, os: ubuntu-20.04} - - {std: 20, cxx: g++-10, bt: Release, vg: ON, os: ubuntu-20.04} - # - - {std: 11, cxx: g++-9, bt: Debug , os: ubuntu-20.04} - - {std: 11, cxx: g++-9, bt: Release, os: ubuntu-20.04} - - {std: 11, cxx: g++-8, bt: Debug , os: ubuntu-20.04} - - {std: 11, cxx: g++-8, bt: Release, os: ubuntu-20.04} - - {std: 11, cxx: g++-7, bt: Debug , os: ubuntu-20.04} - - {std: 11, cxx: g++-7, bt: Release, os: ubuntu-20.04} - - {std: 11, cxx: g++-6, bt: Debug , os: ubuntu-18.04} - - {std: 11, cxx: g++-6, bt: Release, os: ubuntu-18.04} - - {std: 11, cxx: g++-5, bt: Debug , os: ubuntu-18.04} - - {std: 11, cxx: g++-5, bt: Release, os: ubuntu-18.04} - - {std: 11, cxx: g++-4.8, bt: Debug, os: ubuntu-18.04} - - {std: 11, cxx: g++-4.8, bt: Release, os: ubuntu-18.04} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - #---------------------------------------------------------------------------- - arm: - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - # these jobs take much longer, so run only one bitlink pair per job to profit from parallelism - - {std: 11, bt: Debug , toolchain: cmake/Toolchain-Arm-ubuntu.cmake, cxx: arm-linux-gnueabihf-gcc, os: ubuntu-18.04} - - {std: 11, bt: Release, toolchain: cmake/Toolchain-Arm-ubuntu.cmake, cxx: arm-linux-gnueabihf-gcc, os: ubuntu-18.04} - - {std: 14, bt: Debug , toolchain: cmake/Toolchain-Arm-ubuntu.cmake, cxx: arm-linux-gnueabihf-gcc, os: ubuntu-18.04} - - {std: 14, bt: Release, toolchain: cmake/Toolchain-Arm-ubuntu.cmake, cxx: arm-linux-gnueabihf-gcc, os: ubuntu-18.04} - - {std: 17, bt: Debug , toolchain: cmake/Toolchain-Arm-ubuntu.cmake, cxx: arm-linux-gnueabihf-gcc, os: ubuntu-18.04} - - {std: 17, bt: Release, toolchain: cmake/Toolchain-Arm-ubuntu.cmake, cxx: arm-linux-gnueabihf-gcc, os: ubuntu-18.04} - env: {TOOLCHAIN: "${{matrix.toolchain}}", STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test arm - - {name: build, run: source .github/setenv.sh && c4_build_test arm} - - {name: run, run: source .github/setenv.sh && c4_run_test arm} - - {name: pack, run: source .github/setenv.sh && c4_package arm} diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/libcxx.yml b/thirdparty/ryml/ext/c4core/.github/workflows/libcxx.yml deleted file mode 100644 index f55fe6adc..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/libcxx.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: libcxx - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -# ubuntu-20.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.2.0 -# clang: 8.0.1, 9.0.1, 10.0.0 -# ubuntu-18.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.1.0 -# clang: 6.0.0, 8.0.0, 9.0.0 -# macos-11.0: macOS Big Sur 11.0 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 10.0.1 -# gcc-8 gcc-9 -# macos-10.15: macOS Catalina 10.15 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 11.0.0 -# gcc-8 gcc-9 -# windows-2019: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md -# vs2019 -# windows-2016: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md -# vs2017 - -jobs: - - #---------------------------------------------------------------------------- - libcxx: - name: libc++/${{matrix.cxx}}/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 20, cxx: clang++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 20, cxx: clang++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 17, cxx: clang++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 17, cxx: clang++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 14, cxx: clang++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 14, cxx: clang++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 11, cxx: clang++-10 , bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 11, cxx: clang++-10 , bt: Release, os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 17, cxx: clang++-6.0, bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 17, cxx: clang++-6.0, bt: Release, os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 14, cxx: clang++-6.0, bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 14, cxx: clang++-6.0, bt: Release, os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 11, cxx: clang++-6.0, bt: Debug , os: ubuntu-20.04, bitlinks: shared64 static64} - - {std: 11, cxx: clang++-6.0, bt: Release, os: ubuntu-20.04, bitlinks: shared64 static64} - env: - LIBCXX: ON # <---- enable libc++ - STD: "${{matrix.std}}" - CXX_: "${{matrix.cxx}}" - BT: "${{matrix.bt}}" - BITLINKS: "${{matrix.bitlinks}}" - VG: "${{matrix.vg}}" - SAN: "${{matrix.san}}" - LINT: "${{matrix.lint}}" - OS: "${{matrix.os}}" - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/macosx.yml b/thirdparty/ryml/ext/c4core/.github/workflows/macosx.yml deleted file mode 100644 index 33145e5c7..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/macosx.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: macosx - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -# ubuntu-20.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.2.0 -# clang: 8.0.1, 9.0.1, 10.0.0 -# ubuntu-18.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.1.0 -# clang: 6.0.0, 8.0.0, 9.0.0 -# macos-11.0: macOS Big Sur 11.0 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 10.0.1 -# gcc-8 gcc-9 -# macos-10.15: macOS Catalina 10.15 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 11.0.0 -# gcc-8 gcc-9 -# windows-2019: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md -# vs2019 -# windows-2016: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md -# vs2017 - -jobs: - - #---------------------------------------------------------------------------- - xcode: - name: xcode${{matrix.xcver}}/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 11, cxx: xcode, xcver: 13, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 11, cxx: xcode, xcver: 13, bt: Release, os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 13, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 13, bt: Release, os: macos-11, bitlinks: shared64 static64} - # - - {std: 11, cxx: xcode, xcver: 12, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 11, cxx: xcode, xcver: 12, bt: Release, os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 12, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 12, bt: Release, os: macos-11, bitlinks: shared64 static64} - # - - {std: 11, cxx: xcode, xcver: 11, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 11, cxx: xcode, xcver: 11, bt: Release, os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 11, bt: Debug , os: macos-11, bitlinks: shared64 static64} - - {std: 17, cxx: xcode, xcver: 11, bt: Release, os: macos-11, bitlinks: shared64 static64} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: xcode, uses: maxim-lobanov/setup-xcode@v1, with: {xcode-version: "${{matrix.xcver}}" }} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/release.yml b/thirdparty/ryml/ext/c4core/.github/workflows/release.yml deleted file mode 100644 index 0f5a13873..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/release.yml +++ /dev/null @@ -1,197 +0,0 @@ -name: release - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - tags: - - v0.* - - v1.* - - v2.* - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PKG_NAME: c4core- - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -# useful to iterate when fixing the release: -# ver=0.2.1 ; ( set -x ; git tag -d v$ver ; git push origin :v$ver ) ; (set -x ; set -e ; tbump --only-patch --non-interactive $ver ; git add -u ; git commit --amend --no-edit ; git tag --annotate --message "v$ver" "v$ver" ; git push -f --tags origin ) - -jobs: - - gettag: - runs-on: ubuntu-latest - steps: - # use fetch-depth to ensure all tags are fetched - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive, fetch-depth: 0}} - - name: Variables (from tag) - if: contains(github.ref, 'tags/v') - run: | - # https://github.community/t/how-to-get-just-the-tag-name/16241/11 - SRC_TAG=${GITHUB_REF#refs/tags/} - SRC_VERSION=${GITHUB_REF#refs/tags/v} - cat < vars.sh - export SRC_TAG=$SRC_TAG - export SRC_VERSION=$SRC_VERSION - EOF - - name: Variables (from commit, no tag) - if: ${{ !contains(github.ref, 'tags/v') }} - run: | - set -x - branch_name=${GITHUB_REF#refs/heads/} - # builds triggered from PRs have the branch_name like this: refs/pull/150/merge - # so filter to eg pr0150_merge - branch_name=`echo $branch_name | sed "s:refs/pull/\([0-9]*\)/\(.*\):pr0\1_\2:"` - # sanitize the branch name; eg merge/foo-bar -> merge_foo_bar - branch_name=`echo $branch_name | sed 's:[/.-]:_:g'` - SRC_TAG=$(git describe || git rev-parse --short HEAD) # eg v0.2.0-110-gda837e0 - SRC_VERSION="${branch_name}-${SRC_TAG}" - cat < vars.sh - export SRC_TAG=$SRC_TAG - export SRC_VERSION=$SRC_VERSION - EOF - - name: Verify vars.sh - run: cat vars.sh ; source vars.sh ; echo $SRC_TAG ; echo $SRC_VERSION - - name: Save vars.sh - uses: actions/upload-artifact@v3 - with: {name: vars.sh, path: ./vars.sh} - - #---------------------------------------------------------------------------- - # create source packages - src: - needs: gettag - runs-on: ubuntu-latest - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - name: Download vars.sh - uses: actions/download-artifact@v3 - with: {name: vars.sh, path: ./} - - name: Install python 3.9 - uses: actions/setup-python@v4 - with: { python-version: 3.9 } - - name: Install requirements - run: | - sudo -E pip install git-archive-all - - name: Create source packages - run: | - pwd - ls -lFhp - source vars.sh - echo SRC_TAG=$SRC_TAG - echo SRC_VERSION=$SRC_VERSION - id=${PROJ_PKG_NAME}${SRC_VERSION} - name=${id}-src - mkdir -p assets - git-archive-all --prefix $name assets/$name.tgz - git-archive-all --prefix $name assets/$name.zip - python --version - python tools/amalgamate.py assets/$id.hpp - - name: Save source artifacts - uses: actions/upload-artifact@v3 - with: {name: assets, path: assets} - - #---------------------------------------------------------------------------- - # create c++ packages - cpp: - name: cpp/${{matrix.config.os}}/${{matrix.config.gen}} - needs: gettag - runs-on: ${{matrix.config.os}} - env: {DEV: OFF, BT: Release, OS: "${{matrix.config.os}}", CXX_: "${{matrix.config.cxx}}", GEN: "${{matrix.config.gen}}"} - strategy: - fail-fast: false - matrix: - config: - # name of the artifact | suffix (gen) | suffix (package) | cpack gen | mime type | os | cxx - - {name: Ubuntu 20.04 deb , sfxg: unix64-shared-Release.deb, sfxp: ubuntu-20.04.deb , gen: DEB , mime: vnd.debian.binary-package, os: ubuntu-20.04 } - - {name: Windows VS2019 zip, sfxg: win64-shared-Release.zip , sfxp: windows-vs2019.zip , gen: ZIP , mime: zip , os: windows-2019, cxx: vs2019} - - {name: MacOSX sh , sfxg: apple64-shared-Release.sh, sfxp: macosx-xcode.sh , gen: STGZ , mime: x-sh , os: macos-11.0 , cxx: xcode } - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - name: Download vars.sh - uses: actions/download-artifact@v3 - with: {name: vars.sh, path: ./} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info } - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_target shared64} - - name: shared64-pack - run: source .github/setenv.sh && c4_package shared64 $GEN - - name: shared64-normalize - run: | - set -x - source vars.sh - mkdir -p assets - asset_src=`ls -1 ./build/shared64/${PROJ_PFX_TARGET}*-${{matrix.config.sfxg}}` - asset_dst=./assets/${PROJ_PKG_NAME}${SRC_VERSION}-${{matrix.config.sfxp}} - [ ! -f $asset_src ] && exit 1 - cp -fav $asset_src $asset_dst - - name: Save artifacts - uses: actions/upload-artifact@v3 - with: {name: assets, path: assets} - - #---------------------------------------------------------------------------- - release: - runs-on: ubuntu-latest - needs: - - src - - cpp - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - name: Gather artifacts - ./assets - uses: actions/download-artifact@v3 - with: {name: assets, path: assets} - - name: Verify existing artifacts - run: | - ls -lFhp assets/ - # - # Github - - name: Restore vars.sh - if: contains(github.ref, 'tags/v') - uses: actions/download-artifact@v3 - with: {name: vars.sh, path: ./} - - name: Save vars for following steps - if: contains(github.ref, 'tags/v') - id: vars - run: | - source vars.sh - version_body=${{github.workspace}}/changelog/$SRC_VERSION.md - if [ ! -f $version_body ] ; then - echo "version body file was not found: $version_body" - exit 1 - fi - echo "VERSION=$SRC_VERSION >> $GITHUB_OUTPUT" - echo "VERSION_BODY=$version_body >> $GITHUB_OUTPUT" - - name: Create Github Release - if: contains(github.ref, 'tags/v') - id: create_release - uses: actions/create-release@v1 - env: { GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" } - with: - tag_name: ${{github.ref}} - release_name: Release ${{steps.vars.outputs.VERSION}} - body_path: ${{steps.vars.outputs.VERSION_BODY}} - draft: true - prerelease: ${{contains(github.ref, 'rc')}} - - name: Upload assets to Github Release - if: contains(github.ref, 'tags/v') - uses: dwenegar/upload-release-assets@v1 - env: { GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" } - with: - release_id: ${{steps.create_release.outputs.id}} - assets_path: ./assets/ diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/test_install.yml b/thirdparty/ryml/ext/c4core/.github/workflows/test_install.yml deleted file mode 100644 index 7f28e7578..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/test_install.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: test_install - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - -jobs: - - #---------------------------------------------------------------------------- - install_tests: - name: ${{matrix.name}}/${{matrix.bt}} - # if: github.ref == 'refs/heads/master' - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {name: find_package/linux , sdir: test/test_install , os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Release, vars: "-Dc4core_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/c4core -DC4CORE_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/linux , sdir: test/test_install , os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Debug , vars: "-Dc4core_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/c4core -DC4CORE_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/linux/libcxx, sdir: test/test_install , os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Release, vars: "-Dc4core_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/c4core -DC4CORE_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: "-DC4CORE_USE_LIBCXX=ON"} - - {name: find_package/linux/libcxx, sdir: test/test_install , os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Debug , vars: "-Dc4core_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/c4core -DC4CORE_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: "-DC4CORE_USE_LIBCXX=ON"} - - {name: find_package/macos , sdir: test/test_install , os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Release, vars: "-Dc4core_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/c4core -DC4CORE_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/macos , sdir: test/test_install , os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Debug , vars: "-Dc4core_DIR=$GITHUB_WORKSPACE/$PDIR/lib/cmake/c4core -DC4CORE_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/win , sdir: test/test_install , os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Release, vars: "-Dc4core_DIR=$GITHUB_WORKSPACE/$PDIR/cmake -DC4CORE_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - - {name: find_package/win , sdir: test/test_install , os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Debug , vars: "-Dc4core_DIR=$GITHUB_WORKSPACE/$PDIR/cmake -DC4CORE_TEST_INSTALL_PACKAGE_MODE=ON", commonvars: } - # - - {name: find_library/linux , sdir: test/test_install , os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Release, vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DC4CORE_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/linux , sdir: test/test_install , os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Debug , vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DC4CORE_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/linux/libcxx, sdir: test/test_install , os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Release, vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DC4CORE_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: "-DC4CORE_USE_LIBCXX=ON"} - - {name: find_library/linux/libcxx, sdir: test/test_install , os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Debug , vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DC4CORE_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: "-DC4CORE_USE_LIBCXX=ON"} - - {name: find_library/macos , sdir: test/test_install , os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Release, vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DC4CORE_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/macos , sdir: test/test_install , os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Debug , vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DC4CORE_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/win , sdir: test/test_install , os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Release, vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DC4CORE_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - - {name: find_library/win , sdir: test/test_install , os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Debug , vars: "-DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/$PDIR -DC4CORE_TEST_INSTALL_PACKAGE_MODE=OFF", commonvars: } - # - - {name: singleheader/linux , sdir: test/test_singleheader, os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Release, vars: , commonvars: } - - {name: singleheader/linux , sdir: test/test_singleheader, os: ubuntu-20.04, cxx: g++-10 , gen: "-DCMAKE_CXX_COMPILER=g++-10" , tgt: all , bt: Debug , vars: , commonvars: } - - {name: singleheader/linux/libcxx, sdir: test/test_singleheader, os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Release, vars: , commonvars: "-DC4CORE_USE_LIBCXX=ON"} - - {name: singleheader/linux/libcxx, sdir: test/test_singleheader, os: ubuntu-20.04, cxx: clang++-9, gen: "-DCMAKE_CXX_COMPILER=clang++-9" , tgt: all , bt: Debug , vars: , commonvars: "-DC4CORE_USE_LIBCXX=ON"} - - {name: singleheader/macos , sdir: test/test_singleheader, os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Release, vars: , commonvars: } - - {name: singleheader/macos , sdir: test/test_singleheader, os: macos-11.0 , cxx: xcode , gen: "-G Xcode -DCMAKE_OSX_ARCHITECTURES=x86_64", tgt: ALL_BUILD, bt: Debug , vars: , commonvars: } - - {name: singleheader/win , sdir: test/test_singleheader, os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Release, vars: , commonvars: } - - {name: singleheader/win , sdir: test/test_singleheader, os: windows-2019, cxx: vs2019 , gen: "-G 'Visual Studio 16 2019' -A x64" , tgt: ALL_BUILD, bt: Debug , vars: , commonvars: } - env: - CXX_: "${{matrix.cxx}}" - BT: "${{matrix.bt}}" - OS: "${{matrix.os}}" - BDIR: "build/${{matrix.name}}-${{matrix.bt}}" - IDIR: "install/${{matrix.name}}-${{matrix.bt}}" - PDIR: "prefix/${{matrix.name}}-${{matrix.bt}}" - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: Install python 3.9 - uses: actions/setup-python@v4 - with: { python-version: 3.9 } - - name: preinstall - run: | - if [ "${{matrix.sdir}}" == "test/test_install" ] ; then - mkdir -p $BDIR-staging - cmake -S . -B $BDIR-staging -DCMAKE_INSTALL_PREFIX=$PDIR -DCMAKE_BUILD_TYPE=${{matrix.bt}} ${{matrix.gen}} ${{matrix.commonvars}} - cmake --build $BDIR-staging --config ${{matrix.bt}} --target ${{matrix.tgt}} -j - cmake --build $BDIR-staging --config ${{matrix.bt}} --target install - fi - - name: configure - run: | - mkdir -p $BDIR - mkdir -p $IDIR - cmake -S ${{matrix.sdir}} -B $BDIR \ - -DC4CORE_BUILD_TESTS=ON \ - -DC4CORE_VALGRIND=OFF \ - -DCMAKE_BUILD_TYPE=${{matrix.bt}} \ - -DCMAKE_INSTALL_PREFIX=$IDIR \ - ${{matrix.gen}} \ - ${{matrix.vars}} \ - ${{matrix.commonvars}} - - name: build - run: | - cmake --build $BDIR --config ${{matrix.bt}} --target c4core-test-build -j - - name: run - run: | - cmake --build $BDIR --config ${{matrix.bt}} --target c4core-test-run diff --git a/thirdparty/ryml/ext/c4core/.github/workflows/windows.yml b/thirdparty/ryml/ext/c4core/.github/workflows/windows.yml deleted file mode 100644 index 11a54f185..000000000 --- a/thirdparty/ryml/ext/c4core/.github/workflows/windows.yml +++ /dev/null @@ -1,157 +0,0 @@ -name: windows - -defaults: - #if: "!contains(github.event.head_commit.message, 'skip ci')" # SKIP - run: - # Use a bash shell so we can use the same syntax for environment variable - # access regardless of the host operating system - shell: bash -e -x {0} - -on: - # https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 - workflow_dispatch: - push: - branches: - - master - pull_request: - branches: - - master - -env: - PROJ_PFX_TARGET: c4core- - PROJ_PFX_CMAKE: C4CORE_ - CMAKE_FLAGS: - NUM_JOBS_BUILD: # 4 - - -# ubuntu-20.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.2.0 -# clang: 8.0.1, 9.0.1, 10.0.0 -# ubuntu-18.04: -# # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md -# gcc: 7.5.0, 8.4.0, 9.3.0, 10.1.0 -# clang: 6.0.0, 8.0.0, 9.0.0 -# macos-11.0: macOS Big Sur 11.0 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-11.0-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 10.0.1 -# gcc-8 gcc-9 -# macos-10.15: macOS Catalina 10.15 -# # https://github.com/actions/virtual-environments/blob/main/images/macos/macos-10.15-Readme.md -# Xcode 12.1 11.7 -# clang/LLVM 11.0.0 -# gcc-8 gcc-9 -# windows-2019: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md -# vs2019 -# windows-2016: -# # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2016-Readme.md -# vs2017 - -jobs: - - #---------------------------------------------------------------------------- - windows: - name: win/${{matrix.cxx}}/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - # github retired windows-2016 - #- {std: 11, cxx: vs2017, bt: Debug , os: windows-2016, bitlinks: shared64 static32} - #- {std: 11, cxx: vs2017, bt: Release, os: windows-2016, bitlinks: shared64 static32} - #- {std: 14, cxx: vs2017, bt: Debug , os: windows-2016, bitlinks: shared64 static32} - #- {std: 14, cxx: vs2017, bt: Release, os: windows-2016, bitlinks: shared64 static32} - # - - {std: 11, cxx: vs2019, bt: Debug , os: windows-2019, bitlinks: shared64 static32} - - {std: 11, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: shared64 static32} - - {std: 14, cxx: vs2019, bt: Debug , os: windows-2019, bitlinks: shared64 static32} - - {std: 14, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: shared64 static32} - - {std: 17, cxx: vs2019, bt: Debug , os: windows-2019, bitlinks: shared64 static32} - - {std: 17, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: shared64 static32} - # - - {std: 11, cxx: vs2022, bt: Debug , os: windows-2022, bitlinks: shared64 static32} - - {std: 11, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: shared64 static32} - - {std: 14, cxx: vs2022, bt: Debug , os: windows-2022, bitlinks: shared64 static32} - - {std: 14, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: shared64 static32} - - {std: 17, cxx: vs2022, bt: Debug , os: windows-2022, bitlinks: shared64 static32} - - {std: 17, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: shared64 static32} - - {std: 20, cxx: vs2022, bt: Debug , os: windows-2022, bitlinks: shared64 static32} - - {std: 20, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: shared64 static32} - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64 - - {name: shared64-build, run: source .github/setenv.sh && c4_build_test shared64} - - {name: shared64-run, run: source .github/setenv.sh && c4_run_test shared64} - - {name: shared64-pack, run: source .github/setenv.sh && c4_package shared64} - - name: static64-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64 - - {name: static64-build, run: source .github/setenv.sh && c4_build_test static64} - - {name: static64-run, run: source .github/setenv.sh && c4_run_test static64} - - {name: static64-pack, run: source .github/setenv.sh && c4_package static64} - - name: shared32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32 - - {name: shared32-build, run: source .github/setenv.sh && c4_build_test shared32} - - {name: shared32-run, run: source .github/setenv.sh && c4_run_test shared32} - - {name: shared32-pack, run: source .github/setenv.sh && c4_package shared32} - - name: static32-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32 - - {name: static32-build, run: source .github/setenv.sh && c4_build_test static32} - - {name: static32-run, run: source .github/setenv.sh && c4_run_test static32} - - {name: static32-pack, run: source .github/setenv.sh && c4_package static32} - - #---------------------------------------------------------------------------- - # TODO how to run? - windows_arm: - name: win_arm/${{matrix.cxx}}/c++${{matrix.std}}/${{matrix.bt}} - continue-on-error: true - if: always() # https://stackoverflow.com/questions/62045967/github-actions-is-there-a-way-to-continue-on-error-while-still-getting-correct - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - include: - - {std: 11, cxx: vs2019, bt: Debug , os: windows-2019, bitlinks: shared64arm static32arm} - - {std: 11, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: shared64arm static32arm} - - {std: 17, cxx: vs2019, bt: Debug , os: windows-2019, bitlinks: shared64arm static32arm} - - {std: 17, cxx: vs2019, bt: Release, os: windows-2019, bitlinks: shared64arm static32arm} - # - # vs2022 has an internal compiler error on iarm32 Release builds: - # https://github.com/biojppm/c4core/runs/5593534734?check_suite_focus=true#step:15:126 - - {std: 11, cxx: vs2022, bt: Debug , os: windows-2022, bitlinks: shared64arm static32arm} - - {std: 11, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: shared64arm } - - {std: 20, cxx: vs2022, bt: Debug , os: windows-2022, bitlinks: shared64arm static32arm} - - {std: 20, cxx: vs2022, bt: Release, os: windows-2022, bitlinks: shared64arm } - env: {STD: "${{matrix.std}}", CXX_: "${{matrix.cxx}}", BT: "${{matrix.bt}}", BITLINKS: "${{matrix.bitlinks}}", VG: "${{matrix.vg}}", SAN: "${{matrix.san}}", LINT: "${{matrix.lint}}", OS: "${{matrix.os}}"} - steps: - - {name: checkout, uses: actions/checkout@v3, with: {submodules: recursive}} - - {name: install requirements, run: source .github/reqs.sh && c4_install_test_requirements $OS} - - {name: show info, run: source .github/setenv.sh && c4_show_info} - - name: shared64arm-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared64arm - - {name: shared64arm-build, run: source .github/setenv.sh && c4_build_test shared64arm} - #- {name: shared64arm-run, run: source .github/setenv.sh && c4_run_test shared64arm} - - {name: shared64arm-pack, run: source .github/setenv.sh && c4_package shared64arm} - - name: static64arm-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static64arm - - {name: static64arm-build, run: source .github/setenv.sh && c4_build_test static64arm} - #- {name: static64arm-run, run: source .github/setenv.sh && c4_run_test static64arm} - - {name: static64arm-pack, run: source .github/setenv.sh && c4_package static64arm} - - name: shared32arm-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test shared32arm - - {name: shared32arm-build, run: source .github/setenv.sh && c4_build_test shared32arm} - #- {name: shared32arm-run, run: source .github/setenv.sh && c4_run_test shared32arm} - - {name: shared32arm-pack, run: source .github/setenv.sh && c4_package shared32arm} - - name: static32arm-configure--------------------------------------------------- - run: source .github/setenv.sh && c4_cfg_test static32arm - - {name: static32arm-build, run: source .github/setenv.sh && c4_build_test static32arm} - #- {name: static32arm-run, run: source .github/setenv.sh && c4_run_test static32arm} - - {name: static32arm-pack, run: source .github/setenv.sh && c4_package static32arm} diff --git a/thirdparty/ryml/ext/c4core/.gitignore b/thirdparty/ryml/ext/c4core/.gitignore deleted file mode 100644 index 9c258ea3b..000000000 --- a/thirdparty/ryml/ext/c4core/.gitignore +++ /dev/null @@ -1,34 +0,0 @@ -# text editor files -*.bck -\#* -*~ -.ccls-cache/ -.clangd/ -.cache/ -.cquery_cached_index/ -__pycache__/ - -# Visual Studio files -.vs/ -.vscode/ -# QtCreator files -CMakeLists.txt.user -# Eclipse -.project -.cproject -/.settings/ - -# build files -build/ -install/ -.python-version -compile_commands.json - -# test files -/Testing/ - -# continuous integration files -.github/vagrant/*.log -.github/vagrant/.vagrant -.github/vagrant/macos/.vagrant -src_singleheader/ \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/.gitmodules b/thirdparty/ryml/ext/c4core/.gitmodules deleted file mode 100644 index 77f89eae3..000000000 --- a/thirdparty/ryml/ext/c4core/.gitmodules +++ /dev/null @@ -1,9 +0,0 @@ -[submodule "cmake"] - path = cmake - url = https://github.com/biojppm/cmake -[submodule "extern/debugbreak"] - path = src/c4/ext/debugbreak - url = https://github.com/biojppm/debugbreak -[submodule "src/c4/ext/fast_float"] - path = src/c4/ext/fast_float - url = https://github.com/fastfloat/fast_float diff --git a/thirdparty/ryml/ext/c4core/LICENSE-BOOST.txt b/thirdparty/ryml/ext/c4core/LICENSE-BOOST.txt deleted file mode 100644 index 689a5fa00..000000000 --- a/thirdparty/ryml/ext/c4core/LICENSE-BOOST.txt +++ /dev/null @@ -1,26 +0,0 @@ -src/c4/ext/sg14/inplace_function.h is distributed under the following terms: ----------------------------------------------------------------------------- - -Boost Software License - Version 1.0 - August 17th, 2003 - -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/thirdparty/ryml/ext/c4core/LICENSE.txt b/thirdparty/ryml/ext/c4core/LICENSE.txt deleted file mode 100644 index 47b6b4394..000000000 --- a/thirdparty/ryml/ext/c4core/LICENSE.txt +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2018, Joao Paulo Magalhaes - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - diff --git a/thirdparty/ryml/ext/c4core/README.md b/thirdparty/ryml/ext/c4core/README.md deleted file mode 100644 index cd420954d..000000000 --- a/thirdparty/ryml/ext/c4core/README.md +++ /dev/null @@ -1,381 +0,0 @@ -# c4core - C++ core utilities - -[![MIT Licensed](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/biojppm/c4core/blob/master/LICENSE.txt) -[![Docs](https://img.shields.io/badge/docs-docsforge-blue)](https://c4core.docsforge.com/) -[![ci](https://github.com/biojppm/c4core/workflows/ci/badge.svg)](https://github.com/biojppm/c4core/actions?query=ci) - -[![Codecov](https://codecov.io/gh/biojppm/c4core/branch/master/graph/badge.svg)](https://codecov.io/gh/biojppm/c4core) -[![LGTM alerts](https://img.shields.io/lgtm/alerts/g/biojppm/c4core.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/biojppm/c4core/alerts/) -[![LGTM grade: C/C++](https://img.shields.io/lgtm/grade/cpp/g/biojppm/c4core.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/biojppm/c4core/context:cpp) - - -c4core is a library of low-level C++ utilities, written with low-latency -projects in mind. - -Some of the utilities provided by c4core have already equivalent -functionality in the C++ standard, but they are provided as the existing C++ -equivalent may be insufficient (eg, std::string_view), inefficient (eg, -std::string), heavy (eg streams), or plainly unusable on some -platforms/projects, (eg exceptions); some other utilities have equivalent -under consideration for C++ standardisation; and yet some other utilities -have (to my knowledge) no equivalent under consideration. Be that as it may, -I've been using these utilities in this or similar forms for some years now, -and I've found them incredibly useful in my projects. I'm packing these as a -separate library, as all of my projects use it. - -c4core is [extensively unit-tested in Linux, Windows and -MacOS](https://github.com/biojppm/c4core/actions). The tests cover -x64, x86, arm, wasm (emscripten), aarch64, ppc64le and s390x -architectures, and include analysing c4core with: - * valgrind - * clang-tidy - * clang sanitizers: - * memory - * address - * undefined behavior - * thread - * [LGTM.com](https://lgtm.com/projects/g/biojppm/c4core) - -c4core also works [in -bare-metal](https://github.com/biojppm/c4core/issues/63) as well as -[in RISC-V](https://github.com/biojppm/c4core/pull/69) but at the -moment it's not easy to add automated tests to the CI, so for now -these are not in the list of official architectures. - - -- [c4core - C++ core utilities](#c4core---c-core-utilities) - - [Obtaining c4core](#obtaining-c4core) - - [Using c4core in your project](#using-c4core-in-your-project) - - [CMake](#cmake) - - [Bazel](#bazel) - - [Header-only](#header-only) - - [Package managers](#package-managers) - - [Quick tour](#quick-tour) - - [Writeable string views: c4::substr and c4::csubstr](#writeable-string-views-c4substr-and-c4csubstr) - - [Value <-> character interoperation](#value---character-interoperation) - - [String formatting and parsing](#string-formatting-and-parsing) - - [`c4::span` and `c4::blob`](#c4span-and-c4blob) - - [Enums and enum symbols](#enums-and-enum-symbols) - - [Bitmasks and bitmask symbols](#bitmasks-and-bitmask-symbols) - - [Base64 encoding / decoding](#base64-encoding--decoding) - - [Fuzzy float comparison](#fuzzy-float-comparison) - - [Multi-platform / multi-compiler utilities](#multi-platform--multi-compiler-utilities) - - [Runtime assertions and error handling](#runtime-assertions-and-error-handling) - - [Memory allocation](#memory-allocation) - - [Mass initialization/construction/destruction](#mass-initializationconstructiondestruction) - - - - -## Obtaining c4core - -c4core uses git submodules. It is best to clone c4core with the `--recursive` -option: - -```bash -# using --recursive makes sure git submodules are also cloned at the same time -git clone --recursive https://github.com/biojppm/c4core -``` - -If you ommit the `--recursive` option, then after cloning you will have to -make git checkout the current version of the submodules, using `git submodule -init` followed by `git submodule update`. - - -## Using c4core in your project - -c4core can be built with [cmake](#cmake), or can be used header only. It can also be obtained through some package managers. - -### CMake - -The recommended way to use c4core is by making it part of your project -by using `add_subdirectory(${path_to_c4core_root})` in your -CMakeLists.txt. Doing this is not intrusive to your cmake project -because c4core is fast to build, also prefixes every cmake -variable with `C4CORE_`. But more importantly, this will enable you to -compile c4core with the exact same compile settings used by your -project. - -Here's a very quick complete example of setting up your project to use -c4core as a cmake subproject: -```cmake -project(foo) - -add_subdirectory(c4core) - -add_library(foo foo.cpp) -target_link_libraries(foo PUBLIC c4core) # that's it! -``` -Note above that the call to `target_link_libraries()` is using PUBLIC -linking. This is required to make sure the include directories from `c4core` -are transitively used by clients of `foo`. - - -### Header-only - -If you prefer to pick a single header to get you quickly going, [there is an amalgamation tool](tools/amalgamate.py) which generates this header: -```console -[user@host c4core]$ python tools/amalgamate.py -h -usage: amalgamate.py [-h] [--fastfloat | --no-fastfloat] [--stl | --no-stl] [output] - -positional arguments: - output output file. defaults to stdout - -options: - -h, --help show this help message and exit - --fastfloat enable fastfloat library. this is the default. - --no-fastfloat enable fastfloat library. the default is --fastfloat. - --stl enable stl interop. this is the default. - --no-stl enable stl interop. the default is --stl. -``` - - -### Package managers - -c4core is available through the following package managers: - - * [vcpkg](https://vcpkg.io/en/packages.html): `vcpkg install c4core` - * Arch Linux/Manjaro: - * [rapidyaml](https://aur.archlinux.org/packages/rapidyaml/) - - - - - -## Quick tour - -All of the utilities in this library are under the namespace `c4`; any -exposed macros use the prefix `C4_`: eg `C4_ASSERT()`. - - -### Writeable string views: c4::substr and c4::csubstr - -Here: [`#include `](src/c4/substr.hpp) - - -### Value <-> character interoperation - -Here: [`#include `](src/c4/charconv.hpp) - -```c++ -// TODO: elaborate on the topics: - -c4::digits_dec(), c4::read_dec(), c4::write_dec() -c4::digits_hex(), c4::read_hex(), c4::write_hex() -c4::digits_oct(), c4::read_oct(), c4::write_oct() -c4::digits_bin(), c4::read_bin(), c4::write_bin() - -c4::utoa(), c4::atou() -c4::itoa(), c4::atoi() -c4::ftoa(), c4::atof() -c4::dtoa(), c4::atod() -c4::xtoa(), c4::atox() - -c4::to_chars(), c4::from_chars() -c4::to_chars_sub() -c4::to_chars_first() -``` - -The charconv funcions above are very fast; even faster than C++'s fastest facility `std::from_chars()`, `std::to_chars()`. For continuous benchmark results, browse through c4core's [github CI benchmark runs](https://github.com/biojppm/c4core/actions/workflows/benchmarks.yml). For example, a benchmark run on Linux/g++11.2 shows that: -- `c4::to_chars()` can be expected to be roughly... - - ~40% to 2x faster than `std::to_chars()` - - ~10x-30x faster than `sprintf()` - - ~50x-100x faster than a naive `stringstream::operator<<()` followed by `stringstream::str()` -- `c4::from_chars()` can be expected to be roughly... - - ~10%-30% faster than `std::from_chars()` - - ~10x faster than `scanf()` - - ~30x-50x faster than a naive `stringstream::str()` followed by `stringstream::operator>>()` - -Here are the results: - -| Write throughput | | Read throughput | | -|:-------------------------|--------:|:-------------------------|---------:| -| **write `uint8_t`** | **MB/s**| **read `uint8_t`** | **MB/s**| -| `c4::to_chars` | 526.86 | `c4::from_chars` | 163.06 | -| `std::to_chars` | 379.03 | `std::from_chars` | 154.85 | -| `std::sprintf` | 20.49 | `std::scanf` | 15.75 | -| `std::stringstream` | 3.82 | `std::stringstream` | 3.83 | -| **write `int8_t`** | **MB/s**| **read `int8_t`** | **MB/s**| -| `c4::to_chars` | 599.98 | `c4::from_chars` | 184.20 | -| `std::to_chars` | 246.32 | `std::from_chars` | 156.40 | -| `std::sprintf` | 19.15 | `std::scanf` | 16.44 | -| `std::stringstream` | 3.83 | `std::stringstream` | 3.89 | -| **write `uint16_t`** | **MB/s**| **read `uint16_t`** | **MB/s**| -| `c4::to_chars` | 486.40 | `c4::from_chars` | 349.48 | -| `std::to_chars` | 454.24 | `std::from_chars` | 319.13 | -| `std::sprintf` | 38.74 | `std::scanf` | 28.12 | -| `std::stringstream` | 7.08 | `std::stringstream`| 6.73 | -| **write `int16_t`** | **MB/s**| **read `int16_t`** | **MB/s**| -| `c4::to_chars` | 507.44 | `c4::from_chars` | 282.95 | -| `std::to_chars` | 297.49 | `std::from_chars` | 186.18 | -| `std::sprintf` | 39.03 | `std::scanf` | 28.45 | -| `std::stringstream` | 6.98 | `std::stringstream`| 6.49 | -| **write `uint32_t`** | **MB/s**| **read `uint32_t`** | **MB/s**| -| `c4::to_chars` | 730.12 | `c4::from_chars` | 463.95 | -| `std::to_chars` | 514.76 | `std::from_chars` | 329.42 | -| `std::sprintf` | 71.19 | `std::scanf` | 44.97 | -| `std::stringstream` | 14.05 | `std::stringstream`| 12.57 | -| **write `int32_t`** | **MB/s**| **read `int32_t`** | **MB/s**| -| `c4::to_chars` | 618.76 | `c4::from_chars` | 345.53 | -| `std::to_chars` | 394.72 | `std::from_chars` | 224.46 | -| `std::sprintf` | 71.14 | `std::scanf` | 43.49 | -| `std::stringstream` | 13.91 | `std::stringstream`| 12.03 | -| **write `uint64_t`** | **MB/s**| **read `uint64_t`** | **MB/s**| -| `c4::to_chars` | 1118.87 | `c4::from_chars` | 928.49 | -| `std::to_chars` | 886.58 | `std::from_chars` | 759.03 | -| `std::sprintf` | 140.96 | `std::scanf` | 91.60 | -| `std::stringstream` | 28.01 | `std::stringstream`| 25.00 | -| **write `int64_t`** | **MB/s**| **read `int64_t`** | **MB/s**| -| `c4::to_chars` | 1198.78 | `c4::from_chars` | 713.76 | -| `std::to_chars` | 882.17 | `std::from_chars` | 646.18 | -| `std::sprintf` | 138.79 | `std::scanf` | 90.07 | -| `std::stringstream` | 27.62 | `std::stringstream`| 25.12 | - - -Or here are plots for g++12.1 and VS2019 (from the same computer): - -| Linux gxx12.1 | Windows VS2019 | -|---------------|----------------| -| ![atox-u32-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-atox-mega_bytes_per_second-u32.png "atox-u32-linux") | ![atox-u32-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-atox-mega_bytes_per_second-u32.png "atox-u32-windows") | -| ![atox-i32-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-atox-mega_bytes_per_second-i32.png "atox-i32-linux") | ![atox-i32-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-atox-mega_bytes_per_second-i32.png "atox-i32-windows") | -| ![atox-u64-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-atox-mega_bytes_per_second-u64.png "atox-u64-linux") | ![atox-u64-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-atox-mega_bytes_per_second-u64.png "atox-u64-windows") | -| ![atox-i64-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-atox-mega_bytes_per_second-i64.png "atox-i64-linux") | ![atox-i64-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-atox-mega_bytes_per_second-i64.png "atox-i64-windows") | -| ![atof-double-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-atof-mega_bytes_per_second-double.png "atof-double-linux") | ![atof-double-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-atof-mega_bytes_per_second-double.png "atof-double-windows") | -| ![atof-float-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-atof-mega_bytes_per_second-float.png "atof-float-linux") | ![atof-float-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-atof-mega_bytes_per_second-float.png "atof-float-windows") | -| | | -| ![xtoa-u32-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-xtoa-mega_bytes_per_second-u32.png "xtoa-u32-linux") | ![xtoa-u32-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-xtoa-mega_bytes_per_second-u32.png "xtoa-u32-windows") | -| ![xtoa-i32-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-xtoa-mega_bytes_per_second-i32.png "xtoa-i32-linux") | ![xtoa-i32-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-xtoa-mega_bytes_per_second-i32.png "xtoa-i32-windows") | -| ![xtoa-u64-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-xtoa-mega_bytes_per_second-u64.png "xtoa-u64-linux") | ![xtoa-u64-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-xtoa-mega_bytes_per_second-u64.png "xtoa-u64-windows") | -| ![xtoa-i64-linux](doc/img/linux-x86_64-gxx12.1-Release/c4core-bm-charconv-xtoa-mega_bytes_per_second-i64.png "xtoa-i64-linux") | ![xtoa-i64-windows](doc/img/windows-x86_64-vs2019-Release/c4core-bm-charconv-xtoa-mega_bytes_per_second-i64.png "xtoa-i64-windows") | - - - -### String formatting and parsing - -* [`#include `](src/c4/format.hpp) - -```c++ -// TODO: elaborate on the topics: - -c4::cat(), c4::uncat() -c4::catsep(), c4::uncatsep() -c4::format(), c4::unformat() - -c4::catrs() -c4::catseprs() -c4::formatrs() - -// formatting: -c4::fmt::overflow_checked -c4::fmt::real -c4::fmt::boolalpha -c4::fmt::dec -c4::fmt::hex -c4::fmt::oct -c4::fmt::bin -c4::fmt::zpad -c4::fmt::right -c4::fmt::left -c4::fmt::raw, c4::fmt::craw -c4::fmt::base64, c4::fmt::cbase64 -``` - -### `c4::span` and `c4::blob` - -* [`#include `](src/c4/span.hpp) -* [`#include `](src/c4/blob.hpp) - - -### Enums and enum symbols - -[`#include `](src/c4/enum.hpp) - -```c++ -// TODO: elaborate on the topics: - -c4::e2str(), c4::str2e() -``` - -### Bitmasks and bitmask symbols - -[`#include `](src/c4/bitmask.hpp) - -```c++ -// TODO: elaborate on the topics: - -c4::bm2str(), c4::str2bm() -``` - -### Base64 encoding / decoding - -[`#include `](src/c4/base64.hpp) - -### Fuzzy float comparison - - -### Multi-platform / multi-compiler utilities - -```c++ -// TODO: elaborate on the topics: -#include - -C4_RESTRICT, $, c$, $$, c$$ -#include -#include - -#include -#include - -C4_UNREACHABLE() - -c4::type_name() - -// portable attributes -C4_LIKELY()/C4_UNLIKELY() -C4_ALWAYS_INLINE -C4_CONST -C4_PURE -C4_HOT -C4_COLD -``` - -### Runtime assertions and error handling - -```c++ -// TODO: elaborate on the topics: - -error callback - -C4_ASSERT() -C4_XASSERT() -C4_CHECK() - -C4_ERROR() -C4_NOT_IMPLEMENTED() -``` - -### Memory allocation - -```c++ -// TODO: elaborate on the topics: - -c4::aalloc(), c4::afree() // aligned allocation - -c4::MemoryResource // global and scope - -c4::Allocator -``` - -### Mass initialization/construction/destruction - -```c++ -// TODO: elaborate on the topics: - -c4::make_room()/c4::destroy_room() -c4::construct()/c4::construct_n() -c4::destroy()/c4::destroy_n() -c4::copy_construct()/c4::copy_construct_n() -c4::copy_assign()/c4::copy_assign_n() -c4::move_construct()/c4::move_construct_n() -c4::move_assign()/c4::move_assign_n() -``` diff --git a/thirdparty/ryml/ext/c4core/ROADMAP.md b/thirdparty/ryml/ext/c4core/ROADMAP.md deleted file mode 100644 index 9857514e4..000000000 --- a/thirdparty/ryml/ext/c4core/ROADMAP.md +++ /dev/null @@ -1,23 +0,0 @@ -# ROADMAP - -## New features - -These changes will provide new features, and client code can be kept -unchanged. - - -## API changes - -These changes will require client code to be updated. - -* [breaking] drop use of C-style sprintf() formats in error messages and - assertions. Change the implementation to use c4::format() - ```c++ - C4_ASSERT_MSG(sz > s.size(), "sz=%zu s.size()=%zu", sz, s.size()); - // ... the above changes to: - C4_ASSERT_MSG(sz > s.size(), "sz={} s.size()={}", sz, s.size()); - ``` - -## Implementation changes - -* drop calls to sprintf() in charconv.hpp. diff --git a/thirdparty/ryml/ext/c4core/bm/bm.yml b/thirdparty/ryml/ext/c4core/bm/bm.yml deleted file mode 100644 index 885dcf7db..000000000 --- a/thirdparty/ryml/ext/c4core/bm/bm.yml +++ /dev/null @@ -1,32 +0,0 @@ - -prefix: c4core-bm - -# TODO https://stackoverflow.com/questions/12360547/tool-for-retrieving-the-list-of-functions-and-methods-in-a-c-code-base - -bm: - charconv-atox: - desc: read string to arithmetic value - src: bm_charconv.cpp - template_types: [uint8, int8, uint16, int16, uint32, int32, uint64, int64, float, double] - charconv-xtoa: - desc: write arithmetic value to string - src: bm_charconv.cpp - template_types: [uint8, int8, uint16, int16, uint32, int32, uint64, int64, float, double] - format-cat: - desc: compares stringification of a sequence of heterogeneous general types - src: bm_format.cpp - format-catfile: - desc: compares stringification to a file of a sequence of heterogeneous general types - src: bm_format.cpp - format-catsep: - desc: compares stringification of a sequence of heterogeneous general types - src: bm_format.cpp - format-catsepfile: - desc: compares stringification to a file of a sequence of heterogeneous general types - src: bm_format.cpp - format-format: - desc: compares formatting of a sequence of heterogeneous general types - src: bm_format.cpp - format-formatfile: - desc: compares stringification to a file of a sequence of heterogeneous general types - src: bm_format.cpp diff --git a/thirdparty/ryml/ext/c4core/bm/bm_atox.cpp b/thirdparty/ryml/ext/c4core/bm/bm_atox.cpp deleted file mode 100644 index b4ca2b609..000000000 --- a/thirdparty/ryml/ext/c4core/bm/bm_atox.cpp +++ /dev/null @@ -1,478 +0,0 @@ -#include "./bm_charconv.hpp" - - -// this is an exploratory benchmark to compare the possible -// combinations for all the components of the read_dec() algorithm - - -template -bool range_based_restrictvar0(c4::csubstr s, T * v) -{ - *v = 0; - for(char c : s) - { - if(C4_UNLIKELY(c < '0' || c > '9')) - return false; - *v = (*v) * T(10) + (T(c) - T('0')); - } - return true; -} - -template -bool range_based_restrictvar1(c4::csubstr s, T *C4_RESTRICT v) -{ - *v = 0; - for(char c : s) - { - if(C4_UNLIKELY(c < '0' || c > '9')) - return false; - *v = (*v) * T(10) + (T(c) - T('0')); - } - return true; -} - -template -bool indexloop_restrictvar0(c4::csubstr s, T * v) -{ - *v = 0; - for(size_t i = 0; i < s.len; ++i) - { - const char c = s.str[i]; - if(C4_UNLIKELY(c < '0' || c > '9')) - return false; - *v = (*v) * T(10) + (T(c) - T('0')); - } - return true; -} - -template -bool indexloop_restrictvar1(c4::csubstr s, T *C4_RESTRICT v) -{ - *v = 0; - for(size_t i = 0; i < s.len; ++i) - { - const char c = s.str[i]; - if(C4_UNLIKELY(c < '0' || c > '9')) - return false; - *v = (*v) * T(10) + (T(c) - T('0')); - } - return true; -} - -template -bool prefer_likely(c4::csubstr s, T * v) -{ - *v = 0; - for(char c : s) - { - if(C4_LIKELY(c >= '0' && c <= '9')) - *v = (*v) * T(10) + (T(c) - T('0')); - else - return false; - } - return true; -} - -template -bool no_early_return(c4::csubstr s, T *C4_RESTRICT v) -{ - *v = 0; - bool stat = true; - for(char c : s) - { - if(C4_LIKELY(c >= '0' && c <= '9')) - *v = (*v) * T(10) + (T(c) - T('0')); - else - { - stat = false; - break; - } - } - return stat; -} - -template -bool no_early_return_auto_type(c4::csubstr s, T *C4_RESTRICT v) -{ - *v = 0; - bool stat = true; - for(char c : s) - { - if(C4_LIKELY(c >= '0' && c <= '9')) - *v = (*v) * T(10) + (T)(c - '0'); - else - { - stat = false; - break; - } - } - return stat; -} - -template -bool no_early_return_auto_type2(c4::csubstr s, T *C4_RESTRICT v) -{ - *v = 0; - bool stat = true; - for(char c : s) - { - if(C4_LIKELY(c >= '0' && c <= '9')) - { - *v *= 10; - *v += (T)(c - '0'); - } - else - { - stat = false; - break; - } - } - return stat; -} - -#define _(i) (T)(s.str[i] - '0') -C4_SUPPRESS_WARNING_GCC_CLANG_WITH_PUSH("-Wimplicit-fallthrough") - -template -C4_ALWAYS_INLINE auto unroll_switch_nocheck(c4::csubstr s, T *C4_RESTRICT v) - -> typename std::enable_if::type -{ - *v = 0; - switch(s.len) - { - case 1: - *v = _(0); - return true; - case 2: - *v = T(10) * _(0) + _(1); - return true; - case 3: - *v = T(100) * _(0) + T(10) * _(1) + _(2); - return true; - } - return false; -} - -template -C4_ALWAYS_INLINE auto unroll_switch_nocheck(c4::csubstr s, T *C4_RESTRICT v) - -> typename std::enable_if::type -{ - *v = 0; - switch(s.len) - { - case 1: - *v = _(0); - return true; - case 2: - *v = T(10) * _(0) + _(1); - return true; - case 3: - *v = T(100) * _(0) + T(10) * _(1) + _(2); - return true; - case 4: - *v = T(1000) * _(0) + T(100) * _(1) + T(10) * _(2) + _(3); - return true; - case 5: - *v = T(10000) * _(0) + T(1000) * _(1) + T(100) * _(2) + T(10) * _(3) + _(4); - return true; - } - return false; -} - -template -C4_ALWAYS_INLINE auto unroll_switch_nocheck(c4::csubstr s, T *C4_RESTRICT v) - -> typename std::enable_if::type -{ - switch(s.len) - { - case 1: - *v = _(0); - return true; - case 2: - *v = T(10) * _(0) + _(1); - return true; - case 3: - *v = T(100) * _(0) + T(10) * _(1) + _(2); - return true; - case 4: - *v = T(1000) * _(0) + T(100) * _(1) + T(10) * _(2) + _(3); - return true; - case 5: - *v = T(10000) * _(0) + T(1000) * _(1) + T(100) * _(2) + T(10) * _(3) + _(4); - return true; - case 6: - *v = T(100000) * _(0) + T(10000) * _(1) + T(1000) * _(2) + T(100) * _(3) + T(10) * _(4) + _(5); - return true; - case 7: - *v = T(1000000) * _(0) + T(100000) * _(1) + T(10000) * _(2) + T(1000) * _(3) + T(100) * _(4) + T(10) * _(5) + _(6); - return true; - case 8: - *v = T(10000000) * _(0) + T(1000000) * _(1) + T(100000) * _(2) + T(10000) * _(3) + T(1000) * _(4) + T(100) * _(5) + T(10) * _(6) + _(7); - return true; - case 9: - *v = T(100000000) * _(0) + T(10000000) * _(1) + T(1000000) * _(2) + T(100000) * _(3) + T(10000) * _(4) + T(1000) * _(5) + T(100) * _(6) + T(10) * _(7) + _(8); - return true; - case 10: - *v = T(1000000000) * _(0) + T(100000000) * _(1) + T(10000000) * _(2) + T(1000000) * _(3) + T(100000) * _(4) + T(10000) * _(5) + T(1000) * _(6) + T(100) * _(7) + T(10) * _(8) + _(9); - return true; - } - return false; -} - -template -C4_ALWAYS_INLINE auto unroll_switch_nocheck(c4::csubstr s, T *C4_RESTRICT v) - -> typename std::enable_if::type -{ - switch(s.len) - { - case 1: - *v = _(0); - return true; - case 2: - *v = T(10) * _(0) + _(1); - return true; - case 3: - *v = T(100) * _(0) + T(10) * _(1) + _(2); - return true; - case 4: - *v = T(1000) * _(0) + T(100) * _(1) + T(10) * _(2) + _(3); - return true; - case 5: - *v = T(10000) * _(0) + T(1000) * _(1) + T(100) * _(2) + T(10) * _(3) + _(4); - return true; - case 6: - *v = T(100000) * _(0) + T(10000) * _(1) + T(1000) * _(2) + T(100) * _(3) + T(10) * _(4) + _(5); - return true; - case 7: - *v = T(1000000) * _(0) + T(100000) * _(1) + T(10000) * _(2) + T(1000) * _(3) + T(100) * _(4) + T(10) * _(5) + _(6); - return true; - case 8: - *v = T(10000000) * _(0) + T(1000000) * _(1) + T(100000) * _(2) + T(10000) * _(3) + T(1000) * _(4) + T(100) * _(5) + T(10) * _(6) + _(7); - return true; - case 9: - *v = T(100000000) * _(0) + T(10000000) * _(1) + T(1000000) * _(2) + T(100000) * _(3) + T(10000) * _(4) + T(1000) * _(5) + T(100) * _(6) + T(10) * _(7) + _(8); - return true; - case 10: - *v = T(1000000000) * _(0) + T(100000000) * _(1) + T(10000000) * _(2) + T(1000000) * _(3) + T(100000) * _(4) + T(10000) * _(5) + T(1000) * _(6) + T(100) * _(7) + T(10) * _(8) + _(9); - return true; - default: - { - size_t i = 0; - *v = 0; - for( ; i + 10 < s.len; ++i) - *v = *v * T(10) + _(i); - *v = T(1000000000) * _(i) + T(100000000) * _(i+1) + T(10000000) * _(i+2) + T(1000000) * _(i+3) + T(100000) * _(i+4) + T(10000) * _(i+5) + T(1000) * _(i+6) + T(100) * _(i+7) + T(10) * _(i+8) + _(i+9); - return true; - } - } - return false; -} - -#define ok(i) (s.str[i] >= '0' && s.str[i] <= '9') - -template -C4_ALWAYS_INLINE auto unroll_switch(c4::csubstr s, T *C4_RESTRICT v) - -> typename std::enable_if::type -{ - *v = 0; - switch(s.len) - { - case 1: - *v = _(0); - return ok(0); - case 2: - *v = T(10) * _(0) + _(1); - return ok(0) && ok(1); - case 3: - *v = T(100) * _(0) + T(10) * _(1) + _(2); - return ok(0) && ok(1) && ok(2); - } - return false; -} - -template -C4_ALWAYS_INLINE auto unroll_switch(c4::csubstr s, T *C4_RESTRICT v) - -> typename std::enable_if::type -{ - *v = 0; - switch(s.len) - { - case 1: - *v = _(0); - return true; - case 2: - *v = T(10) * _(0) + _(1); - return true; - case 3: - *v = T(100) * _(0) + T(10) * _(1) + _(2); - return ok(0) && ok(1) && ok(2); - case 4: - *v = T(1000) * _(0) + T(100) * _(1) + T(10) * _(2) + _(3); - return ok(0) && ok(1) && ok(2) && ok(3); - case 5: - *v = T(10000) * _(0) + T(1000) * _(1) + T(100) * _(2) + T(10) * _(3) + _(4); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4); - } - return false; -} - -template -C4_ALWAYS_INLINE auto unroll_switch(c4::csubstr s, T *C4_RESTRICT v) - -> typename std::enable_if::type -{ - switch(s.len) - { - case 1: - *v = _(0); - return ok(0); - case 2: - *v = T(10) * _(0) + _(1); - return ok(0) && ok(1); - case 3: - *v = T(100) * _(0) + T(10) * _(1) + _(2); - return ok(0) && ok(1) && ok(2); - case 4: - *v = T(1000) * _(0) + T(100) * _(1) + T(10) * _(2) + _(3); - return ok(0) && ok(1) && ok(2) && ok(3); - case 5: - *v = T(10000) * _(0) + T(1000) * _(1) + T(100) * _(2) + T(10) * _(3) + _(4); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4); - case 6: - *v = T(100000) * _(0) + T(10000) * _(1) + T(1000) * _(2) + T(100) * _(3) + T(10) * _(4) + _(5); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5); - case 7: - *v = T(1000000) * _(0) + T(100000) * _(1) + T(10000) * _(2) + T(1000) * _(3) + T(100) * _(4) + T(10) * _(5) + _(6); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5) && ok(6); - case 8: - *v = T(10000000) * _(0) + T(1000000) * _(1) + T(100000) * _(2) + T(10000) * _(3) + T(1000) * _(4) + T(100) * _(5) + T(10) * _(6) + _(7); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5) && ok(6) && ok(7); - case 9: - *v = T(100000000) * _(0) + T(10000000) * _(1) + T(1000000) * _(2) + T(100000) * _(3) + T(10000) * _(4) + T(1000) * _(5) + T(100) * _(6) + T(10) * _(7) + _(8); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5) && ok(6) && ok(7) && ok(8); - case 10: - *v = T(1000000000) * _(0) + T(100000000) * _(1) + T(10000000) * _(2) + T(1000000) * _(3) + T(100000) * _(4) + T(10000) * _(5) + T(1000) * _(6) + T(100) * _(7) + T(10) * _(8) + _(9); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5) && ok(6) && ok(7) && ok(8) && ok(9); - } - return false; -} - -template -C4_ALWAYS_INLINE auto unroll_switch(c4::csubstr s, T *C4_RESTRICT v) - -> typename std::enable_if::type -{ - switch(s.len) - { - case 1: - *v = _(0); - return ok(0); - case 2: - *v = T(10) * _(0) + _(1); - return ok(0) && ok(1); - case 3: - *v = T(100) * _(0) + T(10) * _(1) + _(2); - return ok(0) && ok(1) && ok(2); - case 4: - *v = T(1000) * _(0) + T(100) * _(1) + T(10) * _(2) + _(3); - return ok(0) && ok(1) && ok(2) && ok(3); - case 5: - *v = T(10000) * _(0) + T(1000) * _(1) + T(100) * _(2) + T(10) * _(3) + _(4); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4); - case 6: - *v = T(100000) * _(0) + T(10000) * _(1) + T(1000) * _(2) + T(100) * _(3) + T(10) * _(4) + _(5); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5); - case 7: - *v = T(1000000) * _(0) + T(100000) * _(1) + T(10000) * _(2) + T(1000) * _(3) + T(100) * _(4) + T(10) * _(5) + _(6); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5) && ok(6); - case 8: - *v = T(10000000) * _(0) + T(1000000) * _(1) + T(100000) * _(2) + T(10000) * _(3) + T(1000) * _(4) + T(100) * _(5) + T(10) * _(6) + _(7); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5) && ok(6) && ok(7); - case 9: - *v = T(100000000) * _(0) + T(10000000) * _(1) + T(1000000) * _(2) + T(100000) * _(3) + T(10000) * _(4) + T(1000) * _(5) + T(100) * _(6) + T(10) * _(7) + _(8); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5) && ok(6) && ok(7) && ok(8); - case 10: - *v = T(1000000000) * _(0) + T(100000000) * _(1) + T(10000000) * _(2) + T(1000000) * _(3) + T(100000) * _(4) + T(10000) * _(5) + T(1000) * _(6) + T(100) * _(7) + T(10) * _(8) + _(9); - return ok(0) && ok(1) && ok(2) && ok(3) && ok(4) && ok(5) && ok(6) && ok(7) && ok(8) && ok(9); - default: - { - size_t i = 0; - *v = 0; - for( ; i + 10 < s.len; ++i) - { - *v = *v * T(10) + _(i); - if(C4_UNLIKELY(!ok(i))) - return false; - } - *v = T(1000000000) * _(i) + T(100000000) * _(i+1) + T(10000000) * _(i+2) + T(1000000) * _(i+3) + T(100000) * _(i+4) + T(10000) * _(i+5) + T(1000) * _(i+6) + T(100) * _(i+7) + T(10) * _(i+8) + _(i+9); - return ok(i) && ok(i+1) && ok(i+2) && ok(i+3) && ok(i+4) && ok(i+5) && ok(i+6) && ok(i+7) && ok(i+8) && ok(i+9); - } - } - return false; -} - -C4_SUPPRESS_WARNING_GCC_CLANG_POP -#undef _ - - - - -#define DECLARE_BM(func) \ -template \ -void func(bm::State &st) \ -{ \ - random_strings_cref values = mkstrings_positive(); \ - T val = {}; \ - T sum = {}; \ - for(auto _ : st) \ - { \ - C4DOALL(kNumValues) \ - { \ - const bool ok = func(values.next(), &val); \ - sum += (T)(T(ok) + val); \ - } \ - } \ - bm::DoNotOptimize(sum); \ - report(st, kNumValues); \ -} - -#define DEFINE_BM(ty) \ - C4BM_TEMPLATE(unroll_switch_nocheck, ty); \ - C4BM_TEMPLATE(unroll_switch, ty); \ - C4BM_TEMPLATE(indexloop_restrictvar0, ty); \ - C4BM_TEMPLATE(indexloop_restrictvar1, ty); \ - C4BM_TEMPLATE(range_based_restrictvar0, ty); \ - C4BM_TEMPLATE(range_based_restrictvar1, ty); \ - C4BM_TEMPLATE(prefer_likely, ty); \ - C4BM_TEMPLATE(no_early_return, ty); \ - C4BM_TEMPLATE(no_early_return_auto_type, ty); \ - C4BM_TEMPLATE(no_early_return_auto_type2, ty); \ - - -DECLARE_BM(unroll_switch_nocheck) -DECLARE_BM(unroll_switch) -DECLARE_BM(indexloop_restrictvar0) -DECLARE_BM(indexloop_restrictvar1) -DECLARE_BM(range_based_restrictvar0) -DECLARE_BM(range_based_restrictvar1) -DECLARE_BM(prefer_likely) -DECLARE_BM(no_early_return) -DECLARE_BM(no_early_return_auto_type) -DECLARE_BM(no_early_return_auto_type2) - - -DEFINE_BM(uint8_t) -DEFINE_BM(int8_t) -DEFINE_BM(uint16_t) -DEFINE_BM(int16_t) -DEFINE_BM(uint32_t) -DEFINE_BM(int32_t) -DEFINE_BM(uint64_t) -DEFINE_BM(int64_t) - - -int main(int argc, char *argv[]) -{ - //do_test(); - bm::Initialize(&argc, argv); - bm::RunSpecifiedBenchmarks(); - return 0; -} diff --git a/thirdparty/ryml/ext/c4core/bm/bm_charconv.cpp b/thirdparty/ryml/ext/c4core/bm/bm_charconv.cpp deleted file mode 100644 index 888fb91b6..000000000 --- a/thirdparty/ryml/ext/c4core/bm/bm_charconv.cpp +++ /dev/null @@ -1,1617 +0,0 @@ -#include "./bm_charconv.hpp" - -#include -#if C4_CPP >= 17 -#include -#if defined(__cpp_lib_to_chars) -#define C4CORE_BM_HAVE_TOCHARS 1 -#endif -#include -#endif - -#ifdef C4CORE_BM_USE_RYU -#include -#include -#endif - -#ifdef C4CORE_BM_USE_FP -#include -#endif - - -// some of the benchmarks do not need to be templates, -// but it helps in the naming scheme. - -// xtoa means to string -// atox means string to - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -C4FOR(T, isint) -c4_digits_dec(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += c4::digits_dec(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -c4_digits_hex(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += c4::digits_hex(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -c4_digits_oct(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += c4::digits_oct(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -c4_digits_bin(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += c4::digits_bin(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -C4FOR(T, isint) -atox_c4_read_dec(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::read_dec(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atoxhex_c4_read_hex(bm::State& st) -{ - random_strings_cref strings = mkstrings_hex_positive(/*with_prefix*/false); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::read_hex(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atoxoct_c4_read_oct(bm::State& st) -{ - random_strings_cref strings = mkstrings_oct_positive(/*with_prefix*/false); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::read_oct(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atoxbin_c4_read_bin(bm::State& st) -{ - random_strings_cref strings = mkstrings_bin_positive(/*with_prefix*/false); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::read_bin(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -C4FOR(T, isint) -xtoa_c4_write_dec(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals_positive(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::write_dec(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isint) -xtoahex_c4_write_hex(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals_positive(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::write_hex(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isint) -xtoaoct_c4_write_oct(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals_positive(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::write_oct(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isint) -xtoabin_c4_write_bin(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals_positive(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::write_bin(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -C4FOR(T, isiint) -xtoa_c4_itoa(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals_positive(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::itoa(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isuint) -xtoa_c4_utoa(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals_positive(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::utoa(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isfloat) -xtoa_c4_ftoa(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::ftoa(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isdouble) -xtoa_c4_dtoa(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::dtoa(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isint) -xtoa_c4_xtoa(bm::State& st) -{ - string_buffer buf; - random_values values = mkvals_positive(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::xtoa(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isreal) -xtoa_c4_xtoa(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals(); - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::xtoa(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- - -C4FOR(T, isiint) -atox_c4_atoi(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::atoi(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isuint) -atox_c4_atou(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::atou(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isfloat) -atox_c4_atof(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::atof(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isdouble) -atox_c4_atod(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::atod(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_c4_atox(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::atox(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_c4_atox(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::atox(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- - -C4FOR(T, isint) -atox_std_atoi(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - val = (T) std::atoi(strings.next_s().c_str()); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_std_atol(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - val = (T) std::atol(strings.next_s().c_str()); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_std_atof(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - val = (T) std::atof(strings.next_s().c_str()); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- - -C4FOR(T, isint) -atox_std_strtol(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::string const& s = strings.next_s(); - val = (T) std::strtol(s.data(), nullptr, 10); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_std_strtoll(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::string const& s = strings.next_s(); - val = (T) std::strtoll(s.data(), nullptr, 10); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_std_strtoul(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::string const& s = strings.next_s(); - val = (T) std::strtoul(s.data(), nullptr, 10); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_std_strtoull(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::string const& s = strings.next_s(); - val = (T) std::strtoull(s.data(), nullptr, 10); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_std_strtof(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::string const& s = strings.next_s(); - val = (T) std::strtof(s.data(), nullptr); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_std_strtod(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::string const& s = strings.next_s(); - val = (T) std::strtod(s.data(), nullptr); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_std_stof(bm::State &st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - val = std::stof(strings.next_s()); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_std_stod(bm::State &st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - val = std::stod(strings.next_s()); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- - -#ifdef C4CORE_BM_USE_RYU -C4FOR(T, isfloat) -atox_ryu_s2f(bm::State &st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::csubstr s = strings.next(); - s2f_n(s.data(), (int) s.size(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isdouble) -atox_ryu_s2d(bm::State &st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::csubstr s = strings.next(); - s2d_n(s.data(), (int) s.size(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isfloat) -xtoa_ryu_f2s(bm::State &st) -{ - string_buffer buf; - random_values_cref values = mkvals(); - for(auto _ : st) - { - C4DOALL(kNumValues) - f2s_buffered_n(values.next(), buf.buf.str); - } - bm::DoNotOptimize(buf.buf); - report(st, kNumValues); -} - -C4FOR(T, isdouble) -xtoa_ryu_d2s(bm::State &st) -{ - string_buffer buf; - random_values_cref values = mkvals(); - for(auto _ : st) - { - C4DOALL(kNumValues) - d2s_buffered_n(values.next(), buf.buf.str); - } - bm::DoNotOptimize(buf.buf); - report(st, kNumValues); -} -#endif // C4CORE_BM_USE_RYU - - -//----------------------------------------------------------------------------- - -// fp is still experimental and undocumented; -// some assertions are firing in debug builds -// so we make these benchmarks available only with NDEBUG -#if (defined(C4CORE_BM_USE_FP)) && (!defined(NDEBUG)) -#undef C4CORE_BM_USE_FP -#endif - -#ifdef C4CORE_BM_USE_FP -C4FOR(T, isreal) -atox_fp_from_chars_limited(bm::State &st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::csubstr s = strings.next(); - val = jkj::fp::from_chars_limited(s.begin(), s.end()).to_float(); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} -#endif // C4CORE_BM_USE_FP - -#ifdef C4CORE_BM_USE_FP -C4FOR(T, isreal) -atox_fp_from_chars_unlimited(bm::State &st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::csubstr s = strings.next(); - val = jkj::fp::from_chars_unlimited(s.begin(), s.end()).to_float(); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} -#endif // C4CORE_BM_USE_FP - - -//----------------------------------------------------------------------------- - -C4FOR(T, isreal) -atox_fast_float(bm::State &st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::csubstr s = strings.next(); - fast_float::from_chars(s.begin(), s.end(), val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- - -template struct fmtspec; -template<> struct fmtspec< uint8_t> { static const char w[], r[]; }; -template<> struct fmtspec< int8_t> { static const char w[], r[]; }; -template<> struct fmtspec { static const char w[], r[]; }; -template<> struct fmtspec< int16_t> { static const char w[], r[]; }; -template<> struct fmtspec { static const char w[], r[]; }; -template<> struct fmtspec< int32_t> { static const char w[], r[]; }; -template<> struct fmtspec { static const char w[], r[]; }; -template<> struct fmtspec< int64_t> { static const char w[], r[]; }; -template<> struct fmtspec< float > { static const char w[], r[]; }; -template<> struct fmtspec< double > { static const char w[], r[]; }; - -constexpr const char fmtspec< uint8_t>::w[] = "%" PRIu8 ; -constexpr const char fmtspec< int8_t>::w[] = "%" PRIi8 ; -constexpr const char fmtspec::w[] = "%" PRIu16; -constexpr const char fmtspec< int16_t>::w[] = "%" PRIi16; -constexpr const char fmtspec::w[] = "%" PRIu32; -constexpr const char fmtspec< int32_t>::w[] = "%" PRIi32; -constexpr const char fmtspec::w[] = "%" PRIu64; -constexpr const char fmtspec< int64_t>::w[] = "%" PRIi64; -constexpr const char fmtspec< float >::w[] = "%g" ; -constexpr const char fmtspec< double >::w[] = "%lg" ; - -constexpr const char fmtspec< uint8_t>::r[] = "%" SCNu8 ; -constexpr const char fmtspec< int8_t>::r[] = "%" SCNi8 ; -constexpr const char fmtspec::r[] = "%" SCNu16; -constexpr const char fmtspec< int16_t>::r[] = "%" SCNi16; -constexpr const char fmtspec::r[] = "%" SCNu32; -constexpr const char fmtspec< int32_t>::r[] = "%" SCNi32; -constexpr const char fmtspec::r[] = "%" SCNu64; -constexpr const char fmtspec< int64_t>::r[] = "%" SCNi64; -constexpr const char fmtspec< float >::r[] = "%g" ; -constexpr const char fmtspec< double >::r[] = "%lg" ; - -C4FOR(T, isint) -xtoa_sprintf(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals_positive(); - for(auto _ : st) - { - C4DOALL(kNumValues) - ::snprintf(buf.buf.str, buf.buf.len, fmtspec::w, values.next()); - } - bm::DoNotOptimize(buf.buf); - report(st, kNumValues); -} - -C4FOR(T, isreal) -xtoa_sprintf(bm::State& st) -{ - string_buffer buf; - random_values_cref values = mkvals(); - C4_SUPPRESS_WARNING_GCC_CLANG_WITH_PUSH("-Wdouble-promotion") - for(auto _ : st) - { - C4DOALL(kNumValues) - ::snprintf(buf.buf.str, buf.buf.len, fmtspec::w, values.next()); - } - C4_SUPPRESS_WARNING_GCC_CLANG_POP - bm::DoNotOptimize(buf.buf); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_scanf(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - ::sscanf(strings.next_s().c_str(), fmtspec::r, &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_scanf(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - ::sscanf(strings.next_s().c_str(), fmtspec::r, &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- - -C4FOR(T, isint) -xtoa_sstream(bm::State& st) -{ - random_values_cref values = mkvals_positive(); - std::string out; C4_UNUSED(out); - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::stringstream ss; - ss << values.next(); - out = ss.str(); - } - } - bm::DoNotOptimize(out); - report(st, kNumValues); -} - -C4FOR(T, isreal) -xtoa_sstream(bm::State& st) -{ - random_values_cref values = mkvals(); - std::string out; C4_UNUSED(out); - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::stringstream ss; - ss << values.next(); - out = ss.str(); - } - } - bm::DoNotOptimize(out); - report(st, kNumValues); -} - -C4FOR(T, isint) -xtoa_sstream_reuse(bm::State& st) -{ - random_values_cref values = mkvals_positive(); - std::stringstream ss; - std::string out; C4_UNUSED(out); - for(auto _ : st) - { - C4DOALL(kNumValues) - { - ss.clear(); - ss.str(""); - ss << values.next(); - out = ss.str(); - } - } - bm::DoNotOptimize(out); - report(st, kNumValues); -} - -C4FOR(T, isreal) -xtoa_sstream_reuse(bm::State& st) -{ - random_values_cref values = mkvals(); - std::stringstream ss; - std::string out; C4_UNUSED(out); - for(auto _ : st) - { - C4DOALL(kNumValues) - { - ss.clear(); - ss.str(""); - ss << values.next(); - out = ss.str(); - } - } - bm::DoNotOptimize(out); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_sstream(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::stringstream ss(strings.next_s()); - ss >> val; - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_sstream(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::stringstream ss(strings.next_s()); - ss >> val; - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_sstream_reuse(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - std::stringstream ss; - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - ss.clear(); - ss.str(strings.next_s()); - ss >> val; - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_sstream_reuse(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - std::stringstream ss; - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - ss.clear(); - ss.str(strings.next_s()); - ss >> val; - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- - -C4FOR(T, isint) -xtoa_std_to_string(bm::State& st) -{ - random_values_cref values = mkvals_positive(); - std::string out; - for(auto _ : st) - { - C4DOALL(kNumValues) - out = std::to_string(values.next()); - } - bm::DoNotOptimize(out); - report(st, kNumValues); -} - -C4FOR(T, isreal) -xtoa_std_to_string(bm::State& st) -{ - random_values_cref values = mkvals(); - std::string out; - for(auto _ : st) - { - C4DOALL(kNumValues) - out = std::to_string(values.next()); - } - bm::DoNotOptimize(out); - report(st, kNumValues); -} - - -//----------------------------------------------------------------------------- - -C4FOR(T, isint) -xtoa_c4_to_chars(bm::State& st) -{ - random_values_cref values = mkvals_positive(); - string_buffer buf; - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::to_chars(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isreal) -xtoa_c4_to_chars(bm::State& st) -{ - random_values_cref values = mkvals(); - string_buffer buf; - for(auto _ : st) - { - C4DOALL(kNumValues) - c4::to_chars(buf, values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_c4_from_chars(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::from_chars(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isint) -atox_c4_from_chars_checked(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::from_chars(strings.next(), c4::fmt::overflow_checked(val)); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -C4FOR(T, isreal) -atox_c4_from_chars(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::from_chars(strings.next(), &val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -#if (C4_CPP >= 17) -C4FOR(T, isint) -xtoa_std_to_chars(bm::State& st) -{ - random_values_cref values = mkvals_positive(); - string_buffer buf; - for(auto _ : st) - { - C4DOALL(kNumValues) - std::to_chars(buf.begin(), buf.end(), values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} -#endif - -#if C4CORE_BM_HAVE_TOCHARS -C4FOR(T, isreal) -xtoa_std_to_chars(bm::State& st) -{ - random_values_cref values = mkvals(); - string_buffer buf; - for(auto _ : st) - { - C4DOALL(kNumValues) - std::to_chars(buf.begin(), buf.end(), values.next()); - } - bm::DoNotOptimize(buf); - report(st, kNumValues); -} -#endif - -#if (C4_CPP >= 17) -C4FOR(T, isint) -atox_std_from_chars(bm::State& st) -{ - random_strings_cref strings = mkstrings_positive(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::csubstr s = strings.next(); - std::from_chars(s.begin(), s.end(), val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} -#endif - -#if C4CORE_BM_HAVE_TOCHARS -C4FOR(T, isreal) -atox_std_from_chars(bm::State& st) -{ - random_strings_cref strings = mkstrings(); - T val = {}, sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - c4::csubstr s = strings.next(); - std::from_chars(s.begin(), s.end(), val); - sum += val; - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} -#endif - - -//----------------------------------------------------------------------------- - -C4BM_TEMPLATE(c4_digits_dec, uint8_t); -C4BM_TEMPLATE(c4_digits_hex, uint8_t); -C4BM_TEMPLATE(c4_digits_oct, uint8_t); -C4BM_TEMPLATE(c4_digits_bin, uint8_t); - -C4BM_TEMPLATE(c4_digits_dec, int8_t); -C4BM_TEMPLATE(c4_digits_hex, int8_t); -C4BM_TEMPLATE(c4_digits_oct, int8_t); -C4BM_TEMPLATE(c4_digits_bin, int8_t); - -C4BM_TEMPLATE(c4_digits_dec, uint16_t); -C4BM_TEMPLATE(c4_digits_hex, uint16_t); -C4BM_TEMPLATE(c4_digits_oct, uint16_t); -C4BM_TEMPLATE(c4_digits_bin, uint16_t); - -C4BM_TEMPLATE(c4_digits_dec, int16_t); -C4BM_TEMPLATE(c4_digits_hex, int16_t); -C4BM_TEMPLATE(c4_digits_oct, int16_t); -C4BM_TEMPLATE(c4_digits_bin, int16_t); - -C4BM_TEMPLATE(c4_digits_dec, uint32_t); -C4BM_TEMPLATE(c4_digits_hex, uint32_t); -C4BM_TEMPLATE(c4_digits_oct, uint32_t); -C4BM_TEMPLATE(c4_digits_bin, uint32_t); - -C4BM_TEMPLATE(c4_digits_dec, int32_t); -C4BM_TEMPLATE(c4_digits_hex, int32_t); -C4BM_TEMPLATE(c4_digits_oct, int32_t); -C4BM_TEMPLATE(c4_digits_bin, int32_t); - -C4BM_TEMPLATE(c4_digits_dec, uint64_t); -C4BM_TEMPLATE(c4_digits_hex, uint64_t); -C4BM_TEMPLATE(c4_digits_oct, uint64_t); -C4BM_TEMPLATE(c4_digits_bin, uint64_t); - -C4BM_TEMPLATE(c4_digits_dec, int64_t); -C4BM_TEMPLATE(c4_digits_hex, int64_t); -C4BM_TEMPLATE(c4_digits_oct, int64_t); -C4BM_TEMPLATE(c4_digits_bin, int64_t); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -C4BM_TEMPLATE(xtoahex_c4_write_hex, uint8_t); -C4BM_TEMPLATE(xtoaoct_c4_write_oct, uint8_t); -C4BM_TEMPLATE(xtoabin_c4_write_bin, uint8_t); - -C4BM_TEMPLATE(xtoahex_c4_write_hex, int8_t); -C4BM_TEMPLATE(xtoaoct_c4_write_oct, int8_t); -C4BM_TEMPLATE(xtoabin_c4_write_bin, int8_t); - -C4BM_TEMPLATE(xtoahex_c4_write_hex, uint16_t); -C4BM_TEMPLATE(xtoaoct_c4_write_oct, uint16_t); -C4BM_TEMPLATE(xtoabin_c4_write_bin, uint16_t); - -C4BM_TEMPLATE(xtoahex_c4_write_hex, int16_t); -C4BM_TEMPLATE(xtoaoct_c4_write_oct, int16_t); -C4BM_TEMPLATE(xtoabin_c4_write_bin, int16_t); - -C4BM_TEMPLATE(xtoahex_c4_write_hex, uint32_t); -C4BM_TEMPLATE(xtoaoct_c4_write_oct, uint32_t); -C4BM_TEMPLATE(xtoabin_c4_write_bin, uint32_t); - -C4BM_TEMPLATE(xtoahex_c4_write_hex, int32_t); -C4BM_TEMPLATE(xtoaoct_c4_write_oct, int32_t); -C4BM_TEMPLATE(xtoabin_c4_write_bin, int32_t); - -C4BM_TEMPLATE(xtoahex_c4_write_hex, uint64_t); -C4BM_TEMPLATE(xtoaoct_c4_write_oct, uint64_t); -C4BM_TEMPLATE(xtoabin_c4_write_bin, uint64_t); - -C4BM_TEMPLATE(xtoahex_c4_write_hex, int64_t); -C4BM_TEMPLATE(xtoaoct_c4_write_oct, int64_t); -C4BM_TEMPLATE(xtoabin_c4_write_bin, int64_t); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -C4BM_TEMPLATE(xtoa_c4_write_dec, uint8_t); -C4BM_TEMPLATE(xtoa_c4_utoa, uint8_t); -C4BM_TEMPLATE(xtoa_c4_xtoa, uint8_t); -C4BM_TEMPLATE(xtoa_c4_to_chars, uint8_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(xtoa_std_to_chars, uint8_t); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, uint8_t); -C4BM_TEMPLATE(xtoa_sprintf, uint8_t); -C4BM_TEMPLATE(xtoa_sstream_reuse, uint8_t); -C4BM_TEMPLATE(xtoa_sstream, uint8_t); - -C4BM_TEMPLATE(xtoa_c4_write_dec, int8_t); -C4BM_TEMPLATE(xtoa_c4_itoa, int8_t); -C4BM_TEMPLATE(xtoa_c4_xtoa, int8_t); -C4BM_TEMPLATE(xtoa_c4_to_chars, int8_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(xtoa_std_to_chars, int8_t); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, int8_t); -C4BM_TEMPLATE(xtoa_sprintf, int8_t); -C4BM_TEMPLATE(xtoa_sstream_reuse, int8_t); -C4BM_TEMPLATE(xtoa_sstream, int8_t); - -C4BM_TEMPLATE(xtoa_c4_write_dec, uint16_t); -C4BM_TEMPLATE(xtoa_c4_utoa, uint16_t); -C4BM_TEMPLATE(xtoa_c4_xtoa, uint16_t); -C4BM_TEMPLATE(xtoa_c4_to_chars, uint16_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(xtoa_std_to_chars, uint16_t); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, uint16_t); -C4BM_TEMPLATE(xtoa_sprintf, uint16_t); -C4BM_TEMPLATE(xtoa_sstream_reuse, uint16_t); -C4BM_TEMPLATE(xtoa_sstream, uint16_t); - -C4BM_TEMPLATE(xtoa_c4_write_dec, int16_t); -C4BM_TEMPLATE(xtoa_c4_itoa, int16_t); -C4BM_TEMPLATE(xtoa_c4_xtoa, int16_t); -C4BM_TEMPLATE(xtoa_c4_to_chars, int16_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(xtoa_std_to_chars, int16_t); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, int16_t); -C4BM_TEMPLATE(xtoa_sprintf, int16_t); -C4BM_TEMPLATE(xtoa_sstream_reuse, int16_t); -C4BM_TEMPLATE(xtoa_sstream, int16_t); - -C4BM_TEMPLATE(xtoa_c4_write_dec, uint32_t); -C4BM_TEMPLATE(xtoa_c4_utoa, uint32_t); -C4BM_TEMPLATE(xtoa_c4_xtoa, uint32_t); -C4BM_TEMPLATE(xtoa_c4_to_chars, uint32_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(xtoa_std_to_chars, uint32_t); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, uint32_t); -C4BM_TEMPLATE(xtoa_sprintf, uint32_t); -C4BM_TEMPLATE(xtoa_sstream_reuse, uint32_t); -C4BM_TEMPLATE(xtoa_sstream, uint32_t); - -C4BM_TEMPLATE(xtoa_c4_write_dec, int32_t); -C4BM_TEMPLATE(xtoa_c4_itoa, int32_t); -C4BM_TEMPLATE(xtoa_c4_xtoa, int32_t); -C4BM_TEMPLATE(xtoa_c4_to_chars, int32_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(xtoa_std_to_chars, int32_t); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, int32_t); -C4BM_TEMPLATE(xtoa_sprintf, int32_t); -C4BM_TEMPLATE(xtoa_sstream_reuse, int32_t); -C4BM_TEMPLATE(xtoa_sstream, int32_t); - -C4BM_TEMPLATE(xtoa_c4_write_dec, uint64_t); -C4BM_TEMPLATE(xtoa_c4_utoa, uint64_t); -C4BM_TEMPLATE(xtoa_c4_xtoa, uint64_t); -C4BM_TEMPLATE(xtoa_c4_to_chars, uint64_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(xtoa_std_to_chars, uint64_t); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, uint64_t); -C4BM_TEMPLATE(xtoa_sprintf, uint64_t); -C4BM_TEMPLATE(xtoa_sstream_reuse, uint64_t); -C4BM_TEMPLATE(xtoa_sstream, uint64_t); - -C4BM_TEMPLATE(xtoa_c4_write_dec, int64_t); -C4BM_TEMPLATE(xtoa_c4_itoa, int64_t); -C4BM_TEMPLATE(xtoa_c4_xtoa, int64_t); -C4BM_TEMPLATE(xtoa_c4_to_chars, int64_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(xtoa_std_to_chars, int64_t); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, int64_t); -C4BM_TEMPLATE(xtoa_sprintf, int64_t); -C4BM_TEMPLATE(xtoa_sstream_reuse, int64_t); -C4BM_TEMPLATE(xtoa_sstream, int64_t); - -C4BM_TEMPLATE(xtoa_c4_ftoa, float); -C4BM_TEMPLATE(xtoa_c4_xtoa, float); -C4BM_TEMPLATE(xtoa_c4_to_chars, float); -#ifdef C4CORE_BM_USE_RYU -C4BM_TEMPLATE(xtoa_ryu_f2s, float); -#endif -#ifdef C4CORE_BM_HAVE_TOCHARS -C4BM_TEMPLATE_TO_CHARS_FLOAT(xtoa_std_to_chars, float); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, float); -C4BM_TEMPLATE(xtoa_sprintf, float); -C4BM_TEMPLATE(xtoa_sstream_reuse, float); -C4BM_TEMPLATE(xtoa_sstream, float); - -C4BM_TEMPLATE(xtoa_c4_dtoa, double); -C4BM_TEMPLATE(xtoa_c4_xtoa, double); -C4BM_TEMPLATE(xtoa_c4_to_chars, double); -#ifdef C4CORE_BM_USE_RYU -C4BM_TEMPLATE(xtoa_ryu_d2s, double); -#endif -#ifdef C4CORE_BM_HAVE_TOCHARS -C4BM_TEMPLATE_TO_CHARS_FLOAT(xtoa_std_to_chars, double); -#endif -C4BM_TEMPLATE(xtoa_std_to_string, double); -C4BM_TEMPLATE(xtoa_sprintf, double); -C4BM_TEMPLATE(xtoa_sstream_reuse, double); -C4BM_TEMPLATE(xtoa_sstream, double); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -C4BM_TEMPLATE(atoxhex_c4_read_hex, uint8_t); -C4BM_TEMPLATE(atoxoct_c4_read_oct, uint8_t); -C4BM_TEMPLATE(atoxbin_c4_read_bin, uint8_t); - -C4BM_TEMPLATE(atoxhex_c4_read_hex, int8_t); -C4BM_TEMPLATE(atoxoct_c4_read_oct, int8_t); -C4BM_TEMPLATE(atoxbin_c4_read_bin, int8_t); - -C4BM_TEMPLATE(atoxhex_c4_read_hex, uint16_t); -C4BM_TEMPLATE(atoxoct_c4_read_oct, uint16_t); -C4BM_TEMPLATE(atoxbin_c4_read_bin, uint16_t); - -C4BM_TEMPLATE(atoxhex_c4_read_hex, int16_t); -C4BM_TEMPLATE(atoxoct_c4_read_oct, int16_t); -C4BM_TEMPLATE(atoxbin_c4_read_bin, int16_t); - -C4BM_TEMPLATE(atoxhex_c4_read_hex, uint32_t); -C4BM_TEMPLATE(atoxoct_c4_read_oct, uint32_t); -C4BM_TEMPLATE(atoxbin_c4_read_bin, uint32_t); - -C4BM_TEMPLATE(atoxhex_c4_read_hex, int32_t); -C4BM_TEMPLATE(atoxoct_c4_read_oct, int32_t); -C4BM_TEMPLATE(atoxbin_c4_read_bin, int32_t); - -C4BM_TEMPLATE(atoxhex_c4_read_hex, uint64_t); -C4BM_TEMPLATE(atoxoct_c4_read_oct, uint64_t); -C4BM_TEMPLATE(atoxbin_c4_read_bin, uint64_t); - -C4BM_TEMPLATE(atoxhex_c4_read_hex, int64_t); -C4BM_TEMPLATE(atoxoct_c4_read_oct, int64_t); -C4BM_TEMPLATE(atoxbin_c4_read_bin, int64_t); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -C4BM_TEMPLATE(atox_c4_read_dec, uint8_t); -C4BM_TEMPLATE(atox_c4_atou, uint8_t); -C4BM_TEMPLATE(atox_c4_atox, uint8_t); -C4BM_TEMPLATE(atox_c4_from_chars, uint8_t); -C4BM_TEMPLATE(atox_c4_from_chars_checked, uint8_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(atox_std_from_chars, uint8_t); -#endif -C4BM_TEMPLATE(atox_std_atoi, uint8_t); -C4BM_TEMPLATE(atox_std_strtoul, uint8_t); -C4BM_TEMPLATE(atox_scanf, uint8_t); -C4BM_TEMPLATE(atox_sstream_reuse, uint8_t); -C4BM_TEMPLATE(atox_sstream, uint8_t); - -C4BM_TEMPLATE(atox_c4_read_dec, int8_t); -C4BM_TEMPLATE(atox_c4_atoi, int8_t); -C4BM_TEMPLATE(atox_c4_atox, int8_t); -C4BM_TEMPLATE(atox_c4_from_chars, int8_t); -C4BM_TEMPLATE(atox_c4_from_chars_checked, int8_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(atox_std_from_chars, int8_t); -#endif -C4BM_TEMPLATE(atox_std_atoi, int8_t); -C4BM_TEMPLATE(atox_std_strtol, int8_t); -C4BM_TEMPLATE(atox_scanf, int8_t); -C4BM_TEMPLATE(atox_sstream_reuse, int8_t); -C4BM_TEMPLATE(atox_sstream, int8_t); - -C4BM_TEMPLATE(atox_c4_read_dec, uint16_t); -C4BM_TEMPLATE(atox_c4_atou, uint16_t); -C4BM_TEMPLATE(atox_c4_atox, uint16_t); -C4BM_TEMPLATE(atox_c4_from_chars, uint16_t); -C4BM_TEMPLATE(atox_c4_from_chars_checked, uint16_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(atox_std_from_chars, uint16_t); -#endif -C4BM_TEMPLATE(atox_std_atoi, uint16_t); -C4BM_TEMPLATE(atox_std_strtoul, uint16_t); -C4BM_TEMPLATE(atox_scanf, uint16_t); -C4BM_TEMPLATE(atox_sstream_reuse, uint16_t); -C4BM_TEMPLATE(atox_sstream, uint16_t); - -C4BM_TEMPLATE(atox_c4_read_dec, int16_t); -C4BM_TEMPLATE(atox_c4_atoi, int16_t); -C4BM_TEMPLATE(atox_c4_atox, int16_t); -C4BM_TEMPLATE(atox_c4_from_chars, int16_t); -C4BM_TEMPLATE(atox_c4_from_chars_checked, int16_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(atox_std_from_chars, int16_t); -#endif -C4BM_TEMPLATE(atox_std_atoi, int16_t); -C4BM_TEMPLATE(atox_std_strtol, int16_t); -C4BM_TEMPLATE(atox_scanf, int16_t); -C4BM_TEMPLATE(atox_sstream_reuse, int16_t); -C4BM_TEMPLATE(atox_sstream, int16_t); - -C4BM_TEMPLATE(atox_c4_read_dec, uint32_t); -C4BM_TEMPLATE(atox_c4_atou, uint32_t); -C4BM_TEMPLATE(atox_c4_atox, uint32_t); -C4BM_TEMPLATE(atox_c4_from_chars, uint32_t); -C4BM_TEMPLATE(atox_c4_from_chars_checked, uint32_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(atox_std_from_chars, uint32_t); -#endif -C4BM_TEMPLATE(atox_std_atoi, uint32_t); -C4BM_TEMPLATE(atox_std_strtoul, uint32_t); -C4BM_TEMPLATE(atox_scanf, uint32_t); -C4BM_TEMPLATE(atox_sstream_reuse, uint32_t); -C4BM_TEMPLATE(atox_sstream, uint32_t); - -C4BM_TEMPLATE(atox_c4_read_dec, int32_t); -C4BM_TEMPLATE(atox_c4_atoi, int32_t); -C4BM_TEMPLATE(atox_c4_atox, int32_t); -C4BM_TEMPLATE(atox_c4_from_chars, int32_t); -C4BM_TEMPLATE(atox_c4_from_chars_checked, int32_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(atox_std_from_chars, int32_t); -#endif -C4BM_TEMPLATE(atox_std_atoi, int32_t); -C4BM_TEMPLATE(atox_std_strtol, int32_t); -C4BM_TEMPLATE(atox_scanf, int32_t); -C4BM_TEMPLATE(atox_sstream_reuse, int32_t); -C4BM_TEMPLATE(atox_sstream, int32_t); - -C4BM_TEMPLATE(atox_c4_read_dec, uint64_t); -C4BM_TEMPLATE(atox_c4_atou, uint64_t); -C4BM_TEMPLATE(atox_c4_atox, uint64_t); -C4BM_TEMPLATE(atox_c4_from_chars, uint64_t); -C4BM_TEMPLATE(atox_c4_from_chars_checked, uint64_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(atox_std_from_chars, uint64_t); -#endif -C4BM_TEMPLATE(atox_std_atol, uint64_t); -C4BM_TEMPLATE(atox_std_strtoull, uint64_t); -C4BM_TEMPLATE(atox_scanf, uint64_t); -C4BM_TEMPLATE(atox_sstream_reuse, uint64_t); -C4BM_TEMPLATE(atox_sstream, uint64_t); - -C4BM_TEMPLATE(atox_c4_read_dec, int64_t); -C4BM_TEMPLATE(atox_c4_atoi, int64_t); -C4BM_TEMPLATE(atox_c4_atox, int64_t); -C4BM_TEMPLATE(atox_c4_from_chars, int64_t); -C4BM_TEMPLATE(atox_c4_from_chars_checked, int64_t); -#if (C4_CPP >= 17) -C4BM_TEMPLATE_TO_CHARS_INT(atox_std_from_chars, int64_t); -#endif -C4BM_TEMPLATE(atox_std_atol, int64_t); -C4BM_TEMPLATE(atox_std_strtoll, int64_t); -C4BM_TEMPLATE(atox_scanf, int64_t); -C4BM_TEMPLATE(atox_sstream_reuse, int64_t); -C4BM_TEMPLATE(atox_sstream, int64_t); - -C4BM_TEMPLATE(atox_c4_atof, float); -C4BM_TEMPLATE(atox_c4_atox, float); -C4BM_TEMPLATE(atox_c4_from_chars, float); -C4BM_TEMPLATE(atox_fast_float, float); -#ifdef C4CORE_BM_HAVE_TOCHARS -C4BM_TEMPLATE_TO_CHARS_FLOAT(atox_std_from_chars, float); -#endif -#ifdef C4CORE_BM_USE_RYU -C4BM_TEMPLATE(atox_ryu_s2f, float); -#endif -#ifdef C4CORE_BM_USE_FP -C4BM_FP_BENCHMARK(atox_fp_from_chars_limited, float); -C4BM_FP_BENCHMARK(atox_fp_from_chars_unlimited, float); -#endif -C4BM_TEMPLATE(atox_std_atof, float); -C4BM_TEMPLATE(atox_std_strtof, float); -C4BM_TEMPLATE(atox_std_stof, float); -C4BM_TEMPLATE(atox_scanf, float); -C4BM_TEMPLATE(atox_sstream_reuse, float); -C4BM_TEMPLATE(atox_sstream, float); - -C4BM_TEMPLATE(atox_c4_atod, double); -C4BM_TEMPLATE(atox_c4_atox, double); -C4BM_TEMPLATE(atox_c4_from_chars, double); -C4BM_TEMPLATE(atox_fast_float, double); -#ifdef C4CORE_BM_HAVE_TOCHARS -C4BM_TEMPLATE_TO_CHARS_FLOAT(atox_std_from_chars, double); -#endif -#ifdef C4CORE_BM_USE_RYU -C4BM_TEMPLATE(atox_ryu_s2d, double); -#endif -#ifdef C4CORE_BM_USE_FP -C4BM_FP_BENCHMARK(atox_fp_from_chars_limited, double); -C4BM_FP_BENCHMARK(atox_fp_from_chars_unlimited, double); -#endif -C4BM_TEMPLATE(atox_std_atof, double); -C4BM_TEMPLATE(atox_std_strtod, double); -C4BM_TEMPLATE(atox_std_stod, double); -C4BM_TEMPLATE(atox_scanf, double); -C4BM_TEMPLATE(atox_sstream_reuse, double); -C4BM_TEMPLATE(atox_sstream, double); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -int main(int argc, char *argv[]) -{ - bm::Initialize(&argc, argv); - bm::RunSpecifiedBenchmarks(); - return 0; -} - -#include diff --git a/thirdparty/ryml/ext/c4core/bm/bm_charconv.hpp b/thirdparty/ryml/ext/c4core/bm/bm_charconv.hpp deleted file mode 100644 index 3a2b7dc13..000000000 --- a/thirdparty/ryml/ext/c4core/bm/bm_charconv.hpp +++ /dev/null @@ -1,454 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -inline double getmax(std::vector const& v) -{ - return *(std::max_element(std::begin(v), std::end(v))); -} -inline double getmin(std::vector const& v) -{ - return *(std::min_element(std::begin(v), std::end(v))); -} -inline double getrange(std::vector const& v) -{ - auto min_max = std::minmax_element(std::begin(v), std::end(v)); - return *min_max.second - *min_max.first; -} - -#define _c4bm_stats \ - /*->Repetitions(20)*/ \ - ->DisplayAggregatesOnly(true) \ - ->ComputeStatistics("range", &getrange) \ - ->ComputeStatistics("max", &getmax) \ - ->ComputeStatistics("min", &getmin) - -#define C4BM_TEMPLATE(fn, ...) BENCHMARK_TEMPLATE(fn, __VA_ARGS__) _c4bm_stats - - -// benchmarks depending on c++17 features are disabled using the -// preprocessor. google benchmark has state.SkipWithError() but it -// makes the program return a nonzero exit code when it finishes. So -// we must resort to the preprocessor to conditionally disable these -// benchmarks -#if defined(__cpp_lib_to_chars) || (C4_CPP >= 17) -#define C4BM_TEMPLATE_TO_CHARS_INT(fn, ...) BENCHMARK_TEMPLATE(fn, __VA_ARGS__) _c4bm_stats -#define C4BM_TEMPLATE_TO_CHARS_FLOAT(fn, ...) BENCHMARK_TEMPLATE(fn, __VA_ARGS__) _c4bm_stats -#else -#define C4BM_TEMPLATE_TO_CHARS_INT(fn, ...) void shutup_extra_semicolon() -#define C4BM_TEMPLATE_TO_CHARS_FLOAT(fn, ...) void shutup_extra_semicolon() -#endif - -C4_SUPPRESS_WARNING_GCC_CLANG_PUSH -C4_SUPPRESS_WARNING_GCC_CLANG("-Wdouble-promotion") -C4_SUPPRESS_WARNING_GCC_CLANG("-Wdeprecated") -C4_SUPPRESS_WARNING_GCC_CLANG("-Wsign-conversion") -C4_SUPPRESS_WARNING_GCC_CLANG("-Wconversion") - -#include - - -namespace bm = benchmark; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// utilities for use in the benchmarks below - - -// facilities to deuglify SFINAE -#define C4FOR(ty, condition) \ - template \ - typename std::enable_if::type -#define C4FOR2(ty1, ty2, condition) \ - template \ - typename std::enable_if::type - -#define isint(ty) std::is_integral::value -#define isiint(ty) std::is_integral::value && !std::is_unsigned::value -#define isuint(ty) std::is_integral::value && std::is_unsigned::value -#define isreal(ty) std::is_floating_point::value -#define isfloat(ty) std::is_same::value -#define isdouble(ty) std::is_same::value - - -/** convenience wrapper to avoid boilerplate code */ -template -void report(bm::State &st, size_t numvals=1) -{ - int64_t iters = (int64_t)(st.iterations() * numvals); - int64_t bytes = (int64_t)(st.iterations() * numvals * sizeof(T)); - st.SetBytesProcessed(bytes); - st.SetItemsProcessed(iters); -} -template -void report_threadavg(bm::State &st, size_t numvals=1) -{ - int64_t iters = (int64_t)(st.iterations() * numvals); - int64_t bytes = (int64_t)(st.iterations() * numvals * sizeof(T)); - st.SetBytesProcessed(bytes / (int64_t)st.threads()); - st.SetItemsProcessed(iters / (int64_t)st.threads()); -} - -template -constexpr bool is_pot(T val) -{ - return val > 0 && (val & (val-T(1))) == 0; -} - -constexpr const uint64_t kSeed = 37; -constexpr const size_t kNumValues = 1u<<20; // 1.049M -C4_STATIC_ASSERT(is_pot(kNumValues)); - - -template -T gen(Dist &dist, Eng &eng) -{ - return static_cast(dist(eng)); -} - -template -T gen_pos(Dist &dist, Eng &eng) -{ - T val = static_cast(dist(eng)); - while(val <= T(0)) - val = static_cast(dist(eng)); - return val; -} - -/** generate in place a random sequence of values: integral version*/ -C4FOR(T, isint) -generate_n(T *begin, T *end) -{ - // do not use T in the distribution: - // N4659 29.6.1.1 [rand.req.genl]/1e requires one of short, int, long, long long, unsigned short, unsigned int, unsigned long, or unsigned long long - std::uniform_int_distribution idist; - c4::rng::pcg rng(kSeed); - for(; begin != end; ++begin) - *begin = gen(idist, rng); -} - -C4FOR(T, isint) -generate_n_positive(T *begin, T *end) -{ - // do not use T in the distribution: - // N4659 29.6.1.1 [rand.req.genl]/1e requires one of short, int, long, long long, unsigned short, unsigned int, unsigned long, or unsigned long long - std::uniform_int_distribution idist; - c4::rng::pcg rng(kSeed); - for(; begin != end; ++begin) - *begin = gen_pos(idist, rng); -} - -/** generate in place a random sequence of values: real-number version*/ -C4FOR(T, isreal) -generate_n(T *begin, T *end) -{ - c4::rng::pcg rng(kSeed); - // make sure we also have some integral numbers in the real sequence - T *rstart = begin + (std::distance(begin, end) / 20); // 5% integral numbers - std::uniform_int_distribution idist; - std::uniform_real_distribution rdist; - for(; begin != rstart; ++begin) - *begin = gen(idist, rng); - for(; begin != end; ++begin) - *begin = gen(rdist, rng); -} - -/** generate in place a random sequence of values: real-number version*/ -C4FOR(T, isreal) -generate_n_positive(T *begin, T *end) -{ - c4::rng::pcg rng(kSeed); - // make sure we also have some integral numbers in the real sequence - T *rstart = begin + (std::distance(begin, end) / 20); // 5% integral numbers - std::uniform_int_distribution idist; - std::uniform_real_distribution rdist; - for(; begin != rstart; ++begin) - *begin = gen_pos(idist, rng); - for(; begin != end; ++begin) - *begin = gen_pos(rdist, rng); -} - - -/** a ring buffer with input values for xtoa benchmarks */ -template -struct random_values -{ - std::vector v; - mutable size_t curr; - size_t szm1; - T next() const { T f = v[curr]; curr = (curr + 1) & szm1; return f; } - random_values(bool positive_only, size_t sz) : v(sz), curr(0), szm1(sz) - { - C4_CHECK(is_pot(sz)); - if(positive_only) - generate_n_positive(&v.front(), &v.back()); - else - generate_n(&v.front(), &v.back()); - } -}; -template -using random_values_cref = random_values const&; - -template -random_values_cref mkvals() -{ - static random_values vals(/*positive_only*/false, kNumValues); - return vals; -} -template -random_values_cref mkvals_positive() -{ - static random_values vals(/*positive_only*/true, kNumValues); - return vals; -} - - -/** a ring buffer with input strings for atox benchmarks */ -struct random_strings -{ - std::vector v_s; - std::vector v; - std::vector arena; - mutable size_t curr; - size_t szm1; - - C4_HOT C4_ALWAYS_INLINE c4::csubstr next() const noexcept { c4::csubstr f = v[curr]; curr = (curr + 1) & szm1; return f; } - C4_HOT C4_ALWAYS_INLINE std::string const& next_s() const noexcept { std::string const& f = v_s[curr]; curr = (curr + 1) & szm1; return f; } - - random_strings() = default; - - template - void _init(random_values const& tmp) - { - C4_CHECK(is_pot(tmp.v.size())); - v.resize(tmp.v.size()); - v_s.resize(tmp.v.size()); - curr = 0; - szm1 = tmp.v.size() - 1; - } - void _build_arena() - { - size_t sum = 0; - for(std::string const& s : v_s) - sum += s.size(); - sum += v_s.size(); - v.resize(v_s.size()); - arena.resize(sum); - size_t pos = 0; - size_t i = 0; - for(std::string const& s : v_s) - { - memcpy(&arena[pos], s.data(), s.size()); - v[i++] = c4::csubstr(&arena[pos], s.size()); - pos += s.size(); - arena[pos++] = '\0'; - } - } - - template - void init_as(random_values const& tmp) - { - _init(tmp); - for(size_t i = 0; i < v.size(); ++i) - c4::catrs(&v_s[i], tmp.v[i]); - _build_arena(); - } - template - void init_as_hex(random_values const& tmp, bool with_prefix) - { - _init(tmp); - for(size_t i = 0; i < v.size(); ++i) - { - c4::catrs(&v_s[i], c4::fmt::hex(tmp.v[i])); - if(!with_prefix) - _erase_radix_prefix(&v_s[i]); - } - _build_arena(); - } - template - void init_as_oct(random_values const& tmp, bool with_prefix) - { - _init(tmp); - for(size_t i = 0; i < v.size(); ++i) - { - c4::catrs(&v_s[i], c4::fmt::oct(tmp.v[i])); - if(!with_prefix) - _erase_radix_prefix(&v_s[i]); - } - _build_arena(); - } - template - void init_as_bin(random_values const& tmp, bool with_prefix) - { - _init(tmp); - for(size_t i = 0; i < v.size(); ++i) - { - c4::catrs(&v_s[i], c4::fmt::bin(tmp.v[i])); - if(!with_prefix) - _erase_radix_prefix(&v_s[i]); - } - _build_arena(); - } - - static void _erase_radix_prefix(std::string *s) - { - C4_ASSERT(s->front() != '-'); - s->erase(0, 2); - } -}; -using random_strings_cref = random_strings const&; - -template -random_strings_cref mkstrings() -{ - static random_strings rs; - if(rs.v.empty()) - rs.init_as(mkvals()); - return rs; -} -template -random_strings_cref mkstrings_positive() -{ - static random_strings rs; - if(rs.v.empty()) - rs.init_as(mkvals_positive()); - return rs; -} -template -random_strings_cref mkstrings_hex(bool with_prefix=true) -{ - static random_strings rs; - static random_strings rs_wo_prefix; - if(with_prefix) - { - if(rs.v.empty()) - rs.init_as_hex(mkvals()); - return rs; - } - else - { - if(rs_wo_prefix.v.empty()) - rs_wo_prefix.init_as_hex(mkvals(), false); - return rs_wo_prefix; - } -} -template -random_strings_cref mkstrings_hex_positive(bool with_prefix=true) -{ - static random_strings rs; - static random_strings rs_wo_prefix; - if(with_prefix) - { - if(rs.v.empty()) - rs.init_as_hex(mkvals_positive(), true); - return rs; - } - else - { - if(rs_wo_prefix.v.empty()) - rs_wo_prefix.init_as_hex(mkvals_positive(), false); - return rs_wo_prefix; - } -} -template -random_strings_cref mkstrings_oct(bool with_prefix=true) -{ - static random_strings rs; - static random_strings rs_wo_prefix; - if(with_prefix) - { - if(rs.v.empty()) - rs.init_as_oct(mkvals(), true); - return rs; - } - else - { - if(rs_wo_prefix.v.empty()) - rs_wo_prefix.init_as_oct(mkvals(), false); - return rs_wo_prefix; - } -} -template -random_strings_cref mkstrings_oct_positive(bool with_prefix=true) -{ - static random_strings rs; - static random_strings rs_wo_prefix; - if(with_prefix) - { - if(rs.v.empty()) - rs.init_as_oct(mkvals_positive(), true); - return rs; - } - else - { - if(rs_wo_prefix.v.empty()) - rs_wo_prefix.init_as_oct(mkvals_positive(), false); - return rs_wo_prefix; - } -} -template -random_strings_cref mkstrings_bin(bool with_prefix=true) -{ - static random_strings rs; - static random_strings rs_wo_prefix; - if(with_prefix) - { - if(rs.v.empty()) - rs.init_as_bin(mkvals(), true); - return rs; - } - else - { - if(rs_wo_prefix.v.empty()) - rs_wo_prefix.init_as_bin(mkvals(), false); - return rs_wo_prefix; - } -} -template -random_strings_cref mkstrings_bin_positive(bool with_prefix=true) -{ - static random_strings rs; - static random_strings rs_wo_prefix; - if(with_prefix) - { - if(rs.v.empty()) - rs.init_as_bin(mkvals_positive(), true); - return rs; - } - else - { - if(rs_wo_prefix.v.empty()) - rs_wo_prefix.init_as_bin(mkvals_positive(), false); - return rs_wo_prefix; - } -} - - -/** a character buffer, easily convertible to c4::substr */ -template -struct sbuf -{ - char buf_[Dim]; - c4::substr buf; - sbuf() : buf_(), buf(buf_) {} - C4_HOT C4_ALWAYS_INLINE operator c4::substr& () { return buf; } - char* begin() { return buf.begin(); } - char* end() { return buf.end(); } -}; - -using string_buffer = sbuf<>; - -#define C4DOALL(n) for(size_t elm##__LINE__ = 0; elm##__LINE__ < n; ++elm##__LINE__) diff --git a/thirdparty/ryml/ext/c4core/bm/bm_format.cpp b/thirdparty/ryml/ext/c4core/bm/bm_format.cpp deleted file mode 100644 index 64d23cbf3..000000000 --- a/thirdparty/ryml/ext/c4core/bm/bm_format.cpp +++ /dev/null @@ -1,900 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace bm = benchmark; - -double getmax(std::vector const& v) -{ - return *(std::max_element(std::begin(v), std::end(v))); -} -double getmin(std::vector const& v) -{ - return *(std::min_element(std::begin(v), std::end(v))); -} -double getrange(std::vector const& v) -{ - auto min_max = std::minmax_element(std::begin(v), std::end(v)); - return *min_max.second - *min_max.first; -} - -#define _c4bm_stats \ - /*->Repetitions(20)*/ \ - ->DisplayAggregatesOnly(true) \ - ->ComputeStatistics("range", &getrange) \ - ->ComputeStatistics("max", &getmax) \ - ->ComputeStatistics("min", &getmin) - -#define C4BM(fn) BENCHMARK(fn) _c4bm_stats - -/** convenience wrapper to avoid boilerplate code */ -void report(bm::State &st, size_t sz) -{ - st.SetBytesProcessed(st.iterations() * static_cast(sz)); - st.SetItemsProcessed(st.iterations()); -} - - -const c4::csubstr sep = " --- "; - - -#define _c4argbundle_fmt "hello here you have some numbers: "\ - "1={}, 2={}, 3={}, 4={}, 5={}, 6={}, 7={}, 8={}, 9={}, size_t(283482349)={}, "\ - "\" \"=\"{}\", \"haha\"=\"{}\", std::string(\"hehe\")=\"{}\", "\ - "str=\"{}\"" - -#define _c4argbundle_fmt_printf "hello here you have some numbers: "\ - "1=%d, 2=%d, 3=%d, 4=%d, 5=%d, 6=%d, 7=%d, 8=%d, 9=%d, size_t(283482349)=%zu, "\ - "\" \"=\"%s\", \"haha\"=\"%s\", std::string(\"hehe\")=\"%s\", "\ - "str=\"%s\"" - -#define _c4argbundle_fmt_printf_sep "hello here you have some numbers: "\ - "1=%d%s2=%d%s3=%d%s4=%d%s5=%d%s6=%d%s7=%d%s8=%d%s9=%d%ssize_t(283482349)=%zu%s"\ - "\" \"=\"%s\"%s\"haha\"=\"%s\"%sstd::string(\"hehe\")=\"%s\"%s"\ - "str=\"%s\"" - -#define _c4argbundle \ - 1, 2, 3, 4, 5, 6, 7, 8, 9, size_t(283482349),\ - " ", "haha", std::string("hehe"),\ - std::string("asdlklkasdlkjasd asdlkjasdlkjasdlkjasdoiasdlkjasldkj") - -#define _c4argbundle_printf \ - 1, 2, 3, 4, 5, 6, 7, 8, 9, size_t(283482349),\ - " ", "haha", std::string("hehe").c_str(),\ - std::string("asdlklkasdlkjasd asdlkjasdlkjasdlkjasdoiasdlkjasldkj").c_str() - -#define _c4argbundle_printf_sep \ - 1, sep.str, 2, sep.str, 3, sep.str, 4, sep.str, 5, sep.str, 6, sep.str, 7, sep.str, 8, sep.str, 9, sep.str, size_t(283482349), sep.str,\ - " ", sep.str, "haha", sep.str, std::string("hehe").c_str(), sep.str,\ - std::string("asdlklkasdlkjasd asdlkjasdlkjasdlkjasdoiasdlkjasldkj").c_str() - -#define _c4argbundle_lshift \ - 1 << 2 << 3 << 4 << 5 << 6 << 7 << 8 << 9 << size_t(283482349)\ - << " " << "haha" << std::string("hehe")\ - << std::string("asdlklkasdlkjasd asdlkjasdlkjasdlkjasdoiasdlkjasldkj") - -#define _c4argbundle_lshift_sep \ - 1 << sep << 2 << sep << 3 << sep << 4 << sep << 5 << sep << 6 << sep << 7 << sep << 8 << sep << 9 << sep << size_t(283482349)\ - << sep << " " << sep << "haha" << sep << std::string("hehe")\ - << sep << std::string("asdlklkasdlkjasd asdlkjasdlkjasdlkjasdoiasdlkjasldkj") - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace dump2str { -std::string c_style_subject; -void c_style(c4::csubstr s) { c_style_subject.append(s.str, s.len); } -struct cpp_style -{ - std::string subject = {}; - void operator() (c4::csubstr s) { subject.append(s.str, s.len); } -}; -struct lambda_style -{ - std::string subject = {}; -}; -} // namespace dump2str - -namespace dump2file { -FILE * c_style_subject; -void c_style(c4::csubstr s) { fwrite(s.str, 1, s.len, c_style_subject); } -struct cpp_style -{ - FILE * subject; - cpp_style() : subject(fopen("asdkjhasdkjhsdfoiefkjn", "wb")) {} - ~cpp_style() { fclose(subject); } - void operator() (c4::csubstr s) { fwrite(s.str, 1, s.len, subject); } -}; -struct lambda_style -{ - lambda_style() : subject(fopen("asdkjhasdkjhsdfoiefkjn", "wb")) {} - ~lambda_style() { fclose(subject); } - FILE * subject; -}; -} // namespace dump2fil - -template -C4_ALWAYS_INLINE typename std::enable_if::value, std::string>::type -std_to_string(T const& a) -{ - return std::to_string(a); -} - -template -C4_ALWAYS_INLINE typename std::enable_if::value, std::string const&>::type -std_to_string(std::string const& a) -{ - return a; -} - -C4_ALWAYS_INLINE std::string std_to_string(c4::csubstr a) -{ - return std::string(a.str, a.len); -} - -template -C4_ALWAYS_INLINE typename std::enable_if< ! std::is_arithmetic::value, std::string>::type -std_to_string(T const& a) -{ - return std::string(a); -} - -C4_ALWAYS_INLINE void cat_std_string_impl(std::string *) -{ -} - -C4_ALWAYS_INLINE void catsep_std_string_impl(std::string *) -{ -} - -template -void cat_std_string_impl(std::string *s, Arg const& a, Args const& ...args) -{ - *s += std_to_string(a); - cat_std_string_impl(s, args...); -} - -template -void catsep_std_string_impl(std::string *s, Arg const& a, Args const& ...args) -{ - *s += std_to_string(a); - if(sizeof...(args) > 0) - { - s->append(sep.str, sep.len); - catsep_std_string_impl(s, args...); - } -} - -void cat_std_stringstream_impl(std::stringstream &) -{ -} -void catsep_std_stringstream_impl(std::stringstream &) -{ -} - -template -void cat_std_stringstream_impl(std::stringstream &ss, Arg const& a, Args const& ...args) -{ - ss << a; - cat_std_stringstream_impl(ss, args...); -} - -template -void catsep_std_stringstream_impl(std::stringstream &ss, Arg const& a, Args const& ...args) -{ - ss << sep << a; - cat_std_stringstream_impl(ss, args...); -} - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void cat_c4cat_substr(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = 0; - for(auto _ : st) - { - sz = cat(buf, _c4argbundle); - } - report(st, sz); -} - -void cat_c4catrs_reuse(bm::State &st) -{ - std::string buf; - size_t sz = 0; - for(auto _ : st) - { - c4::catrs(&buf, _c4argbundle); - sz = buf.size(); - } - report(st, sz); -} - -void cat_c4catrs_no_reuse(bm::State &st) -{ - size_t sz = 0; - for(auto _ : st) - { - auto buf = c4::catrs(_c4argbundle); - sz = buf.size(); - } - report(st, sz); -} - -void cat_c4catdump_c_style_static_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::cat(buf, _c4argbundle); - for(auto _ : st) - { - c4::cat_dump<&dump2str::c_style>(buf, _c4argbundle); - } - report(st, sz); -} - -void cat_c4catdump_c_style_dynamic_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::cat(buf, _c4argbundle); - for(auto _ : st) - { - sz = c4::cat_dump(&dump2str::c_style, buf, _c4argbundle); - } - report(st, sz); -} - -void cat_c4catdump_cpp_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::cat(buf, _c4argbundle); - dump2str::cpp_style dumper; - for(auto _ : st) - { - sz = c4::cat_dump(dumper, buf, _c4argbundle); - } - report(st, sz); -} - -void cat_c4catdump_lambda_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::cat(buf, _c4argbundle); - dump2str::lambda_style dumper; - auto lambda = [&dumper](c4::csubstr s) { dumper.subject.append(s.str, s.len); }; - for(auto _ : st) - { - sz = c4::cat_dump(lambda, buf, _c4argbundle); - } - report(st, sz); -} - -void cat_stdsstream_reuse(bm::State &st) -{ - size_t sz = 0; - std::stringstream ss; - for(auto _ : st) - { - ss.clear(); - ss.str(""); - cat_std_stringstream_impl(ss, _c4argbundle); - sz = ss.str().size(); - } - report(st, sz); -} - -void cat_stdsstream_no_reuse(bm::State &st) -{ - size_t sz = 0; - for(auto _ : st) - { - std::stringstream ss; - cat_std_stringstream_impl(ss, _c4argbundle); - sz = ss.str().size(); - } - report(st, sz); -} - -void cat_std_to_string_reuse(bm::State &st) -{ - size_t sz = 0; - std::string s; - for(auto _ : st) - { - s.clear(); - cat_std_string_impl(&s, _c4argbundle); - sz = s.size(); - } - report(st, sz); -} - -void cat_std_to_string_no_reuse(bm::State &st) -{ - size_t sz = 0; - for(auto _ : st) - { - std::string s; - cat_std_string_impl(&s, _c4argbundle); - sz = s.size(); - } - report(st, sz); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void catfile_c4catdump_c_style_static_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - dump2file::cpp_style fileowner; - dump2file::c_style_subject = fileowner.subject; - size_t sz = c4::cat(buf, _c4argbundle); - for(auto _ : st) - { - c4::cat_dump<&dump2file::c_style>(buf, _c4argbundle); - } - report(st, sz); -} - -void catfile_c4catdump_c_style_dynamic_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - dump2file::cpp_style fileowner; - dump2file::c_style_subject = fileowner.subject; - size_t sz = c4::cat(buf, _c4argbundle); - for(auto _ : st) - { - c4::cat_dump(&dump2file::c_style, buf, _c4argbundle); - } - report(st, sz); -} - -void catfile_c4catdump_cpp_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::cat(buf, _c4argbundle); - dump2file::cpp_style dumper; - for(auto _ : st) - { - sz = c4::cat_dump(dumper, buf, _c4argbundle); - } - report(st, sz); -} - -void catfile_c4catdump_lambda_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::cat(buf, _c4argbundle); - dump2file::lambda_style dumper; - auto lambda = [&dumper](c4::csubstr s) { fwrite(s.str, 1, s.len, dumper.subject); }; - for(auto _ : st) - { - sz = c4::cat_dump(lambda, buf, _c4argbundle); - } - report(st, sz); -} - -void catfile_fprintf(bm::State &st) -{ - char buf[256]; - size_t sz = c4::cat(buf, _c4argbundle); - dump2file::cpp_style dumper; - for(auto _ : st) - { - fprintf(dumper.subject, _c4argbundle_fmt_printf, _c4argbundle_printf); - } - report(st, sz); -} - -void catfile_ofstream(bm::State &st) -{ - char buf[256]; - size_t sz = c4::cat(buf, _c4argbundle); - std::ofstream ofs("ddofgufgbmn4g0rtglf", std::ios::out|std::ios::binary); - for(auto _ : st) - { - ofs << _c4argbundle_lshift; - } - report(st, sz); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void catsep_c4cat_substr(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = 0; - for(auto _ : st) - { - sz = catsep(buf, _c4argbundle); - } - report(st, sz); -} - -void catsep_c4catrs_reuse(bm::State &st) -{ - std::string buf; - size_t sz = 0; - for(auto _ : st) - { - c4::catseprs(&buf, _c4argbundle); - sz = buf.size(); - } - report(st, sz); -} - -void catsep_c4catrs_no_reuse(bm::State &st) -{ - size_t sz = 0; - for(auto _ : st) - { - auto buf = c4::catseprs(sep, _c4argbundle); - sz = buf.size(); - } - report(st, sz); -} - -void catsep_c4catdump_c_style_static_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::catsep(buf, _c4argbundle); - for(auto _ : st) - { - c4::catsep_dump<&dump2str::c_style>(buf, sep, _c4argbundle); - } - report(st, sz); -} - -void catsep_c4catdump_c_style_dynamic_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::catsep(buf, _c4argbundle); - for(auto _ : st) - { - sz = c4::catsep_dump(&dump2str::c_style, buf, sep, _c4argbundle); - } - report(st, sz); -} - -void catsep_c4catdump_cpp_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::catsep(buf, _c4argbundle); - dump2str::cpp_style dumper; - for(auto _ : st) - { - sz = c4::catsep_dump(dumper, buf, sep, _c4argbundle); - } - report(st, sz); -} - -void catsep_c4catdump_lambda_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::catsep(buf, sep, _c4argbundle); - dump2str::lambda_style dumper; - auto lambda = [&dumper](c4::csubstr s) { dumper.subject.append(s.str, s.len); }; - for(auto _ : st) - { - sz = c4::catsep_dump(lambda, buf, _c4argbundle); - } - report(st, sz); -} - -void catsep_stdsstream_reuse(bm::State &st) -{ - size_t sz = 0; - std::stringstream ss; - for(auto _ : st) - { - ss.clear(); - ss.str(""); - catsep_std_stringstream_impl(ss, sep, _c4argbundle); - sz = ss.str().size(); - } - report(st, sz); -} - -void catsep_stdsstream_no_reuse(bm::State &st) -{ - size_t sz = 0; - for(auto _ : st) - { - std::stringstream ss; - catsep_std_stringstream_impl(ss, sep, _c4argbundle); - sz = ss.str().size(); - } - report(st, sz); -} - -void catsep_std_to_string_reuse(bm::State &st) -{ - size_t sz = 0; - std::string s; - for(auto _ : st) - { - s.clear(); - catsep_std_string_impl(&s, sep, _c4argbundle); - sz = s.size(); - } - report(st, sz); -} - -void catsep_std_to_string_no_reuse(bm::State &st) -{ - size_t sz = 0; - for(auto _ : st) - { - std::string s; - catsep_std_string_impl(&s, sep, _c4argbundle); - sz = s.size(); - } - report(st, sz); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void catsepfile_c4catsepdump_c_style_static_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - dump2file::cpp_style fileowner; - dump2file::c_style_subject = fileowner.subject; - size_t sz = c4::catsep(buf, sep, _c4argbundle); - for(auto _ : st) - { - c4::catsep_dump<&dump2file::c_style>(buf, sep, _c4argbundle); - } - report(st, sz); -} - -void catsepfile_c4catsepdump_c_style_dynamic_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - dump2file::cpp_style fileowner; - dump2file::c_style_subject = fileowner.subject; - size_t sz = c4::catsep(buf, sep, _c4argbundle); - for(auto _ : st) - { - c4::catsep_dump(&dump2file::c_style, buf, sep, _c4argbundle); - } - report(st, sz); -} - -void catsepfile_c4catsepdump_cpp_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::catsep(buf, sep, _c4argbundle); - dump2file::cpp_style dumper; - for(auto _ : st) - { - c4::catsep_dump(dumper, buf, sep, _c4argbundle); - } - report(st, sz); -} - -void catsepfile_c4catsepdump_lambda_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::catsep(buf, sep, _c4argbundle); - dump2file::lambda_style dumper; - auto lambda = [&dumper](c4::csubstr s) { fwrite(s.str, 1, s.len, dumper.subject); }; - for(auto _ : st) - { - c4::catsep_dump(lambda, buf, sep, _c4argbundle); - } - report(st, sz); -} - -void catsepfile_fprintf(bm::State &st) -{ - char buf[256]; - size_t sz = c4::catsep(buf, sep, _c4argbundle); - dump2file::cpp_style dumper; - for(auto _ : st) - { - fprintf(dumper.subject, _c4argbundle_fmt_printf_sep, _c4argbundle_printf_sep); - } - report(st, sz); -} - -void catsepfile_ofstream(bm::State &st) -{ - char buf[256]; - size_t sz = c4::catsep(buf, sep, _c4argbundle); - std::ofstream ofs("ddofgufgbmn4g0rtglf", std::ios::out|std::ios::binary); - for(auto _ : st) - { - ofs << _c4argbundle_lshift_sep; - } - report(st, sz); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void format_c4format(bm::State &st) -{ - char buf_[512]; - c4::substr buf(buf_); - size_t sz = 0; - for(auto _ : st) - { - sz = format(buf, _c4argbundle_fmt, _c4argbundle); - } - report(st, sz); -} - -void format_c4formatrs_reuse(bm::State &st) -{ - std::string buf; - size_t sz = 0; - for(auto _ : st) - { - c4::formatrs(&buf, _c4argbundle_fmt, _c4argbundle); - sz = buf.size(); - } - report(st, sz); -} - -void format_c4formatrs_no_reuse(bm::State &st) -{ - size_t sz = 0; - for(auto _ : st) - { - auto buf = c4::formatrs(_c4argbundle_fmt, _c4argbundle); - sz = buf.size(); - } - report(st, sz); -} - -void format_c4formatdump_c_style_static_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::format(buf, _c4argbundle_fmt, _c4argbundle); - for(auto _ : st) - { - c4::format_dump<&dump2str::c_style>(buf, _c4argbundle_fmt, _c4argbundle); - } - report(st, sz); -} - -void format_c4formatdump_c_style_dynamic_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::format(buf, _c4argbundle_fmt, _c4argbundle); - for(auto _ : st) - { - c4::format_dump(&dump2str::c_style, buf, _c4argbundle_fmt, _c4argbundle); - } - report(st, sz); -} - -void format_c4formatdump_cpp_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::format(buf, _c4argbundle_fmt, _c4argbundle); - dump2str::cpp_style dumper; - for(auto _ : st) - { - c4::format_dump(dumper, buf, _c4argbundle_fmt, _c4argbundle); - } - report(st, sz); -} - -void format_c4formatdump_lambda_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::format(buf, _c4argbundle_fmt, _c4argbundle); - dump2str::lambda_style dumper; - auto lambda = [&dumper](c4::csubstr s) { dumper.subject.append(s.str, s.len); }; - for(auto _ : st) - { - c4::format_dump(lambda, buf, _c4argbundle_fmt, _c4argbundle); - } - report(st, sz); -} - -void format_snprintf(bm::State &st) -{ - char buf_[512]; - c4::substr buf(buf_); - size_t sz = 0; - for(auto _ : st) - { - sz = (size_t) snprintf(buf.str, buf.len, _c4argbundle_fmt_printf, _c4argbundle_printf); - } - report(st, sz); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void formatfile_c4formatdump_c_style_static_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - dump2file::cpp_style fileowner; - dump2file::c_style_subject = fileowner.subject; - size_t sz = c4::format(buf, _c4argbundle_fmt, _c4argbundle); - for(auto _ : st) - { - c4::format_dump<&dump2file::c_style>(buf, _c4argbundle_fmt, _c4argbundle); - } - report(st, sz); -} - -void formatfile_c4formatdump_c_style_dynamic_dispatch(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - dump2file::cpp_style fileowner; - dump2file::c_style_subject = fileowner.subject; - size_t sz = c4::format(buf, _c4argbundle_fmt, _c4argbundle); - for(auto _ : st) - { - c4::format_dump(&dump2file::c_style, buf, _c4argbundle_fmt, _c4argbundle); - } - report(st, sz); -} - -void formatfile_c4formatdump_cpp_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::format(buf, _c4argbundle_fmt, _c4argbundle); - dump2file::cpp_style dumper; - for(auto _ : st) - { - c4::format_dump(dumper, buf, _c4argbundle_fmt, _c4argbundle); - } - report(st, sz); -} - -void formatfile_c4formatdump_lambda_style(bm::State &st) -{ - char buf_[256]; - c4::substr buf(buf_); - size_t sz = c4::format(buf, _c4argbundle_fmt, _c4argbundle); - dump2file::lambda_style dumper; - auto lambda = [&dumper](c4::csubstr s) { fwrite(s.str, 1, s.len, dumper.subject); }; - for(auto _ : st) - { - c4::format_dump(lambda, buf, _c4argbundle_fmt, _c4argbundle); - } - report(st, sz); -} - -void formatfile_fprintf(bm::State &st) -{ - char buf[256]; - size_t sz = c4::format(buf, _c4argbundle_fmt, _c4argbundle); - dump2file::cpp_style dumper; - for(auto _ : st) - { - fprintf(dumper.subject, _c4argbundle_fmt_printf, _c4argbundle_printf); - } - report(st, sz); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -C4BM(cat_c4cat_substr); -C4BM(cat_c4catrs_reuse); -C4BM(cat_c4catrs_no_reuse); -C4BM(cat_c4catdump_c_style_static_dispatch); -C4BM(cat_c4catdump_c_style_dynamic_dispatch); -C4BM(cat_c4catdump_cpp_style); -C4BM(cat_c4catdump_lambda_style); -C4BM(cat_std_to_string_reuse); -C4BM(cat_std_to_string_no_reuse); -C4BM(cat_stdsstream_reuse); -C4BM(cat_stdsstream_no_reuse); - -C4BM(catfile_c4catdump_c_style_static_dispatch); -C4BM(catfile_c4catdump_c_style_dynamic_dispatch); -C4BM(catfile_c4catdump_cpp_style); -C4BM(catfile_c4catdump_lambda_style); -C4BM(catfile_fprintf); -C4BM(catfile_ofstream); - - -C4BM(catsep_c4cat_substr); -C4BM(catsep_c4catrs_reuse); -C4BM(catsep_c4catrs_no_reuse); -C4BM(catsep_c4catdump_c_style_static_dispatch); -C4BM(catsep_c4catdump_c_style_dynamic_dispatch); -C4BM(catsep_c4catdump_cpp_style); -C4BM(catsep_c4catdump_lambda_style); -C4BM(catsep_std_to_string_reuse); -C4BM(catsep_std_to_string_no_reuse); -C4BM(catsep_stdsstream_reuse); -C4BM(catsep_stdsstream_no_reuse); - -C4BM(catsepfile_c4catsepdump_c_style_static_dispatch); -C4BM(catsepfile_c4catsepdump_c_style_dynamic_dispatch); -C4BM(catsepfile_c4catsepdump_cpp_style); -C4BM(catsepfile_c4catsepdump_lambda_style); -C4BM(catsepfile_fprintf); -C4BM(catsepfile_ofstream); - - -C4BM(format_c4format); -C4BM(format_c4formatrs_reuse); -C4BM(format_c4formatrs_no_reuse); -C4BM(format_c4formatdump_c_style_static_dispatch); -C4BM(format_c4formatdump_c_style_dynamic_dispatch); -C4BM(format_c4formatdump_cpp_style); -C4BM(format_c4formatdump_lambda_style); -C4BM(format_snprintf); - -C4BM(formatfile_c4formatdump_c_style_static_dispatch); -C4BM(formatfile_c4formatdump_c_style_dynamic_dispatch); -C4BM(formatfile_c4formatdump_cpp_style); -C4BM(formatfile_c4formatdump_lambda_style); -C4BM(formatfile_fprintf); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -int main(int argc, char *argv[]) -{ - bm::Initialize(&argc, argv); - bm::RunSpecifiedBenchmarks(); - return 0; -} - - -#include - - diff --git a/thirdparty/ryml/ext/c4core/bm/bm_itoa_threads.cpp b/thirdparty/ryml/ext/c4core/bm/bm_itoa_threads.cpp deleted file mode 100644 index 0e102065a..000000000 --- a/thirdparty/ryml/ext/c4core/bm/bm_itoa_threads.cpp +++ /dev/null @@ -1,356 +0,0 @@ -#include "./bm_charconv.hpp" - -#include -#include -#include -#include -#include -#include -#include -#if defined(__cpp_lib_to_chars) || (C4_CPP >= 17) -#define C4_HAS_STDTOCHARS 1 -#else -#define C4_HAS_STDTOCHARS 0 -#endif -#if C4_HAS_STDTOCHARS -#include -#endif - -C4_SUPPRESS_WARNING_GCC_CLANG_PUSH -C4_SUPPRESS_WARNING_GCC_CLANG("-Wcast-align") -C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") -#define FMT_HEADER_ONLY -#include - -#define STB_SPRINTF_IMPLEMENTATION -#include -C4_SUPPRESS_WARNING_GCC_POP - -#if C4_CXX >= 20 -#include -#define C4_HAS_STD_FORMAT (__has_cpp_attribute(__cpp_lib_format)) -#else -#define C4_HAS_STD_FORMAT (0) -#endif -#if C4_HAS_STD_FORMAT -#include -#endif - - -#define BMTHREADS(func) \ - BENCHMARK(func) \ - ->Threads(1) \ - ->Threads(2) \ - ->Threads(3) \ - ->Threads(4) \ - ->Threads(5) \ - ->Threads(6) \ - ->Threads(7) \ - ->Threads(8) \ - ->Threads(9) \ - ->Threads(10) \ - ->UseRealTime() \ - - -void snprintf(bm::State &st) -{ - size_t sum = {}; - char buf[100]; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - int ret = snprintf(buf, sizeof(buf), "%i", i++); - sum += (size_t)ret + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(snprintf); - - -#ifndef __linux__ -void snprintf_l(bm::State &st) -{ - size_t sum = {}; - char buf[100]; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - #if defined(_MSC_VER) - int ret = _snprintf_l(buf, 100, "%i", NULL, i++); - #else - int ret = snprintf_l(buf, 100, NULL, "%i", i++); - #endif - sum += (size_t)ret + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(snprintf_l); -#endif - - -void stb_snprintf(bm::State &st) -{ - size_t sum = {}; - char buf[100]; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - stbsp_snprintf(buf, 100, "%i", i++); - sum += strlen(buf) + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(stb_snprintf); - - - -void sstream(bm::State &st) -{ - size_t sum = {}; - std::stringstream buf; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - buf.seekp(0); - buf << i++; - size_t len = (size_t)buf.tellp(); - buf.seekg(0); - int firstchar = buf.get(); - sum += len + firstchar; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(sstream); - - -void sstream_naive(bm::State &st) -{ - size_t sum = {}; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::stringstream buf; - buf << i++; - std::string ret = buf.str(); - sum += ret.size() + ret[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(sstream_naive); - - -void sstream_naive_reuse(bm::State &st) -{ - size_t sum = {}; - int i = 0; - std::stringstream buf; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - buf.clear(); - buf.str(""); - buf << i++; - std::string ret = buf.str(); - sum += ret.size() + ret[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(sstream_naive_reuse); - - -#ifdef _MSC_VER -void itoa(bm::State &st) -{ - char buf[100]; - size_t sum = {}; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - ::_itoa(i++, buf, 10); - sum += strlen(buf) + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(itoa); -#endif - - -#if C4_HAS_STD_FORMAT -static void std_format_to(bm::State &st) -{ - size_t sum = {}; - char buf[100]; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - const auto res = std::format_to_n(buf, sizeof(buf), /*loc,*/ "{}", i++); - sum += (res.out - buf) + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(std_format_to); -#endif - - -void fmtlib_format_to(bm::State &st) -{ - size_t sum = {}; - fmt::memory_buffer buf; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - buf.clear(); - fmt::format_to(fmt::appender(buf), "{}", i++); - sum += buf.size() + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(fmtlib_format_to); - - -#if C4_HAS_STDTOCHARS -void std_to_chars(bm::State &st) -{ - size_t sum = {}; - char buf[100]; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - std::to_chars_result res = std::to_chars(buf, buf+sizeof(buf), i++); - sum += (res.ptr - buf) + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(std_to_chars); -#endif - - -void c4_write_dec(bm::State &st) -{ - size_t sum = {}; - char buf_[100]; - c4::substr buf = buf_; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - size_t len = c4::write_dec(buf, i++); - sum += len + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(c4_write_dec); - - -void c4_itoa(bm::State &st) -{ - size_t sum = {}; - char buf_[100]; - c4::substr buf = buf_; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - size_t len = c4::itoa(buf, i++); - sum += len + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(c4_itoa); - - -void c4_xtoa(bm::State &st) -{ - size_t sum = {}; - char buf_[100]; - c4::substr buf = buf_; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - size_t len = c4::xtoa(buf, i++); - sum += len + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(c4_xtoa); - - -void c4_to_chars(bm::State &st) -{ - size_t sum = {}; - char buf_[100]; - c4::substr buf = buf_; - int i = 0; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - size_t len = c4::to_chars(buf, i++); - sum += len + buf[0]; - } - } - bm::DoNotOptimize(sum); - report_threadavg(st, kNumValues); -} -BMTHREADS(c4_to_chars); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -int main(int argc, char *argv[]) -{ - bm::Initialize(&argc, argv); - bm::RunSpecifiedBenchmarks(); - return 0; -} diff --git a/thirdparty/ryml/ext/c4core/bm/bm_plot_c4core.py b/thirdparty/ryml/ext/c4core/bm/bm_plot_c4core.py deleted file mode 100644 index 23fa55ba3..000000000 --- a/thirdparty/ryml/ext/c4core/bm/bm_plot_c4core.py +++ /dev/null @@ -1,266 +0,0 @@ -import sys -import os -import re - -thisdir = os.path.dirname(os.path.abspath(__file__)) -moddir = os.path.abspath(f"{thisdir}/../cmake/bm-xp") -sys.path.insert(0, moddir) -import bm_plot as bm -from bm_util import first - -from dataclasses import dataclass -import prettytable - - -def get_function_benchmark(function_name, run: bm.BenchmarkRun): - for rbm in run.entries: - if rbm.meta.function == function_name: - return rbm - raise Exception(f"function not found: {function_name}. Existing: {[rbm.meta.function for rbm in run.entries]}") - - -@dataclass -class CharconvMeta: # also for atox - title: str - subject: str - function: str - data_type: bm.FundamentalTypes - - @classmethod - def make(cls, bm_title: str): - # eg: - # xtoa_c4_write_dec - # xtoa_c4_utoa - # xtoa_c4_xtoa - # xtoa_c4_to_chars - # xtoa_std_to_chars - # xtoa_std_to_string - # xtoa_sprintf - # xtoa_sstream_reuse - # xtoa_sstream - rx = re.compile(r'(atox|xtoa|xtoahex|xtoaoct|xtoabin)_(.*?)<(u?int\d+_t|float|double)>') - if not rx.fullmatch(bm_title): - raise Exception(f"cannot understand bm title: {bm_title}") - subject = rx.sub(r'\1', bm_title) - function = rx.sub(r'\2', bm_title) - data_type = rx.sub(r'\3', bm_title) - return cls( - title=bm_title, - subject=subject, - function=function.replace("c4_", "c4::").replace("std_", "std::"), - data_type=bm.FundamentalTypes.make(data_type) - ) - - @property - def shortname(self): - return self.function - - @property - def shortparams(self): - return str(self.data_type.short) - - @property - def shorttitle(self): - return f"{self.shortname}<{self.shortparams}>" - - -@dataclass -class CharconvThreadsMeta: - function: str - num_threads: int - - @classmethod - def make(cls, bm_title: str): - # eg: - # c4_itoa/real_time/threads:4 - rx = re.compile(r'(.*?)/real_time/threads:(\d+)') - if not rx.fullmatch(bm_title): - raise Exception(f"cannot understand bm title: {bm_title}") - function = rx.sub(r'\1', bm_title) - num_threads = int(rx.sub(r'\2', bm_title)) - return cls( - function=function.replace("c4_", "c4::").replace("std_", "std::"), - num_threads=num_threads - ) - - def checkbox_groups(self): - return {} - - @property - def shortname(self): - return self.function - - @property - def shorttitle(self): - return self.shortname - - -def plot_charconv_bars(bm_panel: bm.BenchmarkPanel, getref, - panel_title_human: str, - outputfile_prefix: str): - assert os.path.isabs(outputfile_prefix), outputfile_prefix - # make a comparison table - anchor = lambda run: f"{os.path.basename(outputfile_prefix)}-{first(run.meta).data_type}" - anchorlink = lambda run: f"
{first(run.meta).data_type}
" - with open(f"{outputfile_prefix}.txt", "w") as tablefile: - with open(f"{outputfile_prefix}.md", "w") as mdfile: - print(f"## {panel_title_human}\n\n

Data type benchmark results:

\n
    \n", - "\n".join([f"
  • {anchorlink(run)}
  • " for run in bm_panel.runs]), - "
\n\n", file=mdfile) - for run in bm_panel.runs: - data_type = first(run.meta).data_type - tabletitle = f"{outputfile_prefix}-{data_type.short}" - table = prettytable.PrettyTable(title=f"{panel_title_human}: {data_type}") - table.add_column("function", [m.shorttitle for m in run.meta], align="l") - for prop in ("mega_bytes_per_second", "cpu_time_ms"): - ref = getref(run) - bar_values = list(run.extract_plot_series(prop)) - bar_values_rel = list(run.extract_plot_series(prop, relative_to_entry=ref)) - bar_values_pc = list(run.extract_plot_series(prop, percent_of_entry=ref)) - pd = bm_panel.first_run.property_plot_data(prop) - hns = pd.human_name_short - table.add_column(hns, [f"{v_:7.2f}" for v_ in bar_values], align="r") - hns = hns.replace(" (ms)", "") - table.add_column(f"{hns}(x)", [f"{v_:5.2f}x" for v_ in bar_values_rel], align="r") - table.add_column(f"{hns}(%)", [f"{v_:7.2f}%" for v_ in bar_values_pc], align="r") - print(table, "\n\n") - print(table, "\n\n", file=tablefile) - pfx_bps = f"{os.path.basename(outputfile_prefix)}-mega_bytes_per_second-{data_type.short}" - pfx_cpu = f"{os.path.basename(outputfile_prefix)}-cpu_time_ms-{data_type.short}" - print(f""" -
-
- ---- - - - -### {panel_title_human}: `{data_type}` - -* Interactive html graphs for `{data_type}`: - * [MB/s](./{pfx_bps}.html) - * [CPU time](./{pfx_cpu}.html) - -[![{data_type}: MB/s](./{pfx_bps}.png)](./{pfx_bps}.png) -[![{data_type}: CPU time](./{pfx_cpu}.png)](./{pfx_cpu}.png) - -``` -{table} -``` -""", file=mdfile) - # make plots - for prop in ("mega_bytes_per_second", "cpu_time_ms"): - ps, ps_ = [], [] - pd = bm_panel.first_run.property_plot_data(prop) - bar_label = f"{pd.human_name_short}{pd.qty_type.comment}" - outfilename = f"{outputfile_prefix}-{prop}" - for run in bm_panel.runs: - data_type = first(run.meta).data_type - bar_names = [m.shorttitle for m in run.meta] - bar_values = list(run.extract_plot_series(prop)) - runtitle = f"{outfilename}-{data_type.short}" - # to save each bokeh plot separately and also - # a grid plot with all of them, we have to plot - # twice because bokeh does not allow saving twice - # the same plot from multiple pictures. - plotit = lambda: bm.plot_benchmark_run_as_bars(run, title=f"{panel_title_human}: {data_type}\n{bar_label}", - bar_names=bar_names, bar_values=bar_values, bar_label=bar_label) - # make one plot to save: - p, p_ = plotit() - bm._bokeh_save_html(f"{runtitle}.html", p) - bm._plt_save_png(f"{runtitle}.png") - bm._plt_clear() - # and another to gather: - p, p_ = plotit() - ps.append(p) - ps_.append(p_) - bm._plt_clear() - bm.bokeh_plot_many(ps, f"{outfilename}.html") - - -def plot_itoa_threads_(bm_panel: bm.BenchmarkPanel, getref, - panel_title_human: str, - outputfile_prefix: str): - assert os.path.isabs(outputfile_prefix), outputfile_prefix - orig = lambda yprop, **kw: lambda run: list(run.extract_plot_series(yprop, **kw)) - divnt = lambda yprop, **kw: lambda run: [v / n for v, n in run.extract_plot_series_with_threads(yprop, **kw)] - mulnt = lambda yprop, **kw: lambda run: [v * n for v, n in run.extract_plot_series_with_threads(yprop, **kw)] - xprop = "threads" - xpd = bm_panel.first_run.property_plot_data(xprop) - xlabel = f"{xpd.human_name_short}" - for yprop, ylog, yget in ( - #("mega_items_per_second", False, orig), - ("mega_bytes_per_second", False, orig), - #("iterations", False, divnt), - #("real_time_ms", True, mulnt), - ("cpu_time_ms", True, orig),): - ypd = bm_panel.first_run.property_plot_data(yprop) - ylabel = f"{ypd.human_name_short}{ypd.qty_type.comment}" - p = bm.plot_benchmark_panel_as_lines( - bm_panel, f"{panel_title_human}\n{ylabel}", - xget=orig("threads"), - yget=yget(yprop), - nameget=lambda run: first(run.meta).function, - ylog=ylog, - xlabel=xlabel, - ylabel=ylabel - ) - name = f"{outputfile_prefix}-lines-{yprop}" - # save png using matplotlib - bm._plt_save_png(f"{name}.png") - bm._plt_clear() - # save html using bokeh - bm._bokeh_save_html(f"{name}.html", p) - #bkp.show(p) - return p - - -def plot_itoa_threads(dir_: str, json_files): - panel = bm.BenchmarkPanel(json_files, CharconvThreadsMeta) - ref = lambda bmrun: get_function_benchmark("std::to_chars", run=bmrun) - plot_itoa_threads_(panel, ref, - f"itoa benchmark: convert 2M 32b integers to string", - f"{dir_}/c4core-bm-charconv_threads") - - -def plot_charconv_xtoa(dir_: str, json_files, is_ftoa: bool): - fcase = "ftoa" if is_ftoa else "xtoa" - panel = bm.BenchmarkPanel(json_files, CharconvMeta) - ref = lambda bmrun: get_function_benchmark("sprintf", run=bmrun) - plot_charconv_bars(panel, ref, - f"xtoa benchmark: convert 1M numbers to strings", - f"{dir_}/c4core-bm-charconv-{fcase}") - - -def plot_charconv_atox(dir_: str, json_files, is_atof: bool): - fcase = "atof" if is_atof else "atox" - panel = bm.BenchmarkPanel(json_files, CharconvMeta) - ref = lambda bmrun: get_function_benchmark("scanf", run=bmrun) - plot_charconv_bars(panel, ref, - f"atox benchmark: convert 1M strings to numbers", - f"{dir_}/c4core-bm-charconv-{fcase}") - - -if __name__ == '__main__': - args = sys.argv[1:] - if len(args) < 2: - raise Exception(f"usage: {sys.executable} {sys.argv[0]} benchmarkfile.json[,benchmarkfile2.json,...]") - cmd = args[0] - json_files = args[1:] - dir_ = os.path.dirname(json_files[0]) - for jf in json_files: - print("jf:", jf, flush=True) - assert os.path.dirname(jf) == dir_, (os.path.dirname(jf), dir_) - assert os.path.exists(jf), jf - if cmd == "itoa_threads": - plot_itoa_threads(dir_, json_files) - elif cmd == "xtoa" or cmd == "ftoa": - plot_charconv_xtoa(dir_, json_files, (cmd == "ftoa")) - elif cmd == "atox" or cmd == "atof": - plot_charconv_atox(dir_, json_files, (cmd == "atof")) - elif cmd == "format": - raise Exception(f"not implemented: {cmd}") - elif cmd == "digits": - pass # nothing to do - else: - raise Exception(f"not implemented: {cmd}") diff --git a/thirdparty/ryml/ext/c4core/bm/bm_xtoa.cpp b/thirdparty/ryml/ext/c4core/bm/bm_xtoa.cpp deleted file mode 100644 index 681e443c6..000000000 --- a/thirdparty/ryml/ext/c4core/bm/bm_xtoa.cpp +++ /dev/null @@ -1,1538 +0,0 @@ -#include "./bm_charconv.hpp" -#include -#include - - -// this is an exploratory benchmark to compare the possible -// combinations for all the components of the write_dec() algorithm - - -template using msb_func = unsigned (*)(T val); - -#if defined(__GNUC__) -# pragma GCC diagnostic push -# if __GNUC__ >= 8 -# pragma GCC diagnostic ignored "-Wstringop-truncation" -# pragma GCC diagnostic ignored "-Wstringop-overflow" -# endif -#endif - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// WIP. -// TODO: _BitscanReverse() in MSVC -C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wuseless-cast") -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto msb_intrinsic(unsigned v) noexcept - -> typename std::enable_if<__has_builtin(__builtin_clz), unsigned>::type -{ - using I = unsigned; - enum : I { total = (I)(I(8) * sizeof(I) - 1) }; - return (total - (I) __builtin_clz(v)); -} - -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto msb_intrinsic_64bit(unsigned long v) noexcept --> typename std::enable_if<__has_builtin(__builtin_clzl), unsigned>::type -{ - using I = unsigned long; - enum : I { total = (I)(I(8) * sizeof(I) - 1) }; - return (unsigned)(total - (I) __builtin_clzl(v)); -} - -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto msb_intrinsic_64bit(unsigned long long v) noexcept - -> typename std::enable_if<__has_builtin(__builtin_clzll), unsigned>::type -{ - using I = unsigned long long; - enum : I { total = (I)(I(8) * sizeof(I) - 1) }; - return (unsigned)(total - (I) __builtin_clzll(v)); -} -C4_SUPPRESS_WARNING_GCC_POP - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned msb_loop(I v) noexcept -{ - // https://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10Obvious - unsigned r = 0; - while (v >>= 1) // unroll for more speed... - r++; - return r; -} - -// https://graphics.stanford.edu/~seander/bithacks.html#IntegerLogLookup -constexpr static const int8_t LogTable256[256] = { -#define LT(n) n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n - -1, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, - LT(4), LT(5), LT(5), LT(6), LT(6), LT(6), LT(6), - LT(7), LT(7), LT(7), LT(7), LT(7), LT(7), LT(7), LT(7) -#undef LT -}; -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -unsigned msb_de_bruijn_32bit(I v) noexcept -{ - I t, tt; // temporaries - tt = v >> 16; - if (tt) - return (unsigned)((t = tt >> 8) ? 24 + LogTable256[t] : 16 + LogTable256[tt]); - return (unsigned)((t = v >> 8) ? 8 + LogTable256[t] : LogTable256[v]); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -unsigned msb_intrinsic_8bit(I v) noexcept -{ - return msb_intrinsic((unsigned)v); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -unsigned msb_divconq_8bit(I v) noexcept -{ - C4_STATIC_ASSERT(sizeof(I) == 1); - unsigned n = 0; - if(v & I(0xf0)) v >>= 4, n |= I(4); - if(v & I(0x0c)) v >>= 2, n |= I(2); - if(v & I(0x02)) v >>= 1, n |= I(1); - return n; -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned msb_intrinsic_16bit(I v) noexcept -{ - return msb_intrinsic((unsigned)v); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned msb_divconq_16bit(I v) noexcept -{ - C4_STATIC_ASSERT(sizeof(I) == 2); - unsigned n = 0; - if(v & I(0xff00)) v >>= 8, n |= I(8); - if(v & I(0x00f0)) v >>= 4, n |= I(4); - if(v & I(0x000c)) v >>= 2, n |= I(2); - if(v & I(0x0002)) v >>= 1, n |= I(1); - return n; -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned msb_intrinsic_32bit(I v) noexcept -{ - return msb_intrinsic((unsigned)v); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned msb_divconq_32bit(I v) noexcept -{ - C4_STATIC_ASSERT(sizeof(I) == 4); - unsigned n = 0; - if(v & I(0xffff0000)) v >>= 16, n |= I(16); - if(v & I(0x0000ff00)) v >>= 8, n |= I(8); - if(v & I(0x000000f0)) v >>= 4, n |= I(4); - if(v & I(0x0000000c)) v >>= 2, n |= I(2); - if(v & I(0x00000002)) v >>= 1, n |= I(1); - return n; -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned msb_divconq_branchless_32bit(I v) noexcept -{ - C4_STATIC_ASSERT(sizeof(I) == 4); - unsigned r = (unsigned)(v > 0xFFFF) << 4; v >>= r; - unsigned shift = (unsigned)(v > 0xFF ) << 3; v >>= shift; r |= shift; - shift = (unsigned)(v > 0xF ) << 2; v >>= shift; r |= shift; - shift = (unsigned)(v > 0x3 ) << 1; v >>= shift; r |= shift; - r |= (unsigned)(v >> 1); - return r; -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned msb_intrinsic_64bit(I v) noexcept -{ - return msb_intrinsic_64bit((uint64_t)v); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned msb_divconq_64bit(I v) noexcept -{ - C4_STATIC_ASSERT(sizeof(I) == 8); - unsigned n = 0; - if(v & I(0xffffffff00000000)) v >>= 32, n |= I(32); - if(v & I(0x00000000ffff0000)) v >>= 16, n |= I(16); - if(v & I(0x000000000000ff00)) v >>= 8, n |= I(8); - if(v & I(0x00000000000000f0)) v >>= 4, n |= I(4); - if(v & I(0x000000000000000c)) v >>= 2, n |= I(2); - if(v & I(0x0000000000000002)) v >>= 1, n |= I(1); - return n; -} - - -template msbfunc> -void msb(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += msbfunc(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - -C4BM_TEMPLATE(msb, uint8_t, msb_intrinsic_8bit); -C4BM_TEMPLATE(msb, uint8_t, msb_divconq_8bit); -C4BM_TEMPLATE(msb, uint8_t, msb_loop); - -C4BM_TEMPLATE(msb, int8_t, msb_intrinsic_8bit); -C4BM_TEMPLATE(msb, int8_t, msb_divconq_8bit); -C4BM_TEMPLATE(msb, int8_t, msb_loop); - -C4BM_TEMPLATE(msb, uint16_t, msb_intrinsic_16bit); -C4BM_TEMPLATE(msb, uint16_t, msb_divconq_16bit); -C4BM_TEMPLATE(msb, uint16_t, msb_loop); - -C4BM_TEMPLATE(msb, int16_t, msb_intrinsic_16bit); -C4BM_TEMPLATE(msb, int16_t, msb_divconq_16bit); -C4BM_TEMPLATE(msb, int16_t, msb_loop); - -C4BM_TEMPLATE(msb, uint32_t, msb_intrinsic_32bit); -C4BM_TEMPLATE(msb, uint32_t, msb_de_bruijn_32bit); -C4BM_TEMPLATE(msb, uint32_t, msb_divconq_32bit); -C4BM_TEMPLATE(msb, uint32_t, msb_divconq_branchless_32bit); -C4BM_TEMPLATE(msb, uint32_t, msb_loop); - -C4BM_TEMPLATE(msb, int32_t, msb_intrinsic_32bit); -C4BM_TEMPLATE(msb, int32_t, msb_de_bruijn_32bit); -C4BM_TEMPLATE(msb, int32_t, msb_divconq_32bit); -C4BM_TEMPLATE(msb, int32_t, msb_divconq_branchless_32bit); -C4BM_TEMPLATE(msb, int32_t, msb_loop); - -C4BM_TEMPLATE(msb, uint64_t, msb_intrinsic_64bit); -C4BM_TEMPLATE(msb, uint64_t, msb_divconq_64bit); -C4BM_TEMPLATE(msb, uint64_t, msb_loop); - -C4BM_TEMPLATE(msb, int64_t, msb_intrinsic_64bit); -C4BM_TEMPLATE(msb, int64_t, msb_divconq_64bit); -C4BM_TEMPLATE(msb, int64_t, msb_loop); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace impl { - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto digits_glibc(_Tp __value) noexcept - -> typename std::enable_if::value, unsigned>::type -{ - static_assert(std::is_integral<_Tp>::value, "implementation bug"); - static_assert(std::is_unsigned<_Tp>::value, "implementation bug"); - unsigned __n = 1; - const unsigned __b2 = __base * __base; - const unsigned __b3 = __b2 * __base; - const unsigned long __b4 = __b3 * __base; - for (;;) - { - if (__value < (unsigned)__base) return __n; - if (__value < __b2) return __n + 1; - if (__value < __b3) return __n + 2; - if (__value < __b4) return __n + 3; - __value /= (_Tp) __b4; - __n += 4; - } -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto digits_glibc(_Tp __value, unsigned __base) noexcept - -> typename std::enable_if::value, unsigned>::type -{ - static_assert(std::is_integral<_Tp>::value, "implementation bug"); - static_assert(std::is_unsigned<_Tp>::value, "implementation bug"); - unsigned __n = 1; - const unsigned __b2 = __base * __base; - const unsigned __b3 = __b2 * __base; - const unsigned long __b4 = __b3 * __base; - for (;;) - { - if (__value < (unsigned)__base) return __n; - if (__value < __b2) return __n + 1; - if (__value < __b3) return __n + 2; - if (__value < __b4) return __n + 3; - __value /= (_Tp)__b4; - __n += 4; - } -} - -template -constexpr C4_ALWAYS_INLINE -auto digits_glibc(_Tp __value) noexcept - -> typename std::enable_if::value, unsigned>::type -{ - using U = typename std::make_unsigned<_Tp>::type; - return digits_glibc((U)__value); -} - -template -constexpr C4_ALWAYS_INLINE -auto digits_glibc(_Tp __value, unsigned __base) noexcept - -> typename std::enable_if::value, unsigned>::type -{ - using U = typename std::make_unsigned<_Tp>::type; - return digits_glibc((U)__value, __base); -} - - -//------------------------------------------- -// https://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10Obvious - -// these functions assume the numbers are positive even when the type -// is signed - -template -constexpr C4_ALWAYS_INLINE -auto digits_dec_naive_hifirst(T v) noexcept - -> typename std::enable_if::type -{ - // best when the numbers are uniformly distributed over the whole range - return ((v >= 100) ? 3u : ((v >= 10) ? 2u : 1u)); -} - -template -constexpr C4_ALWAYS_INLINE -auto digits_dec_naive_lofirst(T v) noexcept - -> typename std::enable_if::type -{ - // best when lower numbers are more likely - return ((v < 10) ? 1u : ((v < 100) ? 2u : 3u)); -} - - -// 16 bit - -template -constexpr C4_ALWAYS_INLINE -auto digits_dec_naive_hifirst(T v) noexcept - -> typename std::enable_if::type -{ - // best when the numbers are uniformly distributed over the whole range - return ((v >= 10000) ? 5u : (v >= 1000) ? 4u : (v >= 100) ? 3u : (v >= 10) ? 2u : 1u); -} - -template -constexpr C4_ALWAYS_INLINE -auto digits_dec_naive_lofirst(T v) noexcept - -> typename std::enable_if::type -{ - // best when lower numbers are more likely - return ((v < 100) ? ((v >= 10) ? 2u : 1u) : ((v < 1000) ? 3u : ((v < 10000) ? 4u : 5u))); -} - - -// 32 bit - -template -constexpr C4_ALWAYS_INLINE -auto digits_dec_naive_hifirst(T v) noexcept - -> typename std::enable_if::type -{ - return ((v >= 1000000000) ? 10u : (v >= 100000000) ? 9u : (v >= 10000000) ? 8u : - (v >= 1000000) ? 7u : (v >= 100000) ? 6u : (v >= 10000) ? 5u : - (v >= 1000) ? 4u : (v >= 100) ? 3u : (v >= 10) ? 2u : 1u); -} - -template -constexpr C4_ALWAYS_INLINE -auto digits_dec_naive_lofirst(T v) noexcept - -> typename std::enable_if::type -{ - return ((v < 10) ? 1u : (v < 100) ? 2u : (v < 1000) ? 3u : (v < 10000) ? 4u : - (v < 100000) ? 5u : (v < 1000000) ? 6u : (v < 10000000) ? 7u : - (v < 100000000) ? 8u : (v < 1000000000) ? 9u : 10u); -} - - -// 64 bit - -template -constexpr C4_ALWAYS_INLINE -auto digits_dec_naive_hifirst(T v) noexcept - -> typename std::enable_if::type -{ - return ((std::is_unsigned::value && v >= T(10000000000000000000u)) ? 20u : - (v >= 1000000000000000000) ? 19u : (v >= 100000000000000000) ? 18u : (v >= 10000000000000000) ? 17u : - (v >= 1000000000000000) ? 16u : (v >= 100000000000000) ? 15u : (v >= 10000000000000) ? 14u : - (v >= 1000000000000) ? 13u : (v >= 100000000000) ? 12u : (v >= 10000000000) ? 11u : - (v >= 1000000000) ? 10u : (v >= 100000000) ? 9u : (v >= 10000000) ? 8u : - (v >= 1000000) ? 7u : (v >= 100000) ? 6u : (v >= 10000) ? 5u : - (v >= 1000) ? 4u : (v >= 100) ? 3u : (v >= 10) ? 2u : 1u); -} - -template -constexpr C4_ALWAYS_INLINE -auto digits_dec_naive_lofirst(T v) noexcept - -> typename std::enable_if::type -{ - return ((v < 10) ? 1u : (v < 100) ? 2u : (v < 1000) ? 3u : - (v < 10000) ? 4u : (v < 100000) ? 5u : (v < 1000000) ? 6u : - (v < 10000000) ? 7u : (v < 100000000) ? 8u : (v < 1000000000) ? 9u : - (v < 10000000000) ? 10u : (v < 100000000000) ? 11u : (v < 1000000000000) ? 12u : - (v < 10000000000000) ? 13u : (v < 100000000000000) ? 14u : (v < 1000000000000000) ? 15u : - (v < 10000000000000000) ? 16u : (v < 100000000000000000) ? 17u : (v < 1000000000000000000) ? 18u : - ((typename std::make_unsigned::type)v < 10000000000000000000u) ? 19u : 20u); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto digits_dec_naive_hifirst64fallback32(T v) noexcept - -> typename std::enable_if::type -{ - if(v >= std::numeric_limits::max()) - return digits_glibc(v); - else - return ((v >= 1000000000) ? 10u : (v >= 100000000) ? 9u : (v >= 10000000) ? 8u : - (v >= 1000000) ? 7u : (v >= 100000) ? 6u : (v >= 10000) ? 5u : - (v >= 1000) ? 4u : (v >= 100) ? 3u : (v >= 10) ? 2u : 1u); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto digits_dec_naive_lofirst64fallback32(T v) noexcept - -> typename std::enable_if::type -{ - if(v < std::numeric_limits::max()) - return ((v < 10) ? 1u : (v < 100) ? 2u : (v < 1000) ? 3u : (v < 10000) ? 4u : - (v < 100000) ? 5u : (v < 1000000) ? 6u : (v < 10000000) ? 7u : - (v < 100000000) ? 8u : (v < 1000000000) ? 9u : 10u); - else - return digits_glibc(v); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto digits_dec_naive_fargies(T v) noexcept - -> typename std::enable_if::type -{ - // https://github.com/biojppm/c4core/pull/77#issuecomment-1063753568 - if(v >= 1000000000) // 10 - { - if(v >= 100000000000000) // 15 [15-20] range - { - if(v >= 100000000000000000) // 18 (15 + (20 - 15) / 2) - { - if((typename std::make_unsigned::type)v >= 10000000000000000000u) // 20 - return 20u; - else - return (v >= 1000000000000000000) ? 19u : 18u; - } - else if(v >= 10000000000000000) // 17 - return 17u; - else - return(v >= 1000000000000000) ? 16u : 15u; - } - else if(v >= 1000000000000) // 13 - return (v >= 10000000000000) ? 14u : 13u; - else if(v >= 100000000000) // 12 - return 12; - else - return(v >= 10000000000) ? 11u : 10u; - } - else if(v >= 10000) // 5 [5-9] range - { - if(v >= 10000000) // 8 - return (v >= 100000000) ? 9u : 8u; - else if(v >= 1000000) // 7 - return 7; - else - return (v >= 100000) ? 6u : 5u; - } - else if(v >= 100) - return (v >= 1000) ? 4u : 3u; - else - return (v >= 10) ? 2u : 1u; -} - - -//------------------------------------------- -namespace c4 { -namespace detail { -template -struct powers_of_10; - -#define _C4_POWERS_OF_10_FOR(cond, ...) \ -template \ -struct powers_of_10::type> \ -{ \ - static C4_INLINE_CONSTEXPR const T values[] = {__VA_ARGS__}; \ - static C4_INLINE_CONSTEXPR const T values_size = C4_COUNTOF(values); \ -}; \ -template \ -C4_INLINE_CONSTEXPR const T powers_of_10::type>::values[] - -_C4_POWERS_OF_10_FOR(sizeof(T)==1u, 1, 10, 100 ); -_C4_POWERS_OF_10_FOR(sizeof(T)==2u, 1, 10, 100, 1000, 10000 ); -_C4_POWERS_OF_10_FOR(sizeof(T)==4u, 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 ); -_C4_POWERS_OF_10_FOR(std::is_signed::value && - sizeof(T)==8u, 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, 1000000000000, 10000000000000, 100000000000000, 1000000000000000, 10000000000000000, 100000000000000000, 1000000000000000000 ); -_C4_POWERS_OF_10_FOR(std::is_unsigned::value && - sizeof(T)==8u, 1u, 10u, 100u, 1000u, 10000u, 100000u, 1000000u, 10000000u, 100000000u, 1000000000u, 10000000000u, 100000000000u, 1000000000000u, 10000000000000u, 100000000000000u, 1000000000000000u, 10000000000000000u, 100000000000000000u, 1000000000000000000u, 10000000000000000000u ); -} // namespace detail -} // namespace c4 - - -template msbfunc> -C4_CONSTEXPR14 C4_ALWAYS_INLINE -unsigned digits_dec_log10_nocheck(T v) noexcept -{ - // https://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10 - const unsigned mag = ((unsigned)msbfunc(v) + 1u) * 1233u >> 12; - C4_ASSERT(mag < c4::detail::powers_of_10::values_size); - return 1u + mag - (v < c4::detail::powers_of_10::values[mag]); -} - -template msbfunc> -C4_CONSTEXPR14 C4_ALWAYS_INLINE -unsigned digits_dec_log10(T v) noexcept -{ - if(v) - { - // https://graphics.stanford.edu/~seander/bithacks.html#IntegerLog10 - const unsigned mag = ((unsigned)msbfunc(v) + 1u) * 1233u >> 12; - C4_ASSERT(mag < c4::detail::powers_of_10::values_size); - return 1u + mag - (v < c4::detail::powers_of_10::values[mag]); - } - return 1u; -} - -} // namespace impl - - -template -void digits_dec_naive_fargies(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += impl::digits_dec_naive_fargies(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -template -void digits_dec_naive_hifirst(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += impl::digits_dec_naive_hifirst(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -template -void digits_dec_naive_lofirst(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += impl::digits_dec_naive_lofirst(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -template -void digits_dec_naive_hifirst64fallback32(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += impl::digits_dec_naive_hifirst64fallback32(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -template -void digits_dec_naive_lofirst64fallback32(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += impl::digits_dec_naive_lofirst64fallback32(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - - -template -void digits_dec_glibc_tpl(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += impl::digits_glibc(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -template -void digits_dec_glibc(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += impl::digits_glibc(values.next(), 10); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -template msbfunc> -void digits_dec_log10_nocheck(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += impl::digits_dec_log10_nocheck(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -template msbfunc> -void digits_dec_log10(bm::State &st) -{ - random_values_cref values = mkvals_positive(); - unsigned sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += impl::digits_dec_log10(values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -#define C4_INSTANTIATE_DIGITS_DEC_BENCHMARKS(ty, num) \ -C4BM_TEMPLATE(digits_dec_naive_hifirst, ty); \ -C4BM_TEMPLATE(digits_dec_naive_lofirst, ty); \ -C4BM_TEMPLATE(digits_dec_log10_nocheck, ty, msb_intrinsic_##num##bit); \ -C4BM_TEMPLATE(digits_dec_log10_nocheck, ty, msb_divconq_##num##bit); \ -C4BM_TEMPLATE(digits_dec_log10_nocheck, ty, msb_loop); \ -C4BM_TEMPLATE(digits_dec_log10, ty, msb_intrinsic_##num##bit); \ -C4BM_TEMPLATE(digits_dec_log10, ty, msb_divconq_##num##bit); \ -C4BM_TEMPLATE(digits_dec_log10, ty, msb_loop); \ -C4BM_TEMPLATE(digits_dec_glibc_tpl, ty); \ -C4BM_TEMPLATE(digits_dec_glibc, ty) - - -C4_INSTANTIATE_DIGITS_DEC_BENCHMARKS(uint8_t, 8); -C4_INSTANTIATE_DIGITS_DEC_BENCHMARKS(int8_t, 8); -C4_INSTANTIATE_DIGITS_DEC_BENCHMARKS(uint16_t, 16); -C4_INSTANTIATE_DIGITS_DEC_BENCHMARKS(int16_t, 16); -C4_INSTANTIATE_DIGITS_DEC_BENCHMARKS(uint32_t, 32); -C4_INSTANTIATE_DIGITS_DEC_BENCHMARKS(int32_t, 32); -C4BM_TEMPLATE(digits_dec_naive_fargies, uint64_t); -C4BM_TEMPLATE(digits_dec_naive_hifirst64fallback32, uint64_t); -C4BM_TEMPLATE(digits_dec_naive_lofirst64fallback32, uint64_t); -C4_INSTANTIATE_DIGITS_DEC_BENCHMARKS(uint64_t, 64); -C4BM_TEMPLATE(digits_dec_naive_fargies, int64_t); -C4BM_TEMPLATE(digits_dec_naive_hifirst64fallback32, int64_t); -C4BM_TEMPLATE(digits_dec_naive_lofirst64fallback32, int64_t); -C4_INSTANTIATE_DIGITS_DEC_BENCHMARKS(int64_t, 64); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace impl { - -// LEGEND: - -// checkall: check buffer length on every insertion -// checkonce: check buffer length only once when entering the function -// checkoncemax: as above, and compare against a compile-time maxdigits for the type -// checkoncelog: as above, and compare against the exact digits each from the actual number - -// divrem: compute div with operator/ and rem with operator% -// singlediv: compute div with operator/ but rem without using operator% (explicitly compute the remainder) - -// write1: write 1 digit per division (divide by 10 on each step) -// write2: write 2 digits per division (divide by 100 on each step) - - -static constexpr const char digits0099[201] = - "0001020304050607080910111213141516171819" - "2021222324252627282930313233343536373839" - "4041424344454647484950515253545556575859" - "6061626364656667686970717273747576777879" - "8081828384858687888990919293949596979899"; - - -//------------------------------------------------------------------- - -template -C4_ALWAYS_INLINE size_t write_dec_checkall_divrem_write1(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - size_t pos = 0; - do { - if(C4_LIKELY(pos < buf.len)) - buf.str[pos] = (char)('0' + (v % T(10))); - ++pos; - v /= T(10); - } while(v); - buf.reverse_range(0, pos); - return pos; -} - -template -C4_ALWAYS_INLINE size_t write_dec_checkall_divrem_write2(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - size_t pos = 0; - while(v >= T(100)) - { - const auto num = (v % T(100)) << 1u; - v /= T(100); - if(C4_LIKELY(pos + 2 < buf.len)) - { - buf.str[pos++] = digits0099[num + 1]; - buf.str[pos++] = digits0099[num]; - } - } - if(v >= T(10)) - { - const auto num = v << 1u; - if(C4_LIKELY(pos + 2 < buf.len)) - { - buf.str[pos++] = digits0099[num + 1]; - buf.str[pos++] = digits0099[num]; - } - } - else - { - if(C4_LIKELY(pos < buf.len)) - buf.str[pos++] = (char)('0' + v); - } - buf.reverse_range(0, pos); - return pos; -} - - -//------------------------------------------------------------------- - -template -C4_ALWAYS_INLINE size_t write_dec_checkall_singlediv_write1(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - size_t pos = 0; - do { - const T quo = v / T(10); - const auto rem = v - quo * T(10); - v = quo; - if(C4_LIKELY(pos < buf.len)) - buf.str[pos] = (char)('0' + rem); - ++pos; - } while(v); - buf.reverse_range(0, pos); - return pos; -} - -template -C4_ALWAYS_INLINE size_t write_dec_checkall_singlediv_write2(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - size_t pos = 0; - while(v >= T(100)) - { - const T quo = v / T(100); - const auto num = (v - quo * T(100)) << 1u; - v = quo; - if(C4_LIKELY(pos+2 < buf.len)) - { - buf.str[pos++] = digits0099[num + 1]; - buf.str[pos++] = digits0099[num]; - } - } - if(v >= T(10)) - { - const auto num = v << 1u; - if(C4_LIKELY(pos+2 < buf.len)) - { - buf.str[pos++] = digits0099[num + 1]; - buf.str[pos++] = digits0099[num ]; - } - } - else - { - if(C4_LIKELY(pos < buf.len)) - buf.str[pos++] = (char)('0' + v); - } - buf.reverse_range(0, pos); - return pos; -} - - -//------------------------------------------------------------------- - -template -C4_ALWAYS_INLINE size_t write_dec_checkoncemax_divrem_write1(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - if(C4_UNLIKELY(buf.len < c4::detail::charconv_digits::maxdigits_dec)) - return c4::detail::charconv_digits::maxdigits_dec; - size_t pos = 0; - do { - buf.str[pos++] = (char)('0' + (v % T(10))); - v /= T(10); - } while(v); - buf.reverse_range(0, pos); - return pos; -} - -template -C4_ALWAYS_INLINE size_t write_dec_checkoncemax_divrem_write2(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - if(C4_UNLIKELY(buf.len < c4::detail::charconv_digits::maxdigits_dec)) - return c4::detail::charconv_digits::maxdigits_dec; - size_t pos = 0; - while(v >= T(100)) - { - const auto num = (v % T(100)) << 1u; - v /= T(100); - buf.str[pos++] = digits0099[num + 1]; - buf.str[pos++] = digits0099[num]; - } - if(v >= T(10)) - { - const auto num = v << 1u; - buf.str[pos++] = digits0099[num + 1]; - buf.str[pos++] = digits0099[num ]; - } - else - { - buf.str[pos++] = (char)('0' + v); - } - buf.reverse_range(0, pos); - return pos; -} - - -//------------------------------------------------------------------- - -template -C4_ALWAYS_INLINE size_t write_dec_checkoncemax_singlediv_write1(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - if(C4_UNLIKELY(buf.len < c4::detail::charconv_digits::maxdigits_dec)) - return c4::detail::charconv_digits::maxdigits_dec; - size_t pos = 0; - do { - const T quo = v / T(10); - const auto rem = (v - quo * T(10)); - v = quo; - buf.str[pos++] = (char)('0' + rem); - } while(v); - buf.reverse_range(0, pos); - return pos; -} - -template -C4_ALWAYS_INLINE size_t write_dec_checkoncemax_singlediv_write2(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - size_t pos = 0; - if(C4_UNLIKELY(buf.len < c4::detail::charconv_digits::maxdigits_dec)) - return c4::detail::charconv_digits::maxdigits_dec; - while(v >= T(100)) - { - const T quo = v / T(100); - const auto num = (v - quo * T(100)) << 1u; - v = quo; - buf.str[pos++] = digits0099[num + 1]; - buf.str[pos++] = digits0099[num]; - } - if(v >= T(10)) - { - const auto num = v << 1u; - buf.str[pos++] = digits0099[num + 1]; - buf.str[pos++] = digits0099[num ]; - } - else - { - buf.str[pos++] = (char)('0' + v); - } - buf.reverse_range(0, pos); - return pos; -} - - -//------------------------------------------------------------------- - -template digitsfunc> -C4_ALWAYS_INLINE size_t write_dec_checkoncelog_divrem_write1(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - unsigned digits = digitsfunc(v); - if(C4_UNLIKELY(buf.len < digits)) - return digits; - size_t pos = digits; - do { - buf.str[--pos] = (char)('0' + (v % T(10))); - v /= T(10); - } while(v); - return digits; -} - -template digitsfunc> -C4_ALWAYS_INLINE size_t write_dec_checkoncelog_divrem_write2(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - unsigned digits = digitsfunc(v); - if(C4_UNLIKELY(buf.len < digits)) - return digits; - size_t pos = digits; - while(v >= T(100)) - { - const auto num = (v % T(100)) << 1u; - v /= T(100); - buf.str[--pos] = digits0099[num + 1]; - buf.str[--pos] = digits0099[num]; - } - if(v >= T(10)) - { - const auto num = v << 1u; - buf.str[1] = digits0099[num + 1]; - buf.str[0] = digits0099[num]; - } - else - { - buf.str[0] = (char)('0' + v); - } - return digits; -} - - -//------------------------------------------------------------------- - -template digitsfunc> -C4_ALWAYS_INLINE size_t write_dec_checkoncelog_singlediv_write1(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - unsigned digits = digitsfunc(v); - if(C4_UNLIKELY(buf.len < digits)) - return digits; - size_t pos = digits; - do { - const T quo = v / T(10); - const auto rem = (v - quo * T(10)); - v = quo; - buf.str[--pos] = (char)('0' + rem); - } while(v); - return digits; -} - -template digitsfunc> -C4_ALWAYS_INLINE size_t write_dec_checkoncelog_singlediv_write2(c4::substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - unsigned digits = digitsfunc(v); - if(C4_UNLIKELY(buf.len < digits)) - return digits; - size_t pos = digits; - while(v >= T(100)) - { - const T quo = v / T(100); - const auto num = (v - quo * T(100)) << 1u; - v = quo; - buf.str[--pos] = digits0099[num + 1]; - buf.str[--pos] = digits0099[num]; - } - if(v >= T(10)) - { - const auto num = v << 1u; - buf.str[1] = digits0099[num + 1]; - buf.str[0] = digits0099[num ]; - } - else - { - buf.str[0] = (char)('0' + v); - } - return digits; -} -} // namespace impl - - - -#define C4_DEFINE_WRITE_DEC_BM(name) \ -template \ -void write_dec_##name(bm::State &st) \ -{ \ - random_values_cref values = mkvals_positive(); \ - string_buffer buf_ = {}; \ - c4::substr buf = buf_; \ - C4_ASSERT(buf.len > 11); \ - size_t sum = {}; \ - for(auto _ : st) \ - { \ - C4DOALL(kNumValues) \ - sum += impl::write_dec_##name(buf, values.next()); \ - } \ - bm::DoNotOptimize(sum); \ - report(st, kNumValues); \ -} - -#define C4_DEFINE_WRITE_DEC_BM_FUNC(name) \ -template msbfunc> \ -void write_dec_##name(bm::State &st) \ -{ \ - random_values_cref values = mkvals_positive(); \ - string_buffer buf_ = {}; \ - c4::substr buf = buf_; \ - C4_ASSERT(buf.len > 11); \ - size_t sum = {}; \ - for(auto _ : st) \ - { \ - C4DOALL(kNumValues) \ - sum += impl::write_dec_##name(buf, values.next()); \ - } \ - bm::DoNotOptimize(sum); \ - report(st, kNumValues); \ -} - - -C4FOR(T, isint) -write_dec_c4_write_dec(bm::State& st) -{ - random_values_cref values = mkvals_positive(); - string_buffer buf_ = {}; - c4::substr buf = buf_; - size_t sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - sum += c4::write_dec(buf, values.next()); - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} - -#if defined(__cpp_lib_to_chars) || (C4_CPP >= 17) -#define C4_TO_CHARS_BM(ty) C4BM_TEMPLATE(write_dec_std_to_chars, ty) -C4FOR(T, isint) -write_dec_std_to_chars(bm::State& st) -{ - random_values_cref values = mkvals_positive(); - string_buffer buf_ = {}; - c4::substr buf = buf_; - size_t sum = {}; - for(auto _ : st) - { - C4DOALL(kNumValues) - { - auto result = std::to_chars(buf.begin(), buf.end(), values.next()); - sum += (size_t)(result.ptr - buf.str); - } - } - bm::DoNotOptimize(sum); - report(st, kNumValues); -} -#else -#define C4_TO_CHARS_BM(ty) -#endif - - -C4_DEFINE_WRITE_DEC_BM_FUNC(checkoncelog_singlediv_write2) -C4_DEFINE_WRITE_DEC_BM_FUNC(checkoncelog_singlediv_write1) -C4_DEFINE_WRITE_DEC_BM_FUNC(checkoncelog_divrem_write2) -C4_DEFINE_WRITE_DEC_BM_FUNC(checkoncelog_divrem_write1) - -C4_DEFINE_WRITE_DEC_BM(checkoncemax_singlediv_write2) -C4_DEFINE_WRITE_DEC_BM(checkoncemax_singlediv_write1) -C4_DEFINE_WRITE_DEC_BM(checkoncemax_divrem_write2) -C4_DEFINE_WRITE_DEC_BM(checkoncemax_divrem_write1) - -C4_DEFINE_WRITE_DEC_BM(checkall_singlediv_write2) -C4_DEFINE_WRITE_DEC_BM(checkall_singlediv_write1) -C4_DEFINE_WRITE_DEC_BM(checkall_divrem_write2) -C4_DEFINE_WRITE_DEC_BM(checkall_divrem_write1) - - - -#define C4_INSTANTIATE_WRITE_DEC_BENCHMARKS(ty, num) \ - \ -/*compare against std::to_chars()*/ \ -C4_TO_CHARS_BM(ty); \ - \ -/*our versions*/ \ -C4BM_TEMPLATE(write_dec_c4_write_dec, ty); \ - \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_naive_hifirst); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_naive_lofirst); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_glibc); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_log10>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_log10>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_log10>); \ - \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_naive_hifirst); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_naive_lofirst); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_glibc); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_log10>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_log10>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_log10>); \ - \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_naive_hifirst); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_naive_lofirst); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_glibc); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_log10>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_log10>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_log10>); \ - \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_naive_hifirst); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_naive_lofirst); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_glibc); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_log10_nocheck>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_log10>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_log10>); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_log10>); \ - \ -C4BM_TEMPLATE(write_dec_checkoncemax_singlediv_write2, ty); \ -C4BM_TEMPLATE(write_dec_checkoncemax_singlediv_write1, ty); \ -C4BM_TEMPLATE(write_dec_checkoncemax_divrem_write2, ty); \ -C4BM_TEMPLATE(write_dec_checkoncemax_divrem_write1, ty); \ - \ -C4BM_TEMPLATE(write_dec_checkall_singlediv_write2, ty); \ -C4BM_TEMPLATE(write_dec_checkall_singlediv_write1, ty); \ -C4BM_TEMPLATE(write_dec_checkall_divrem_write2, ty); \ -C4BM_TEMPLATE(write_dec_checkall_divrem_write1, ty) - - - -#define C4_INSTANTIATE_WRITE_DEC_BENCHMARKS64(ty) \ - \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_naive_fargies); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_naive_hifirst64fallback32); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write2, ty, impl::digits_dec_naive_lofirst64fallback32); \ - \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_naive_fargies); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_naive_hifirst64fallback32); \ -C4BM_TEMPLATE(write_dec_checkoncelog_singlediv_write1, ty, impl::digits_dec_naive_lofirst64fallback32); \ - \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_naive_fargies); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_naive_hifirst64fallback32); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write2, ty, impl::digits_dec_naive_lofirst64fallback32); \ - \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_naive_fargies); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_naive_hifirst64fallback32); \ -C4BM_TEMPLATE(write_dec_checkoncelog_divrem_write1, ty, impl::digits_dec_naive_lofirst64fallback32) - - -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS(uint8_t, 8); -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS(int8_t, 8); -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS(uint16_t, 16); -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS(int16_t, 16); -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS(uint32_t, 32); -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS(int32_t, 32); -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS64(uint64_t); -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS(uint64_t, 64); -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS64(int64_t); -C4_INSTANTIATE_WRITE_DEC_BENCHMARKS(int64_t, 64); - - -#if defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template struct xtoacase { c4::csubstr str; T val; }; -#define _(num) {c4::csubstr(#num), num##u} -xtoacase cases_8bit[] = { - _(0), - _(1), - _(9), - _(10), - _(11), - _(98), - _(99), - _(100), - _(101), -}; -xtoacase cases_16bit[] = { - _(999), - _(1000), - _(1001), - _(9999), - _(10000), - _(10001), -}; -xtoacase cases_32bit[] = { - _(99999), - _(100000), - _(100001), - _(999999), - _(1000000), - _(1000001), - _(9999999), - _(10000000), - _(10000001), - _(99999999), - _(100000000), - _(100000001), - _(999999999), - _(1000000000), - _(1000000001), -}; -xtoacase cases_64bit[] = { - _(9999999999), - _(10000000000), - _(10000000001), - _(99999999999), - _(100000000000), - _(100000000001), - _(999999999999), - _(1000000000000), - _(1000000000001), - _(9999999999999), - _(10000000000000), - _(10000000000001), - _(99999999999999), - _(100000000000000), - _(100000000000001), - _(999999999999999), - _(1000000000000000), - _(1000000000000001), - _(9999999999999999), - _(10000000000000000), - _(10000000000000001), - _(99999999999999999), - _(100000000000000000), - _(100000000000000001), - _(999999999999999999), - _(1000000000000000000), - _(1000000000000000001), - _(9223372036854775807), -}; -xtoacase cases_64bitu[] = { - _(9999999999999999999), - _(10000000000000000000), - _(18446744073709551615), -}; -#undef _ - -bool logtest = true; -bool printok = false; -bool testfail = false; -#define C4_CHECK_(lhs, op, rhs, ...) \ - { \ - if(!((lhs) op (rhs))) \ - { \ - std::cout << __FILE__ << ":" << __LINE__ \ - << ": failed! " << #lhs " " #op " " #rhs "\n" \ - << " " #lhs "=" << (lhs) << "\n" \ - << " " #rhs "=" << (rhs) << "\n" \ - << " " << __VA_ARGS__ << "\n"; \ - testfail = true; \ - } \ - else if(printok) \ - { \ - std::cout << __FILE__ << ":" << __LINE__ \ - << ": ok! " << #lhs " " #op " " #rhs \ - << " " << __VA_ARGS__ << "\n"; \ - } \ - } - -C4_SUPPRESS_WARNING_CLANG("-Wgnu-zero-variadic-macro-arguments") -#define C4_CHECK_LT(lhs, rhs, ...) C4_CHECK_(lhs, <, rhs, ## __VA_ARGS__) -#define C4_CHECK_LE(lhs, rhs, ...) C4_CHECK_(lhs, <=, rhs, ## __VA_ARGS__) -#define C4_CHECK_GT(lhs, rhs, ...) C4_CHECK_(lhs, >, rhs, ## __VA_ARGS__) -#define C4_CHECK_GE(lhs, rhs, ...) C4_CHECK_(lhs, >=, rhs, ## __VA_ARGS__) -#define C4_CHECK_EQ(lhs, rhs, ...) C4_CHECK_(lhs, ==, rhs, ## __VA_ARGS__) -#define C4_CHECK_NE(lhs, rhs, ...) C4_CHECK_(lhs, !=, rhs, ## __VA_ARGS__) - -#define DO_TEST_DIGITS_(ty, fn, num) \ - if(logtest) std::cout << "\ntesting: " #fn "\n"; \ - test_digits##num(); \ - if(logtest) std::cout << "success: " #fn "\n" - -#define DO_TEST_WRITE_(ty, fn, num) \ - if(logtest) std::cout << "\ntesting: " #fn "\n"; \ - test_write##num(&fn); \ - if(logtest) std::cout << "success: " #fn "\n" - - -template -void test_write(xtoacase c, Func fn) -{ - if(c.val == 0) - return; - C4_STATIC_ASSERT(sizeof(T) >= sizeof(U)); - char buf_[32] = {}; - c4::substr buf = buf_; - C4_CHECK_GT(c.val, 0, c.str << "/" << (uint64_t)c.val); - C4_CHECK_LE((U)c.val, (U)std::numeric_limits::max(), c.str << "/" << (uint64_t)c.val); - T val = (T)c.val; - size_t ret = fn(buf, val); - C4_CHECK_EQ(ret, c.str.len, c.str << "/" << (uint64_t)c.val << ": " << buf.first(ret)); - C4_CHECK_EQ(buf.first(ret), c.str, c.str << "/" << (uint64_t)c.val); -} - -template digitsfunc> -void test_digits8() -{ - for(auto c : cases_8bit) - C4_CHECK_EQ(digitsfunc((T)c.val), c.str.len, (uint64_t)c.val); -} -template -void test_write8(Func func) -{ - for(auto c : cases_8bit) - test_write(c, func); -} - -template digitsfunc> -void test_digits16() -{ - test_digits8(); - for(auto c : cases_16bit) - C4_CHECK_EQ(digitsfunc((T)c.val), c.str.len, (uint64_t)c.val); -} -template -void test_write16(Func func) -{ - test_write8(func); - for(auto c : cases_16bit) - test_write(c, func); -} - -template digitsfunc> -void test_digits32() -{ - test_digits8(); - test_digits16(); - for(auto c : cases_32bit) - C4_CHECK_EQ(digitsfunc((T)c.val), c.str.len, (uint64_t)c.val); -} -template -void test_write32(Func func) -{ - test_write8(func); - test_write16(func); - for(auto c : cases_32bit) - test_write(c, func); -} - -template digitsfunc> -void test_digits64() -{ - test_digits8(); - test_digits16(); - test_digits32(); - for(auto c : cases_64bit) - C4_CHECK_EQ(digitsfunc((T)c.val), c.str.len, (uint64_t)c.val); - if(std::is_unsigned::value) - for(auto c : cases_64bitu) - C4_CHECK_EQ(digitsfunc((T)c.val), c.str.len, (uint64_t)c.val << "/" << c.str); -} -template -auto test_write64(Func func) - -> typename std::enable_if::value, void>::type -{ - test_write8(func); - test_write16(func); - test_write32(func); - for(auto c : cases_64bit) - test_write(c, func); - for(auto c : cases_64bitu) - test_write(c, func); -} -template -auto test_write64(Func func) - -> typename std::enable_if::value, void>::type -{ - test_write8(func); - test_write16(func); - test_write32(func); - for(auto c : cases_64bit) - test_write(c, func); -} - -#define DO_TEST_DIGITS(ty, num) \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_hifirst, num); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_lofirst, num); \ -DO_TEST_DIGITS_(ty, impl::digits_glibc, num); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_log10>, num); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_log10>, num); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_log10>, num) - -#define DO_TEST_DIGITS_64(ty) \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_fargies, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_hifirst64fallback32, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_lofirst64fallback32, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_fargies, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_hifirst64fallback32, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_lofirst64fallback32, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_fargies, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_hifirst64fallback32, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_lofirst64fallback32, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_fargies, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_hifirst64fallback32, 64); \ -DO_TEST_DIGITS_(ty, impl::digits_dec_naive_lofirst64fallback32, 64) - - -#define DO_TEST_WRITE(ty, num) \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>>, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncemax_singlediv_write2, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncemax_singlediv_write1, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncemax_divrem_write2, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncemax_divrem_write1, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkall_singlediv_write2, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkall_singlediv_write1, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkall_divrem_write2, num);\ -DO_TEST_WRITE_(ty, impl::write_dec_checkall_divrem_write1, num) - - -#define DO_TEST_WRITE_64(ty) \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write2>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_singlediv_write1>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write2>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>, 64); \ -DO_TEST_WRITE_(ty, impl::write_dec_checkoncelog_divrem_write1>, 64) - - -void do_test() -{ - DO_TEST_DIGITS(uint8_t, 8); - DO_TEST_DIGITS(int8_t, 8); - DO_TEST_DIGITS(uint16_t, 16); - DO_TEST_DIGITS(int16_t, 16); - DO_TEST_DIGITS(uint32_t, 32); - DO_TEST_DIGITS(int32_t, 32); - DO_TEST_DIGITS(uint64_t, 64); - DO_TEST_DIGITS(int64_t, 64); - DO_TEST_DIGITS_64(uint64_t); - DO_TEST_DIGITS_64(int64_t); - printf("\n"); - DO_TEST_WRITE(uint8_t, 8); - DO_TEST_WRITE(int8_t, 8); - DO_TEST_WRITE(uint16_t, 16); - DO_TEST_WRITE(int16_t, 16); - DO_TEST_WRITE(uint32_t, 32); - DO_TEST_WRITE(int32_t, 32); - DO_TEST_WRITE(uint64_t, 64); - DO_TEST_WRITE(int64_t, 64); - DO_TEST_WRITE_64(uint64_t); - DO_TEST_WRITE_64(int64_t); - printf("\n"); - C4_CHECK(!testfail); -} - - -int main(int argc, char *argv[]) -{ - //do_test(); - bm::Initialize(&argc, argv); - bm::RunSpecifiedBenchmarks(); - return 0; -} diff --git a/thirdparty/ryml/ext/c4core/bm/float/measure.py b/thirdparty/ryml/ext/c4core/bm/float/measure.py deleted file mode 100644 index 253d03a45..000000000 --- a/thirdparty/ryml/ext/c4core/bm/float/measure.py +++ /dev/null @@ -1,65 +0,0 @@ -import time -import argparse -import os -import subprocess -import shlex -from ruamel import yaml - - -def runcmd(cmd, *cmd_args, **subprocess_args): - cmd = shlex.split(cmd) + list(cmd_args) - #print(" ".join([f"'{a}'" for a in cmd]), flush=True) - proc = subprocess.run(cmd, **subprocess_args) - return proc - - -def getoutput(cmd, *cmd_args, **subprocess_args): - proc = runcmd(cmd, *cmd_args, **subprocess_args, check=True, - stdout=subprocess.PIPE) - return proc.stdout.decode("utf8") - - -def start_build(args): - ts = time.time() - with open(args.out, "w") as f: - f.write(str(ts)) - - -def finish_build(args): - ts = time.time() - with open(args.out, "r") as f: - start = float(f.read()) - duration = ts - start - results = { - 'compile': f"{duration:.3f}s", - 'file_size': f"{os.path.getsize(args.exe)}B" - } - s = yaml.dump({args.target: results}) - print(s, flush=True, end="") - ## too much output: - #if args.unix: - # # https://stackoverflow.com/questions/35485 - # results['size'] = getoutput('size', args.exe) - # #results['symbols'] = getoutput('nm -t d -l -S --size-sort', args.exe) - s = yaml.dump({args.target: results}) - with open(args.out, "w") as f: - f.write(s) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers() - # - sp = subparsers.add_parser("start") - sp.set_defaults(func=start_build) - sp.add_argument('target', type=str, help='the target name') - # - sp = subparsers.add_parser("finish") - sp.set_defaults(func=finish_build) - sp.add_argument('target', type=str, help='the target name') - sp.add_argument('exe', type=str, help='the executable file') - sp.add_argument('-u', '--unix', action="store_true", help='use unix style size reporters') - # - args = parser.parse_args() - args.out = f"{args.target}.dat" - args.func(args) diff --git a/thirdparty/ryml/ext/c4core/bm/float/read.cpp b/thirdparty/ryml/ext/c4core/bm/float/read.cpp deleted file mode 100644 index 5401ff1bb..000000000 --- a/thirdparty/ryml/ext/c4core/bm/float/read.cpp +++ /dev/null @@ -1,170 +0,0 @@ -#include - - -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable : 4430) -# pragma warning(disable : 4305) -# pragma warning(disable : 4309) -# pragma warning(disable : 4838) -# pragma warning(disable : 4996) -#elif defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wsign-conversion" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wsign-conversion" -#endif - - -#if C4FLOAT_STD_ATOF -#include -double doit(const char *s) { return atof(s); } -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_SSCANF_F -float doit(const char *s) { float val; sscanf(s, "%f", &val); return val; } -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_SSCANF_D -double doit(const char *s) { double val; sscanf(s, "%lf", &val); return val; } -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_IOSTREAM_F -#include -float doit(const char *s) { std::stringstream ss; ss << s; float val; ss >> val; return val; } -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_IOSTREAM_D -#include -double doit(const char *s) { std::stringstream ss; ss << s; double val; ss >> val; return val; } -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_IOSTREAM_D -#include -double doit(const char *s) { std::stringstream ss; ss << s; double val; ss >> val; return val; } -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_FP_F_LIMITED -#include -float doit(const char *s) -{ - auto result = jkj::fp::from_chars_limited(s, s+strlen(s)); - return result.to_float(); -} -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_FP_D_LIMITED -#include -double doit(const char *s) -{ - auto result = jkj::fp::from_chars_limited(s, s+strlen(s)); - return result.to_float(); -} -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_FP_F_UNLIMITED -#include -float doit(const char *s) -{ - auto result = jkj::fp::from_chars_unlimited(s, s+strlen(s)); - return result.to_float(); -} -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_FP_D_UNLIMITED -#include -double doit(const char *s) -{ - auto result = jkj::fp::from_chars_unlimited(s, s+strlen(s)); - return result.to_float(); -} -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_FASTFLOAT_F -#include -#include -float doit(const char *s) -{ - float result; - fast_float::from_chars(s, s+strlen(s), result); - return result; -} -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_FASTFLOAT_D -#include -#include -double doit(const char *s) -{ - double result; - fast_float::from_chars(s, s+strlen(s), result); - return result; -} -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_STD_FROM_CHARS_F -#include -#include -float doit(const char *s) -{ - float result; - std::from_chars(s, s+strlen(s), result); - return result; -} -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_STD_FROM_CHARS_D -#include -#include -double doit(const char *s) -{ - double result; - std::from_chars(s, s+strlen(s), result); - return result; -} -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_RYU_F -#include -float doit(const char *s) -{ - float result; - s2f(s, &result); - return result; -} -#define C4_TO_REAL(s) doit(s) - -#elif C4FLOAT_RYU_D -#include -double doit(const char *s) -{ - double result; - s2d(s, &result); - return result; -} -#define C4_TO_REAL(s) doit(s) - -#else -#define C4_TO_REAL(s) 0 -#endif - -int main() -{ - #define BUFSIZE 128 - char buf[BUFSIZE]; - while(fgets(buf, BUFSIZE, stdin)) - { - fputs(buf, stdout); - (void) C4_TO_REAL(buf); - } -} - - -#ifdef _MSC_VER -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/ext/c4core/bm/ryu.cmake b/thirdparty/ryml/ext/c4core/bm/ryu.cmake deleted file mode 100644 index 93079a489..000000000 --- a/thirdparty/ryml/ext/c4core/bm/ryu.cmake +++ /dev/null @@ -1,37 +0,0 @@ -# ryu does not have a cmakelists - -if(C4CORE_BM_USE_RYU) - enable_language(C) - - c4_download_remote_proj(ryu RYU_DIR - GIT_REPOSITORY https://github.com/ulfjack/ryu - GIT_TAG master GIT_SHALLOW ON) - set(RYU_HDR - ${RYU_DIR}/ryu/common.h - ${RYU_DIR}/ryu/d2fixed_full_table.h - ${RYU_DIR}/ryu/d2s_full_table.h - ${RYU_DIR}/ryu/d2s_intrinsics.h - ${RYU_DIR}/ryu/d2s_small_table.h - ${RYU_DIR}/ryu/digit_table.h - ${RYU_DIR}/ryu/f2s_full_table.h - ${RYU_DIR}/ryu/f2s_intrinsics.h - ${RYU_DIR}/ryu/ryu.h - ${RYU_DIR}/ryu/ryu_parse.h - ) - set(RYU_SRC - ${RYU_DIR}/ryu/d2fixed.c - ${RYU_DIR}/ryu/d2s.c - ${RYU_DIR}/ryu/f2s.c - ${RYU_DIR}/ryu/s2d.c - ${RYU_DIR}/ryu/s2f.c - ) - add_library(ryu_c4 ${RYU_SRC} ${RYU_HDR}) - target_include_directories(ryu_c4 PUBLIC $) - set_target_properties(ryu_c4 PROPERTIES LINKER_LANGUAGE CXX) - if(CMAKE_CXX_COMPILER_ID STREQUAL GNU) - target_compile_options(ryu_c4 PRIVATE -Wno-sign-conversion) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL GNU) - target_compile_options(ryu_c4 -Wno-deprecated) - endif() - _c4_set_target_folder(ryu_c4 ext) -endif() diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.0.md b/thirdparty/ryml/ext/c4core/changelog/0.1.0.md deleted file mode 100644 index 7e9e8a67b..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.0.md +++ /dev/null @@ -1,3 +0,0 @@ -# 0.1.0 - -First release. diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.1.md b/thirdparty/ryml/ext/c4core/changelog/0.1.1.md deleted file mode 100644 index 934c65f30..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.1.md +++ /dev/null @@ -1,5 +0,0 @@ -# 0.1.1 - -- Fix parsing of hexadecimal floats ([2d5c3f0](https://github.com/biojppm/c4core/commits/2d5c3f0)) -- Fix `csubstr::reverse_sub()` ([902c5b9](https://github.com/biojppm/c4core/commits/902c5b9)) -- Fix [#35](https://github.com/biojppm/c4core/issues/35): add SO_VERSION diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.10.md b/thirdparty/ryml/ext/c4core/changelog/0.1.10.md deleted file mode 100644 index ce95aaee1..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.10.md +++ /dev/null @@ -1,106 +0,0 @@ -### Changes - -Improved the performance of `c4/charconv.hpp` functions ([PR#77](https://github.com/biojppm/c4core/pull/77)): - - Added `digits_dec/hex/oct/bin()`. - - Optimized `write_dec/hex/oct/bin()`: - - these functions now return immediately without entering the loop if the output buffer is smaller than respectively `digits_dec/hex/oct/bin()`. This enables both: - - writing every character in its final position without having to revert the string at the end - - the need to check the buffer size on appending every character. - - `write_dec()` now writes two digits at once, thus halving the number of integer divisions. - - Added `write_dec/hex/oct/bin_unchecked()`, which receive precomputed `digits_dec/hex/oct/bin()`, thus speeding up the radix `itoa()/utoa()` overloads. - - Added `xtoa()` radix+digits overloads: - - `size_t xtoa(substr s, T v, T radix)` - - `size_t xtoa(substr s, T v, T radix, size_t num_digits)` - - `read_dec/hex/oct/bin()`: these functions no longer allow an empty input buffer. - - Use intrinsic functions `__builtin_clz()` (gcc) / `_BitScanReverse()` (msvc) in `c4::msb()` and `__builtin_ctz()` (gcc) / `_BitScanForward()` (msvc) in `c4::lsb()` when they are available. `msb()` is used by `digits_hex()/digits_bin()`. - - Refactored the charconv tests to improve consistency and thoroughness. - - Improved the charconv benchmarks to ensure full consistency across benchmarks. - - Special thanks and kudos to @fargies for being attentive and pinpointing several issues throughout the PR! - - Finding the best approach involved [writing a R&D benchmark for the several algorithm components](https://github.com/biojppm/c4core/tree/master/bm/bm_xtoa.cpp). This benchmark is disabled by default, and can be enabled with the flag `C4CORE_BM_XTOA_RND`. - - With the changes from this PR, the [charconv benchmark](https://github.com/biojppm/c4core/tree/master/bm_charconv.cpp) results show that on Linux/g++11.2, with integral types: - - `c4::to_chars()` can be expected to be roughly... - - ~40% to 2x faster than `std::to_chars()` - - ~10x-30x faster than `sprintf()` - - ~50x-100x faster than a naive `stringstream::operator<<()` followed by `stringstream::str()` - - `c4::from_chars()` can be expected to be roughly... - - ~10%-30% faster than `std::from_chars()` - - ~10x faster than `scanf()` - - ~30x-50x faster than a naive `stringstream::str()` followed by `stringstream::operator>>()` - - Here are the results from the run: - | Write throughput | | Read throughput | | - |:-------------------------|--------:|:-------------------------|---------:| - | **write `uint8_t`** | **MB/s**| **read `uint8_t`** | **MB/s**| - | `c4::to_chars` | 526.86 | `c4::from_chars` | 163.06 | - | `std::to_chars` | 379.03 | `std::from_chars` | 154.85 | - | `std::sprintf` | 20.49 | `std::scanf` | 15.75 | - | `std::stringstream` | 3.82 | `std::stringstream` | 3.83 | - | **write `int8_t`** | **MB/s**| **read `int8_t`** | **MB/s**| - | `c4::to_chars` | 599.98 | `c4::from_chars` | 184.20 | - | `std::to_chars` | 246.32 | `std::from_chars` | 156.40 | - | `std::sprintf` | 19.15 | `std::scanf` | 16.44 | - | `std::stringstream` | 3.83 | `std::stringstream` | 3.89 | - | **write `uint16_t`** | **MB/s**| **read `uint16_t`** | **MB/s**| - | `c4::to_chars` | 486.40 | `c4::from_chars` | 349.48 | - | `std::to_chars` | 454.24 | `std::from_chars` | 319.13 | - | `std::sprintf` | 38.74 | `std::scanf` | 28.12 | - | `std::stringstream` | 7.08 | `std::stringstream`| 6.73 | - | **write `int16_t`** | **MB/s**| **read `int16_t`** | **MB/s**| - | `c4::to_chars` | 507.44 | `c4::from_chars` | 282.95 | - | `std::to_chars` | 297.49 | `std::from_chars` | 186.18 | - | `std::sprintf` | 39.03 | `std::scanf` | 28.45 | - | `std::stringstream` | 6.98 | `std::stringstream`| 6.49 | - | **write `uint32_t`** | **MB/s**| **read `uint32_t`** | **MB/s**| - | `c4::to_chars` | 730.12 | `c4::from_chars` | 463.95 | - | `std::to_chars` | 514.76 | `std::from_chars` | 329.42 | - | `std::sprintf` | 71.19 | `std::scanf` | 44.97 | - | `std::stringstream` | 14.05 | `std::stringstream`| 12.57 | - | **write `int32_t`** | **MB/s**| **read `int32_t`** | **MB/s**| - | `c4::to_chars` | 618.76 | `c4::from_chars` | 345.53 | - | `std::to_chars` | 394.72 | `std::from_chars` | 224.46 | - | `std::sprintf` | 71.14 | `std::scanf` | 43.49 | - | `std::stringstream` | 13.91 | `std::stringstream`| 12.03 | - | **write `uint64_t`** | **MB/s**| **read `uint64_t`** | **MB/s**| - | `c4::to_chars` | 1118.87 | `c4::from_chars` | 928.49 | - | `std::to_chars` | 886.58 | `std::from_chars` | 759.03 | - | `std::sprintf` | 140.96 | `std::scanf` | 91.60 | - | `std::stringstream` | 28.01 | `std::stringstream`| 25.00 | - | **write `int64_t`** | **MB/s**| **read `int64_t`** | **MB/s**| - | `c4::to_chars` | 1198.78 | `c4::from_chars` | 713.76 | - | `std::to_chars` | 882.17 | `std::from_chars` | 646.18 | - | `std::sprintf` | 138.79 | `std::scanf` | 90.07 | - | `std::stringstream` | 27.62 | `std::stringstream`| 25.12 | - -If you feel suspicious about these bold claims, you can browse through [c4core's CI benchmark results](https://github.com/biojppm/c4core/actions/workflows/benchmarks.yml) which will hopefully give these more substance. - - -### New features - -- Added `bool c4::overflows(csubstr s)` for detecting whether a string overflows a given integral type. See [PR#78](https://github.com/biojppm/c4core/pull/78). - - Also, added `c4::fmt::overflow_checked()` (and the corresponding `from_chars()` overload) to enable a check for overflow before parsing from string: - ```c++ - c4::from_chars(str, &val); // no overflow check - c4::from_chars(str, c4::fmt::overflow_checked(val)); // enable overflow check - // as an example, the implementation looks like: - template - bool c4::from_chars(c4::csubstr str, c4::fmt::overflow_checked oc) - { - if(overflows(str)) - return false; - return c4::from_chars(str, oc.val); - } - ``` - -### Fixes - -- Fix missing endianess macro on windows arm/arm64 compilations [PR #76](https://github.com/biojppm/c4core/pull/76) -- Add missing `#define` for the include guard of the amalgamated header (see [rapidyaml#246](https://github.com/biojppm/rapidyaml/issues/246)). -- Fix CPU detection with ARMEL [PR #86](https://github.com/biojppm/c4core/pull/86). -- Fix GCC version detection [PR #87](https://github.com/biojppm/c4core/pull/87). -- Fix [cmake#8](https://github.com/biojppm/cmake/issues/8): `SOVERSION` missing from shared libraries. -- Update fastfloat to 3.5.1. - -### Thanks - -- @fargies -- @daichifukui -- @janisozaur diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.11.md b/thirdparty/ryml/ext/c4core/changelog/0.1.11.md deleted file mode 100644 index 7c5ce8600..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.11.md +++ /dev/null @@ -1,59 +0,0 @@ - -### Breaking changes - -- `csubstr::operator==(std::nullptr_t)` now strictly checks if the pointer is null and no longer looks at the length ([rapidyaml#264](https://github.com/biojppm/rapidyaml/pull/264)): - ```diff - -bool csubstr::operator== (std::nullptr_t) const noexcept { return str == nullptr || len == 0; } - -bool csubstr::operator!= (std::nullptr_t) const noexcept { return str != nullptr || len == 0; } - +bool csubstr::operator== (std::nullptr_t) const noexcept { return str == nullptr; } - +bool csubstr::operator!= (std::nullptr_t) const noexcept { return str != nullptr; } - ``` -- `to_substr(std::string &s)` and `to_csubstr(std::string const& s)` now point at the first element when the string is empty ([rapidyaml#264](https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1264421024)): - ```diff - - return c4::substr(!s.empty() ? &s[0] : nullptr, s.size()); - + return c4::substr(&s[0], s.size()); - ``` - This is OK because an empty `std::string` is guaranteed to have storage, so calling `s[0]` is safe. - - -### New features - -- `charconv.hpp`: added `xtoa()` floating-point overloads accepting precision and format ([PR#88](https://github.com/biojppm/c4core/pull/88)): - ```c++ - size_t xtoa(substr s, float v, int precision, RealFormat_e formatting=FTOA_FLEX) noexcept; - size_t xtoa(substr s, double v, int precision, RealFormat_e formatting=FTOA_FLEX) noexcept; - ``` -- `memory_util.hpp`: added `ipow()` overloads for computing powers with integral exponents ([PR#88](https://github.com/biojppm/c4core/pull/88)). -- Add `C4_NO_DEBUG_BREAK` preprocessor check to disable calls to `c4::debug_break()` (see [rapidyaml#326](https://github.com/biojppm/rapidyaml/issues/326)) - - The cmake project conditionally enables this macro if the cmake option `C4CORE_NO_DEBUG_BREAK` is set to `ON`. - - -### Fixes - -- `substr`, `to_chars()`, charconv: ensure `memcpy()` is not called when the length is zero. Doing this is UB and enabled the optimizer to wreak havoc in the branches of calling code. See comments at [rapidyaml#264](https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637) for an example and fix. See [Raymond Chen's blog](https://devblogs.microsoft.com/oldnewthing/20140627-00/?p=633) for an explanation. -- `atof()` and `atod()` ([PR#88](https://github.com/biojppm/c4core/pull/88)): - - Always use the fastest implementation available: `std::from_chars()` if available (C++17 or higher standard, with later compilers), `fast_float::from_chars()` otherwise. On Visual Studio, `fast_float::from_chars()` is preferred over `std::from_chars()`. - - If `std::from_chars()` is not available and `C4CORE_NO_FAST_FLOAT` is defined, then the fallback is based on `sscanf()`. - - Ensure hexadecimal floats are accepted. The current fast_float implementation does not accept hexadecimal floats, so an hexfloat scanner was added. -- Likewise for `ftoa()` and `dtoa()`. Prefer the fastest implementation available: `std::to_chars()`->`snprintf()`. - - Change the `FTOA_*` enum values and type to save a function call when converting format. From now on, only the symbols of this enum can be relied on; the values or type will change depending on the selected implementation (`std::to_chars()` or `snprintf()`) ([PR#91](https://github.com/biojppm/c4core/pull/91)). -- Fix [#84](https://github.com/biojppm/c4core/issues/84): `csubstr::compare(char)`: refactor to avoid false-positive warning from VS2022. -- `csubstr` methods: add `noexcept` and annotations `C4_PURE` and `C4_ALWAYS_INLINE` -- `csubstr`: add `C4_RESTRICT` to incoming string on `csubstr::compare()` -- `csubstr::first_real_span()` ([PR#89](https://github.com/biojppm/c4core/pull/89)): - - Refactor to fix number matching rules. Now fully valid for floating point numbers in decimal (eg `0.123/1.23e+01`), hexadecimal (eg `0x123.abc/0x1.23abcp+01`), binary (eg `0b101.10/0b1.0110p+01`) and octal format (eg `0o701.10/0o7.0110p+01`) , with or without exponent or power, in lower or upper case. - - Also, make the number parsing stateful to fix cases where repeated characters occur, (like e.g. `0.1.0` or `1.23e+e10`) which are no longer reported as numbers (see [biojppm/rapidyaml#291](https://github.com/biojppm/rapidyaml/issues/291)). -- `csubstr::first_int_span()`, `csubstr::first_uint_span()`: fix edge cases like e.g. `0xzz` which were wrongly reported as numbers. -- Add fully qualified ARM detection macros: - - `__ARM_ARCH_7EM__` ([PR#90](https://github.com/biojppm/c4core/pull/90)). - - `__ARM_ARCH_6KZ__` ([PR#93](https://github.com/biojppm/c4core/pull/93)). - - `__ARM_ARCH_8A__` ([#94](https://github.com/biojppm/c4core/issues/94)). -- Improve linux and unix platform detection: detect both `__linux` and `__linux__` ([PR#92](https://github.com/biojppm/c4core/pull/92)). - - -### Thanks - -- @mlondono74 -- @musicinmybrain -- @pkubaj -- @Gei0r diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.2.md b/thirdparty/ryml/ext/c4core/changelog/0.1.2.md deleted file mode 100644 index d9c2e9c6c..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.2.md +++ /dev/null @@ -1,4 +0,0 @@ -- Fix error macros (ie `C4_ERROR()`, `C4_CHECK()`, `C4_ASSERT()`, etc) such that they are a single statement -- `is_debugger_attached()`: add MacOSX version -- Add support for Visual Studio 2022 -- Ensure `C4_LITTLE_ENDIAN` is always defined, even with mixed endianness diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.3.md b/thirdparty/ryml/ext/c4core/changelog/0.1.3.md deleted file mode 100644 index f98b88f75..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.3.md +++ /dev/null @@ -1 +0,0 @@ -- Update fast_float to [3.2.1](https://github.com/fastfloat/fast_float/releases/tag/v3.2.0) diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.4.md b/thirdparty/ryml/ext/c4core/changelog/0.1.4.md deleted file mode 100644 index b8e88203e..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.4.md +++ /dev/null @@ -1,6 +0,0 @@ -- [PR #38](https://github.com/biojppm/c4core/pull/38): add s390x architecture feature macros. -- Fix compiler warnings after update of fast_float to [3.2.1](https://github.com/fastfloat/fast_float/releases/tag/v3.2.0). - -### Thanks - -@musicinmybrain diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.5.md b/thirdparty/ryml/ext/c4core/changelog/0.1.5.md deleted file mode 100644 index e4884472e..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.5.md +++ /dev/null @@ -1,2 +0,0 @@ -- Add support for aarch64, s390x, ppc64le CPU architectures -- Update debugbreak header (added support for the above architectures) diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.6.md b/thirdparty/ryml/ext/c4core/changelog/0.1.6.md deleted file mode 100644 index 296de1398..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.6.md +++ /dev/null @@ -1,2 +0,0 @@ -- Fix wrong version names in version 0.1.5 (was saying 0.1.4, should be 0.1.5) - diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.7.md b/thirdparty/ryml/ext/c4core/changelog/0.1.7.md deleted file mode 100644 index 2b053a069..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.7.md +++ /dev/null @@ -1,5 +0,0 @@ -- Fix build with C4CORE_NO_FAST_FLOAT ([#42](https://github.com/biojppm/c4core/pull/42)). -- Fix clang warning in AIX/xlclang ([#44](https://github.com/biojppm/c4core/pull/44)). - -### Thanks ---- @mbs-c diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.8.md b/thirdparty/ryml/ext/c4core/changelog/0.1.8.md deleted file mode 100644 index db77aaf97..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.8.md +++ /dev/null @@ -1,45 +0,0 @@ - -### New features - -- Add amalgamation into a single header file ([PR #48](https://github.com/biojppm/c4core/pull/48)): - - The amalgamated header will be available together with the deliverables from each release. - - To generate the amalgamated header: - ``` - $ python tools/amalgamate.py c4core_all.hpp - ``` - - To use the amalgamated header: - - Include at will in any header of your project. - - In one - and only one - of your project source files, `#define C4CORE_SINGLE_HDR_DEFINE_NOW` and then `#include `. This will enable the function and class definitions in the header file. For example, here's a sample program: - ```c++ - #include - #define C4CORE_SINGLE_HDR_DEFINE_NOW // do this before the include - #include - int main() - { - for(c4::csubstr s : c4::csubstr("a/b/c/d").split('/')) - std::cout << s << "\n"; - } - ``` -- Add `csubstr::is_unsigned_integer()` and `csubstr::is_real()` ([PR #49](https://github.com/biojppm/c4core/pull/49)). -- CMake: add alias target c4core::c4core, guaranteeing that the same code can be used with `add_subdirectory()` and `find_package()`. (see [rapidyaml #173](https://github.com/biojppm/rapidyaml/issues/173)) -- Add support for compilation with emscripten (WebAssembly+javascript) ([PR #52](https://github.com/biojppm/c4core/pull/52)). - - -### Fixes - -- Fix edge cases with empty strings in `span::first()`, `span::last()` and `span::range()` ([PR #49](https://github.com/biojppm/c4core/pull/49)). -- Accept octal numbers in `substr::first_real_span()` and `substr::is_real()` ([PR #49](https://github.com/biojppm/c4core/pull/49)). -- `substr`: fix coverage misses in number query methods ([PR #49](https://github.com/biojppm/c4core/pull/49)). -- Use single-header version of fast_float ([PR #49](https://github.com/biojppm/c4core/pull/47)). -- Suppress warnings triggered from fast_float in clang (`-Wfortify-source`) ([PR #49](https://github.com/biojppm/c4core/pull/47)). -- Add missing `inline` in [src/c4/ext/rng/rng.hpp](src/c4/ext/rng/rng.hpp) ([PR #49](https://github.com/biojppm/c4core/pull/47)). -- Fix compilation of [src/c4/ext/rng/inplace_function.h](src/c4/ext/inplace_function.h) in C++11 ([PR #49](https://github.com/biojppm/c4core/pull/47)). -- Change order of headers, notably in `windows_push.hpp` ([PR #47](https://github.com/biojppm/c4core/pull/47)). -- In `c4/charconv.hpp`: do not use C4_ASSERT in `to_c_fmt()`, which is `constexpr`. -- Fix [#53](https://github.com/biojppm/c4core/issues/53): cmake install targets were missing call to `export()` ([PR #55](https://github.com/biojppm/c4core/pull/55)). -- Fix linking of subprojects with libc++: flags should be forwarded through `CMAKE_***_FLAGS` instead of being set explicitly per-target ([PR #54](https://github.com/biojppm/c4core/pull/54)). - - -### Thanks - -- @cschreib diff --git a/thirdparty/ryml/ext/c4core/changelog/0.1.9.md b/thirdparty/ryml/ext/c4core/changelog/0.1.9.md deleted file mode 100644 index 1ac5aa842..000000000 --- a/thirdparty/ryml/ext/c4core/changelog/0.1.9.md +++ /dev/null @@ -1,31 +0,0 @@ -### Breaking changes - -- fix [#63](https://github.com/biojppm/c4core/issues/63): remove `c4/time.hpp` and `c4/time.cpp` which prevented compilation in bare-metal mode ([PR #64](https://github.com/biojppm/c4core/issues/64)). - -### New features - -- Added decoding of UTF codepoints: `c4::decode_code_point()` ([PR #65](https://github.com/biojppm/c4core/issues/65)). -- Experimental feature: add formatted-dumping facilities: using semantics like `c4::cat()`, `c4::catsep()` and `c4::format()`, where the subject is not a string buffer but a dump callback accepting strings. This still requires a string buffer for serialization of non-string types, but the buffer's required size is now limited to the max serialized size of non-string arguments, in contrast to the requirement in `c4::cat()` et al which is the total serialized size of every argument. This enables very efficient and generic printf-like semantics with reuse of a single small buffer, and allows direct-printing to terminal or file ([PR #67](https://github.com/biojppm/c4core/issues/67)). This feature is still experimental and a minor amount of changes to the API is possible. -- Added macro `C4_IF_CONSTEXPR` resolving to `if constexpr (...)` if the c++ standard is at least c++17. -- `csubstr`: add `count(csubstr)` overload. -- Add support for RISC-V architectures ([PR #69](https://github.com/biojppm/c4core/issues/69)). -- Add support for bare-metal compilation ([PR #64](https://github.com/biojppm/c4core/issues/64)). -- gcc >= 4.8 support using polyfills for missing templates and features ([PR #74](https://github.com/biojppm/c4core/pull/74) and [PR #68](https://github.com/biojppm/c4core/pull/68)). - -### Fixes - -- `csubstr::operator==(std::nullptr_t)` now returns true if either `.str==nullptr` or `.len==0`. -- Fix: `bool operator==(const char (&s)[N], csubstr)` and `operator==(const char (&s)[N], substr)`. The template declaration for these functions had an extra `const` which prevented these functions to participate in overload resolution, which in some cases resulted in calls resolving to `operator==(std::string const&, csubstr)` if that header was visible ([PR #64](https://github.com/biojppm/c4core/issues/64)). -- Fix `csubstr::last_not_of()`: optional positional parameter was ignored [PR #62](https://github.com/biojppm/c4core/pull/62). -- `atof()`, `atod()`, `atox()`, `substr::is_real()`, `substr::first_real_span()`: accept `infinity`, `inf` and `nan` as valid reals [PR #60](https://github.com/biojppm/c4core/pull/60). -- Add missing export symbols [PR #56](https://github.com/biojppm/c4core/pull/56), [PR #57](https://github.com/biojppm/c4core/pull/57). -- `c4/substr_fwd.hpp`: fix compilation failure in Xcode 12 and earlier, where the forward declaration for `std::allocator` is inside the `inline namespace __1`, unlike later versions [PR #61](https://github.com/biojppm/c4core/pull/61), reported in [rapidyaml#185](https://github.com/biojppm/rapidyaml/issues/185). -- `c4/error.hpp`: fix compilation failure in debug mode in Xcode 12 and earlier: `__clang_major__` does not mean the same as in the common clang, and as a result the warning `-Wgnu-inline-cpp-without-extern` does not exist there. - - -### Thanks - -- @danngreen -- @Xeonacid -- @aviktorov -- @fargies diff --git a/thirdparty/ryml/ext/c4core/changelog/current.md b/thirdparty/ryml/ext/c4core/changelog/current.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/thirdparty/ryml/ext/c4core/cmake/.gitignore b/thirdparty/ryml/ext/c4core/cmake/.gitignore deleted file mode 100644 index ed8ebf583..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__ \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/cmake/ConfigurationTypes.cmake b/thirdparty/ryml/ext/c4core/cmake/ConfigurationTypes.cmake deleted file mode 100644 index 45395ad1b..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/ConfigurationTypes.cmake +++ /dev/null @@ -1,120 +0,0 @@ - - -# this function works both with multiconfig and single-config generators. -function(set_default_build_type which) - # CMAKE_CONFIGURATION_TYPES is available only for multiconfig generators. - # so set the build type only if CMAKE_CONFIGURATION_TYPES does not exist. - if(NOT CMAKE_CONFIGURATION_TYPES) # not a multiconfig generator? - if(NOT CMAKE_BUILD_TYPE) - if(NOT which) - set(which RelWithDebInfo) - endif() - message("Defaulting to ${which} build.") - set(CMAKE_BUILD_TYPE ${which} CACHE STRING "") - endif() - endif() -endfunction() - - -# https://stackoverflow.com/questions/31546278/where-to-set-cmake-configuration-types-in-a-project-with-subprojects -function(setup_configuration_types) - set(options0arg - ) - set(options1arg - DEFAULT - ) - set(optionsnarg - TYPES - ) - cmake_parse_arguments("" "${options0arg}" "${options1arg}" "${optionsnarg}" ${ARGN}) - - if(NOT TYPES) - set(TYPES Release Debug RelWithDebInfo MinSizeRel) - endif() - - # make it safe to call repeatedly - if(NOT _setup_configuration_types_done) - set(_setup_configuration_types_done 1 CACHE INTERNAL "") - - # No reason to set CMAKE_CONFIGURATION_TYPES if it's not a multiconfig generator - # Also no reason mess with CMAKE_BUILD_TYPE if it's a multiconfig generator. - - if(CMAKE_CONFIGURATION_TYPES) # multiconfig generator? - set(CMAKE_CONFIGURATION_TYPES "${TYPES}" CACHE STRING "") - else() # single-config generator - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY HELPSTRING "Choose the type of build") - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "${TYPES}") - # set the valid options for cmake-gui drop-down list - endif() - endif() -endfunction() - - -# https://stackoverflow.com/questions/31546278/where-to-set-cmake-configuration-types-in-a-project-with-subprojects -function(add_configuration_type name) - set(flag_vars - C_FLAGS - CXX_FLAGS - SHARED_LINKER_FLAGS - STATIC_LINKER_FLAGS - MODULE_LINKER_FLAGS - EXE_LINKER_FLAGS - RC_FLAGS - ) - - set(options0arg - PREPEND # when defaulting to a config, prepend to it instead of appending to it - SET_MAIN_FLAGS # eg, set CMAKE_CXX_FLAGS from CMAKE_CXX_FLAGS_${name} - ) - set(options1arg - DEFAULT_FROM # take the initial value of the flags from this config - ) - set(optionsnarg - C_FLAGS - CXX_FLAGS - SHARED_LINKER_FLAGS - STATIC_LINKER_FLAGS - MODULE_LINKER_FLAGS - EXE_LINKER_FLAGS - RC_FLAGS - ) - cmake_parse_arguments(_act "${options0arg}" "${options1arg}" "${optionsnarg}" ${ARGN}) - - string(TOUPPER ${name} UNAME) - - # make it safe to call repeatedly - if(NOT _add_configuration_type_${name}) - set(_add_configuration_type_${name} 1 CACHE INTERNAL "") - - setup_configuration_types() - - if(CMAKE_CONFIGURATION_TYPES) # multiconfig generator? - set(CMAKE_CONFIGURATION_TYPES "${CMAKE_CONFIGURATION_TYPES};${name}" CACHE STRING "" FORCE) - else() # single-config generator - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY HELPSTRING "Choose the type of build" FORCE) - set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "${CMAKE_BUILD_TYPES};${name}" FORCE) - # set the valid options for cmake-gui drop-down list - endif() - - # now set up the configuration - message(STATUS "config: CMAKE_${f}_${UNAME} --- ${val}") - foreach(f ${flag_vars}) - set(val ${_act_${f}}) - message(STATUS "config: ${name}: ${f} --- ${val}") - if(_act_DEFAULT_FROM) - if(_act_PREPEND) - set(val "${val} ${CMAKE_${f}_${_act_DEFAULT_FROM}}") - else() - set(val "${CMAKE_${f}_${_act_DEFAULT_FROM}} ${val}") - endif() - endif() - message(STATUS "config: CMAKE_${f}_${UNAME} --- ${val}") - set(CMAKE_${f}_${UNAME} "${val}" CACHE STRING "" FORCE) - mark_as_advanced(CMAKE_${f}_${UNAME}) - if(_act_SET_MAIN_FLAGS) - set(CMAKE_${f} "${CMAKE_${f}_${UNAME}}" CACHE STRING "" FORCE) - endif() - endforeach() - endif() - -endfunction() diff --git a/thirdparty/ryml/ext/c4core/cmake/CreateSourceGroup.cmake b/thirdparty/ryml/ext/c4core/cmake/CreateSourceGroup.cmake deleted file mode 100644 index e8e144184..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/CreateSourceGroup.cmake +++ /dev/null @@ -1,31 +0,0 @@ -# create hierarchical source groups based on a dir tree -# -# EXAMPLE USAGE: -# -# create_source_group("src" "${SRC_ROOT}" "${SRC_LIST}") -# -# Visual Studio usually has the equivalent to this: -# -# create_source_group("Header Files" ${PROJ_SRC_DIR} "${PROJ_HEADERS}") -# create_source_group("Source Files" ${PROJ_SRC_DIR} "${PROJ_SOURCES}") -# -# TODO: this was taken from a stack overflow answer. Need to find it -# and add a link here. - -macro(create_source_group GroupPrefix RootDir ProjectSources) - set(DirSources ${ProjectSources}) - foreach(Source ${DirSources}) - #message(STATUS "s=${Source}") - string(REGEX REPLACE "([\\^\\$*+?|])" "\\\\\\1" RootDirRegex "${RootDir}") - string(REGEX REPLACE "${RootDirRegex}" "" RelativePath "${Source}") - #message(STATUS " ${RelativePath}") - string(REGEX REPLACE "[\\\\/][^\\\\/]*$" "" RelativePath "${RelativePath}") - #message(STATUS " ${RelativePath}") - string(REGEX REPLACE "^[\\\\/]" "" RelativePath "${RelativePath}") - #message(STATUS " ${RelativePath}") - string(REGEX REPLACE "/" "\\\\\\\\" RelativePath "${RelativePath}") - #message(STATUS " ${RelativePath}") - source_group("${GroupPrefix}\\${RelativePath}" FILES ${Source}) - #message(STATUS " ${Source}") - endforeach(Source) -endmacro(create_source_group) diff --git a/thirdparty/ryml/ext/c4core/cmake/Doxyfile.full.in b/thirdparty/ryml/ext/c4core/cmake/Doxyfile.full.in deleted file mode 100644 index f444cb742..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/Doxyfile.full.in +++ /dev/null @@ -1,2566 +0,0 @@ -# Doxyfile 1.8.15 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project. -# -# All text after a double hash (##) is considered a comment and is placed in -# front of the TAG it is preceding. -# -# All text after a single hash (#) is considered a comment and will be ignored. -# The format is: -# TAG = value [value, ...] -# For lists, items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (\" \"). - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- - -# This tag specifies the encoding used for all characters in the configuration -# file that follow. The default is UTF-8 which is also the encoding used for all -# text before the first occurrence of this tag. Doxygen uses libiconv (or the -# iconv built into libc) for the transcoding. See -# https://www.gnu.org/software/libiconv/ for the list of possible encodings. -# The default value is: UTF-8. - -DOXYFILE_ENCODING = UTF-8 - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by -# double-quotes, unless you are using Doxywizard) that should identify the -# project for which the documentation is generated. This name is used in the -# title of most generated pages and in a few other places. -# The default value is: My Project. - -PROJECT_NAME = @_PROJ@ - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. This -# could be handy for archiving the generated documentation or if some version -# control system is used. - -PROJECT_NUMBER = @_VERSION@ - -# Using the PROJECT_BRIEF tag one can provide an optional one line description -# for a project that appears at the top of each page and should give viewer a -# quick idea about the purpose of the project. Keep the description short. - -PROJECT_BRIEF = @_PROJ_BRIEF@ - -# With the PROJECT_LOGO tag one can specify a logo or an icon that is included -# in the documentation. The maximum height of the logo should not exceed 55 -# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy -# the logo to the output directory. - -PROJECT_LOGO = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path -# into which the generated documentation will be written. If a relative path is -# entered, it will be relative to the location where doxygen was started. If -# left blank the current directory will be used. - -OUTPUT_DIRECTORY = @_OUTPUT_DIR@ - -# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- -# directories (in 2 levels) under the output directory of each output format and -# will distribute the generated files over these directories. Enabling this -# option can be useful when feeding doxygen a huge amount of source files, where -# putting all generated files in the same directory would otherwise causes -# performance problems for the file system. -# The default value is: NO. - -CREATE_SUBDIRS = NO - -# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII -# characters to appear in the names of generated files. If set to NO, non-ASCII -# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode -# U+3044. -# The default value is: NO. - -ALLOW_UNICODE_NAMES = YES - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, -# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), -# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, -# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), -# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, -# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, -# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, -# Ukrainian and Vietnamese. -# The default value is: English. - -OUTPUT_LANGUAGE = English - -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - -# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member -# descriptions after the members that are listed in the file and class -# documentation (similar to Javadoc). Set to NO to disable this. -# The default value is: YES. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief -# description of a member or function before the detailed description -# -# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. -# The default value is: YES. - -REPEAT_BRIEF = YES - -# This tag implements a quasi-intelligent brief description abbreviator that is -# used to form the text in various listings. Each string in this list, if found -# as the leading text of the brief description, will be stripped from the text -# and the result, after processing the whole list, is used as the annotated -# text. Otherwise, the brief description is used as-is. If left blank, the -# following values are used ($name is automatically replaced with the name of -# the entity):The $name class, The $name widget, The $name file, is, provides, -# specifies, contains, represents, a, an and the. - -ABBREVIATE_BRIEF = "The $name class" \ - "The $name widget" \ - "The $name file" \ - is \ - provides \ - specifies \ - contains \ - represents \ - a \ - an \ - the - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# doxygen will generate a detailed section even if there is only a brief -# description. -# The default value is: NO. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all -# inherited members of a class in the documentation of that class as if those -# members were ordinary class members. Constructors, destructors and assignment -# operators of the base classes will not be shown. -# The default value is: NO. - -INLINE_INHERITED_MEMB = YES - -# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path -# before files name in the file list and in the header files. If set to NO the -# shortest path that makes the file name unique will be used -# The default value is: YES. - -FULL_PATH_NAMES = YES - -# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. -# Stripping is only done if one of the specified strings matches the left-hand -# part of the path. The tag can be used to show relative paths in the file list. -# If left blank the directory from which doxygen is run is used as the path to -# strip. -# -# Note that you can specify absolute paths here, but also relative paths, which -# will be relative from the directory where doxygen is started. -# This tag requires that the tag FULL_PATH_NAMES is set to YES. - -STRIP_FROM_PATH = @_STRIP_FROM_PATH@ - -# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the -# path mentioned in the documentation of a class, which tells the reader which -# header file to include in order to use a class. If left blank only the name of -# the header file containing the class definition is used. Otherwise one should -# specify the list of include paths that are normally passed to the compiler -# using the -I flag. - -STRIP_FROM_INC_PATH = @_STRIP_FROM_INC_PATH@ - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but -# less readable) file names. This can be useful is your file systems doesn't -# support long names like on DOS, Mac, or CD-ROM. -# The default value is: NO. - -SHORT_NAMES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the -# first line (until the first dot) of a Javadoc-style comment as the brief -# description. If set to NO, the Javadoc-style will behave just like regular Qt- -# style comments (thus requiring an explicit @brief command for a brief -# description.) -# The default value is: NO. - -JAVADOC_AUTOBRIEF = YES - -# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first -# line (until the first dot) of a Qt-style comment as the brief description. If -# set to NO, the Qt-style will behave just like regular Qt-style comments (thus -# requiring an explicit \brief command for a brief description.) -# The default value is: NO. - -QT_AUTOBRIEF = YES - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a -# multi-line C++ special comment block (i.e. a block of //! or /// comments) as -# a brief description. This used to be the default behavior. The new default is -# to treat a multi-line C++ comment block as a detailed description. Set this -# tag to YES if you prefer the old behavior instead. -# -# Note that setting this tag to YES also means that rational rose comments are -# not recognized any more. -# The default value is: NO. - -MULTILINE_CPP_IS_BRIEF = YES - -# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the -# documentation from any documented member that it re-implements. -# The default value is: YES. - -INHERIT_DOCS = YES - -# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new -# page for each member. If set to NO, the documentation of a member will be part -# of the file/class/namespace that contains it. -# The default value is: NO. - -SEPARATE_MEMBER_PAGES = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen -# uses this value to replace tabs by spaces in code fragments. -# Minimum value: 1, maximum value: 16, default value: 4. - -TAB_SIZE = 4 - -# This tag can be used to specify a number of aliases that act as commands in -# the documentation. An alias has the form: -# name=value -# For example adding -# "sideeffect=@par Side Effects:\n" -# will allow you to put the command \sideeffect (or @sideeffect) in the -# documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) - -ALIASES = - -# This tag can be used to specify a number of word-keyword mappings (TCL only). -# A mapping has the form "name=value". For example adding "class=itcl::class" -# will allow you to use the command class in the itcl::class meaning. - -TCL_SUBST = - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. For -# instance, some of the names that are used will be different. The list of all -# members will be omitted, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or -# Python sources only. Doxygen will then generate output that is more tailored -# for that language. For instance, namespaces will be presented as packages, -# qualified scopes will look different, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran -# sources. Doxygen will then generate output that is tailored for Fortran. -# The default value is: NO. - -OPTIMIZE_FOR_FORTRAN = NO - -# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL -# sources. Doxygen will then generate output that is tailored for VHDL. -# The default value is: NO. - -OPTIMIZE_OUTPUT_VHDL = NO - -# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice -# sources only. Doxygen will then generate output that is more tailored for that -# language. For instance, namespaces will be presented as modules, types will be -# separated into more groups, etc. -# The default value is: NO. - -OPTIMIZE_OUTPUT_SLICE = NO - -# Doxygen selects the parser to use depending on the extension of the files it -# parses. With this tag you can assign which parser to use for a given -# extension. Doxygen has a built-in mapping, but you can override or extend it -# using this tag. The format is ext=language, where ext is a file extension, and -# language is one of the parsers supported by doxygen: IDL, Java, Javascript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: -# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser -# tries to guess whether the code is fixed or free formatted code, this is the -# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat -# .inc files as Fortran files (default is PHP), and .f files as C (default is -# Fortran), use: inc=Fortran f=C. -# -# Note: For files without extension you can use no_extension as a placeholder. -# -# Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. - -EXTENSION_MAPPING = - -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments -# according to the Markdown format, which allows for more readable -# documentation. See https://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in -# case of backward compatibilities issues. -# The default value is: YES. - -MARKDOWN_SUPPORT = YES - -# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up -# to that level are automatically included in the table of contents, even if -# they do not have an id attribute. -# Note: This feature currently applies only to Markdown headings. -# Minimum value: 0, maximum value: 99, default value: 0. -# This tag requires that the tag MARKDOWN_SUPPORT is set to YES. - -TOC_INCLUDE_HEADINGS = 4 - -# When enabled doxygen tries to link words that correspond to documented -# classes, or namespaces to their corresponding documentation. Such a link can -# be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. -# The default value is: YES. - -AUTOLINK_SUPPORT = YES - -# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want -# to include (a tag file for) the STL sources as input, then you should set this -# tag to YES in order to let doxygen match functions declarations and -# definitions whose arguments contain STL classes (e.g. func(std::string); -# versus func(std::string) {}). This also make the inheritance and collaboration -# diagrams that involve STL classes more complete and accurate. -# The default value is: NO. - -BUILTIN_STL_SUPPORT = YES - -# If you use Microsoft's C++/CLI language, you should set this option to YES to -# enable parsing support. -# The default value is: NO. - -CPP_CLI_SUPPORT = NO - -# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: -# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen -# will parse them like normal C++ but will assume all classes use public instead -# of private inheritance when no explicit protection keyword is present. -# The default value is: NO. - -SIP_SUPPORT = NO - -# For Microsoft's IDL there are propget and propput attributes to indicate -# getter and setter methods for a property. Setting this option to YES will make -# doxygen to replace the get and set methods by a property in the documentation. -# This will only work if the methods are indeed getting or setting a simple -# type. If this is not the case, or you want to show the methods anyway, you -# should set this option to NO. -# The default value is: YES. - -IDL_PROPERTY_SUPPORT = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. -# The default value is: NO. - -DISTRIBUTE_GROUP_DOC = NO - -# If one adds a struct or class to a group and this option is enabled, then also -# any nested class or struct is added to the same group. By default this option -# is disabled and one has to add nested compounds explicitly via \ingroup. -# The default value is: NO. - -GROUP_NESTED_COMPOUNDS = YES - -# Set the SUBGROUPING tag to YES to allow class member groups of the same type -# (for instance a group of public functions) to be put as a subgroup of that -# type (e.g. under the Public Functions section). Set it to NO to prevent -# subgrouping. Alternatively, this can be done per class using the -# \nosubgrouping command. -# The default value is: YES. - -SUBGROUPING = YES - -# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions -# are shown inside the group in which they are included (e.g. using \ingroup) -# instead of on a separate page (for HTML and Man pages) or section (for LaTeX -# and RTF). -# -# Note that this feature does not work in combination with -# SEPARATE_MEMBER_PAGES. -# The default value is: NO. - -INLINE_GROUPED_CLASSES = NO - -# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions -# with only public data fields or simple typedef fields will be shown inline in -# the documentation of the scope in which they are defined (i.e. file, -# namespace, or group documentation), provided this scope is documented. If set -# to NO, structs, classes, and unions are shown on a separate page (for HTML and -# Man pages) or section (for LaTeX and RTF). -# The default value is: NO. - -INLINE_SIMPLE_STRUCTS = YES - -# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or -# enum is documented as struct, union, or enum with the name of the typedef. So -# typedef struct TypeS {} TypeT, will appear in the documentation as a struct -# with name TypeT. When disabled the typedef will appear as a member of a file, -# namespace, or class. And the struct will be named TypeS. This can typically be -# useful for C code in case the coding convention dictates that all compound -# types are typedef'ed and only the typedef is referenced, never the tag name. -# The default value is: NO. - -TYPEDEF_HIDES_STRUCT = NO - -# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This -# cache is used to resolve symbols given their name and scope. Since this can be -# an expensive process and often the same symbol appears multiple times in the -# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small -# doxygen will become slower. If the cache is too large, memory is wasted. The -# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range -# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 -# symbols. At the end of a run doxygen will report the cache usage and suggest -# the optimal cache size from a speed point of view. -# Minimum value: 0, maximum value: 9, default value: 0. - -LOOKUP_CACHE_SIZE = 0 - -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- - -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in -# documentation are documented, even if no documentation was available. Private -# class members and static file members will be hidden unless the -# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. -# Note: This will also disable the warnings about undocumented members that are -# normally produced when WARNINGS is set to YES. -# The default value is: NO. - -EXTRACT_ALL = YES - -# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will -# be included in the documentation. -# The default value is: NO. - -EXTRACT_PRIVATE = YES - -# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal -# scope will be included in the documentation. -# The default value is: NO. - -EXTRACT_PACKAGE = YES - -# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be -# included in the documentation. -# The default value is: NO. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined -# locally in source files will be included in the documentation. If set to NO, -# only classes defined in header files are included. Does not have any effect -# for Java sources. -# The default value is: YES. - -EXTRACT_LOCAL_CLASSES = YES - -# This flag is only useful for Objective-C code. If set to YES, local methods, -# which are defined in the implementation section but not in the interface are -# included in the documentation. If set to NO, only methods in the interface are -# included. -# The default value is: NO. - -EXTRACT_LOCAL_METHODS = YES - -# If this flag is set to YES, the members of anonymous namespaces will be -# extracted and appear in the documentation as a namespace called -# 'anonymous_namespace{file}', where file will be replaced with the base name of -# the file that contains the anonymous namespace. By default anonymous namespace -# are hidden. -# The default value is: NO. - -EXTRACT_ANON_NSPACES = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all -# undocumented members inside documented classes or files. If set to NO these -# members will be included in the various overviews, but no documentation -# section is generated. This option has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. If set -# to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend -# (class|struct|union) declarations. If set to NO, these declarations will be -# included in the documentation. -# The default value is: NO. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any -# documentation blocks found inside the body of a function. If set to NO, these -# blocks will be appended to the function's detailed documentation block. -# The default value is: NO. - -HIDE_IN_BODY_DOCS = NO - -# The INTERNAL_DOCS tag determines if documentation that is typed after a -# \internal command is included. If the tag is set to NO then the documentation -# will be excluded. Set it to YES to include the internal documentation. -# The default value is: NO. - -INTERNAL_DOCS = YES - -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# and Mac users are advised to set this option to NO. -# The default value is: system dependent. - -CASE_SENSE_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with -# their full class and namespace scopes in the documentation. If set to YES, the -# scope will be hidden. -# The default value is: NO. - -HIDE_SCOPE_NAMES = NO - -# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will -# append additional text to a page's title, such as Class Reference. If set to -# YES the compound reference will be hidden. -# The default value is: NO. - -HIDE_COMPOUND_REFERENCE= NO - -# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of -# the files that are included by a file in the documentation of that file. -# The default value is: YES. - -SHOW_INCLUDE_FILES = YES - -# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each -# grouped member an include statement to the documentation, telling the reader -# which file to include in order to use the member. -# The default value is: NO. - -SHOW_GROUPED_MEMB_INC = YES - -# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include -# files with double quotes in the documentation rather than with sharp brackets. -# The default value is: NO. - -FORCE_LOCAL_INCLUDES = NO - -# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the -# documentation for inline members. -# The default value is: YES. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the -# (detailed) documentation of file and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. -# The default value is: YES. - -SORT_MEMBER_DOCS = NO - -# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief -# descriptions of file, namespace and class members alphabetically by member -# name. If set to NO, the members will appear in declaration order. Note that -# this will also influence the order of the classes in the class list. -# The default value is: NO. - -SORT_BRIEF_DOCS = NO - -# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the -# (brief and detailed) documentation of class members so that constructors and -# destructors are listed first. If set to NO the constructors will appear in the -# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. -# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief -# member documentation. -# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting -# detailed member documentation. -# The default value is: NO. - -SORT_MEMBERS_CTORS_1ST = NO - -# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy -# of group names into alphabetical order. If set to NO the group names will -# appear in their defined order. -# The default value is: NO. - -SORT_GROUP_NAMES = NO - -# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by -# fully-qualified names, including namespaces. If set to NO, the class list will -# be sorted only by class name, not including the namespace part. -# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. -# Note: This option applies only to the class list, not to the alphabetical -# list. -# The default value is: NO. - -SORT_BY_SCOPE_NAME = NO - -# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper -# type resolution of all parameters of a function it will reject a match between -# the prototype and the implementation of a member function even if there is -# only one candidate or it is obvious which candidate to choose by doing a -# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still -# accept a match between prototype and implementation in such cases. -# The default value is: NO. - -STRICT_PROTO_MATCHING = NO - -# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo -# list. This list is created by putting \todo commands in the documentation. -# The default value is: YES. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test -# list. This list is created by putting \test commands in the documentation. -# The default value is: YES. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug -# list. This list is created by putting \bug commands in the documentation. -# The default value is: YES. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) -# the deprecated list. This list is created by putting \deprecated commands in -# the documentation. -# The default value is: YES. - -GENERATE_DEPRECATEDLIST= YES - -# The ENABLED_SECTIONS tag can be used to enable conditional documentation -# sections, marked by \if ... \endif and \cond -# ... \endcond blocks. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the -# initial value of a variable or macro / define can have for it to appear in the -# documentation. If the initializer consists of more lines than specified here -# it will be hidden. Use a value of 0 to hide initializers completely. The -# appearance of the value of individual variables and macros / defines can be -# controlled using \showinitializer or \hideinitializer command in the -# documentation regardless of this setting. -# Minimum value: 0, maximum value: 10000, default value: 30. - -MAX_INITIALIZER_LINES = 30 - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at -# the bottom of the documentation of classes and structs. If set to YES, the -# list will mention the files that were used to generate the documentation. -# The default value is: YES. - -SHOW_USED_FILES = YES - -# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This -# will remove the Files entry from the Quick Index and from the Folder Tree View -# (if specified). -# The default value is: YES. - -SHOW_FILES = YES - -# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces -# page. This will remove the Namespaces entry from the Quick Index and from the -# Folder Tree View (if specified). -# The default value is: YES. - -SHOW_NAMESPACES = YES - -# The FILE_VERSION_FILTER tag can be used to specify a program or script that -# doxygen should invoke to get the current version for each file (typically from -# the version control system). Doxygen will invoke the program by executing (via -# popen()) the command command input-file, where command is the value of the -# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided -# by doxygen. Whatever the program writes to standard output is used as the file -# version. For an example see the documentation. - -FILE_VERSION_FILTER = - -# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed -# by doxygen. The layout file controls the global structure of the generated -# output files in an output format independent way. To create the layout file -# that represents doxygen's defaults, run doxygen with the -l option. You can -# optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. -# -# Note that if you run doxygen from a directory containing a file called -# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE -# tag is left empty. - -LAYOUT_FILE = - -# The CITE_BIB_FILES tag can be used to specify one or more bib files containing -# the reference definitions. This must be a list of .bib files. The .bib -# extension is automatically appended if omitted. This requires the bibtex tool -# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info. -# For LaTeX the style of the bibliography can be controlled using -# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the -# search path. See also \cite for info how to create references. - -CITE_BIB_FILES = - -#--------------------------------------------------------------------------- -# Configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated to -# standard output by doxygen. If QUIET is set to YES this implies that the -# messages are off. -# The default value is: NO. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES -# this implies that the warnings are on. -# -# Tip: Turn warnings on while writing the documentation. -# The default value is: YES. - -WARNINGS = YES - -# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate -# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag -# will automatically be disabled. -# The default value is: YES. - -WARN_IF_UNDOCUMENTED = YES - -# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. -# The default value is: YES. - -WARN_IF_DOC_ERROR = YES - -# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that -# are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. -# The default value is: NO. - -WARN_NO_PARAMDOC = NO - -# If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. -# The default value is: NO. - -WARN_AS_ERROR = NO - -# The WARN_FORMAT tag determines the format of the warning messages that doxygen -# can produce. The string should contain the $file, $line, and $text tags, which -# will be replaced by the file and line number from which the warning originated -# and the warning text. Optionally the format may contain $version, which will -# be replaced by the version of the file (if it could be obtained via -# FILE_VERSION_FILTER) -# The default value is: $file:$line: $text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning and error -# messages should be written. If left blank the output is written to standard -# error (stderr). - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# Configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag is used to specify the files and/or directories that contain -# documented source files. You may enter file names like myfile.cpp or -# directories like /usr/src/myproject. Separate the files or directories with -# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING -# Note: If this tag is empty the current directory is searched. - -INPUT = @_INPUT@ - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses -# libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: https://www.gnu.org/software/libiconv/) for the list of -# possible encodings. -# The default value is: UTF-8. - -INPUT_ENCODING = UTF-8 - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and -# *.h) to filter out the source-files in the directories. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. -# -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, -# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. - -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cpp \ - *.c++ \ - *.java \ - *.ii \ - *.ixx \ - *.ipp \ - *.i++ \ - *.inl \ - *.idl \ - *.ddl \ - *.odl \ - *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ - *.cs \ - *.d \ - *.php \ - *.php4 \ - *.php5 \ - *.phtml \ - *.inc \ - *.m \ - *.markdown \ - *.md \ - *.mm \ - *.dox \ - *.py \ - *.pyw \ - *.f90 \ - *.f95 \ - *.f03 \ - *.f08 \ - *.f \ - *.for \ - *.tcl \ - *.vhd \ - *.vhdl \ - *.ucf \ - *.qsf \ - *.ice \ - @_FILE_PATTERNS@ - -# The RECURSIVE tag can be used to specify whether or not subdirectories should -# be searched for input files as well. -# The default value is: NO. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = @_EXCLUDE@ - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. -# The default value is: NO. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories for example use the pattern */test/* - -EXCLUDE_PATTERNS = @_EXCLUDE_PATTERNS@ - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* - -EXCLUDE_SYMBOLS = @_EXCLUDE_SYMBOLS@ - -# The EXAMPLE_PATH tag can be used to specify one or more files or directories -# that contain example code fragments that are included (see the \include -# command). - -EXAMPLE_PATH = @_EXAMPLE_PATH@ - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and -# *.h) to filter out the source-files in the directories. If left blank all -# files are included. - -EXAMPLE_PATTERNS = * - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude commands -# irrespective of the value of the RECURSIVE tag. -# The default value is: NO. - -EXAMPLE_RECURSIVE = YES - -# The IMAGE_PATH tag can be used to specify one or more files or directories -# that contain images that are to be included in the documentation (see the -# \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command: -# -# -# -# where is the value of the INPUT_FILTER tag, and is the -# name of an input file. Doxygen will then use the output that the filter -# program writes to standard output. If FILTER_PATTERNS is specified, this tag -# will be ignored. -# -# Note that the filter must not add or remove lines; it is applied before the -# code is scanned, but not when the output code is generated. If lines are added -# or removed, the anchors will not be placed correctly. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -INPUT_FILTER = - -# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern -# basis. Doxygen will compare the file name with each pattern and apply the -# filter if there is a match. The filters are a list of the form: pattern=filter -# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how -# filters are used. If the FILTER_PATTERNS tag is empty or if none of the -# patterns match the file name, INPUT_FILTER is applied. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# properly processed by doxygen. - -FILTER_PATTERNS = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will also be used to filter the input files that are used for -# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). -# The default value is: NO. - -FILTER_SOURCE_FILES = NO - -# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file -# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and -# it is also possible to disable source filtering for a specific pattern using -# *.ext= (so without naming a filter). -# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. - -FILTER_SOURCE_PATTERNS = - -# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that -# is part of the input, its contents will be placed on the main page -# (index.html). This can be useful if you have a project on for instance GitHub -# and want to reuse the introduction page also for the doxygen output. - -USE_MDFILE_AS_MAINPAGE = - -#--------------------------------------------------------------------------- -# Configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will be -# generated. Documented entities will be cross-referenced with these sources. -# -# Note: To get rid of all source code in the generated output, make sure that -# also VERBATIM_HEADERS is set to NO. -# The default value is: NO. - -SOURCE_BROWSER = YES - -# Setting the INLINE_SOURCES tag to YES will include the body of functions, -# classes and enums directly into the documentation. -# The default value is: NO. - -INLINE_SOURCES = YES - -# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any -# special comment blocks from generated source code fragments. Normal C, C++ and -# Fortran comments will always remain visible. -# The default value is: YES. - -STRIP_CODE_COMMENTS = NO - -# If the REFERENCED_BY_RELATION tag is set to YES then for each documented -# entity all documented functions referencing it will be listed. -# The default value is: NO. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES then for each documented function -# all documented entities called/used by that function will be listed. -# The default value is: NO. - -REFERENCES_RELATION = YES - -# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set -# to YES then the hyperlinks from functions in REFERENCES_RELATION and -# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will -# link to the documentation. -# The default value is: YES. - -REFERENCES_LINK_SOURCE = YES - -# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the -# source code will show a tooltip with additional information such as prototype, -# brief description and links to the definition and documentation. Since this -# will make the HTML file larger and loading of large files a bit slower, you -# can opt to disable this feature. -# The default value is: YES. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -SOURCE_TOOLTIPS = YES - -# If the USE_HTAGS tag is set to YES then the references to source code will -# point to the HTML generated by the htags(1) tool instead of doxygen built-in -# source browser. The htags tool is part of GNU's global source tagging system -# (see https://www.gnu.org/software/global/global.html). You will need version -# 4.8.6 or higher. -# -# To use it do the following: -# - Install the latest version of global -# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file -# - Make sure the INPUT points to the root of the source tree -# - Run doxygen as normal -# -# Doxygen will invoke htags (and that will in turn invoke gtags), so these -# tools must be available from the command line (i.e. in the search path). -# -# The result: instead of the source browser generated by doxygen, the links to -# source code will now point to the output of htags. -# The default value is: NO. -# This tag requires that the tag SOURCE_BROWSER is set to YES. - -USE_HTAGS = NO - -# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a -# verbatim copy of the header file for each class for which an include is -# specified. Set to NO to disable this. -# See also: Section \class. -# The default value is: YES. - -VERBATIM_HEADERS = YES - -# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the -# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the -# cost of reduced performance. This can be particularly helpful with template -# rich C++ code for which doxygen's built-in parser lacks the necessary type -# information. -# Note: The availability of this option depends on whether or not doxygen was -# generated with the -Duse_libclang=ON option for CMake. -# The default value is: NO. - -CLANG_ASSISTED_PARSING = YES - -# If clang assisted parsing is enabled you can provide the compiler with command -# line options that you would normally use when invoking the compiler. Note that -# the include paths will already be set by doxygen for the files and directories -# specified with INPUT and INCLUDE_PATH. -# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. - -CLANG_OPTIONS = - -# If clang assisted parsing is enabled you can provide the clang parser with the -# path to the compilation database (see: -# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) used when the files -# were built. This is equivalent to specifying the "-p" option to a clang tool, -# such as clang-check. These options will then be passed to the parser. -# Note: The availability of this option depends on whether or not doxygen was -# generated with the -Duse_libclang=ON option for CMake. - -CLANG_DATABASE_PATH = @_CLANG_DATABASE_PATH@ - -#--------------------------------------------------------------------------- -# Configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all -# compounds will be generated. Enable this if the project contains a lot of -# classes, structs, unions or interfaces. -# The default value is: YES. - -ALPHABETICAL_INDEX = YES - -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all classes will -# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag -# can be used to specify a prefix (or a list of prefixes) that should be ignored -# while generating the index headers. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output -# The default value is: YES. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a -# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of -# it. -# The default directory is: html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each -# generated HTML page (for example: .htm, .php, .asp). -# The default value is: .html. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a user-defined HTML header file for -# each generated HTML page. If the tag is left blank doxygen will generate a -# standard header. -# -# To get valid HTML the header file that includes any scripts and style sheets -# that doxygen needs, which is dependent on the configuration options used (e.g. -# the setting GENERATE_TREEVIEW). It is highly recommended to start with a -# default header using -# doxygen -w html new_header.html new_footer.html new_stylesheet.css -# YourConfigFile -# and then modify the file new_header.html. See also section "Doxygen usage" -# for information on how to generate the default header that doxygen normally -# uses. -# Note: The header is subject to change so you typically have to regenerate the -# default header when upgrading to a newer version of doxygen. For a description -# of the possible markers and block names see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each -# generated HTML page. If the tag is left blank doxygen will generate a standard -# footer. See HTML_HEADER for more information on how to generate a default -# footer and what special commands can be used inside the footer. See also -# section "Doxygen usage" for information on how to generate the default footer -# that doxygen normally uses. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style -# sheet that is used by each HTML page. It can be used to fine-tune the look of -# the HTML output. If left blank doxygen will generate a default style sheet. -# See also section "Doxygen usage" for information on how to generate the style -# sheet that doxygen normally uses. -# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as -# it is more robust and this tag (HTML_STYLESHEET) will in the future become -# obsolete. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_STYLESHEET = - -# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined -# cascading style sheets that are included after the standard style sheets -# created by doxygen. Using this option one can overrule certain style aspects. -# This is preferred over using HTML_STYLESHEET since it does not replace the -# standard style sheet and is therefore more robust against future updates. -# Doxygen will copy the style sheet files to the output directory. -# Note: The order of the extra style sheet files is of importance (e.g. the last -# style sheet in the list overrules the setting of the previous ones in the -# list). For an example see the documentation. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_STYLESHEET = - -# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or -# other source files which should be copied to the HTML output directory. Note -# that these files will be copied to the base HTML output directory. Use the -# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these -# files. In the HTML_STYLESHEET file, use the file name only. Also note that the -# files will be copied as-is; there are no commands or markers available. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_EXTRA_FILES = - -# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen -# will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see -# https://en.wikipedia.org/wiki/Hue for more information. For instance the value -# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 -# purple, and 360 is red again. -# Minimum value: 0, maximum value: 359, default value: 220. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_HUE = 220 - -# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A -# value of 255 will produce the most vivid colors. -# Minimum value: 0, maximum value: 255, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_SAT = 100 - -# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the -# luminance component of the colors in the HTML output. Values below 100 -# gradually make the output lighter, whereas values above 100 make the output -# darker. The value divided by 100 is the actual gamma applied, so 80 represents -# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not -# change the gamma. -# Minimum value: 40, maximum value: 240, default value: 80. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_COLORSTYLE_GAMMA = 80 - -# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML -# page will contain the date and time when the page was generated. Setting this -# to YES can help to show when doxygen was last run and thus if the -# documentation is up to date. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_TIMESTAMP = YES - -# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML -# documentation will contain a main index with vertical navigation menus that -# are dynamically created via Javascript. If disabled, the navigation index will -# consists of multiple levels of tabs that are statically embedded in every HTML -# page. Disable this option to support browsers that do not have Javascript, -# like the Qt help browser. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_MENUS = YES - -# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML -# documentation will contain sections that can be hidden and shown after the -# page has loaded. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_DYNAMIC_SECTIONS = NO - -# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries -# shown in the various tree structured indices initially; the user can expand -# and collapse entries dynamically later on. Doxygen will expand the tree to -# such a level that at most the specified number of entries are visible (unless -# a fully collapsed tree already exceeds this amount). So setting the number of -# entries 1 will produce a full collapsed tree by default. 0 is a special value -# representing an infinite number of entries and will result in a full expanded -# tree by default. -# Minimum value: 0, maximum value: 9999, default value: 100. -# This tag requires that the tag GENERATE_HTML is set to YES. - -HTML_INDEX_NUM_ENTRIES = 100 - -# If the GENERATE_DOCSET tag is set to YES, additional index files will be -# generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: https://developer.apple.com/xcode/), introduced with OSX -# 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in -# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at -# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy -# genXcode/_index.html for more information. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_DOCSET = NO - -# This tag determines the name of the docset feed. A documentation feed provides -# an umbrella under which multiple documentation sets from a single provider -# (such as a company or product suite) can be grouped. -# The default value is: Doxygen generated docs. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_FEEDNAME = "Doxygen generated docs" - -# This tag specifies a string that should uniquely identify the documentation -# set bundle. This should be a reverse domain-name style string, e.g. -# com.mycompany.MyDocSet. Doxygen will append .docset to the name. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_BUNDLE_ID = org.doxygen.Project - -# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify -# the documentation publisher. This should be a reverse domain-name style -# string, e.g. com.mycompany.MyDocSet.documentation. -# The default value is: org.doxygen.Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_ID = org.doxygen.Publisher - -# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. -# The default value is: Publisher. -# This tag requires that the tag GENERATE_DOCSET is set to YES. - -DOCSET_PUBLISHER_NAME = Publisher - -# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three -# additional HTML index files: index.hhp, index.hhc, and index.hhk. The -# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. -# -# The HTML Help Workshop contains a compiler that can convert all HTML output -# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML -# files are now used as the Windows 98 help format, and will replace the old -# Windows help format (.hlp) on all Windows platforms in the future. Compressed -# HTML files also contain an index, a table of contents, and you can search for -# words in the documentation. The HTML workshop also contains a viewer for -# compressed HTML files. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_HTMLHELP = NO - -# The CHM_FILE tag can be used to specify the file name of the resulting .chm -# file. You can add a path in front of the file if the result should not be -# written to the html output directory. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_FILE = - -# The HHC_LOCATION tag can be used to specify the location (absolute path -# including file name) of the HTML help compiler (hhc.exe). If non-empty, -# doxygen will try to run the HTML help compiler on the generated index.hhp. -# The file has to be specified with full path. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -HHC_LOCATION = - -# The GENERATE_CHI flag controls if a separate .chi index file is generated -# (YES) or that it should be included in the master .chm file (NO). -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -GENERATE_CHI = NO - -# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) -# and project file content. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -CHM_INDEX_ENCODING = - -# The BINARY_TOC flag controls whether a binary table of contents is generated -# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it -# enables the Previous and Next buttons. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members to -# the table of contents of the HTML help documentation and to the tree view. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTMLHELP is set to YES. - -TOC_EXPAND = NO - -# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and -# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that -# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help -# (.qch) of the generated HTML documentation. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_QHP = NO - -# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify -# the file name of the resulting .qch file. The path specified is relative to -# the HTML output folder. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QCH_FILE = - -# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help -# Project output. For more information please see Qt Help Project / Namespace -# (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_NAMESPACE = org.doxygen.Project - -# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt -# Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- -# folders). -# The default value is: doc. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_VIRTUAL_FOLDER = doc - -# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom -# filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_NAME = - -# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the -# custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: http://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_CUST_FILTER_ATTRS = - -# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this -# project's filter section matches. Qt Help Project / Filter Attributes (see: -# http://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes). -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHP_SECT_FILTER_ATTRS = - -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. -# This tag requires that the tag GENERATE_QHP is set to YES. - -QHG_LOCATION = - -# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be -# generated, together with the HTML files, they form an Eclipse help plugin. To -# install this plugin and make it available under the help contents menu in -# Eclipse, the contents of the directory containing the HTML and XML files needs -# to be copied into the plugins directory of eclipse. The name of the directory -# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. -# After copying Eclipse needs to be restarted before the help appears. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_ECLIPSEHELP = NO - -# A unique identifier for the Eclipse help plugin. When installing the plugin -# the directory name containing the HTML and XML files should also have this -# name. Each documentation set should have its own identifier. -# The default value is: org.doxygen.Project. -# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. - -ECLIPSE_DOC_ID = org.doxygen.Project - -# If you want full control over the layout of the generated HTML pages it might -# be necessary to disable the index and replace it with your own. The -# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top -# of each HTML page. A value of NO enables the index and the value YES disables -# it. Since the tabs in the index contain the same information as the navigation -# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -DISABLE_INDEX = NO - -# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index -# structure should be generated to display hierarchical information. If the tag -# value is set to YES, a side panel will be generated containing a tree-like -# index structure (just like the one that is generated for HTML Help). For this -# to work a browser that supports JavaScript, DHTML, CSS and frames is required -# (i.e. any modern browser). Windows users are probably better off using the -# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -GENERATE_TREEVIEW = NO - -# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that -# doxygen will group on one line in the generated HTML documentation. -# -# Note that a value of 0 will completely suppress the enum values from appearing -# in the overview section. -# Minimum value: 0, maximum value: 20, default value: 4. -# This tag requires that the tag GENERATE_HTML is set to YES. - -ENUM_VALUES_PER_LINE = 4 - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used -# to set the initial width (in pixels) of the frame in which the tree is shown. -# Minimum value: 0, maximum value: 1500, default value: 250. -# This tag requires that the tag GENERATE_HTML is set to YES. - -TREEVIEW_WIDTH = 250 - -# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to -# external symbols imported via tag files in a separate window. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -EXT_LINKS_IN_WINDOW = NO - -# Use this tag to change the font size of LaTeX formulas included as images in -# the HTML documentation. When you change the font size after a successful -# doxygen run you need to manually remove any form_*.png images from the HTML -# output directory to force them to be regenerated. -# Minimum value: 8, maximum value: 50, default value: 10. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_FONTSIZE = 10 - -# Use the FORMULA_TRANSPARENT tag to determine whether or not the images -# generated for formulas are transparent PNGs. Transparent PNGs are not -# supported properly for IE 6.0, but are supported on all modern browsers. -# -# Note that when changing this option you need to delete any form_*.png files in -# the HTML output directory before the changes have effect. -# The default value is: YES. -# This tag requires that the tag GENERATE_HTML is set to YES. - -FORMULA_TRANSPARENT = YES - -# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see -# https://www.mathjax.org) which uses client side Javascript for the rendering -# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX -# installed or if you want to formulas look prettier in the HTML output. When -# enabled you may also need to install MathJax separately and configure the path -# to it using the MATHJAX_RELPATH option. -# The default value is: NO. -# This tag requires that the tag GENERATE_HTML is set to YES. - -USE_MATHJAX = NO - -# When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. -# Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. -# The default value is: HTML-CSS. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_FORMAT = HTML-CSS - -# When MathJax is enabled you need to specify the location relative to the HTML -# output directory using the MATHJAX_RELPATH option. The destination directory -# should contain the MathJax.js script. For instance, if the mathjax directory -# is located at the same level as the HTML output directory, then -# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax -# Content Delivery Network so you can quickly see the result without installing -# MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_RELPATH = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/ - -# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax -# extension names that should be enabled during MathJax rendering. For example -# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_EXTENSIONS = - -# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces -# of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an -# example see the documentation. -# This tag requires that the tag USE_MATHJAX is set to YES. - -MATHJAX_CODEFILE = - -# When the SEARCHENGINE tag is enabled doxygen will generate a search box for -# the HTML output. The underlying search engine uses javascript and DHTML and -# should work on any modern browser. Note that when using HTML help -# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) -# there is already a search function so this one should typically be disabled. -# For large projects the javascript based search engine can be slow, then -# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to -# search using the keyboard; to jump to the search box use + S -# (what the is depends on the OS and browser, but it is typically -# , /${specs.projname}`); - bmSpecs = specs; - var toc = $("#toc"); - /*toc.append(`
  • Load all
  • `);*/ - iterBms(function(key, bm) { - toc.append(`
  • ${key}: ${bm.specs.desc}
  • `) - bm.name = key; - }); - // load if required - currBm = getParam("bm", ""); - dbg("params=", _curr_url_params, currBm); - if(currBm != "") { - dbg("loading BM from URL:", currBm) - loadBm(currBm); - } -} - -function normalizeBy(results, column_name, best_fn) -{ - var best = best_fn(results.benchmarks, column_name); - results.benchmarks.forEach(function(item, index){ - item[`${column_name}_normalized`] = item[column_name] / best; - }); -} - - -function loadAll() -{ - var id = "#bm-results"; - $(id).empty(); - var i = 0; - iterBms(function(key, bm){ - if(i++ > 0) $(id).append("

    "); - appendBm(key); - }); -} - - -function loadBm(key) -{ - dbg("loading-.....", key); - /*if(key == "all") { - loadAll(); - }*/ - $("#bm-results").empty(); - var bm = bmSpecs.bm[key]; - if(bm.src != "") { - fileContents(bm.src, function(data){ - dbg(`${key}: got src data!`) - bm.src_data = data; - }); - } - var latestRun = last(bm.entries); - var bmfile = `${latestRun}/${key}.json`; - dbg("bmfile=", bmfile); - fileContents("bm/"+bmfile, function(data){ - dbg(`${key}: got bm data!`) - bm.results_data = new BmResults(JSON.parse(data)); - bm.results_data.benchmarks.forEach(function(item, index){ - item.id = index; - }); - normalizeBy(bm.results_data, 'iterations', colMin); - normalizeBy(bm.results_data, 'real_time', colMin, ); - normalizeBy(bm.results_data, 'cpu_time', colMin); - normalizeBy(bm.results_data, 'bytes_per_second', colMin); - normalizeBy(bm.results_data, 'items_per_second', colMin); - appendBm(latestRun, key, bm); - }); -} - - -function appendBm(run_id, id, bm) -{ - if($(document).find(`bm-results-${id}`).length == 0) - { - $("#bm-results").append(` -
    -

    ${id}

    - -

    Run details

    - -

    Result tables

    -

    Results

    -

    Normalized by column min

    - -

    Chart

    -
    - -

    Code

    -
    -
    -`); - } - var results = bm.results_data; - var code = bm.src_data; - loadDetailsTable(run_id, id, bm, results); - loadTable(id, bm, results); - loadChart(id, bm, results); - loadCode(id, bm, code); -} - - -function loadCode(elmId, bm, code) -{ - var elm = $(`#code-${elmId}`); - elm.text(code); - /* hljs.highlightBlock(elm); // this doesn't work */ - /* ... and this is very inefficient: */ - document.querySelectorAll('pre code').forEach((block) => { - hljs.highlightBlock(block); - }); -} - -function parseRunId(run_id) -{ - // example: - // commit id / cpu id - system id - build id - // git20201204_202919-b3f7fa7/x86_64_b9db3176-linux_4e9326b4-64bit_Debug_gcc10.2.0_10c5d03c - // git20201203_193348-2974fb0/x86_64_16ac0500-win32_59f3579c-64bit_MinSizeRel_msvc19.28.29304.1_32f6fc66 - // to tune the regex: https://regex101.com/r/rdkPi8/1 - // commit / cpu - system - build - var rx = /^(.+?)-([0-9a-f]{7})\/(.+?)_([0-9a-f]{8})-(.+?)_([0-9a-f]{8})-(.+?)_([0-9a-f]{8})$/gim; - var tag = rx.exec(run_id); - dbg("fdx: run_id=", run_id); - dbg("fdx: tag=", tag); - dbg("fdx: len=", tag.length); - return { - commit_id: `${tag[2]}: ${tag[1]}`, - cpu_id: `${tag[4]}: ${tag[3]} `, - system_id: `${tag[6]}: ${tag[5]}`, - build_id: `${tag[8]}: ${tag[7]}`, - }; -} - -function getBuildId(run_id) -{ - return parseRunId(run_id).build_id; -} - -function loadDetailsTable(run_id, id, bm, results) -{ - var url = bmSpecs.url; - var run = bmSpecs.runs[run_id]; - var commit = bmSpecs.commit[run.commit].specs; - var cpu = bmSpecs.cpu[run.cpu].specs; - var system = bmSpecs.system[run.system].specs; - - let other_commit_entries = bmSpecs.commit[run.commit].entries.filter( - entry_run => entry_run != run_id - ).map(entry_run => getBuildId(entry_run)).join('
    '); - - /* https://datatables.net/ */ - $(`#table-details-${id}`).DataTable({ - data: results.benchmarks, - info: false, - paging: false, - searching: false, - retrieve: false, - order: [], - columns: [ - {title: "", data: "desc"}, - {title: "", data: "contents"}, - ], - data: [ - {desc: "benchmark id" , contents: id}, - {desc: "commit" , contents: ahref(`${url}/commit/${commit.sha1}`, commit.sha1)}, - {desc: "commit date" , contents: ahref(`${url}/commit/${commit.sha1}`, commit.committed_datetime)}, - {desc: "commit summary", contents: ahref(`${url}/commit/${commit.sha1}`, commit.summary)}, - {desc: "source tree" , contents: ahref(`${url}/tree/${commit.sha1}`, `tree @ ${commit.sha1}`)}, - {desc: "benchmark" , contents: ahref(`${url}/tree/${commit.sha1}/${bm.specs.src}`, `source @ ${commit.sha1}`)}, - {desc: "cpu used" , contents: `${cpu.arch} ${cpu.brand_raw}`}, - {desc: "system used" , contents: `${system.uname.system} ${system.uname.release}`}, - {desc: "this build" , contents: `
    ${getBuildId(run_id)}
    `}, - {desc: "commit builds" , contents: `
    ${other_commit_entries}
    `}, - ] - }); - function ahref(url, txt) { return `${txt}`; } -} - - -function loadTable(id, bm, results) -{ - function render_int(data, type, row, meta) { return toFixedNumber(data, 0); } - function render_megas(data, type, row, meta) { return toFixedNumber(data / 1.e6, 3); } - function render_fixed(data, type, row, meta) { return toFixedNumber(data, 3); } - function render_human(data, type, row, meta) { return humanReadable(data, 1000, 3); } - - addTable("_pretty" , "" , {ns: render_int, iters: render_megas, rates: render_megas}); - addTable("_normalized", "_normalized", {ns: render_fixed, iters: render_fixed, rates: render_fixed}); - - function addTable(suffix, data_suffix, renderers) { - /* https://datatables.net/ */ - var searching = (results.benchmarks.count > 20); - var ratePrefix = renderers.rates == render_megas ? "M" : ""; - var iterPrefix = renderers.iters == render_megas ? "M" : ""; - var clockSuffix = data_suffix == "_normalized" ? "" : "(ns)"; - $(`#table-${id}${suffix}`).DataTable( { - data: results.benchmarks, - info: false, - paging: false, - searching: searching, - retrieve: searching, - /* https://datatables.net/reference/option/columns.type */ - columns: [ - {title: "ID", data: "id", type: "num"}, - {title: "Name", data: "name", render: function(data, type, row, meta) { return escapeHtml(data); }}, - {title: `${ratePrefix}B/s` , data: `bytes_per_second${data_suffix}`, type: "num", className: "text-right", render: renderers.rates}, - {title: `${ratePrefix}items/s` , data: `items_per_second${data_suffix}`, type: "num", className: "text-right", render: renderers.rates}, - {title: `Clock${clockSuffix}` , data: `real_time${data_suffix}` , type: "num", className: "text-right", render: renderers.ns}, - {title: `CPU${clockSuffix}` , data: `cpu_time${data_suffix}` , type: "num", className: "text-right", render: renderers.ns}, - {title: `${ratePrefix}Iterations`, data: `iterations${data_suffix}` , type: "num", className: "text-right", render: renderers.iters}, - ]}); - } -} - -function loadChart(id, bm, results) -{ - - addChartFromColumn('bytes_per_second_normalized', "B/s", "(more is better)"); - addChartFromColumn('items_per_second_normalized', "items/s", "(more is better)"); - addChartFromColumn('iterations_normalized', "Iterations", "(more is better)"); - addChartFromColumn('real_time_normalized', "Clock time", "(less is better)"); - addChartFromColumn('cpu_time_normalized', "CPU time", "(less is better)"); - - function addChartFromColumn(column, column_name, obs) { - var elmId = `chart-${id}-${column}`; - var canvas = `${elmId}-canvas`; - - $(`#chart-container-${id}`).append(` -
    - -
    -`); - - var chart = new CanvasJS.Chart(elmId, { - animationEnabled: false, - title:{ - fontSize: 24, - /* text: `${id}: ${column_name}\n${obs}` */ - text: `${column_name}\n${obs}` - }, - axisX: { - labelFontSize: 12, - }, - data: [{ - type: "bar", - axisYType: "secondary", - color: "#eb7434",/*"#014D65",*/ - dataPoints: results.benchmarks.map(function(item){ - return { - indexLabelFormatter: function(e) { return e.dataPoint.indexLabel; }, - indexLabelFontSize: 16, - indexLabel: item.name, - /* label: item.name, */ - y: item[column], - /* save the result here: the tooltip will show the full thing */ - benchmark_results: item - }; - }), - }], - toolTip: { - /*content: "{indexLabel}: {y}",*/ - contentFormatter: function(e){ - function hr(val) { return humanReadable(val, 1000, 3); } - function fx(val) { return toFixedNumber(val, 3); } - function fxi(val) { return toFixedNumber(val, 0); } - function getRow(name, abs, rel) { return `${name}${abs}${rel}x min`; } - var r = e.entries[0].dataPoint.benchmark_results; - var hdrRow = `AbsoluteNormalized`; - var bpsRow = getRow("B/s", hr(r.bytes_per_second), fx(r.bytes_per_second_normalized)); - var ipsRow = getRow("items/s", hr(r.items_per_second), fx(r.items_per_second_normalized)); - var cpuRow = getRow("CPU", fxi(r.cpu_time) + "ns", fx(r.cpu_time_normalized)); - var clockRow = getRow("Clock", fxi(r.real_time) + "ns", fx(r.real_time_normalized)); - var itersRow = getRow("Iterations", hr(r.iterations), fx(r.iterations_normalized)); - var table = `${hdrRow}${bpsRow}${ipsRow}${cpuRow}${clockRow}${itersRow}
    `; - return `

    ${escapeHtml(r.name)}

    ${table}`; - } - } - }); - chart.render(); - } -} diff --git a/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_plot.py b/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_plot.py deleted file mode 100644 index 3bcab5f4d..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_plot.py +++ /dev/null @@ -1,746 +0,0 @@ -import os -import sys -import copy -import re -import itertools -import typing -import enum - -# https://stackoverflow.com/questions/11351032/named-tuple-and-default-values-for-optional-keyword-arguments -from dataclasses import dataclass - -from munch import Munch, munchify - -import bokeh.io as bki -import bokeh.models as bkm -import bokeh.plotting as bkp -import bokeh.transform as bkt -import bokeh.layouts as bkl -from bokeh.models.markers import marker_types as bk_markers -# https://docs.bokeh.org/en/latest/docs/reference/palettes.html -from bokeh.palettes import d3 as bk_palette_d3 -bk_palette = bk_palette_d3['Category20c'][20] - -# saving bokeh to png is not working, so we save png using matplotlib -import matplotlib.pyplot as plt -import matplotlib.ticker as plttck -plt_markers = [c for c in ".,ov^<>1234spP*hH+xXDdl"] - -from bm_util import _enum -from bm_util import * -from bm_run import BenchmarkRun, BenchmarkPanel - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - - -# https://stackoverflow.com/questions/11351032/named-tuple-and-default-values-for-optional-keyword-arguments -@dataclass -class BarChartSpecs: - horizontal: bool = True - bar_width: float = 0.9 - - -@dataclass -class LineChartSpecs: - width: int = 1000 - xlog: bool = False - ylog: bool = False - xlabel: str = "" - ylabel: str = "" - - -def _plt_save_png(name): - log(name) - plt.savefig(name, bbox_inches='tight', dpi=100) - - -def _plt_clear(): - plt.clf() - - -def _bokeh_save_html(name, p): - log(name) - bki.save(p, name) - - -def _bokeh_adjust_figure_props(p): - p.toolbar.autohide = True - #p.toolbar.active_inspect = [hover_tool, crosshair_tool] - p.toolbar.active_drag = "auto" - p.toolbar.active_scroll = "auto" - p.legend - p.legend.click_policy = "hide" - p.legend.label_text_font_size = "10px" - - -def bokeh_plot_many(plots, name: str, ncols: int = 2): - layout = bkl.gridplot(plots, ncols=ncols) - _bokeh_save_html(name, layout) - #bkp.show(layout) - - -def plot_benchmark_run_as_bars(bm: BenchmarkRun, title: str, - bar_names, bar_values, bar_label, - **kwargs): - kwargs = BarChartSpecs(**kwargs) - # - palette = itertools.cycle(bk_palette) - colors = [next(palette) for _ in bar_names] - # - fig_args_bokeh = { - "title": title, - #"toolbar_location": None, - #"tools": "" - } - if kwargs.horizontal: - # - # plot with bokeh (interactive, but cannot export png) - rnames = list(reversed(bar_names)) - rvalues = list(reversed(bar_values)) - rcolors = list(reversed(colors)) - p = bkp.figure(y_range=rnames, **fig_args_bokeh) - p.hbar(y=rnames, right=rvalues, fill_color=rcolors, - line_color=rcolors, height=kwargs.bar_width) - p.ygrid.grid_line_color = None - p.x_range.start = 0 - p.xaxis.axis_label = bar_label - # - # plot with matplotlib (to export png) - p_ = plt.barh(y=rnames, width=rvalues, color=rcolors, - height=kwargs.bar_width) - plt.gca().xaxis.grid(True) - plt.gca().xaxis.set_minor_locator(plttck.AutoMinorLocator()) - plt.xlabel(bar_label, fontsize='small') - plt.yticks(fontsize='x-small') - plt.title(title) - else: - # - # plot with bokeh (interactive, but cannot export png) - p = bkp.figure(x_range=bar_names, **fig_args_bokeh) - p.vbar(x=bar_names, top=bar_values, fill_color=colors, - line_color=colors, width=kwargs.bar_width) - p.xaxis.major_label_orientation = 1 - p.xgrid.grid_line_color = None - p.y_range.start = 0 - p.yaxis.axis_label = bar_label - # - # plot with matplotlib (to export png) - p_ = plt.bar(x=bar_names, height=bar_values, color=colors, - width=kwargs.bar_width) - plt.gca().yaxis.grid(True) - plt.gca().yaxis.set_minor_locator(plttck.AutoMinorLocator()) - plt.ylabel(bar_label, fontsize='small') - plt.xticks(fontsize='x-small') - plt.title(title) - _bokeh_adjust_figure_props(p) - return p, p_ - - -def plot_benchmark_panel_as_lines(bm_panel: BenchmarkPanel, title: str, - xget, yget, nameget, - **kwargs): - kwargs = LineChartSpecs(**kwargs) - # - colors = itertools.cycle(bk_palette) - markers = itertools.cycle(bk_markers) - markers_ = itertools.cycle(plt_markers) - # - # plot with bokeh (interactive, but cannot export png) - p = bkp.figure(title=title, - x_axis_type="log" if kwargs.xlog else "linear", - y_axis_type="log" if kwargs.ylog else "linear", - #background_fill_color="#fafafa", - x_axis_label=kwargs.xlabel, - y_axis_label=kwargs.ylabel, - plot_width=kwargs.width, - ) - # plot with matplotlib (to export png) - plt.title(title) - for bm in bm_panel.runs: - x = xget(bm) - y = yget(bm) - line_name = nameget(bm) - color = next(colors) - marker = next(markers) - marker_ = next(markers_) - # plot with bokeh (interactive, but cannot export png) - #legends.append(LegendItem(name=c, label=line_name)) - p.scatter(x, y, marker=marker, size=8, color=color, - legend_label=line_name) - p.line(x, y, color=color, alpha=0.9, - #muted_color=c, muted_alpha=0.05, - legend_label=line_name) - # - # plot with matplotlib (to export png) - plt.plot(x, y, f'-{marker_}', color=color, label=line_name) - plt.gca().xaxis.grid(True) - plt.gca().yaxis.grid(True) - plt.xscale("log" if kwargs.xlog else "linear") - plt.yscale("log" if kwargs.ylog else "linear") - plt.xlabel(kwargs.xlabel, fontsize='small') - plt.ylabel(kwargs.ylabel, fontsize='small') - plt.gca().legend(loc='center left', bbox_to_anchor=(1, 0.5), fontsize='x-small') - _bokeh_adjust_figure_props(p) - return p - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -####### old code: to remove and tidy up - -@dataclass -class CharconvMeta: # also for atox - title: str - subject: str - function: str - data_type: FundamentalTypes - - @classmethod - def make(cls, bm_title: str): - # eg: - # xtoa_c4_write_dec - # xtoa_c4_utoa - # xtoa_c4_xtoa - # xtoa_c4_to_chars - # xtoa_std_to_chars - # xtoa_std_to_string - # xtoa_sprintf - # xtoa_sstream_reuse - # xtoa_sstream - rx = re.compile(r'(atox|xtoa|xtoahex|xtoaoct|xtoabin)_(.*?)<(u?int\d+_t)>') - if not rx.fullmatch(bm_title): - raise Exception(f"cannot understand bm title: {bm_title}") - subject = rx.sub(r'\1', bm_title) - function = rx.sub(r'\2', bm_title) - data_type = rx.sub(r'\3', bm_title) - return cls( - title=bm_title, - subject=subject, - function=function.replace("c4_", "c4::").replace("std_", "std::"), - data_type=FundamentalTypes.make(data_type) - ) - - def checkbox_groups(self): - return { - 'data_type': [t for t in FundamentalTypes], - } - - @property - def shortname(self): - return self.function - - @property - def shortparams(self): - return str(self.data_type.short) - - @property - def shorttitle(self): - return f"{self.shortname}<{self.shortparams}>" - - -@dataclass -class CharconvThreadsMeta: - function: str - num_threads: int - - @classmethod - def make(cls, bm_title: str): - # eg: - # c4_itoa/real_time/threads:4 - rx = re.compile(r'(.*?)/real_time/threads:(\d+)') - if not rx.fullmatch(bm_title): - raise Exception(f"cannot understand bm title: {bm_title}") - function = rx.sub(r'\1', bm_title) - num_threads = int(rx.sub(r'\2', bm_title)) - return cls( - function=function.replace("c4_", "c4::").replace("std_", "std::"), - num_threads=num_threads - ) - - def checkbox_groups(self): - return {} - - @property - def shortname(self): - return self.function - - @property - def shorttitle(self): - return self.shortname - - -def plot_charconv_bars(bm_panel: BenchmarkPanel, panel_title_human: str, outputfile_prefix: str): - assert os.path.isabs(outputfile_prefix), outputfile_prefix - for prop in ("mega_bytes_per_second", "cpu_time_ms"): - ps, ps_ = [], [] - pd = bm_panel.first_run.property_plot_data(prop) - bar_label = f"{pd.human_name_short}{pd.what_is_better}" - outfilename = f"{outputfile_prefix}-{prop}" - for bm in bm_panel.runs: - bar_names = [m.shorttitle for m in bm.meta] - bar_values = list(getattr(bm, prop)) - data_type = first(bm.meta).data_type - # to save each bokeh plot separately and also - # a grid plot with all of them, we have to plot - # twice because bokeh does not allow saving twice - # the same plot from multiple pictures. - plotit = lambda: plot_benchmark_run_as_bars(bm, title=f"{panel_title_human}: {data_type}\n{bar_label}", - bar_names=bar_names, bar_values=bar_values, bar_label=bar_label) - # make one plot to save: - p, p_ = plotit() - _bokeh_save_html(f"{outfilename}-{data_type.short}.html", p) - _plt_save_png_and_clear(f"{outfilename}-{data_type.short}.png") - # and another to gather: - p, p_ = plotit() - ps.append(p) - ps_.append(p_) - layout = bkl.gridplot(ps, ncols=2) - _bokeh_save_html(f"{outfilename}.html", layout) - # now show - #bkp.show(layout) - - -def plot_charconv_threads_(bm_panel: BenchmarkPanel, panel_title_human: str, outputfile_prefix: str): - assert os.path.isabs(outputfile_prefix), outputfile_prefix - orig = lambda yprop: lambda bm: list(bm.extract_plot_series(yprop)) - divnt = lambda yprop: lambda bm: [v / n for v, n in bm.extract_plot_series_with_threads(yprop)] - mulnt = lambda yprop: lambda bm: [v * n for v, n in bm.extract_plot_series_with_threads(yprop)] - xprop = "threads" - xpd = bm_panel.first_run.property_plot_data(xprop) - xlabel = f"{xpd.human_name_short}" - for yprop, ylog, yget in ( - #("mega_items_per_second", False, orig), - ("mega_bytes_per_second", False, orig), - #("iterations", False, divnt), - #("real_time_ms", True, mulnt), - ("cpu_time_ms", True, orig),): - ypd = bm_panel.first_run.property_plot_data(yprop) - ylabel = f"{ypd.human_name_short}{ypd.what_is_better}" - p = plot_benchmark_panel_as_lines( - bm_panel, f"{panel_title_human}\n{ylabel}", - xget=orig("threads"), - yget=yget(yprop), - nameget=lambda bm: first(bm.meta).function, - ylog=ylog, - xlabel=xlabel, - ylabel=ylabel - ) - name = f"{outputfile_prefix}-lines-{yprop}" - # save png using matplotlib - _plt_save_png_and_clear(f"{name}.png") - # save html using bokeh - _bokeh_save_html(f"{name}.html", p) - #bkp.show(p) - return p - - -def plot_charconv_threads(json_files, case: str = ""): - case = f" [{case}]" if case else "" - dir_ = os.path.dirname(first(json_files)) - panel = BenchmarkPanel(json_files, CharconvThreadsMeta) - plot_charconv_threads_(panel, - f"itoa benchmark: convert 2M 32b integers to string{case}", - f"{dir_}/c4core-bm-charconv_threads") - - -def plot_charconv_xtoa(json_files, case: str = ""): - case = f" [{case}]" if case else "" - dir_ = os.path.dirname(first(json_files)) - panel = BenchmarkPanel(json_files, CharconvMeta) - plot_charconv_bars(panel, - f"xtoa benchmark: convert 2M numbers to strings{case}", - f"{dir_}/c4core-bm-charconv-xtoa") - - -def plot_charconv_atox(json_files, case: str = ""): - case = f" [{case}]" if case else "" - dir_ = os.path.dirname(first(json_files)) - panel = BenchmarkPanel(json_files, CharconvMeta) - plot_charconv_bars(panel, - f"atox benchmark: convert 2M strings to numbers{case}", - f"{dir_}/c4core-bm-charconv-atox") - - -def threads_data(dir_: str): - assert os.path.exists(dir_), dir_ - return [ - f"{dir_}/c4core-bm-charconv_threads-c4_write_dec.json", - f"{dir_}/c4core-bm-charconv_threads-c4_itoa.json", - f"{dir_}/c4core-bm-charconv_threads-c4_xtoa.json", - f"{dir_}/c4core-bm-charconv_threads-c4_to_chars.json", - f"{dir_}/c4core-bm-charconv_threads-fmtlib_format_to.json", - f"{dir_}/c4core-bm-charconv_threads-std_to_chars.json", - f"{dir_}/c4core-bm-charconv_threads-snprintf.json", - f"{dir_}/c4core-bm-charconv_threads-stb_snprintf.json", - f"{dir_}/c4core-bm-charconv_threads-sstream.json", - f"{dir_}/c4core-bm-charconv_threads-sstream_naive_reuse.json", - f"{dir_}/c4core-bm-charconv_threads-sstream_naive.json", - ] - - -def xtoa_data(dir_: str): - assert os.path.exists(dir_), dir_ - return [ - f"{dir_}/c4core-bm-charconv-xtoa-int8.json", - f"{dir_}/c4core-bm-charconv-xtoa-uint8.json", - f"{dir_}/c4core-bm-charconv-xtoa-int16.json", - f"{dir_}/c4core-bm-charconv-xtoa-uint16.json", - f"{dir_}/c4core-bm-charconv-xtoa-int32.json", - f"{dir_}/c4core-bm-charconv-xtoa-uint32.json", - f"{dir_}/c4core-bm-charconv-xtoa-int64.json", - f"{dir_}/c4core-bm-charconv-xtoa-uint64.json", - ] - - -def atox_data(dir_: str): - assert os.path.exists(dir_), dir_ - return [ - f"{dir_}/c4core-bm-charconv-atox-int8.json", - f"{dir_}/c4core-bm-charconv-atox-uint8.json", - f"{dir_}/c4core-bm-charconv-atox-int16.json", - f"{dir_}/c4core-bm-charconv-atox-uint16.json", - f"{dir_}/c4core-bm-charconv-atox-int32.json", - f"{dir_}/c4core-bm-charconv-atox-uint32.json", - f"{dir_}/c4core-bm-charconv-atox-int64.json", - f"{dir_}/c4core-bm-charconv-atox-uint64.json", - ] - - -def examples_dir(): - this_dir = os.path.dirname(os.path.abspath(__file__)) - exdir = f"{this_dir}/examples" - assert os.path.exists(exdir), exdir - return exdir - - -if __name__ == '__main__': - xdir = examples_dir() - # - plot_charconv_threads(threads_data(f"{xdir}/lines/gcc11.2"), "gcc11.2") - plot_charconv_threads(threads_data(f"{xdir}/lines/vs2022"), "vs2022") - # - plot_charconv_xtoa(xtoa_data(f"{xdir}/bars/xtoa/gcc11.2"), "gcc11.2") - plot_charconv_xtoa(xtoa_data(f"{xdir}/bars/xtoa/vs2022"), "vs2022") - # - plot_charconv_atox(atox_data(f"{xdir}/bars/atox/gcc11.2"), "gcc11.2") - plot_charconv_atox(atox_data(f"{xdir}/bars/atox/vs2022"), "vs2022") - # - exit() - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - - -def plot_benchmarks_as_lines(title, *bm, transform=None, - line_title_transform=None, - logx=True, logy=True): - import bokeh - from bokeh.plotting import figure, output_file, show - from bokeh.palettes import Dark2_5 as palette - from bokeh.layouts import row, column - from bokeh.models import (Legend, LegendItem, CheckboxGroup, CustomJS, Div, - RadioGroup, Toggle, - ColumnDataSource, DataTable, TableColumn) - from bokeh.models.markers import marker_types - # - ids = entry_ids(*bm, transform=transform) - colors = itertools.cycle(palette) - markers = itertools.cycle(marker_types) - p = figure(title=title, - x_axis_type="log" if logx else "linear", - y_axis_type="log" if logy else "linear", - #background_fill_color="#fafafa", - plot_width=1000, - x_axis_label="Number of pixels", - y_axis_label="Throughput (MB/s)", - ) - p.toolbar.autohide = True - #p.toolbar.active_inspect = [hover_tool, crosshair_tool] - p.toolbar.active_drag = "auto" - p.toolbar.active_scroll = "auto" - # - def dft(v): return v if v else (lambda n: n) - tr = dft(transform) - lttr = dft(line_title_transform) - # - for results in bm: - x = [ids[name] for name in results.names] - y = [bps/1e6 for bps in results.bytes_per_second] - c = next(colors) - marker = next(markers) - next(markers) # advance two - line_name = lttr(results.first) - #legends.append(LegendItem(name=c, label=line_name)) - p.scatter(x, y, marker=marker, size=8, color=c, legend_label=line_name) - p.line(x, y, - color=c, alpha=0.9, - #muted_color=c, muted_alpha=0.05, - legend_label=line_name) - p.legend.click_policy = "hide" - p.legend.label_text_font_size = "10px" - # - def input_title(title): - return Div(text=f"

    {title}

    ") - inputs = [] - first = bm[0].first.meta - for k, g in first.checkbox_groups().items(): - cb = CheckboxGroup(labels=[str(v) for v in g], - active=[i for i in range(len(g))], - inline=True) - inputs.append(input_title(k)) - inputs.append(cb) - # - # https://github.com/bokeh/bokeh/blob/branch-2.3/examples/app/export_csv/main.py - x_axis_values = [f"{m.num_pixels}px" for m in bm[0].meta] - table_sources = [] - for i, px in enumerate(x_axis_values): - c = ColumnDataSource(data={ - 'name': [nth(results.filtered_names, i) for results in bm], - 'bytes_per_second': [nth(results.bytes_per_second, i) for results in bm], - 'items_per_second': [nth(results.items_per_second, i) for results in bm], - 'cpu_time': [nth(results.real_time, i) for results in bm], - 'real_time': [nth(results.real_time, i) for results in bm], - 'iterations': [nth(results.iterations, i) for results in bm], - 'threads': [nth(results.threads, i) for results in bm], - }) - table_sources.append(c) - selected_x_index = 8 # FIXME (currently 2000 pixels) - table_source = copy.deepcopy(table_sources[selected_x_index]) - relvalues = Toggle(label="Table: Relative values") - px_title = input_title("Table: number of pixels") - px_radiogroup = RadioGroup(labels=x_axis_values, active=selected_x_index) - table_inputs = [relvalues, px_title, px_radiogroup] - # - table_cols = [ - TableColumn(field='name', title='Name'), - TableColumn(field='bytes_per_second', title='Bytes/second'), - TableColumn(field='items_per_second', title='Items/second'), - TableColumn(field='cpu_time', title='CPU time'), - TableColumn(field='real_time', title='Real time'), - TableColumn(field='iterations', title='Iterations'), - TableColumn(field='threads', title='Threads'), - ] - data_table = DataTable(source=table_source, columns=table_cols, width=1200) - callback = CustomJS(args=dict( - radiogroup=px_radiogroup, - source=table_source, - table=table_sources - ), code=""" - console.log(`active=${radiogroup.active}`); - /*source.data=table[radiogroup.active];*/ - var nrows = source.data['name'].length; - var ts = table[radiogroup.active].data; - var names = ["name", "bytes_per_second", "items_per_second", "cpu_time", "real_time", "iterations", "threads"]; - var ncols = names.length; - console.log(`names=${names} nrows=${nrows} ncols=${ncols}`); - for(var i = 0; i < nrows; i++) { - for(var j = 0; j < ncols; ++j) { - var name = names[j]; - /*console.log(`i=${i} j=${j} name=${name}`);*/ - source.data[name][i] = ts[name][i]; - } - } - source.change.emit(); - """) - px_radiogroup.js_on_change('active', callback) - # lambda attr, old, new: log(f"attr={attr} old={old} new={new} active={px_radiogroup.active}")) - # - layout = column( - row(column(*inputs), p), - row(column(*table_inputs), data_table)) - show(layout) - - -def entry_ids(*bm, transform=None): - ids = {} - curr = 0 - for results in bm: - log(os.path.basename(results.filename), "------------------------------") - for entry in results.entries: - log(entry.name) - if transform is not None: - ids[entry.name] = transform(entry) - else: - if ids.get(entry.name) is None: - ids[entry.name] = curr - curr += 1 - return ids - - -class MatrixOrder(_enum): - row_major = "row_major" - col_major = "col_major" - @property - def short(self): - return "rm" if self is MatrixOrder.row_major else "cm" - @classmethod - def make(cls, s): - try: - return {"rm": cls.row_major, "cm": cls.col_major}[s] - except: - cls.err_unknown(s) - - -class MatrixLayout(_enum): - compact = "compact" - strided = "strided" - @classmethod - def make(cls, s): - try: - return cls[s] - except: - cls.err_unknown(s) - - -class DimensionBinding(_enum): - compile_time = "compile_time" - run_time = "run_time" - @property - def short(self): - return "ct" if self is DimensionBinding.compile_time else "rt" - @classmethod - def make(cls, s): - try: - return {"ct": cls.compile_time, "rt": cls.run_time}[s] - except: - cls.err_unknown(s) - - -class MultType(_enum): - naive = "naive" - avx2 = "avx2" - avx2_unroll2 = "avx2_unroll2" - avx2_unroll4 = "avx2_unroll4" - avx2_unroll8 = "avx2_unroll8" - @classmethod - def make(cls, s): - try: - s = s.replace("dotprod_", "").replace("_naive", "") - return cls[s] - except: - cls.err_unknown(s) - - -class MatrixMult(typing.NamedTuple): - title: str - num_pixels: int - num_channels: int - num_features: int - mult_type: MultType - layout: MatrixLayout - dim_binding: DimensionBinding - ret_order: MatrixOrder - lhs_order: MatrixOrder - rhs_order: MatrixOrder - - @classmethod - def make(cls, bm_title: str): - # eg: - # mult_naive_strided_ct_rm_cmcm<250, 8, 16> - # mult_naive_compact_rt_rm_rmrm/4000/8/16 - rxline = r'mult_(.*)[") - expect(v, 'title', 'naive_strided_ct_rm_cmcm') - expect(v, 'num_pixels', 250) - expect(v, 'num_channels', 8) - expect(v, 'num_features', 16) - expect(v, 'mult_type', MultType.naive) - expect(v, 'layout', MatrixLayout.strided) - expect(v, 'dim_binding', DimensionBinding.compile_time) - expect(v, 'ret_order', MatrixOrder.row_major) - expect(v, 'lhs_order', MatrixOrder.col_major) - expect(v, 'rhs_order', MatrixOrder.col_major) - v = MatrixMult.make("mult_dotprod_avx2_compact_rt_cm_rmcm/4000/16/8") - expect(v, 'title', 'dotprod_avx2_compact_rt_cm_rmcm') - expect(v, 'num_pixels', 4000) - expect(v, 'num_channels', 16) - expect(v, 'num_features', 8) - expect(v, 'mult_type', MultType.avx2) - expect(v, 'layout', MatrixLayout.compact) - expect(v, 'dim_binding', DimensionBinding.run_time) - expect(v, 'ret_order', MatrixOrder.col_major) - expect(v, 'lhs_order', MatrixOrder.row_major) - expect(v, 'rhs_order', MatrixOrder.col_major) - -_test() - - - -def formatMBps(value): - return value / 1e6 - - - -if __name__ == '__main__': - bms = sorted(sys.argv[2:]) - log(bms) - bms = BenchmarkPanel(bms, bm_meta_cls=MatrixMult.make) - fm = bms.runs[0].first.meta - title = f"Classifier multiplication, {fm.num_channels} channels, {fm.num_features} features: throughput (MB/s)" - bms.plot_all_lines(title) - exit() - main() diff --git a/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_run.py b/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_run.py deleted file mode 100644 index 812b64abe..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_run.py +++ /dev/null @@ -1,248 +0,0 @@ -import copy -import os.path - -# https://stackoverflow.com/questions/11351032/named-tuple-and-default-values-for-optional-keyword-arguments -from dataclasses import dataclass - -from munch import Munch - -from bm_util import load_json, first, _enum - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -class QuantityType(_enum): - neutral = "" - more_is_better = "more is better" - less_is_better = "less is better" - - @property - def comment(self): - return f" ({self.value})" if self.name else "" - - -_more = QuantityType.more_is_better -_less = QuantityType.less_is_better - - -@dataclass -class BenchmarkPropertyPlotData: - human_name: str = "" - human_name_short: str = "" - qty_type: QuantityType = QuantityType.neutral - - -class BenchmarkRun(Munch): - """results of an individual run""" - - def __init__(self, json_file: str, meta_class): - """ - meta_class is a class to extract property values from the benchmark run - """ - self._filename = json_file - props = load_json(json_file) - assert hasattr(props, "context") - assert hasattr(props, "benchmarks") - super().__init__(**props) - setattr(self, 'property_names', list(__class__._properties.keys())) - for bm in self.benchmarks: - if meta_class is not None: - setattr(bm, 'meta', meta_class.make(bm.name)) - else: - setattr(bm, 'meta', None) - - _properties = { - 'filename': None, - 'basename': None, - 'dirname': None, - 'meta': None, - 'shorttitle': None, - 'name': None, - 'run_name': None, - 'run_type': None, - 'repetitions': None, - 'repetition_index': None, - 'threads': BenchmarkPropertyPlotData('number of threads', 'threads'), - 'iterations': BenchmarkPropertyPlotData('number of iterations', 'iterations', _more), - 'real_time': BenchmarkPropertyPlotData('real time', 'real time', _less), - 'cpu_time': BenchmarkPropertyPlotData('CPU time', 'cpu time', _less), - 'real_time_ms': BenchmarkPropertyPlotData('real time', 'real time', _less), - 'cpu_time_ms': BenchmarkPropertyPlotData('CPU time', 'cpu time', _less), - 'time_unit': None, - 'bytes_per_second': BenchmarkPropertyPlotData('Bytes/s', 'B/s', _more), - 'items_per_second': BenchmarkPropertyPlotData('items/s', 'items/s', _more), - 'mega_bytes_per_second': BenchmarkPropertyPlotData('MBytes/s', 'MB/s', _more), - 'mega_items_per_second': BenchmarkPropertyPlotData('Mega items/s', 'Mega items/s', _more), - 'counters': None, - } - - def property_plot_data(self, property_name: str): - pd = copy.deepcopy(__class__._properties.get(property_name, BenchmarkPropertyPlotData())) - if property_name.endswith('_time'): - time_unit = first(self.entries).time_unit - pd.human_name += f' ({time_unit})' - pd.human_name_short += f' ({time_unit})' - elif property_name.endswith('_time_ms'): - pd.human_name += ' (ms)' - pd.human_name_short += ' (ms)' - return pd - - def extract_plot_series(self, property_name_or_getter, - relative_to_entry = None, - percent_of_entry = None, - ): - if isinstance(property_name_or_getter, str): - series = getattr(self, property_name_or_getter) - else: - series = property_name_or_getter(self) - series = list(series) - def getrefval(ref): - assert ref in self.entries, ref.name - pos = self.pos(ref) - assert pos in range(len(series)), (pos, len(series)) - return series[pos] - if relative_to_entry: - refval = getrefval(relative_to_entry) - for v in series: - yield v / refval - elif percent_of_entry: - refval = getrefval(percent_of_entry) - for v in series: - yield 100.0 * ((v - refval) / refval) - else: - for v in series: - yield v - - def extract_plot_series_with_threads(self, property_name_or_getter, - relative_to: str = None, - percent_of: str = None, - ): - series = self.extract_plot_series(property_name_or_getter, relative_to=relative_to, percent_of=percent_of) - for y, n in zip(series, self.threads): - yield y, n - - def pos(self, entry): - for i, e in enumerate(self.entries): - if e == entry: - return i - raise Exception("entry not found") - - @property - def filename(self): - return self._filename - - @property - def basename(self): - return os.path.basename(self._filename) - - @property - def dirname(self): - return os.path.dirname(self._filename) - - @property - def entries(self): - for entry in self.benchmarks: - yield entry - - @property - def meta(self): - for entry in self.benchmarks: - yield entry.meta - - @property - def names(self): - for entry in self.benchmarks: - yield entry.name - - @property - def run_names(self): - for entry in self.benchmarks: - yield entry.run_name - - @property - def run_types(self): - for entry in self.benchmarks: - yield entry.run_type - - @property - def repetitions(self): - for entry in self.benchmarks: - yield entry.repetitions - - @property - def repetition_indices(self): - for entry in self.benchmarks: - yield entry.repetition_index - - @property - def threads(self): - for entry in self.benchmarks: - yield entry.threads - - @property - def iterations(self): - for entry in self.benchmarks: - yield entry.iterations - - @property - def real_time(self): - for entry in self.benchmarks: - yield entry.real_time - - @property - def cpu_time(self): - for entry in self.benchmarks: - yield entry.cpu_time - - @property - def real_time_ms(self): - for entry in self.benchmarks: - assert entry.time_unit == "ns" - yield entry.real_time / 1e6 - - @property - def cpu_time_ms(self): - for entry in self.benchmarks: - assert entry.time_unit == "ns" - yield entry.cpu_time / 1e6 - - @property - def time_unit(self): - for entry in self.benchmarks: - yield entry.time_unit - - @property - def bytes_per_second(self): - for entry in self.benchmarks: - yield entry.bytes_per_second - - @property - def items_per_second(self): - for entry in self.benchmarks: - yield entry.items_per_second - - @property - def mega_bytes_per_second(self): - for entry in self.benchmarks: - yield entry.bytes_per_second / 1e6 - - @property - def mega_items_per_second(self): - for entry in self.benchmarks: - yield entry.items_per_second / 1e6 - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -class BenchmarkPanel: - - def __init__(self, runs, bm_meta_cls=None): - self.runs = [BenchmarkRun(a, bm_meta_cls) for a in runs] - - @property - def first_run(self): - return first(self.runs) diff --git a/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_serve.py b/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_serve.py deleted file mode 100644 index 6fc683b84..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_serve.py +++ /dev/null @@ -1,502 +0,0 @@ -import os -import sys -import argparse -import copy -import requests -import flask -import json -import re -import yaml -import shutil -import mmh3 -import itertools -import typing -import enum - -# https://stackoverflow.com/questions/11351032/named-tuple-and-default-values-for-optional-keyword-arguments -from dataclasses import dataclass - -from munch import Munch, munchify -from flask import render_template, redirect, url_for, send_from_directory -from markupsafe import escape - -from bm_util import * - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -class BenchmarkCollection: - - @staticmethod - def create_new(args): - dir = args.target - filename = os.path.join(dir, "bm.yml") - manifest = os.path.join(dir, "manifest.yml") - if not os.path.exists(dir): - os.makedirs(dir) - shutil.copyfile(args.filename, filename) - dump_yml(load_yml("""{runs: {}, bm: {}}"""), manifest) - return __class__(dir) - - def __init__(self, dir): - if not os.path.exists(dir): - raise Exception(f"not found: {dir}") - self.dir = os.path.abspath(dir) - self.runs_dir = os.path.join(self.dir, "runs") - self.manifest = os.path.join(self.dir, "manifest.yml") - self.filename = os.path.join(self.dir, "bm.yml") - self.specs = munchify(load_yml_file(self.filename)) - self.manif = munchify(load_yml_file(self.manifest)) - - def add(self, results_dir): - results_dir = os.path.abspath(results_dir) - dst_dir, meta = self._read_run(results_dir) - self._add_run(results_dir, dst_dir, meta) - dump_yml(self.manif, self.manifest) - - def _read_run(self, results_dir): - log("adding run...") - id = f"{len(self.manif.runs.keys()):05d}" - log(f"adding run: id={id}") - meta = ResultMeta.load(results_dir) - dst_dir = os.path.join(self.runs_dir, meta.name) - return dst_dir, meta - - def _add_run(self, results_dir, dst_dir, meta): - cats = self._add_meta_categories(meta) - for filename in ("meta.yml", - "CMakeCCompiler.cmake", - "CMakeCXXCompiler.cmake", - "CMakeSystem.cmake", - "compile_commands.json"): - filename = os.path.join(results_dir, filename) - if os.path.exists(filename): - copy_file_to_dir(filename, dst_dir) - else: - if not filename.endswith("compile_commands.json"): - raise Exception(f"wtf???? {filename}") - for name, specs in self.specs.bm.items(): - if not hasattr(specs, 'variants'): - filename = chk(f"{results_dir}/{name}.json") - dst = copy_file_to_dir(filename, dst_dir) - self._add_bm_run(name, specs, meta) - else: - for t in specs.variants: - tname = f"{name}-{t}" - filename = chk(f"{results_dir}/{tname}.json") - dst = copy_file_to_dir(filename, dst_dir) - self._add_bm_run(tname, specs, meta) - - def _add_bm_run(self, name, specs, meta): - if name not in self.manif.bm.keys(): - self.manif.bm[name] = Munch(specs=specs, entries=[]) - entry = self.manif.bm[name] - entry.specs = specs - if meta.name not in entry.entries: - entry.entries.append(meta.name) - - def _add_meta_categories(self, meta): - run = Munch() - for catname in ('commit', 'cpu', 'system', 'build'): - meta_item = getattr(meta, catname) - self._add_item_to_category(meta.name, catname, meta_item) - run[catname] = meta_item.storage_id - # build specs are too verbose; remove them - self.manif.build[meta.build.storage_id].specs = Munch() - self.manif.runs[meta.name] = run - - def _add_item_to_category(self, run, category_name, item): - if not hasattr(self.manif, category_name): - setattr(self.manif, category_name, Munch()) - category = getattr(self.manif, category_name) - if item.storage_id not in category.keys(): - category[item.storage_id] = Munch(specs=item, entries=[]) - entry = category[item.storage_id] - entry.specs = item - if run not in entry.entries: - entry.entries.append(run) - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -class ResultMeta(Munch): - - def __init__(self, results_dir, cmakecache, build_type): - super().__init__(self) - self.date = __class__.get_date() - self.commit = __class__.get_commit(results_dir) - self.cpu = __class__.get_cpu_info() - self.system = __class__.get_sys_info() - self.build = __class__.get_build_info(cmakecache, build_type) - self.name = self._get_name() - - @staticmethod - def load(results_dir): - results_dir = os.path.join(os.path.abspath(results_dir), "meta.yml") - data = load_yml_file(results_dir) - return munchify(data) - - def save(self, results_dir): - out = os.path.join(results_dir, "meta.yml") - log("saving meta:", out) - dump_yml(self, out) - self.build.save(results_dir) - - @staticmethod - def get_date(): - import datetime - now = datetime.datetime.now() - return now.strftime("%Y%m%d-%H%M%S") - - def _get_name(self): - commit = self.commit.storage_name - cpu = self.cpu.storage_name - sys = self.system.storage_name - build = self.build.storage_name - name = f"{commit}/{cpu}-{sys}-{build}" - return name - - @staticmethod - def get_commit(results_dir): - import git - repo = git.Repo(results_dir, search_parent_directories=True) - commit = repo.head.commit - commit = {p: str(getattr(commit, p)) - for p in ('message', 'summary', 'name_rev', - 'author', - 'authored_datetime', - 'committer', - 'committed_datetime',)} - commit = Munch(commit) - commit.message = commit.message.strip() - commit.sha1 = commit.name_rev[:7] - spl = commit.authored_datetime.split(" ") - date = re.sub(r'-', '', spl[0]) - time = re.sub(r'(\d+):(\d+):(\d+).*', r'\1\2\3', spl[1]) - commit.storage_id = commit.sha1 - commit.storage_name = f"git{date}_{time}-{commit.sha1}" - return commit - - @staticmethod - def get_cpu_info(): - import cpuinfo - nfo = cpuinfo.get_cpu_info() - nfo = Munch(nfo) - for a in ('cpu_version', 'cpu_version_string', 'python_version'): - if hasattr(nfo, a): - delattr(nfo, a) - for a in ('arch_string_raw', 'brand_raw', 'hardware_raw', 'vendor_id_raw'): - if not hasattr(nfo, a): - setattr(nfo, a, '') - nfo.storage_id = myhash( - nfo.arch_string_raw, nfo.brand_raw, nfo.hardware_raw, nfo.vendor_id_raw, - nfo.arch, nfo.bits, nfo.count, nfo.family, nfo.model, nfo.stepping, - ",".join(nfo.flags), nfo.hz_advertised_friendly, - nfo.l2_cache_associativity, - nfo.l2_cache_line_size, - nfo.l2_cache_size, - nfo.l3_cache_size, - *optionals('l1_data_cache_size', 'l1_instruction_cache_size') - ) - nfo.storage_name = f"{nfo.arch.lower()}_{nfo.storage_id}" - return nfo - - @staticmethod - def get_sys_info(): - import platform - uname = platform.uname() - nfo = Munch( - sys_platform=sys.platform, - sys=platform.system(), - uname=Munch( - machine=uname.machine, - node=uname.node, - release=uname.release, - system=uname.system, - version=uname.version, - ) - ) - nfo.storage_id = myhash( - nfo.sys_platform, - nfo.uname.machine, - ) - nfo.storage_name = f"{nfo.sys_platform}_{nfo.storage_id}" - return nfo - - @staticmethod - def get_build_info(cmakecache_txt, buildtype): - nfo = CMakeCache(cmakecache_txt) - def _btflags(name): - return (getattr(nfo, name), getattr(nfo, f"{name}_{buildtype.upper()}")) - nfo.storage_id = myhash( - buildtype, - nfo.CMAKE_CXX_COMPILER_ID, - nfo.CMAKE_CXX_COMPILER_VERSION, - nfo.CMAKE_CXX_COMPILER_VERSION_INTERNAL, - nfo.CMAKE_CXX_COMPILER_ABI, - nfo.CMAKE_CXX_SIZEOF_DATA_PTR, - nfo.CMAKE_C_COMPILER_ID, - nfo.CMAKE_C_COMPILER_VERSION, - nfo.CMAKE_C_COMPILER_VERSION_INTERNAL, - nfo.CMAKE_C_COMPILER_ABI, - nfo.CMAKE_C_SIZEOF_DATA_PTR, - *_btflags("CMAKE_CXX_FLAGS"), - *_btflags("CMAKE_C_FLAGS"), - *_btflags("CMAKE_STATIC_LINKER_FLAGS"), - *_btflags("CMAKE_SHARED_LINKER_FLAGS"), - ) - # - ccname = nfo.CMAKE_CXX_COMPILER_ID.lower() - if ccname == "gnu": - ccname = "gcc" - ccname += nfo.CMAKE_CXX_COMPILER_VERSION.lower() - # - if nfo.CMAKE_C_SIZEOF_DATA_PTR == "4": - bits = "32bit" - elif nfo.CMAKE_C_SIZEOF_DATA_PTR == "8": - bits = "64bit" - else: - raise Exception("unknown architecture") - # - nfo.storage_name = f"{bits}_{buildtype}_{ccname}_{nfo.storage_id}" - return nfo - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -class CMakeCache(Munch): - - def __init__(self, cmakecache_txt): - import glob - for line in iter_cmake_lines(cmakecache_txt): - spl = line.split("=") - if len(spl) < 2: - continue - k, ty = spl[0].split(":") - v = "=".join(spl[1:]).strip() - setattr(self, k, v) - bdir = os.path.dirname(os.path.abspath(cmakecache_txt)) - self._c_compiler_file = sorted(glob.glob(f"{bdir}/CMakeFiles/*/CMakeCCompiler.cmake"))[-1] # get the last - self._cxx_compiler_file = sorted(glob.glob(f"{bdir}/CMakeFiles/*/CMakeCXXCompiler.cmake"))[-1] # get the last - self._system_file = sorted(glob.glob(f"{bdir}/CMakeFiles/*/CMakeSystem.cmake"))[-1] # get the last - self._load_cmake_file(self._c_compiler_file) - self._load_cmake_file(self._cxx_compiler_file) - ccomfile = f"{bdir}/compile_commands.json" - self._compile_commands_file = ccomfile if os.path.exists(ccomfile) else None - - def _load_cmake_file(self, filename): - for line in iter_cmake_lines(filename): - if not line.startswith("set("): - continue - k = re.sub(r"set\((.*)\ +(.*)\)", r"\1", line) - v = re.sub(r"set\((.*)\ +(.*)\)", r"\2", line) - v = v.strip('"').strip("'").strip() - setattr(self, k, v) - - def save(self, results_dir): - copy_file_to_dir(self._c_compiler_file, results_dir) - copy_file_to_dir(self._cxx_compiler_file, results_dir) - copy_file_to_dir(self._system_file, results_dir) - if self._compile_commands_file is not None: - copy_file_to_dir(self._compile_commands_file, results_dir) - - -def iter_cmake_lines(filename): - with open(filename) as f: - for line in f.readlines(): - line = line.strip() - if line.startswith("#") or line.startswith("//") or len(line) == 0: - continue - yield line - - -# -------------------------------------------------------- - - -def get_manifest(args): - bmdir = os.path.abspath(args.bmdir) - if not args.manifest: - manifest_yml = os.path.join(bmdir, "manifest.yml") - else: - if not os.path.isabs(args.manifest): - manifest_yml = os.path.join(os.getcwd(), args.manifest) - manifest_json = os.path.join(os.path.dirname(manifest.yml), "manifest.json") - manifest = load_yml_file(manifest_yml) - dump_json(manifest, manifest_json) - return manifest - - -def add_results(args): - log("adding results:", args.results) - col = BenchmarkCollection(args.target) - col.add(args.results) - - -def add_meta(args): - log("adding bm run metadata to results dir:", args.results) - meta = ResultMeta(results_dir=args.results, - cmakecache=args.cmakecache, - build_type=args.build_type) - meta.save(args.results) - log("adding bm run metadata to results dir: success!") - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -app = flask.Flask(__name__, template_folder='template') - -def _setup_app(args): - def _s(prop, val): - assert not hasattr(app, prop), prop - setattr(app, prop, val) - _s('args', args) - _s('manifest', get_manifest(args)) - if args.debug: - app.config["DEBUG"] = True - - -def freeze(args): - "https://pythonhosted.org/Frozen-Flask/" - from flask_frozen import Freezer - _setup_app(args) - freezer = Freezer(app) - freezer.freeze(debug=args.debug) - - -def serve(args): - _setup_app(args) - app.run(host=args.host, port=args.port, debug=args.debug) - - -@app.route("/") -def home(): - log("requested home") - return render_template("index.html") - - -@app.route("/") -def other_(path): - path = escape(path) - d = app.args.bmdir - log("requested other path:", path, "---", os.path.join(d, path)) - return send_from_directory(d, path) - - -@app.route("/static/") -def static_(path): - path = escape(path) - d = os.path.join(app.args.bmdir, "static") - log("requested static path:", path, "---", os.path.join(d, path)) - return send_from_directory(d, path, cache_timeout=1) # timeout in seconds - - -@app.route("/bm///") -def bm_(commit, run, resultjson): - commit = escape(commit) - run = escape(run) - resultjson = escape(resultjson) - d = os.path.join(app.args.bmdir, "runs", commit, run) - log("requested result:", os.path.join(d, resultjson)) - return send_from_directory(d, resultjson, cache_timeout=1) # timeout in seconds - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -def download_deps(): - deps = [ - "https://code.jquery.com/jquery-3.3.1.js", - "https://code.jquery.com/jquery-3.3.1.js", - "https://code.jquery.com/ui/1.12.1/jquery-ui.js", - "https://cdn.datatables.net/1.10.20/js/jquery.dataTables.js", - "https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js", - "https://cdn.datatables.net/1.10.20/css/jquery.dataTables.css", - "https://cdn.datatables.net/1.10.20/css/jquery.dataTables.min.css", - "https://www.chartjs.org/dist/2.9.1/Chart.min.js", - #("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/styles/github.css", "highlight.github.css"), - ("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/styles/github.min.css", "highlight.github.min.css"), - #"https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/highlight.js", - "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/highlight.min.js", - ] - for src in deps: - if type(src) == str: - base = os.path.basename(src) - else: - src, base = src - dst = f"{os.getcwd()}/static/{base}" - download_url(src, dst) - - -def download_url(url, dst): - log("download url:", url, "--->", dst) - req = requests.get(url, stream=True) - if req.status_code == 200: - sz = 0 - with open(dst, 'wb') as f: - for chunk in req: - f.write(chunk) - sz += len(chunk) - log(f"........ finished: {sz}B") - else: - log(f" error:", req.status_code, url) - - - -def main(): - def _common_args(parser): - parser.add_argument("-m", "--manifest", type=str, default="", help="enable debug mode") - parser.add_argument("--debug", action="store_true", help="enable debug mode") - # - parser = argparse.ArgumentParser(description="Browse benchmark results", prog="bm") - _common_args(parser) - subparsers = parser.add_subparsers() - # - sp = subparsers.add_parser("create", help="create benchmark collection") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("filename", type=str, help="the YAML file with the benchmark specs") - sp.add_argument("target", type=str, help="the directory to store the results") - # - sp = subparsers.add_parser("meta", help="get the required meta-information: cpu info, commit data") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("results", type=str, help="the directory with the results") - sp.add_argument("cmakecache", type=str, help="the path to the CMakeCache.txt file used to build the benchmark binaries") - sp.add_argument("build_type", type=str, help="the build type, eg Release Debug MinSizeRel RelWithDebInfo") - # - sp = subparsers.add_parser("add", help="add benchmark results") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("results", type=str, help="the directory with the results") - sp.add_argument("target", type=str, help="the directory to store the results") - # - sp = subparsers.add_parser("serve", help="serve benchmark results") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("bmdir", type=os.path.abspath, default=os.getcwd(), help="the directory with the results. default=.") - sp.add_argument("-H", "--host", type=str, default="localhost", help="host. default=%(default)s") - sp.add_argument("-p", "--port", type=int, default=8000, help="port. default=%(default)s") - # - sp = subparsers.add_parser("export", help="export static html") - sp.set_defaults(func=freeze) - sp.add_argument("bmdir", type=os.path.abspath, default=os.getcwd(), help="the directory with the results. default=.") - _common_args(sp) - # - sp = subparsers.add_parser("deps", help="install server dependencies") - sp.set_defaults(func=lambda _: download_deps()) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - _common_args(sp) - # - args = parser.parse_args(sys.argv[1:] if len(sys.argv) > 1 else ["serve"]) - if args.debug: - log(args) - args.func(args) diff --git a/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_util.py b/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_util.py deleted file mode 100644 index 7c7c2891e..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/bm-xp/bm_util.py +++ /dev/null @@ -1,147 +0,0 @@ -import os -import json -import yaml -import shutil -import mmh3 -import itertools -import munch -import enum - - -# -------------------------------------------------------- - -class _enum(enum.Enum): - def __str__(self): - return str(self.name) - - @property - def short(self): - return self.name - - @classmethod - def make(cls, s): - try: - return cls[s] - except: - cls.err_unknown(s) - - @classmethod - def err_unknown(cls, s): - raise Exception(f"unknown {__class__.__name__}: {s}") - - -class FundamentalTypes(_enum): - float = "float" - double = "double" - int8_t = "int8_t" - uint8_t = "uint8_t" - int16_t = "int16_t" - uint16_t = "uint16_t" - int32_t = "int32_t" - uint32_t = "uint32_t" - int64_t = "int64_t" - uint64_t = "uint64_t" - @property - def short(self): - return self.name.replace("uint", "u").replace("int", "i").replace("_t", "") - - -# -------------------------------------------------------- - -def log(*args, **kwargs): - print(*args, **kwargs, flush=True) - - -def myhash_combine(curr, value): - return curr ^ (value + 0x9e3779b9 + (curr << 6) + (curr >> 2)) - - -def first(iterable): - """Returns the first item""" - if isinstance(iterable, list): - return iterable[0] - return next(iterable) - - -def chain(*iterables): - for it in iterables: - for elm in it: - yield elm - - -def nth(iterable, n, default=None): - """Returns the nth item or a default value""" - return next(itertools.islice(iterable, n, None), default) - - -def optionals(obj, *attrs): - ret = [] - for attr in attrs: - if not hasattr(obj, attr): - log("attr not present:", attr) - continue - ret.append(getattr(obj, attr)) - return ret - - -def myhash(*args): - h = 137597 - for a in args: - if isinstance(a, str): - if a == "": - continue - b = bytes(a, "utf8") - else: - b = bytes(a) - hb = mmh3.hash(b, signed=False) - h = myhash_combine(h, hb) - s = hex(h) - return s[2:min(10, len(s))] - - -def copy_file_to_dir(file, dir): - dir = os.path.abspath(dir) - src = os.path.abspath(file) - dst = f"{dir}/{os.path.basename(src)}" - if not os.path.exists(dir): - os.makedirs(dir) - if os.path.exists(dst): - os.remove(dst) - log("copy:", src, "-->", dst) - shutil.copy(src, dst) - return dst - - -def chk(f): - log("looking for file:", f) - assert os.path.exists(f), f - return f - - -def load_yml_file(filename): - if not os.path.exists(filename): - raise Exception(f"not found: {filename}") - with open(filename) as f: - return load_yml(f.read()) - - -def dump_yml(data, filename): - with open(filename, "w") as f: - yaml.safe_dump(data, f) - - -def load_yml(yml): - return munch.munchify(yaml.safe_load(yml)) - - -def dump_json(data, filename): - with open(filename, "w") as f: - f.write(json.dumps(data, indent=2, sort_keys=True)) - - -def load_json(filename): - with open(filename, "r") as f: - try: - return munch.munchify(json.load(f)) - except Exception as exc: - raise Exception(f"could not load file: {filename}: {exc}") diff --git a/thirdparty/ryml/ext/c4core/cmake/bm-xp/example_c4core.py b/thirdparty/ryml/ext/c4core/cmake/bm-xp/example_c4core.py deleted file mode 100644 index 3db4175cb..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/bm-xp/example_c4core.py +++ /dev/null @@ -1,1061 +0,0 @@ -import os -import sys -import argparse -import copy -import requests -import flask -import json -import re -import yaml -import shutil -import mmh3 -from itertools import islice - -from munch import Munch, munchify -from flask import render_template, redirect, url_for, send_from_directory -from markupsafe import escape - - -def log(*args, **kwargs): - print(*args, **kwargs, flush=True) - - -def myhash_combine(curr, value): - return curr ^ (value + 0x9e3779b9 + (curr<<6) + (curr>>2)) - - -def nth(iterable, n, default=None): - "Returns the nth item or a default value" - return next(islice(iterable, n, None), default) - - -def optionals(obj, *attrs): - ret = [] - for attr in attrs: - if not hasattr(obj, attr): - log("attr not present:", attr) - continue - ret.append(getattr(obj, attr)) - return ret - - -def myhash(*args): - h = 137597 - for a in args: - if isinstance(a, str): - if a == "": - continue - b = bytes(a, "utf8") - else: - b = bytes(a) - hb = mmh3.hash(b, signed=False) - h = myhash_combine(h, hb) - s = hex(h) - return s[2:min(10, len(s))] - - -def copy_file_to_dir(file, dir): - dir = os.path.abspath(dir) - src = os.path.abspath(file) - dst = f"{dir}/{os.path.basename(src)}" - if not os.path.exists(dir): - os.makedirs(dir) - if os.path.exists(dst): - os.remove(dst) - log("copy:", src, "-->", dst) - shutil.copy(src, dst) - return dst - - -def chk(f): - log(f"looking for file:", f) - assert os.path.exists(f), f - return f - - -def load_yml_file(filename): - if not os.path.exists(filename): - raise Exception(f"not found: {filename}") - with open(filename) as f: - return load_yml(f.read()) - - -def dump_yml(data, filename): - with open(filename, "w") as f: - yaml.safe_dump(data, f) - - -def load_yml(yml): - return munchify(yaml.safe_load(yml)) - - -def dump_json(data, filename): - with open(filename, "w") as f: - f.write(json.dumps(data, indent=2, sort_keys=True)) - - -def load_json(filename): - with open(filename, "r") as f: - try: - return munchify(json.load(f)) - except Exception as exc: - raise Exception(f"could not load file: {filename}: {exc}") - - -def main(): - def _common_args(parser): - parser.add_argument("-m", "--manifest", type=str, default="", help="enable debug mode") - parser.add_argument("--debug", action="store_true", help="enable debug mode") - # - parser = argparse.ArgumentParser(description="Browse benchmark results", prog="bm") - _common_args(parser) - subparsers = parser.add_subparsers() - # - sp = subparsers.add_parser("create", help="create benchmark collection") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("filename", type=str, help="the YAML file with the benchmark specs") - sp.add_argument("target", type=str, help="the directory to store the results") - # - sp = subparsers.add_parser("meta", help="get the required meta-information: cpu info, commit data") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("results", type=str, help="the directory with the results") - sp.add_argument("cmakecache", type=str, help="the path to the CMakeCache.txt file used to build the benchmark binaries") - sp.add_argument("build_type", type=str, help="the build type, eg Release Debug MinSizeRel RelWithDebInfo") - # - sp = subparsers.add_parser("add", help="add benchmark results") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("results", type=str, help="the directory with the results") - sp.add_argument("target", type=str, help="the directory to store the results") - # - sp = subparsers.add_parser("serve", help="serve benchmark results") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("bmdir", type=os.path.abspath, default=os.getcwd(), help="the directory with the results. default=.") - sp.add_argument("-H", "--host", type=str, default="localhost", help="host. default=%(default)s") - sp.add_argument("-p", "--port", type=int, default=8000, help="port. default=%(default)s") - # - sp = subparsers.add_parser("export", help="export static html") - sp.set_defaults(func=freeze) - sp.add_argument("bmdir", type=os.path.abspath, default=os.getcwd(), help="the directory with the results. default=.") - _common_args(sp) - # - sp = subparsers.add_parser("deps", help="install server dependencies") - sp.set_defaults(func=lambda _: download_deps()) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - _common_args(sp) - # - args = parser.parse_args(sys.argv[1:] if len(sys.argv) > 1 else ["serve"]) - if args.debug: - log(args) - args.func(args) - - -def get_manifest(args): - bmdir = os.path.abspath(args.bmdir) - if not args.manifest: - manifest_yml = os.path.join(bmdir, "manifest.yml") - else: - if not os.path.isabs(args.manifest): - manifest_yml = os.path.join(os.getcwd(), args.manifest) - manifest_json = os.path.join(os.path.dirname(manifest.yml), "manifest.json") - manifest = load_yml_file(manifest_yml) - dump_json(manifest, manifest_json) - return manifest - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -app = flask.Flask(__name__, - template_folder='template') - - -def _setup_app(args): - def _s(prop, val): - assert not hasattr(app, prop), prop - setattr(app, prop, val) - _s('args', args) - _s('manifest', get_manifest(args)) - if args.debug: - app.config["DEBUG"] = True - - -def freeze(args): - "https://pythonhosted.org/Frozen-Flask/" - from flask_frozen import Freezer - _setup_app(args) - freezer = Freezer(app) - freezer.freeze(debug=args.debug) - - -def serve(args): - _setup_app(args) - app.run(host=args.host, port=args.port, debug=args.debug) - - -@app.route("/") -def home(): - log("requested home") - return render_template("index.html") - - -@app.route("/") -def other_(path): - path = escape(path) - d = app.args.bmdir - log("requested other path:", path, "---", os.path.join(d, path)) - return send_from_directory(d, path) - - -@app.route("/static/") -def static_(path): - path = escape(path) - d = os.path.join(app.args.bmdir, "static") - log("requested static path:", path, "---", os.path.join(d, path)) - return send_from_directory(d, path, cache_timeout=1) # timeout in seconds - - -@app.route("/bm///") -def bm_(commit, run, resultjson): - commit = escape(commit) - run = escape(run) - resultjson = escape(resultjson) - d = os.path.join(app.args.bmdir, "runs", commit, run) - log("requested result:", os.path.join(d, resultjson)) - return send_from_directory(d, resultjson, cache_timeout=1) # timeout in seconds - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -def download_deps(): - deps = [ - "https://code.jquery.com/jquery-3.3.1.js", - "https://code.jquery.com/jquery-3.3.1.js", - "https://code.jquery.com/ui/1.12.1/jquery-ui.js", - "https://cdn.datatables.net/1.10.20/js/jquery.dataTables.js", - "https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js", - "https://cdn.datatables.net/1.10.20/css/jquery.dataTables.css", - "https://cdn.datatables.net/1.10.20/css/jquery.dataTables.min.css", - "https://www.chartjs.org/dist/2.9.1/Chart.min.js", - #("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/styles/github.css", "highlight.github.css"), - ("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/styles/github.min.css", "highlight.github.min.css"), - #"https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/highlight.js", - "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/highlight.min.js", - ] - for src in deps: - if type(src) == str: - base = os.path.basename(src) - else: - src, base = src - dst = f"{os.getcwd()}/static/{base}" - download_url(src, dst) - - -def download_url(url, dst): - log("download url:", url, "--->", dst) - req = requests.get(url, stream=True) - if req.status_code == 200: - sz = 0 - with open(dst, 'wb') as f: - for chunk in req: - f.write(chunk) - sz += len(chunk) - log(f"........ finished: {sz}B") - else: - log(f" error:", req.status_code, url) - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -class BenchmarkCollection: - - @staticmethod - def create_new(args): - dir = args.target - filename = os.path.join(dir, "bm.yml") - manifest = os.path.join(dir, "manifest.yml") - if not os.path.exists(dir): - os.makedirs(dir) - shutil.copyfile(args.filename, filename) - dump_yml(load_yml("""{runs: {}, bm: {}}"""), manifest) - return __class__(dir) - - def __init__(self, dir): - if not os.path.exists(dir): - raise Exception(f"not found: {dir}") - self.dir = os.path.abspath(dir) - self.runs_dir = os.path.join(self.dir, "runs") - self.manifest = os.path.join(self.dir, "manifest.yml") - self.filename = os.path.join(self.dir, "bm.yml") - self.specs = munchify(load_yml_file(self.filename)) - self.manif = munchify(load_yml_file(self.manifest)) - - def add(self, results_dir): - results_dir = os.path.abspath(results_dir) - dst_dir, meta = self._read_run(results_dir) - self._add_run(results_dir, dst_dir, meta) - dump_yml(self.manif, self.manifest) - - def _read_run(self, results_dir): - log("adding run...") - id = f"{len(self.manif.runs.keys()):05d}" - log(f"adding run: id={id}") - meta = ResultMeta.load(results_dir) - dst_dir = os.path.join(self.runs_dir, meta.name) - return dst_dir, meta - - def _add_run(self, results_dir, dst_dir, meta): - cats = self._add_meta_categories(meta) - for filename in ("meta.yml", - "CMakeCCompiler.cmake", - "CMakeCXXCompiler.cmake", - "CMakeSystem.cmake", - "compile_commands.json"): - filename = os.path.join(results_dir, filename) - if os.path.exists(filename): - copy_file_to_dir(filename, dst_dir) - else: - if not filename.endswith("compile_commands.json"): - raise Exception(f"wtf???? {filename}") - for name, specs in self.specs.bm.items(): - if not hasattr(specs, 'variants'): - filename = chk(f"{results_dir}/{name}.json") - dst = copy_file_to_dir(filename, dst_dir) - self._add_bm_run(name, specs, meta) - else: - for t in specs.variants: - tname = f"{name}-{t}" - filename = chk(f"{results_dir}/{tname}.json") - dst = copy_file_to_dir(filename, dst_dir) - self._add_bm_run(tname, specs, meta) - - def _add_bm_run(self, name, specs, meta): - if name not in self.manif.bm.keys(): - self.manif.bm[name] = Munch(specs=specs, entries=[]) - entry = self.manif.bm[name] - entry.specs = specs - if meta.name not in entry.entries: - entry.entries.append(meta.name) - - def _add_meta_categories(self, meta): - run = Munch() - for catname in ('commit', 'cpu', 'system', 'build'): - meta_item = getattr(meta, catname) - self._add_item_to_category(meta.name, catname, meta_item) - run[catname] = meta_item.storage_id - # build specs are too verbose; remove them - self.manif.build[meta.build.storage_id].specs = Munch() - self.manif.runs[meta.name] = run - - def _add_item_to_category(self, run, category_name, item): - if not hasattr(self.manif, category_name): - setattr(self.manif, category_name, Munch()) - category = getattr(self.manif, category_name) - if item.storage_id not in category.keys(): - category[item.storage_id] = Munch(specs=item, entries=[]) - entry = category[item.storage_id] - entry.specs = item - if run not in entry.entries: - entry.entries.append(run) - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -class ResultMeta(Munch): - - def __init__(self, results_dir, cmakecache, build_type): - super().__init__(self) - self.date = __class__.get_date() - self.commit = __class__.get_commit(results_dir) - self.cpu = __class__.get_cpu_info() - self.system = __class__.get_sys_info() - self.build = __class__.get_build_info(cmakecache, build_type) - self.name = self._get_name() - - @staticmethod - def load(results_dir): - results_dir = os.path.join(os.path.abspath(results_dir), "meta.yml") - data = load_yml_file(results_dir) - return munchify(data) - - def save(self, results_dir): - out = os.path.join(results_dir, "meta.yml") - log("saving meta:", out) - dump_yml(self, out) - self.build.save(results_dir) - - @staticmethod - def get_date(): - import datetime - now = datetime.datetime.now() - return now.strftime("%Y%m%d-%H%M%S") - - def _get_name(self): - commit = self.commit.storage_name - cpu = self.cpu.storage_name - sys = self.system.storage_name - build = self.build.storage_name - name = f"{commit}/{cpu}-{sys}-{build}" - return name - - @staticmethod - def get_commit(results_dir): - import git - repo = git.Repo(results_dir, search_parent_directories=True) - commit = repo.head.commit - commit = {p: str(getattr(commit, p)) - for p in ('message', 'summary', 'name_rev', - 'author', - 'authored_datetime', - 'committer', - 'committed_datetime',)} - commit = Munch(commit) - commit.message = commit.message.strip() - commit.sha1 = commit.name_rev[:7] - spl = commit.authored_datetime.split(" ") - date = re.sub(r'-', '', spl[0]) - time = re.sub(r'(\d+):(\d+):(\d+).*', r'\1\2\3', spl[1]) - commit.storage_id = commit.sha1 - commit.storage_name = f"git{date}_{time}-{commit.sha1}" - return commit - - @staticmethod - def get_cpu_info(): - import cpuinfo - nfo = cpuinfo.get_cpu_info() - nfo = Munch(nfo) - for a in ('cpu_version', 'cpu_version_string', 'python_version'): - if hasattr(nfo, a): - delattr(nfo, a) - for a in ('arch_string_raw', 'brand_raw', 'hardware_raw', 'vendor_id_raw'): - if not hasattr(nfo, a): - setattr(nfo, a, '') - nfo.storage_id = myhash( - nfo.arch_string_raw, nfo.brand_raw, nfo.hardware_raw, nfo.vendor_id_raw, - nfo.arch, nfo.bits, nfo.count, nfo.family, nfo.model, nfo.stepping, - ",".join(nfo.flags), nfo.hz_advertised_friendly, - nfo.l2_cache_associativity, - nfo.l2_cache_line_size, - nfo.l2_cache_size, - nfo.l3_cache_size, - *optionals('l1_data_cache_size', 'l1_instruction_cache_size') - ) - nfo.storage_name = f"{nfo.arch.lower()}_{nfo.storage_id}" - return nfo - - @staticmethod - def get_sys_info(): - import platform - uname = platform.uname() - nfo = Munch( - sys_platform=sys.platform, - sys=platform.system(), - uname=Munch( - machine=uname.machine, - node=uname.node, - release=uname.release, - system=uname.system, - version=uname.version, - ) - ) - nfo.storage_id = myhash( - nfo.sys_platform, - nfo.uname.machine, - ) - nfo.storage_name = f"{nfo.sys_platform}_{nfo.storage_id}" - return nfo - - @staticmethod - def get_build_info(cmakecache_txt, buildtype): - nfo = CMakeCache(cmakecache_txt) - def _btflags(name): - return (getattr(nfo, name), getattr(nfo, f"{name}_{buildtype.upper()}")) - nfo.storage_id = myhash( - buildtype, - nfo.CMAKE_CXX_COMPILER_ID, - nfo.CMAKE_CXX_COMPILER_VERSION, - nfo.CMAKE_CXX_COMPILER_VERSION_INTERNAL, - nfo.CMAKE_CXX_COMPILER_ABI, - nfo.CMAKE_CXX_SIZEOF_DATA_PTR, - nfo.CMAKE_C_COMPILER_ID, - nfo.CMAKE_C_COMPILER_VERSION, - nfo.CMAKE_C_COMPILER_VERSION_INTERNAL, - nfo.CMAKE_C_COMPILER_ABI, - nfo.CMAKE_C_SIZEOF_DATA_PTR, - *_btflags("CMAKE_CXX_FLAGS"), - *_btflags("CMAKE_C_FLAGS"), - *_btflags("CMAKE_STATIC_LINKER_FLAGS"), - *_btflags("CMAKE_SHARED_LINKER_FLAGS"), - ) - # - ccname = nfo.CMAKE_CXX_COMPILER_ID.lower() - if ccname == "gnu": - ccname = "gcc" - ccname += nfo.CMAKE_CXX_COMPILER_VERSION.lower() - # - if nfo.CMAKE_C_SIZEOF_DATA_PTR == "4": - bits = "32bit" - elif nfo.CMAKE_C_SIZEOF_DATA_PTR == "8": - bits = "64bit" - else: - raise Exception("unknown architecture") - # - nfo.storage_name = f"{bits}_{buildtype}_{ccname}_{nfo.storage_id}" - return nfo - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -class CMakeCache(Munch): - - def __init__(self, cmakecache_txt): - import glob - for line in iter_cmake_lines(cmakecache_txt): - spl = line.split("=") - if len(spl) < 2: - continue - k, ty = spl[0].split(":") - v = "=".join(spl[1:]).strip() - setattr(self, k, v) - bdir = os.path.dirname(os.path.abspath(cmakecache_txt)) - self._c_compiler_file = sorted(glob.glob(f"{bdir}/CMakeFiles/*/CMakeCCompiler.cmake"))[-1] # get the last - self._cxx_compiler_file = sorted(glob.glob(f"{bdir}/CMakeFiles/*/CMakeCXXCompiler.cmake"))[-1] # get the last - self._system_file = sorted(glob.glob(f"{bdir}/CMakeFiles/*/CMakeSystem.cmake"))[-1] # get the last - self._load_cmake_file(self._c_compiler_file) - self._load_cmake_file(self._cxx_compiler_file) - ccomfile = f"{bdir}/compile_commands.json" - self._compile_commands_file = ccomfile if os.path.exists(ccomfile) else None - - def _load_cmake_file(self, filename): - for line in iter_cmake_lines(filename): - if not line.startswith("set("): - continue - k = re.sub(r"set\((.*)\ +(.*)\)", r"\1", line) - v = re.sub(r"set\((.*)\ +(.*)\)", r"\2", line) - v = v.strip('"').strip("'").strip() - setattr(self, k, v) - - def save(self, results_dir): - copy_file_to_dir(self._c_compiler_file, results_dir) - copy_file_to_dir(self._cxx_compiler_file, results_dir) - copy_file_to_dir(self._system_file, results_dir) - if self._compile_commands_file is not None: - copy_file_to_dir(self._compile_commands_file, results_dir) - - -def iter_cmake_lines(filename): - with open(filename) as f: - for line in f.readlines(): - line = line.strip() - if line.startswith("#") or line.startswith("//") or len(line) == 0: - continue - yield line - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -class BenchmarkRun(Munch): - "results of an individual run" - - def __init__(self, json_file, meta_class): - props = load_json(json_file) - setattr(self, "filename", json_file) - assert hasattr(props, "context") - assert hasattr(props, "benchmarks") - super().__init__(**props) - for e in self.benchmarks: - setattr(e, 'meta', meta_class(e.name)) - setattr(self, 'property_names', ( - 'meta', - 'shorttitle', - 'name', - 'run_name', - 'run_type', - 'repetitions', - 'repetition_index', - 'repetition_index', - 'threads', - 'iterations', - 'real_time', - 'cpu_time', - 'time_unit', - 'bytes_per_second', - 'items_per_second', - 'counters', - )) - - @property - def first(self): - return self.benchmarks[0] - - @property - def entries(self): - for entry in self.benchmarks: - yield entry - - @property - def meta(self): - for entry in self.benchmarks: - yield entry.meta - - @property - def filtered_names(self): - for entry in self.benchmarks: - yield entry.meta.shorttitle - - @property - def names(self): - for entry in self.benchmarks: - yield entry.name - - @property - def run_names(self): - for entry in self.benchmarks: - yield entry.run_name - - @property - def run_types(self): - for entry in self.benchmarks: - yield entry.run_type - - @property - def repetitions(self): - for entry in self.benchmarks: - yield entry.repetitions - - @property - def repetition_indices(self): - for entry in self.benchmarks: - yield entry.repetition_index - - @property - def threads(self): - for entry in self.benchmarks: - yield entry.threads - - @property - def iterations(self): - for entry in self.benchmarks: - yield entry.iterations - - @property - def real_time(self): - for entry in self.benchmarks: - yield entry.real_time - - @property - def cpu_time(self): - for entry in self.benchmarks: - yield entry.cpu_time - - @property - def time_unit(self): - for entry in self.benchmarks: - yield entry.time_unit - - @property - def bytes_per_second(self): - for entry in self.benchmarks: - yield entry.bytes_per_second - - @property - def items_per_second(self): - for entry in self.benchmarks: - yield entry.items_per_second - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -class BenchmarkPanel: - - def __init__(self, runs, bm_meta_cls=None): - self.runs = [BenchmarkRun(a, bm_meta_cls) for a in runs] - - def plot_bars(self, title): - plot_benchmarks_as_lines(title, *self.runs) - - - def plot_all_lines(self, title): - plot_benchmarks_as_lines(title, *self.runs, - transform=lambda r: r.meta.num_pixels, - line_title_transform=lambda r: r.meta.shortname) - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - - -def plot_benchmarks_as_bars(title, *bm, transform=None): - from bokeh.models import ColumnDataSource, FactorRange - from bokeh.plotting import figure, show - from bokeh.transform import factor_cmap - pass - - - - -def plot_benchmarks_as_lines(title, *bm, transform=None, - line_title_transform=None, - logx=True, logy=True): - import bokeh - from bokeh.plotting import figure, output_file, show - from bokeh.palettes import Dark2_5 as palette - from bokeh.layouts import row, column - from bokeh.models import (Legend, LegendItem, CheckboxGroup, CustomJS, Div, - RadioGroup, Toggle, - ColumnDataSource, DataTable, TableColumn) - from bokeh.models.markers import marker_types - import itertools - # - ids = entry_ids(*bm, transform=transform) - colors = itertools.cycle(palette) - markers = itertools.cycle(marker_types) - p = figure(title=title, - x_axis_type="log" if logx else "linear", - y_axis_type="log" if logy else "linear", - #background_fill_color="#fafafa", - plot_width=1000, - x_axis_label="Number of pixels", - y_axis_label="Throughput (MB/s)", - ) - p.toolbar.autohide = True - #p.toolbar.active_inspect = [hover_tool, crosshair_tool] - p.toolbar.active_drag = "auto" - p.toolbar.active_scroll = "auto" - # - def dft(v): return v if v else (lambda n: n) - tr = dft(transform) - lttr = dft(line_title_transform) - # - for results in bm: - x = [ids[name] for name in results.names] - y = [bps/1e6 for bps in results.bytes_per_second] - c = next(colors) - marker = next(markers) - next(markers) # advance two - line_name = lttr(results.first) - #legends.append(LegendItem(name=c, label=line_name)) - p.scatter(x, y, marker=marker, size=8, color=c, legend_label=line_name) - p.line(x, y, - color=c, alpha=0.9, - #muted_color=c, muted_alpha=0.05, - legend_label=line_name) - p.legend.click_policy = "hide" - p.legend.label_text_font_size = "10px" - # - def input_title(title): - return Div(text=f"

    {title}

    ") - inputs = [] - first = bm[0].first.meta - for k, g in first.checkbox_groups().items(): - cb = CheckboxGroup(labels=[str(v) for v in g], - active=[i for i in range(len(g))], - inline=True) - inputs.append(input_title(k)) - inputs.append(cb) - # - # https://github.com/bokeh/bokeh/blob/branch-2.3/examples/app/export_csv/main.py - x_axis_values = [f"{m.num_pixels}px" for m in bm[0].meta] - table_sources = [] - for i, px in enumerate(x_axis_values): - c = ColumnDataSource(data={ - 'name': [nth(results.filtered_names, i) for results in bm], - 'bytes_per_second': [nth(results.bytes_per_second, i) for results in bm], - 'items_per_second': [nth(results.items_per_second, i) for results in bm], - 'cpu_time': [nth(results.real_time, i) for results in bm], - 'real_time': [nth(results.real_time, i) for results in bm], - 'iterations': [nth(results.iterations, i) for results in bm], - 'threads': [nth(results.threads, i) for results in bm], - }) - table_sources.append(c) - selected_x_index = 8 # FIXME (currently 2000 pixels) - table_source = copy.deepcopy(table_sources[selected_x_index]) - relvalues = Toggle(label="Table: Relative values") - px_title = input_title("Table: number of pixels") - px_radiogroup = RadioGroup(labels=x_axis_values, active=selected_x_index) - table_inputs = [relvalues, px_title, px_radiogroup] - # - table_cols = [ - TableColumn(field='name', title='Name'), - TableColumn(field='bytes_per_second', title='Bytes/second'), - TableColumn(field='items_per_second', title='Items/second'), - TableColumn(field='cpu_time', title='CPU time'), - TableColumn(field='real_time', title='Real time'), - TableColumn(field='iterations', title='Iterations'), - TableColumn(field='threads', title='Threads'), - ] - data_table = DataTable(source=table_source, columns=table_cols, width=1200) - callback = CustomJS(args=dict( - radiogroup=px_radiogroup, - source=table_source, - table=table_sources - ), code=""" - console.log(`active=${radiogroup.active}`); - /*source.data=table[radiogroup.active];*/ - var nrows = source.data['name'].length; - var ts = table[radiogroup.active].data; - var names = ["name", "bytes_per_second", "items_per_second", "cpu_time", "real_time", "iterations", "threads"]; - var ncols = names.length; - console.log(`names=${names} nrows=${nrows} ncols=${ncols}`); - for(var i = 0; i < nrows; i++) { - for(var j = 0; j < ncols; ++j) { - var name = names[j]; - /*console.log(`i=${i} j=${j} name=${name}`);*/ - source.data[name][i] = ts[name][i]; - } - } - source.change.emit(); - """) - px_radiogroup.js_on_change('active', callback) - # lambda attr, old, new: log(f"attr={attr} old={old} new={new} active={px_radiogroup.active}")) - # - layout = column( - row(column(*inputs), p), - row(column(*table_inputs), data_table)) - show(layout) - - -def chain(*iterables): - for it in iterables: - for elm in it: - yield elm - - -def entry_ids(*bm, transform=None): - ids = {} - curr = 0 - for results in bm: - log(os.path.basename(results.filename), "------------------------------") - for entry in results.entries: - log(entry.name) - if transform is not None: - ids[entry.name] = transform(entry) - else: - if ids.get(entry.name) is None: - ids[entry.name] = curr - curr += 1 - return ids - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -def add_results(args): - log("adding results:", args.results) - col = BenchmarkCollection(args.target) - col.add(args.results) - - -def add_meta(args): - log("adding bm run metadata to results dir:", args.results) - meta = ResultMeta(results_dir=args.results, - cmakecache=args.cmakecache, - build_type=args.build_type) - meta.save(args.results) - log("adding bm run metadata to results dir: success!") - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -import typing -import enum - - -class _enum(enum.Enum): - def __str__(self): - return str(self.name) - @classmethod - def err_unknown(cls, s): - raise Exception(f"unknown {__class__.__name__}: {s}") - - -class MatrixOrder(_enum): - row_major = "row_major" - col_major = "col_major" - @property - def short(self): - return "rm" if self is MatrixOrder.row_major else "cm" - @classmethod - def make(cls, s): - try: - return {"rm": cls.row_major, "cm": cls.col_major}[s] - except: - cls.err_unknown(s) - - -class MatrixLayout(_enum): - compact = "compact" - strided = "strided" - @classmethod - def make(cls, s): - try: - return cls[s] - except: - cls.err_unknown(s) - - -class DimensionBinding(_enum): - compile_time = "compile_time" - run_time = "run_time" - @property - def short(self): - return "ct" if self is DimensionBinding.compile_time else "rt" - @classmethod - def make(cls, s): - try: - return {"ct": cls.compile_time, "rt": cls.run_time}[s] - except: - cls.err_unknown(s) - - -class MultType(_enum): - naive = "naive" - avx2 = "avx2" - avx2_unroll2 = "avx2_unroll2" - avx2_unroll4 = "avx2_unroll4" - avx2_unroll8 = "avx2_unroll8" - @classmethod - def make(cls, s): - try: - s = s.replace("dotprod_", "").replace("_naive", "") - return cls[s] - except: - cls.err_unknown(s) - - -class MatrixMult(typing.NamedTuple): - title: str - num_pixels: int - num_channels: int - num_features: int - mult_type: MultType - layout: MatrixLayout - dim_binding: DimensionBinding - ret_order: MatrixOrder - lhs_order: MatrixOrder - rhs_order: MatrixOrder - - @classmethod - def make(cls, bm_title: str): - # eg: - # mult_naive_strided_ct_rm_cmcm<250, 8, 16> - # mult_naive_compact_rt_rm_rmrm/4000/8/16 - rxline = r'mult_(.*)[") - expect(v, 'title', 'naive_strided_ct_rm_cmcm') - expect(v, 'num_pixels', 250) - expect(v, 'num_channels', 8) - expect(v, 'num_features', 16) - expect(v, 'mult_type', MultType.naive) - expect(v, 'layout', MatrixLayout.strided) - expect(v, 'dim_binding', DimensionBinding.compile_time) - expect(v, 'ret_order', MatrixOrder.row_major) - expect(v, 'lhs_order', MatrixOrder.col_major) - expect(v, 'rhs_order', MatrixOrder.col_major) - v = MatrixMult.make("mult_dotprod_avx2_compact_rt_cm_rmcm/4000/16/8") - expect(v, 'title', 'dotprod_avx2_compact_rt_cm_rmcm') - expect(v, 'num_pixels', 4000) - expect(v, 'num_channels', 16) - expect(v, 'num_features', 8) - expect(v, 'mult_type', MultType.avx2) - expect(v, 'layout', MatrixLayout.compact) - expect(v, 'dim_binding', DimensionBinding.run_time) - expect(v, 'ret_order', MatrixOrder.col_major) - expect(v, 'lhs_order', MatrixOrder.row_major) - expect(v, 'rhs_order', MatrixOrder.col_major) - -_test() - - - -def formatMBps(value): - return value / 1e6 - - - -if __name__ == '__main__': - bms = sorted(sys.argv[2:]) - log(bms) - bms = BenchmarkPanel(bms, bm_meta_cls=MatrixMult.make) - fm = bms.runs[0].first.meta - title = f"Classifier multiplication, {fm.num_channels} channels, {fm.num_features} features: throughput (MB/s)" - bms.plot_all_lines(title) - exit() - main() diff --git a/thirdparty/ryml/ext/c4core/cmake/bm-xp/example_mintm.py b/thirdparty/ryml/ext/c4core/cmake/bm-xp/example_mintm.py deleted file mode 100644 index 3db4175cb..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/bm-xp/example_mintm.py +++ /dev/null @@ -1,1061 +0,0 @@ -import os -import sys -import argparse -import copy -import requests -import flask -import json -import re -import yaml -import shutil -import mmh3 -from itertools import islice - -from munch import Munch, munchify -from flask import render_template, redirect, url_for, send_from_directory -from markupsafe import escape - - -def log(*args, **kwargs): - print(*args, **kwargs, flush=True) - - -def myhash_combine(curr, value): - return curr ^ (value + 0x9e3779b9 + (curr<<6) + (curr>>2)) - - -def nth(iterable, n, default=None): - "Returns the nth item or a default value" - return next(islice(iterable, n, None), default) - - -def optionals(obj, *attrs): - ret = [] - for attr in attrs: - if not hasattr(obj, attr): - log("attr not present:", attr) - continue - ret.append(getattr(obj, attr)) - return ret - - -def myhash(*args): - h = 137597 - for a in args: - if isinstance(a, str): - if a == "": - continue - b = bytes(a, "utf8") - else: - b = bytes(a) - hb = mmh3.hash(b, signed=False) - h = myhash_combine(h, hb) - s = hex(h) - return s[2:min(10, len(s))] - - -def copy_file_to_dir(file, dir): - dir = os.path.abspath(dir) - src = os.path.abspath(file) - dst = f"{dir}/{os.path.basename(src)}" - if not os.path.exists(dir): - os.makedirs(dir) - if os.path.exists(dst): - os.remove(dst) - log("copy:", src, "-->", dst) - shutil.copy(src, dst) - return dst - - -def chk(f): - log(f"looking for file:", f) - assert os.path.exists(f), f - return f - - -def load_yml_file(filename): - if not os.path.exists(filename): - raise Exception(f"not found: {filename}") - with open(filename) as f: - return load_yml(f.read()) - - -def dump_yml(data, filename): - with open(filename, "w") as f: - yaml.safe_dump(data, f) - - -def load_yml(yml): - return munchify(yaml.safe_load(yml)) - - -def dump_json(data, filename): - with open(filename, "w") as f: - f.write(json.dumps(data, indent=2, sort_keys=True)) - - -def load_json(filename): - with open(filename, "r") as f: - try: - return munchify(json.load(f)) - except Exception as exc: - raise Exception(f"could not load file: {filename}: {exc}") - - -def main(): - def _common_args(parser): - parser.add_argument("-m", "--manifest", type=str, default="", help="enable debug mode") - parser.add_argument("--debug", action="store_true", help="enable debug mode") - # - parser = argparse.ArgumentParser(description="Browse benchmark results", prog="bm") - _common_args(parser) - subparsers = parser.add_subparsers() - # - sp = subparsers.add_parser("create", help="create benchmark collection") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("filename", type=str, help="the YAML file with the benchmark specs") - sp.add_argument("target", type=str, help="the directory to store the results") - # - sp = subparsers.add_parser("meta", help="get the required meta-information: cpu info, commit data") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("results", type=str, help="the directory with the results") - sp.add_argument("cmakecache", type=str, help="the path to the CMakeCache.txt file used to build the benchmark binaries") - sp.add_argument("build_type", type=str, help="the build type, eg Release Debug MinSizeRel RelWithDebInfo") - # - sp = subparsers.add_parser("add", help="add benchmark results") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("results", type=str, help="the directory with the results") - sp.add_argument("target", type=str, help="the directory to store the results") - # - sp = subparsers.add_parser("serve", help="serve benchmark results") - _common_args(sp) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - sp.add_argument("bmdir", type=os.path.abspath, default=os.getcwd(), help="the directory with the results. default=.") - sp.add_argument("-H", "--host", type=str, default="localhost", help="host. default=%(default)s") - sp.add_argument("-p", "--port", type=int, default=8000, help="port. default=%(default)s") - # - sp = subparsers.add_parser("export", help="export static html") - sp.set_defaults(func=freeze) - sp.add_argument("bmdir", type=os.path.abspath, default=os.getcwd(), help="the directory with the results. default=.") - _common_args(sp) - # - sp = subparsers.add_parser("deps", help="install server dependencies") - sp.set_defaults(func=lambda _: download_deps()) - sp.add_argument("--debug", action="store_true", help="enable debug mode") - _common_args(sp) - # - args = parser.parse_args(sys.argv[1:] if len(sys.argv) > 1 else ["serve"]) - if args.debug: - log(args) - args.func(args) - - -def get_manifest(args): - bmdir = os.path.abspath(args.bmdir) - if not args.manifest: - manifest_yml = os.path.join(bmdir, "manifest.yml") - else: - if not os.path.isabs(args.manifest): - manifest_yml = os.path.join(os.getcwd(), args.manifest) - manifest_json = os.path.join(os.path.dirname(manifest.yml), "manifest.json") - manifest = load_yml_file(manifest_yml) - dump_json(manifest, manifest_json) - return manifest - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -app = flask.Flask(__name__, - template_folder='template') - - -def _setup_app(args): - def _s(prop, val): - assert not hasattr(app, prop), prop - setattr(app, prop, val) - _s('args', args) - _s('manifest', get_manifest(args)) - if args.debug: - app.config["DEBUG"] = True - - -def freeze(args): - "https://pythonhosted.org/Frozen-Flask/" - from flask_frozen import Freezer - _setup_app(args) - freezer = Freezer(app) - freezer.freeze(debug=args.debug) - - -def serve(args): - _setup_app(args) - app.run(host=args.host, port=args.port, debug=args.debug) - - -@app.route("/") -def home(): - log("requested home") - return render_template("index.html") - - -@app.route("/") -def other_(path): - path = escape(path) - d = app.args.bmdir - log("requested other path:", path, "---", os.path.join(d, path)) - return send_from_directory(d, path) - - -@app.route("/static/") -def static_(path): - path = escape(path) - d = os.path.join(app.args.bmdir, "static") - log("requested static path:", path, "---", os.path.join(d, path)) - return send_from_directory(d, path, cache_timeout=1) # timeout in seconds - - -@app.route("/bm///") -def bm_(commit, run, resultjson): - commit = escape(commit) - run = escape(run) - resultjson = escape(resultjson) - d = os.path.join(app.args.bmdir, "runs", commit, run) - log("requested result:", os.path.join(d, resultjson)) - return send_from_directory(d, resultjson, cache_timeout=1) # timeout in seconds - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -def download_deps(): - deps = [ - "https://code.jquery.com/jquery-3.3.1.js", - "https://code.jquery.com/jquery-3.3.1.js", - "https://code.jquery.com/ui/1.12.1/jquery-ui.js", - "https://cdn.datatables.net/1.10.20/js/jquery.dataTables.js", - "https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js", - "https://cdn.datatables.net/1.10.20/css/jquery.dataTables.css", - "https://cdn.datatables.net/1.10.20/css/jquery.dataTables.min.css", - "https://www.chartjs.org/dist/2.9.1/Chart.min.js", - #("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/styles/github.css", "highlight.github.css"), - ("https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/styles/github.min.css", "highlight.github.min.css"), - #"https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/highlight.js", - "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/10.3.2/highlight.min.js", - ] - for src in deps: - if type(src) == str: - base = os.path.basename(src) - else: - src, base = src - dst = f"{os.getcwd()}/static/{base}" - download_url(src, dst) - - -def download_url(url, dst): - log("download url:", url, "--->", dst) - req = requests.get(url, stream=True) - if req.status_code == 200: - sz = 0 - with open(dst, 'wb') as f: - for chunk in req: - f.write(chunk) - sz += len(chunk) - log(f"........ finished: {sz}B") - else: - log(f" error:", req.status_code, url) - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -class BenchmarkCollection: - - @staticmethod - def create_new(args): - dir = args.target - filename = os.path.join(dir, "bm.yml") - manifest = os.path.join(dir, "manifest.yml") - if not os.path.exists(dir): - os.makedirs(dir) - shutil.copyfile(args.filename, filename) - dump_yml(load_yml("""{runs: {}, bm: {}}"""), manifest) - return __class__(dir) - - def __init__(self, dir): - if not os.path.exists(dir): - raise Exception(f"not found: {dir}") - self.dir = os.path.abspath(dir) - self.runs_dir = os.path.join(self.dir, "runs") - self.manifest = os.path.join(self.dir, "manifest.yml") - self.filename = os.path.join(self.dir, "bm.yml") - self.specs = munchify(load_yml_file(self.filename)) - self.manif = munchify(load_yml_file(self.manifest)) - - def add(self, results_dir): - results_dir = os.path.abspath(results_dir) - dst_dir, meta = self._read_run(results_dir) - self._add_run(results_dir, dst_dir, meta) - dump_yml(self.manif, self.manifest) - - def _read_run(self, results_dir): - log("adding run...") - id = f"{len(self.manif.runs.keys()):05d}" - log(f"adding run: id={id}") - meta = ResultMeta.load(results_dir) - dst_dir = os.path.join(self.runs_dir, meta.name) - return dst_dir, meta - - def _add_run(self, results_dir, dst_dir, meta): - cats = self._add_meta_categories(meta) - for filename in ("meta.yml", - "CMakeCCompiler.cmake", - "CMakeCXXCompiler.cmake", - "CMakeSystem.cmake", - "compile_commands.json"): - filename = os.path.join(results_dir, filename) - if os.path.exists(filename): - copy_file_to_dir(filename, dst_dir) - else: - if not filename.endswith("compile_commands.json"): - raise Exception(f"wtf???? {filename}") - for name, specs in self.specs.bm.items(): - if not hasattr(specs, 'variants'): - filename = chk(f"{results_dir}/{name}.json") - dst = copy_file_to_dir(filename, dst_dir) - self._add_bm_run(name, specs, meta) - else: - for t in specs.variants: - tname = f"{name}-{t}" - filename = chk(f"{results_dir}/{tname}.json") - dst = copy_file_to_dir(filename, dst_dir) - self._add_bm_run(tname, specs, meta) - - def _add_bm_run(self, name, specs, meta): - if name not in self.manif.bm.keys(): - self.manif.bm[name] = Munch(specs=specs, entries=[]) - entry = self.manif.bm[name] - entry.specs = specs - if meta.name not in entry.entries: - entry.entries.append(meta.name) - - def _add_meta_categories(self, meta): - run = Munch() - for catname in ('commit', 'cpu', 'system', 'build'): - meta_item = getattr(meta, catname) - self._add_item_to_category(meta.name, catname, meta_item) - run[catname] = meta_item.storage_id - # build specs are too verbose; remove them - self.manif.build[meta.build.storage_id].specs = Munch() - self.manif.runs[meta.name] = run - - def _add_item_to_category(self, run, category_name, item): - if not hasattr(self.manif, category_name): - setattr(self.manif, category_name, Munch()) - category = getattr(self.manif, category_name) - if item.storage_id not in category.keys(): - category[item.storage_id] = Munch(specs=item, entries=[]) - entry = category[item.storage_id] - entry.specs = item - if run not in entry.entries: - entry.entries.append(run) - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -class ResultMeta(Munch): - - def __init__(self, results_dir, cmakecache, build_type): - super().__init__(self) - self.date = __class__.get_date() - self.commit = __class__.get_commit(results_dir) - self.cpu = __class__.get_cpu_info() - self.system = __class__.get_sys_info() - self.build = __class__.get_build_info(cmakecache, build_type) - self.name = self._get_name() - - @staticmethod - def load(results_dir): - results_dir = os.path.join(os.path.abspath(results_dir), "meta.yml") - data = load_yml_file(results_dir) - return munchify(data) - - def save(self, results_dir): - out = os.path.join(results_dir, "meta.yml") - log("saving meta:", out) - dump_yml(self, out) - self.build.save(results_dir) - - @staticmethod - def get_date(): - import datetime - now = datetime.datetime.now() - return now.strftime("%Y%m%d-%H%M%S") - - def _get_name(self): - commit = self.commit.storage_name - cpu = self.cpu.storage_name - sys = self.system.storage_name - build = self.build.storage_name - name = f"{commit}/{cpu}-{sys}-{build}" - return name - - @staticmethod - def get_commit(results_dir): - import git - repo = git.Repo(results_dir, search_parent_directories=True) - commit = repo.head.commit - commit = {p: str(getattr(commit, p)) - for p in ('message', 'summary', 'name_rev', - 'author', - 'authored_datetime', - 'committer', - 'committed_datetime',)} - commit = Munch(commit) - commit.message = commit.message.strip() - commit.sha1 = commit.name_rev[:7] - spl = commit.authored_datetime.split(" ") - date = re.sub(r'-', '', spl[0]) - time = re.sub(r'(\d+):(\d+):(\d+).*', r'\1\2\3', spl[1]) - commit.storage_id = commit.sha1 - commit.storage_name = f"git{date}_{time}-{commit.sha1}" - return commit - - @staticmethod - def get_cpu_info(): - import cpuinfo - nfo = cpuinfo.get_cpu_info() - nfo = Munch(nfo) - for a in ('cpu_version', 'cpu_version_string', 'python_version'): - if hasattr(nfo, a): - delattr(nfo, a) - for a in ('arch_string_raw', 'brand_raw', 'hardware_raw', 'vendor_id_raw'): - if not hasattr(nfo, a): - setattr(nfo, a, '') - nfo.storage_id = myhash( - nfo.arch_string_raw, nfo.brand_raw, nfo.hardware_raw, nfo.vendor_id_raw, - nfo.arch, nfo.bits, nfo.count, nfo.family, nfo.model, nfo.stepping, - ",".join(nfo.flags), nfo.hz_advertised_friendly, - nfo.l2_cache_associativity, - nfo.l2_cache_line_size, - nfo.l2_cache_size, - nfo.l3_cache_size, - *optionals('l1_data_cache_size', 'l1_instruction_cache_size') - ) - nfo.storage_name = f"{nfo.arch.lower()}_{nfo.storage_id}" - return nfo - - @staticmethod - def get_sys_info(): - import platform - uname = platform.uname() - nfo = Munch( - sys_platform=sys.platform, - sys=platform.system(), - uname=Munch( - machine=uname.machine, - node=uname.node, - release=uname.release, - system=uname.system, - version=uname.version, - ) - ) - nfo.storage_id = myhash( - nfo.sys_platform, - nfo.uname.machine, - ) - nfo.storage_name = f"{nfo.sys_platform}_{nfo.storage_id}" - return nfo - - @staticmethod - def get_build_info(cmakecache_txt, buildtype): - nfo = CMakeCache(cmakecache_txt) - def _btflags(name): - return (getattr(nfo, name), getattr(nfo, f"{name}_{buildtype.upper()}")) - nfo.storage_id = myhash( - buildtype, - nfo.CMAKE_CXX_COMPILER_ID, - nfo.CMAKE_CXX_COMPILER_VERSION, - nfo.CMAKE_CXX_COMPILER_VERSION_INTERNAL, - nfo.CMAKE_CXX_COMPILER_ABI, - nfo.CMAKE_CXX_SIZEOF_DATA_PTR, - nfo.CMAKE_C_COMPILER_ID, - nfo.CMAKE_C_COMPILER_VERSION, - nfo.CMAKE_C_COMPILER_VERSION_INTERNAL, - nfo.CMAKE_C_COMPILER_ABI, - nfo.CMAKE_C_SIZEOF_DATA_PTR, - *_btflags("CMAKE_CXX_FLAGS"), - *_btflags("CMAKE_C_FLAGS"), - *_btflags("CMAKE_STATIC_LINKER_FLAGS"), - *_btflags("CMAKE_SHARED_LINKER_FLAGS"), - ) - # - ccname = nfo.CMAKE_CXX_COMPILER_ID.lower() - if ccname == "gnu": - ccname = "gcc" - ccname += nfo.CMAKE_CXX_COMPILER_VERSION.lower() - # - if nfo.CMAKE_C_SIZEOF_DATA_PTR == "4": - bits = "32bit" - elif nfo.CMAKE_C_SIZEOF_DATA_PTR == "8": - bits = "64bit" - else: - raise Exception("unknown architecture") - # - nfo.storage_name = f"{bits}_{buildtype}_{ccname}_{nfo.storage_id}" - return nfo - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -class CMakeCache(Munch): - - def __init__(self, cmakecache_txt): - import glob - for line in iter_cmake_lines(cmakecache_txt): - spl = line.split("=") - if len(spl) < 2: - continue - k, ty = spl[0].split(":") - v = "=".join(spl[1:]).strip() - setattr(self, k, v) - bdir = os.path.dirname(os.path.abspath(cmakecache_txt)) - self._c_compiler_file = sorted(glob.glob(f"{bdir}/CMakeFiles/*/CMakeCCompiler.cmake"))[-1] # get the last - self._cxx_compiler_file = sorted(glob.glob(f"{bdir}/CMakeFiles/*/CMakeCXXCompiler.cmake"))[-1] # get the last - self._system_file = sorted(glob.glob(f"{bdir}/CMakeFiles/*/CMakeSystem.cmake"))[-1] # get the last - self._load_cmake_file(self._c_compiler_file) - self._load_cmake_file(self._cxx_compiler_file) - ccomfile = f"{bdir}/compile_commands.json" - self._compile_commands_file = ccomfile if os.path.exists(ccomfile) else None - - def _load_cmake_file(self, filename): - for line in iter_cmake_lines(filename): - if not line.startswith("set("): - continue - k = re.sub(r"set\((.*)\ +(.*)\)", r"\1", line) - v = re.sub(r"set\((.*)\ +(.*)\)", r"\2", line) - v = v.strip('"').strip("'").strip() - setattr(self, k, v) - - def save(self, results_dir): - copy_file_to_dir(self._c_compiler_file, results_dir) - copy_file_to_dir(self._cxx_compiler_file, results_dir) - copy_file_to_dir(self._system_file, results_dir) - if self._compile_commands_file is not None: - copy_file_to_dir(self._compile_commands_file, results_dir) - - -def iter_cmake_lines(filename): - with open(filename) as f: - for line in f.readlines(): - line = line.strip() - if line.startswith("#") or line.startswith("//") or len(line) == 0: - continue - yield line - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -class BenchmarkRun(Munch): - "results of an individual run" - - def __init__(self, json_file, meta_class): - props = load_json(json_file) - setattr(self, "filename", json_file) - assert hasattr(props, "context") - assert hasattr(props, "benchmarks") - super().__init__(**props) - for e in self.benchmarks: - setattr(e, 'meta', meta_class(e.name)) - setattr(self, 'property_names', ( - 'meta', - 'shorttitle', - 'name', - 'run_name', - 'run_type', - 'repetitions', - 'repetition_index', - 'repetition_index', - 'threads', - 'iterations', - 'real_time', - 'cpu_time', - 'time_unit', - 'bytes_per_second', - 'items_per_second', - 'counters', - )) - - @property - def first(self): - return self.benchmarks[0] - - @property - def entries(self): - for entry in self.benchmarks: - yield entry - - @property - def meta(self): - for entry in self.benchmarks: - yield entry.meta - - @property - def filtered_names(self): - for entry in self.benchmarks: - yield entry.meta.shorttitle - - @property - def names(self): - for entry in self.benchmarks: - yield entry.name - - @property - def run_names(self): - for entry in self.benchmarks: - yield entry.run_name - - @property - def run_types(self): - for entry in self.benchmarks: - yield entry.run_type - - @property - def repetitions(self): - for entry in self.benchmarks: - yield entry.repetitions - - @property - def repetition_indices(self): - for entry in self.benchmarks: - yield entry.repetition_index - - @property - def threads(self): - for entry in self.benchmarks: - yield entry.threads - - @property - def iterations(self): - for entry in self.benchmarks: - yield entry.iterations - - @property - def real_time(self): - for entry in self.benchmarks: - yield entry.real_time - - @property - def cpu_time(self): - for entry in self.benchmarks: - yield entry.cpu_time - - @property - def time_unit(self): - for entry in self.benchmarks: - yield entry.time_unit - - @property - def bytes_per_second(self): - for entry in self.benchmarks: - yield entry.bytes_per_second - - @property - def items_per_second(self): - for entry in self.benchmarks: - yield entry.items_per_second - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -class BenchmarkPanel: - - def __init__(self, runs, bm_meta_cls=None): - self.runs = [BenchmarkRun(a, bm_meta_cls) for a in runs] - - def plot_bars(self, title): - plot_benchmarks_as_lines(title, *self.runs) - - - def plot_all_lines(self, title): - plot_benchmarks_as_lines(title, *self.runs, - transform=lambda r: r.meta.num_pixels, - line_title_transform=lambda r: r.meta.shortname) - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - - -def plot_benchmarks_as_bars(title, *bm, transform=None): - from bokeh.models import ColumnDataSource, FactorRange - from bokeh.plotting import figure, show - from bokeh.transform import factor_cmap - pass - - - - -def plot_benchmarks_as_lines(title, *bm, transform=None, - line_title_transform=None, - logx=True, logy=True): - import bokeh - from bokeh.plotting import figure, output_file, show - from bokeh.palettes import Dark2_5 as palette - from bokeh.layouts import row, column - from bokeh.models import (Legend, LegendItem, CheckboxGroup, CustomJS, Div, - RadioGroup, Toggle, - ColumnDataSource, DataTable, TableColumn) - from bokeh.models.markers import marker_types - import itertools - # - ids = entry_ids(*bm, transform=transform) - colors = itertools.cycle(palette) - markers = itertools.cycle(marker_types) - p = figure(title=title, - x_axis_type="log" if logx else "linear", - y_axis_type="log" if logy else "linear", - #background_fill_color="#fafafa", - plot_width=1000, - x_axis_label="Number of pixels", - y_axis_label="Throughput (MB/s)", - ) - p.toolbar.autohide = True - #p.toolbar.active_inspect = [hover_tool, crosshair_tool] - p.toolbar.active_drag = "auto" - p.toolbar.active_scroll = "auto" - # - def dft(v): return v if v else (lambda n: n) - tr = dft(transform) - lttr = dft(line_title_transform) - # - for results in bm: - x = [ids[name] for name in results.names] - y = [bps/1e6 for bps in results.bytes_per_second] - c = next(colors) - marker = next(markers) - next(markers) # advance two - line_name = lttr(results.first) - #legends.append(LegendItem(name=c, label=line_name)) - p.scatter(x, y, marker=marker, size=8, color=c, legend_label=line_name) - p.line(x, y, - color=c, alpha=0.9, - #muted_color=c, muted_alpha=0.05, - legend_label=line_name) - p.legend.click_policy = "hide" - p.legend.label_text_font_size = "10px" - # - def input_title(title): - return Div(text=f"

    {title}

    ") - inputs = [] - first = bm[0].first.meta - for k, g in first.checkbox_groups().items(): - cb = CheckboxGroup(labels=[str(v) for v in g], - active=[i for i in range(len(g))], - inline=True) - inputs.append(input_title(k)) - inputs.append(cb) - # - # https://github.com/bokeh/bokeh/blob/branch-2.3/examples/app/export_csv/main.py - x_axis_values = [f"{m.num_pixels}px" for m in bm[0].meta] - table_sources = [] - for i, px in enumerate(x_axis_values): - c = ColumnDataSource(data={ - 'name': [nth(results.filtered_names, i) for results in bm], - 'bytes_per_second': [nth(results.bytes_per_second, i) for results in bm], - 'items_per_second': [nth(results.items_per_second, i) for results in bm], - 'cpu_time': [nth(results.real_time, i) for results in bm], - 'real_time': [nth(results.real_time, i) for results in bm], - 'iterations': [nth(results.iterations, i) for results in bm], - 'threads': [nth(results.threads, i) for results in bm], - }) - table_sources.append(c) - selected_x_index = 8 # FIXME (currently 2000 pixels) - table_source = copy.deepcopy(table_sources[selected_x_index]) - relvalues = Toggle(label="Table: Relative values") - px_title = input_title("Table: number of pixels") - px_radiogroup = RadioGroup(labels=x_axis_values, active=selected_x_index) - table_inputs = [relvalues, px_title, px_radiogroup] - # - table_cols = [ - TableColumn(field='name', title='Name'), - TableColumn(field='bytes_per_second', title='Bytes/second'), - TableColumn(field='items_per_second', title='Items/second'), - TableColumn(field='cpu_time', title='CPU time'), - TableColumn(field='real_time', title='Real time'), - TableColumn(field='iterations', title='Iterations'), - TableColumn(field='threads', title='Threads'), - ] - data_table = DataTable(source=table_source, columns=table_cols, width=1200) - callback = CustomJS(args=dict( - radiogroup=px_radiogroup, - source=table_source, - table=table_sources - ), code=""" - console.log(`active=${radiogroup.active}`); - /*source.data=table[radiogroup.active];*/ - var nrows = source.data['name'].length; - var ts = table[radiogroup.active].data; - var names = ["name", "bytes_per_second", "items_per_second", "cpu_time", "real_time", "iterations", "threads"]; - var ncols = names.length; - console.log(`names=${names} nrows=${nrows} ncols=${ncols}`); - for(var i = 0; i < nrows; i++) { - for(var j = 0; j < ncols; ++j) { - var name = names[j]; - /*console.log(`i=${i} j=${j} name=${name}`);*/ - source.data[name][i] = ts[name][i]; - } - } - source.change.emit(); - """) - px_radiogroup.js_on_change('active', callback) - # lambda attr, old, new: log(f"attr={attr} old={old} new={new} active={px_radiogroup.active}")) - # - layout = column( - row(column(*inputs), p), - row(column(*table_inputs), data_table)) - show(layout) - - -def chain(*iterables): - for it in iterables: - for elm in it: - yield elm - - -def entry_ids(*bm, transform=None): - ids = {} - curr = 0 - for results in bm: - log(os.path.basename(results.filename), "------------------------------") - for entry in results.entries: - log(entry.name) - if transform is not None: - ids[entry.name] = transform(entry) - else: - if ids.get(entry.name) is None: - ids[entry.name] = curr - curr += 1 - return ids - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -def add_results(args): - log("adding results:", args.results) - col = BenchmarkCollection(args.target) - col.add(args.results) - - -def add_meta(args): - log("adding bm run metadata to results dir:", args.results) - meta = ResultMeta(results_dir=args.results, - cmakecache=args.cmakecache, - build_type=args.build_type) - meta.save(args.results) - log("adding bm run metadata to results dir: success!") - - -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ -# ------------------------------------------------------------------------------ - -import typing -import enum - - -class _enum(enum.Enum): - def __str__(self): - return str(self.name) - @classmethod - def err_unknown(cls, s): - raise Exception(f"unknown {__class__.__name__}: {s}") - - -class MatrixOrder(_enum): - row_major = "row_major" - col_major = "col_major" - @property - def short(self): - return "rm" if self is MatrixOrder.row_major else "cm" - @classmethod - def make(cls, s): - try: - return {"rm": cls.row_major, "cm": cls.col_major}[s] - except: - cls.err_unknown(s) - - -class MatrixLayout(_enum): - compact = "compact" - strided = "strided" - @classmethod - def make(cls, s): - try: - return cls[s] - except: - cls.err_unknown(s) - - -class DimensionBinding(_enum): - compile_time = "compile_time" - run_time = "run_time" - @property - def short(self): - return "ct" if self is DimensionBinding.compile_time else "rt" - @classmethod - def make(cls, s): - try: - return {"ct": cls.compile_time, "rt": cls.run_time}[s] - except: - cls.err_unknown(s) - - -class MultType(_enum): - naive = "naive" - avx2 = "avx2" - avx2_unroll2 = "avx2_unroll2" - avx2_unroll4 = "avx2_unroll4" - avx2_unroll8 = "avx2_unroll8" - @classmethod - def make(cls, s): - try: - s = s.replace("dotprod_", "").replace("_naive", "") - return cls[s] - except: - cls.err_unknown(s) - - -class MatrixMult(typing.NamedTuple): - title: str - num_pixels: int - num_channels: int - num_features: int - mult_type: MultType - layout: MatrixLayout - dim_binding: DimensionBinding - ret_order: MatrixOrder - lhs_order: MatrixOrder - rhs_order: MatrixOrder - - @classmethod - def make(cls, bm_title: str): - # eg: - # mult_naive_strided_ct_rm_cmcm<250, 8, 16> - # mult_naive_compact_rt_rm_rmrm/4000/8/16 - rxline = r'mult_(.*)[") - expect(v, 'title', 'naive_strided_ct_rm_cmcm') - expect(v, 'num_pixels', 250) - expect(v, 'num_channels', 8) - expect(v, 'num_features', 16) - expect(v, 'mult_type', MultType.naive) - expect(v, 'layout', MatrixLayout.strided) - expect(v, 'dim_binding', DimensionBinding.compile_time) - expect(v, 'ret_order', MatrixOrder.row_major) - expect(v, 'lhs_order', MatrixOrder.col_major) - expect(v, 'rhs_order', MatrixOrder.col_major) - v = MatrixMult.make("mult_dotprod_avx2_compact_rt_cm_rmcm/4000/16/8") - expect(v, 'title', 'dotprod_avx2_compact_rt_cm_rmcm') - expect(v, 'num_pixels', 4000) - expect(v, 'num_channels', 16) - expect(v, 'num_features', 8) - expect(v, 'mult_type', MultType.avx2) - expect(v, 'layout', MatrixLayout.compact) - expect(v, 'dim_binding', DimensionBinding.run_time) - expect(v, 'ret_order', MatrixOrder.col_major) - expect(v, 'lhs_order', MatrixOrder.row_major) - expect(v, 'rhs_order', MatrixOrder.col_major) - -_test() - - - -def formatMBps(value): - return value / 1e6 - - - -if __name__ == '__main__': - bms = sorted(sys.argv[2:]) - log(bms) - bms = BenchmarkPanel(bms, bm_meta_cls=MatrixMult.make) - fm = bms.runs[0].first.meta - title = f"Classifier multiplication, {fm.num_channels} channels, {fm.num_features} features: throughput (MB/s)" - bms.plot_all_lines(title) - exit() - main() diff --git a/thirdparty/ryml/ext/c4core/cmake/bm-xp/requirements.txt b/thirdparty/ryml/ext/c4core/cmake/bm-xp/requirements.txt deleted file mode 100644 index 5a1b395ef..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/bm-xp/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -munch -pyyaml -py-cpuinfo -psutil -gitpython -flask -markupsafe -Frozen-Flask -requests -mmh3 -bokeh<3.0 -selenium -matplotlib -prettytable diff --git a/thirdparty/ryml/ext/c4core/cmake/bm-xp/template/index.html b/thirdparty/ryml/ext/c4core/cmake/bm-xp/template/index.html deleted file mode 100644 index fb52b8ba0..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/bm-xp/template/index.html +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - -

    Title

    -
    - Available benchmarks: -
      -
      -
      -
      - - - - - - - - - - - diff --git a/thirdparty/ryml/ext/c4core/cmake/c4CatSources.cmake b/thirdparty/ryml/ext/c4core/cmake/c4CatSources.cmake deleted file mode 100644 index 1633989af..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/c4CatSources.cmake +++ /dev/null @@ -1,105 +0,0 @@ -if(NOT _c4CatSourcesIncluded) -set(_c4CatSourcesIncluded ON) - - -#------------------------------------------------------------------------------ -# concatenate the source files to an output file, adding preprocessor adjustment -# for correct file/line reporting -function(c4_cat_sources files output umbrella_target) - _c4_cat_sources_create_cat(cat) - c4_to_full_path("${files}" full_files) # we must work with full paths - c4_separate_list("${full_files}" sepfiles) # and use a string instead of a list - c4_dbg("${_c4_prefix}: catting sources to ${output}") - if(NOT EXISTS "${output}") - # the cat command is executed at build time, but we need the output - # file to exist to be able to create the target. so to bootstrap, just - # run the command now - c4_dbg("${_c4_prefix}: creating ${output} for the first time") - execute_process( - COMMAND ${cat} "${sepfiles}" "${output}" - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" - ) - else() - c4_dbg("output exists: ${output}") - endif() - # add a custom command invoking our cat script for the input files - add_custom_command(OUTPUT ${output} - COMMAND ${cat} "${sepfiles}" "${output}" - DEPENDS ${files} - WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" - COMMENT "concatenating sources to ${output}") - if(NOT TARGET ${umbrella_target}) - add_custom_target(${umbrella_target} DEPENDS ${output} ${files}) - endif() -endfunction(c4_cat_sources) - - -#------------------------------------------------------------------------------ -# get a cat script -function(_c4_cat_sources_create_cat catfile) - # create a script to concatenate the sources - if(WIN32) - set(cat ${CMAKE_BINARY_DIR}/_c4catfiles.bat) - set(cattmp ${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/_c4catfiles.bat) - else() - set(cat ${CMAKE_BINARY_DIR}/_c4catfiles.sh) - set(cattmp ${CMAKE_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/_c4catfiles.sh) - endif() - set(${catfile} ${cat} PARENT_SCOPE) - if(NOT EXISTS ${cat}) - if(WIN32) - file(WRITE ${cattmp} " -setlocal EnableDelayedExpansion -set \"src_files=%1\" -set \"out_file=%2\" -echo.>\"out_file%\" -for %%f in (%src_files%) do ( - echo.>>\"%out_file%\" - echo.>>\"%out_file%\" - echo \"/*BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB*/\".>>\"%out_file%\" - echo \"/*BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB*/\".>>\"%out_file%\" - echo \"/*BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB*/\".>>\"%out_file%\" - echo \"#line 1 \\\"%%f\\\" // reset __LINE__ and __FILE__ to the correct value\".>>\"%out_file%\" - type %%f>>\"%out_file%\" -) -") - else() - file(WRITE ${cattmp} "#!/bin/sh - -src_files=$1 -out_file=$2 -#echo \"src_files $src_files\" -#echo \"out_file $out_file\" - -cat > $out_file << EOF -// DO NOT EDIT. -// this is an auto-generated file, and will be overwritten -EOF -for f in $src_files ; do - cat >> $out_file <> $out_file -done - -echo \"Wrote output to $out_file\" -") - endif() - # add execute permissions - get_filename_component(catdir ${cat} DIRECTORY) - file(COPY ${cattmp} DESTINATION ${catdir} - FILE_PERMISSIONS - OWNER_READ OWNER_WRITE OWNER_EXECUTE - GROUP_READ GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE - ) - endif() -endfunction() - - -endif(NOT _c4CatSourcesIncluded) diff --git a/thirdparty/ryml/ext/c4core/cmake/c4Doxygen.cmake b/thirdparty/ryml/ext/c4core/cmake/c4Doxygen.cmake deleted file mode 100644 index b5e018872..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/c4Doxygen.cmake +++ /dev/null @@ -1,121 +0,0 @@ -# (C) 2019 Joao Paulo Magalhaes -if(NOT _c4_doxygen_included) -set(_c4_doxygen_included ON) - - -#------------------------------------------------------------------------------ -# TODO use customizations from https://cmake.org/cmake/help/v3.9/module/FindDoxygen.html -function(c4_setup_doxygen umbrella_option) - cmake_dependent_option(${_c4_uprefix}BUILD_DOCS "Enable targets to build documentation for ${prefix}" ON "${umbrella_option}" OFF) - if(${_c4_uprefix}BUILD_DOCS) - find_package(Doxygen QUIET) - if(DOXYGEN_FOUND) - c4_log("enabling documentation targets") - else() - c4_dbg("doxygen not found") - endif() - endif() -endfunction() - -#------------------------------------------------------------------------------ -function(c4_add_doxygen doc_name) - if(NOT ${_c4_uprefix}BUILD_DOCS) - return() - endif() - # - set(opt0 - ) - set(opt1 - DOXYFILE DOXYFILE_IN - PROJ - PROJ_BRIEF - VERSION - OUTPUT_DIR - CLANG_DATABASE_PATH - ) - set(optN - INPUT - FILE_PATTERNS - EXCLUDE - EXCLUDE_PATTERNS - EXCLUDE_SYMBOLS - STRIP_FROM_PATH - STRIP_FROM_INC_PATH - EXAMPLE_PATH - ) - cmake_parse_arguments("" "${opt0}" "${opt1}" "${optN}" ${ARGN}) - # - if(NOT _PROJ) - set(_PROJ ${_c4_ucprefix}) - endif() - if(NOT _DOXYFILE AND NOT _DOXYFILE_IN) - set(_DOXYFILE_IN ${CMAKE_CURRENT_LIST_DIR}/Doxyfile.in) - endif() - if(NOT _OUTPUT_DIR) - if("${doc_name}" MATCHES "^[Dd]oc") - set(_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/${doc_name}) - else() - set(_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/doc/${doc_name}) - endif() - endif() - # - _c4_doxy_fwd_to_cmd(_PROJ OFF) - _c4_doxy_fwd_to_cmd(_PROJ_BRIEF OFF) - _c4_doxy_fwd_to_cmd(_VERSION OFF) - _c4_doxy_fwd_to_cmd(_OUTPUT_DIR OFF) - _c4_doxy_fwd_to_cmd(_CLANG_DATABASE_PATH OFF) - _c4_doxy_fwd_to_cmd(_INPUT ON) - _c4_doxy_fwd_to_cmd(_FILE_PATTERNS ON) - _c4_doxy_fwd_to_cmd(_EXCLUDE ON) - _c4_doxy_fwd_to_cmd(_EXCLUDE_PATTERNS ON) - _c4_doxy_fwd_to_cmd(_EXCLUDE_SYMBOLS ON) - _c4_doxy_fwd_to_cmd(_STRIP_FROM_PATH ON) - _c4_doxy_fwd_to_cmd(_STRIP_FROM_INC_PATH ON) - _c4_doxy_fwd_to_cmd(_EXAMPLE_PATH ON) - # - if("${doc_name}" MATCHES "^[Dd]oc") - set(tgt ${_c4_lcprefix}-${doc_name}) - else() - set(tgt ${_c4_lcprefix}-doc-${doc_name}) - endif() - # - if(_DOXYFILE) - set(doxyfile_out ${_DOXYFILE}) - elseif(_DOXYFILE_IN) - set(doxyfile_out ${_OUTPUT_DIR}/Doxyfile) - set(config_script ${_c4_project_dir}/c4DoxygenConfig.cmake) - add_custom_command(OUTPUT ${doxyfile_out} - COMMAND ${CMAKE_COMMAND} -E remove -f ${doxyfile_out} - COMMAND ${CMAKE_COMMAND} -DDOXYFILE_IN=${_DOXYFILE_IN} -DDOXYFILE_OUT=${doxyfile_out} ${defs} '-DALLVARS=${allvars}' '-DLISTVARS=${listvars}' -P ${config_script} - DEPENDS ${_DOXYFILE_IN} ${config_script} - COMMENT "${tgt}: generating ${doxyfile_out}" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - endif() - # - add_custom_target(${tgt} - COMMAND ${DOXYGEN_EXECUTABLE} ${doxyfile_out} - DEPENDS ${doxyfile_out} - WORKING_DIRECTORY ${_OUTPUT_DIR} - COMMENT "${tgt}: docs will be placed in ${_OUTPUT_DIR}" - VERBATIM) - _c4_set_target_folder(${tgt} doc) -endfunction() - - -macro(_c4_doxy_fwd_to_cmd varname is_list) - if(NOT ("${${varname}}" STREQUAL "")) - if("${defs}" STREQUAL "") - set(li "-D${varname}=${${varname}}") - else() - set(li ${defs}) - list(APPEND li "-D${varname}='${${varname}}'") - endif() - set(defs ${li}) - endif() - set(allvars "${allvars};${varname}") - if(${is_list}) - set(listvars "${listvars};${varname}") - endif() -endmacro() - -endif(NOT _c4_doxygen_included) diff --git a/thirdparty/ryml/ext/c4core/cmake/c4DoxygenConfig.cmake b/thirdparty/ryml/ext/c4core/cmake/c4DoxygenConfig.cmake deleted file mode 100644 index b472cab88..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/c4DoxygenConfig.cmake +++ /dev/null @@ -1,24 +0,0 @@ -function(_c4_doxy_list_to_str var) - set(il) - foreach(i ${${var}}) - if("${il}" STREQUAL "") - set(il "${i}") - else() - set(il "${il} ${i}") - endif() - endforeach() - set(${var} "${il}" PARENT_SCOPE) -endfunction() - -string(REPLACE " " ";" ALLVARS ${ALLVARS}) -string(REPLACE " " ";" LISTVARS ${LISTVARS}) - -foreach(var ${LISTVARS}) - _c4_doxy_list_to_str(${var}) -endforeach() - -foreach(var ${ALLVARS}) - message(STATUS "${var}='${${var}}'") -endforeach() - -configure_file(${DOXYFILE_IN} ${DOXYFILE_OUT} @ONLY) diff --git a/thirdparty/ryml/ext/c4core/cmake/c4GetTargetPropertyRecursive.cmake b/thirdparty/ryml/ext/c4core/cmake/c4GetTargetPropertyRecursive.cmake deleted file mode 100644 index 45a74aa9e..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/c4GetTargetPropertyRecursive.cmake +++ /dev/null @@ -1,186 +0,0 @@ -if(NOT _c4_GTPR_included) -set(_c4_GTPR_included ON) - -function(c4_get_target_property_recursive outputvar target property) - # - # helps for debugging - if(_stack) - set(_stack "${_stack}/${target}") - else() - set(_stack "${property}:${target}") - endif() - # - # what type of target is this? - get_target_property(_rec_target_type ${target} TYPE) - c4_dbg("${_stack} [type=${_rec_target_type}]: get property ${property}") - # - # adjust the property names for interface targets - set(_ept_prop_ll LINK_LIBRARIES) - if(_rec_target_type STREQUAL "INTERFACE_LIBRARY") - set(_ept_prop_ll INTERFACE_LINK_LIBRARIES) - if(property STREQUAL "INCLUDE_DIRECTORIES") - c4_dbg("${_stack} [type=${_rec_target_type}]: property ${property} ---> INTERFACE_INCLUDE_DIRECTORIES") - set(property INTERFACE_INCLUDE_DIRECTORIES) - elseif(property STREQUAL "LINK_LIBRARIES") - c4_dbg("${_stack} [type=${_rec_target_type}]: property ${property} ---> INTERFACE_LINK_LIBRARIES") - set(property INTERFACE_LINK_LIBRARIES) - endif() - endif() - # - get_target_property(_ept_li ${target} ${property}) - c4_dbg("${_stack} [type=${_rec_target_type}]: property ${property}=${_ept_li}") - if(NOT _ept_li) # the property may not be set (ie foo-NOTFOUND) - set(_ept_li) # so clear it in that case - endif() - # - # now descend and append the property for each of the linked libraries - get_target_property(_ept_deps ${target} ${_ept_prop_ll}) - if(_ept_deps) - foreach(_ept_ll ${_ept_deps}) - if(TARGET ${_ept_ll}) - c4_get_target_property_recursive(_ept_out ${_ept_ll} ${property}) - list(APPEND _ept_li ${_ept_out}) - endif() - endforeach() - endif() - # - foreach(le_ ${_ept_li}) - string(STRIP "${le_}" le) - if(NOT le) - elseif("${le}" STREQUAL "") - else() - list(APPEND _ept_li_f ${le}) - endif() - endforeach() - c4_dbg("${_stack} [type=${_rec_target_type}]: final=${_ept_li_f}") - set(${outputvar} ${_ept_li_f} PARENT_SCOPE) -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - - -function(c4_set_transitive_property target prop_name prop_value) - set_target_properties(${target} PROPERTIES "${prop_name}" "${prop_value}") -endfunction() - - -function(c4_append_transitive_property target prop_name prop_value) - get_target_property(curr_value ${target} "${prop_name}") - if(curr_value) - list(APPEND curr_value "${prop_value}") - else() - set(curr_value "${prop_value}") - endif() - c4_set_transitive_property(${target} "${prop_name}" "${curr_value}") -endfunction() - - -# TODO: maybe we can use c4_get_target_property_recursive()? -function(c4_get_transitive_property target prop_name out) - if(NOT TARGET ${target}) - return() - endif() - # these will be the names of the variables we'll use to cache the result - set(_trval _C4_TRANSITIVE_${prop_name}) - set(_trmark _C4_TRANSITIVE_${prop_name}_DONE) - # - get_target_property(cached ${target} ${_trmark}) # is it cached already - if(cached) - get_target_property(p ${target} _C4_TRANSITIVE_${prop_name}) - set(${out} ${p} PARENT_SCOPE) - #c4_dbg("${target}: c4_get_transitive_property ${target} ${prop_name}: cached='${p}'") - else() - #c4_dbg("${target}: gathering transitive property: ${prop_name}...") - set(interleaved) - get_target_property(lv ${target} ${prop_name}) - if(lv) - list(APPEND interleaved ${lv}) - endif() - c4_get_transitive_libraries(${target} LINK_LIBRARIES libs) - c4_get_transitive_libraries(${target} INTERFACE_LINK_LIBRARIES ilibs) - list(APPEND libs ${ilibs}) - foreach(lib ${libs}) - #c4_dbg("${target}: considering ${lib}...") - if(NOT lib) - #c4_dbg("${target}: considering ${lib}: not found, skipping...") - continue() - endif() - if(NOT TARGET ${lib}) - #c4_dbg("${target}: considering ${lib}: not a target, skipping...") - continue() - endif() - get_target_property(lv ${lib} ${prop_name}) - if(lv) - list(APPEND interleaved ${lv}) - endif() - c4_get_transitive_property(${lib} ${prop_name} v) - if(v) - list(APPEND interleaved ${v}) - endif() - #c4_dbg("${target}: considering ${lib}---${interleaved}") - endforeach() - #c4_dbg("${target}: gathering transitive property: ${prop_name}: ${interleaved}") - set(${out} ${interleaved} PARENT_SCOPE) - get_target_property(aliased_target ${target} ALIASED_TARGET) - if(NOT aliased_target) - set_target_properties(${target} PROPERTIES - ${_trmark} ON - ${_trval} "${interleaved}") - endif() - endif() -endfunction() - - -function(c4_get_transitive_libraries target prop_name out) - if(NOT TARGET ${target}) - return() - endif() - # these will be the names of the variables we'll use to cache the result - set(_trval _C4_TRANSITIVE_${prop_name}) - set(_trmark _C4_TRANSITIVE_${prop_name}_DONE) - # - get_target_property(cached ${target} ${_trmark}) - if(cached) - get_target_property(p ${target} ${_trval}) - set(${out} ${p} PARENT_SCOPE) - #c4_dbg("${target}: c4_get_transitive_libraries ${target} ${prop_name}: cached='${p}'") - else() - #c4_dbg("${target}: gathering transitive libraries: ${prop_name}...") - get_target_property(target_type ${target} TYPE) - set(interleaved) - if(NOT ("${target_type}" STREQUAL "INTERFACE_LIBRARY") - AND ("${prop_name}" STREQUAL LINK_LIBRARIES)) - get_target_property(l ${target} ${prop_name}) - foreach(ll ${l}) - #c4_dbg("${target}: considering ${ll}...") - if(NOT ll) - #c4_dbg("${target}: considering ${ll}: not found, skipping...") - continue() - endif() - if(NOT ll) - #c4_dbg("${target}: considering ${ll}: not a target, skipping...") - continue() - endif() - list(APPEND interleaved ${ll}) - c4_get_transitive_libraries(${ll} ${prop_name} v) - if(v) - list(APPEND interleaved ${v}) - endif() - #c4_dbg("${target}: considering ${ll}---${interleaved}") - endforeach() - endif() - #c4_dbg("${target}: gathering transitive libraries: ${prop_name}: result='${interleaved}'") - set(${out} ${interleaved} PARENT_SCOPE) - get_target_property(aliased_target ${target} ALIASED_TARGET) - if(NOT aliased_target) - set_target_properties(${target} PROPERTIES - ${_trmark} ON - ${_trval} "${interleaved}") - endif() - endif() -endfunction() - -endif(NOT _c4_GTPR_included) diff --git a/thirdparty/ryml/ext/c4core/cmake/c4Project.cmake b/thirdparty/ryml/ext/c4core/cmake/c4Project.cmake deleted file mode 100644 index 60c8717fe..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/c4Project.cmake +++ /dev/null @@ -1,3691 +0,0 @@ -if(NOT _c4_project_included) -set(_c4_project_included ON) -set(_c4_project_file ${CMAKE_CURRENT_LIST_FILE}) -set(_c4_project_dir ${CMAKE_CURRENT_LIST_DIR}) - - -# "I didn't have time to write a short letter, so I wrote a long one -# instead." -- Mark Twain -# -# ... Eg, hopefully this code will be cleaned up. There's a lot of -# code here that can be streamlined into a more intuitive arrangement. - - -cmake_minimum_required(VERSION 3.12 FATAL_ERROR) - -list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) -set_property(GLOBAL PROPERTY USE_FOLDERS ON) - -include(ConfigurationTypes) -include(CreateSourceGroup) -include(c4SanitizeTarget) -include(c4StaticAnalysis) -include(PrintVar) -include(c4CatSources) -include(c4Doxygen) -include(PatchUtils) - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -# define c4 project settings - -set(C4_EXTERN_DIR "$ENV{C4_EXTERN_DIR}" CACHE PATH "the directory where imported projects should be looked for (or cloned in when not found)") -set(C4_DBG_ENABLED OFF CACHE BOOL "enable detailed cmake logs in c4Project code") -set(C4_LIBRARY_TYPE "" CACHE STRING "default library type: either \"\"(defer to BUILD_SHARED_LIBS),INTERFACE,STATIC,SHARED,MODULE") -set(C4_SOURCE_TRANSFORM NONE CACHE STRING "global source transform method") -set(C4_HDR_EXTS "h;hpp;hh;h++;hxx" CACHE STRING "list of header extensions for determining which files are headers") -set(C4_SRC_EXTS "c;cpp;cc;c++;cxx;cu;" CACHE STRING "list of compilation unit extensions for determining which files are sources") -set(C4_ADD_EXTS "natvis" CACHE STRING "list of additional file extensions that might be added as sources to targets") -set(C4_GEN_SRC_EXT "cpp" CACHE STRING "the extension of the output source files resulting from concatenation") -set(C4_GEN_HDR_EXT "hpp" CACHE STRING "the extension of the output header files resulting from concatenation") -set(C4_CXX_STANDARDS "20;17;14;11" CACHE STRING "list of CXX standards") -set(C4_CXX_STANDARD_DEFAULT "11" CACHE STRING "the default CXX standard for projects not specifying one") - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -macro(c4_log) - message(STATUS "${_c4_prefix}: ${ARGN}") -endmacro() - - -macro(c4_err) - message(FATAL_ERROR "${_c4_prefix}: ${ARGN}") -endmacro() - - -macro(c4_dbg) - if(C4_DBG_ENABLED) - message(STATUS "${_c4_prefix}: ${ARGN}") - endif() -endmacro() - - -macro(c4_log_var varname) - c4_log("${varname}=${${varname}} ${ARGN}") -endmacro() -macro(c4_log_vars) - set(____s____) - foreach(varname ${ARGN}) - set(____s____ "${____s____}${varname}=${${varname}} ") - endforeach() - c4_log("${____s____}") -endmacro() -macro(c4_dbg_var varname) - c4_dbg("${varname}=${${varname}} ${ARGN}") -endmacro() -macro(c4_log_var_if varname) - if(${varname}) - c4_log("${varname}=${${varname}} ${ARGN}") - endif() -endmacro() -macro(c4_dbg_var_if varname) - if(${varname}) - c4_dbg("${varname}=${${varname}} ${ARGN}") - endif() -endmacro() - - -macro(_c4_show_pfx_vars) - if(NOT ("${ARGN}" STREQUAL "")) - c4_log("prefix vars: ${ARGN}") - endif() - print_var(_c4_prefix) - print_var(_c4_ocprefix) - print_var(_c4_ucprefix) - print_var(_c4_lcprefix) - print_var(_c4_oprefix) - print_var(_c4_uprefix) - print_var(_c4_lprefix) -endmacro() - - -function(c4_zero_pad padded size str) - string(LENGTH "${str}" len) - math(EXPR numchars "${size} - ${len}") - if(numchars EQUAL 0) - set(${padded} "${str}" PARENT_SCOPE) - else() - set(out "${str}") - math(EXPR ncm1 "${numchars} - 1") - foreach(z RANGE ${ncm1}) - set(out "0${out}") - endforeach() - set(${padded} "${out}" PARENT_SCOPE) - endif() -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -# handy macro for dealing with arguments in one single statement. -# look for example usage cases below. -macro(_c4_handle_args) - set(opt0arg - ) - set(opt1arg - _PREFIX - ) - set(optNarg - _ARGS0 - _ARGS1 - _ARGSN - _ARGS - _DEPRECATE - ) - # parse the arguments to this macro to find out the required arguments - cmake_parse_arguments("__c4ha" "${opt0arg}" "${opt1arg}" "${optNarg}" ${ARGN}) - # now parse the required arguments - cmake_parse_arguments("${__c4ha__PREFIX}" "${__c4ha__ARGS0}" "${__c4ha__ARGS1}" "${__c4ha__ARGSN}" ${__c4ha__ARGS}) - # raise an error on deprecated arguments - foreach(a ${__c4ha__DEPRECATE}) - list(FIND __c4ha__ARGS ${a} contains) - if(NOT (${contains} EQUAL -1)) - c4err("${a} is deprecated") - endif() - endforeach() -endmacro() - -# fallback to provided default(s) if argument is not set -macro(_c4_handle_arg argname) - if("${_${argname}}" STREQUAL "") - set(_${argname} "${ARGN}") - else() - set(_${argname} "${_${argname}}") - endif() -endmacro() -macro(_c4_handle_arg_no_pfx argname) - if("${${argname}}" STREQUAL "") - set(${argname} "${ARGN}") - else() - set(${argname} "${${argname}}") - endif() -endmacro() - - -# if ${_${argname}} is non empty, return it -# otherwise, fallback to ${_c4_uprefix}${argname} -# otherwise, fallback to C4_${argname} -# otherwise, fallback to provided default through ${ARGN} -macro(_c4_handle_arg_or_fallback argname) - if(NOT ("${_${argname}}" STREQUAL "")) - c4_dbg("handle arg ${argname}: picking explicit value _${argname}=${_${argname}}") - else() - foreach(_c4haf_varname "${_c4_uprefix}${argname}" "C4_${argname}" "${argname}" "CMAKE_${argname}") - set(v ${${_c4haf_varname}}) - if("${v}" STREQUAL "") - c4_dbg("handle arg ${argname}: ${_c4haf_varname}: empty, continuing") - else() - c4_dbg("handle arg ${argname}: ${_c4haf_varname}=${v} not empty!") - c4_setg(_${argname} "${v}") - break() - endif() - endforeach() - if("${_${argname}}" STREQUAL "") - c4_dbg("handle arg ${argname}: picking default: ${ARGN}") - c4_setg(_${argname} "${ARGN}") - endif() - endif() -endmacro() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -function(c4_get_config var name) - c4_dbg("get_config: ${var} ${name}") - c4_get_from_first_of(config ${ARGN} VARS ${_c4_uprefix}${name} C4_${name} ${name}) - c4_dbg("get_config: ${var} ${name}=${config}") - set(${var} ${config} PARENT_SCOPE) -endfunction() - - -function(c4_get_from_first_of var) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 - REQUIRED # raise an error if no set variable was found - ENV # if none of the provided vars is given, - # then search next on environment variables - # of the same name, using the same sequence - _ARGS1 - DEFAULT - _ARGSN - VARS - ) - c4_dbg("get_from_first_of(): searching ${var}") - foreach(_var ${_VARS}) - set(val ${${_var}}) - c4_dbg("${var}: searching ${_var}=${val}") - if(NOT ("${val}" STREQUAL "")) - set(${var} "${val}" PARENT_SCOPE) - return() - endif() - endforeach() - if(_ENV) - foreach(_envvar ${_VARS}) - set(val $ENV{${_envvar}}) - c4_dbg("${var}: searching environment variable ${_envvar}=${val}") - if(NOT ("${val}" STREQUAL "")) - c4_dbg("${var}: picking ${val} from ${_envvar}") - set(${var} "${val}" PARENT_SCOPE) - return() - endif() - endforeach() - endif() - if(_REQUIRED) - c4_err("could not find a value for the variable ${var}") - endif() - set(${var} ${_DEFAULT} PARENT_SCOPE) -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -# assumes a prior call to project() -function(c4_project) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 # zero-value macro arguments - STANDALONE # Declare that targets from this project MAY be - # compiled in standalone mode. In this mode, any - # designated libraries on which a target depends - # will be incorporated into the target instead of - # being linked with it. The effect is to "flatten" - # those libraries into the requesting library, with - # their sources now becoming part of the requesting - # library; their dependencies are transitively handled. - # Note that requesting targets must explicitly - # opt-in to this behavior via the INCORPORATE - # argument to c4_add_library() or - # c4_add_executable(). Note also that this behavior - # is only enabled if this project's option - # ${prefix}_STANDALONE or C4_STANDALONE is set to ON. - _ARGS1 # one-value macro arguments - AUTHOR # specify author(s); used in cpack - VERSION # cmake does not accept semantic versioning so we provide - # that here (see https://gitlab.kitware.com/cmake/cmake/-/issues/16716) - CXX_STANDARD # one of latest;${C4_VALID_CXX_STANDARDS} - # if this is not provided, falls back on - # ${uprefix}CXX_STANDARD, then C4_CXX_STANDARD, - # then CXX_STANDARD. if none are provided, - # defaults to 11 - _ARGSN # multi-value macro arguments - ) - # get the prefix from the call to project() - set(prefix ${PROJECT_NAME}) - string(TOUPPER "${prefix}" ucprefix) # ucprefix := upper case prefix - string(TOLOWER "${prefix}" lcprefix) # lcprefix := lower case prefix - if(NOT _c4_prefix) - c4_setg(_c4_is_root_proj ON) - c4_setg(_c4_root_proj ${prefix}) - c4_setg(_c4_root_uproj ${ucprefix}) - c4_setg(_c4_root_lproj ${lcprefix}) - c4_setg(_c4_curr_path "") - else() - c4_setg(_c4_is_root_proj OFF) - if(_c4_curr_path) - c4_setg(_c4_curr_path "${_c4_curr_path}/${prefix}") - else() - c4_setg(_c4_curr_path "${prefix}") - endif() - endif() - c4_setg(_c4_curr_subproject ${prefix}) - # get the several prefix flavors - c4_setg(_c4_ucprefix ${ucprefix}) - c4_setg(_c4_lcprefix ${lcprefix}) - c4_setg(_c4_ocprefix ${prefix}) # ocprefix := original case prefix - c4_setg(_c4_prefix ${prefix}) # prefix := original prefix - c4_setg(_c4_oprefix ${prefix}) # oprefix := original prefix - c4_setg(_c4_uprefix ${_c4_ucprefix}) # upper prefix: for variables - c4_setg(_c4_lprefix ${_c4_lcprefix}) # lower prefix: for targets - if(_c4_oprefix) - c4_setg(_c4_oprefix "${_c4_oprefix}_") - endif() - if(_c4_uprefix) - c4_setg(_c4_uprefix "${_c4_uprefix}_") - endif() - if(_c4_lprefix) - c4_setg(_c4_lprefix "${_c4_lprefix}-") - endif() - # - if(_STANDALONE) - option(${_c4_uprefix}STANDALONE - "Enable compilation of opting-in targets from ${_c4_lcprefix} in standalone mode (ie, incorporate subprojects as specified in the INCORPORATE clause to c4_add_library/c4_add_target)" - ${_c4_is_root_proj}) - c4_setg(_c4_root_proj_standalone ${_c4_uprefix}STANDALONE) - endif() - _c4_handle_arg_or_fallback(CXX_STANDARD ${C4_CXX_STANDARD_DEFAULT}) - _c4_handle_arg(VERSION 0.0.0-pre0) - _c4_handle_arg(AUTHOR "") - _c4_handle_semantic_version(${_VERSION}) - # - # make sure project-wide settings are defined -- see cmake's - # documentation for project(), which defines these and other - # variables - if("${PROJECT_DESCRIPTION}" STREQUAL "") - c4_setg(PROJECT_DESCRIPTION "${prefix}") - c4_setg(${prefix}_DESCRIPTION "${prefix}") - endif() - if("${PROJECT_HOMEPAGE_URL}" STREQUAL "") - c4_setg(PROJECT_HOMEPAGE_URL "") - c4_setg(${prefix}_HOMEPAGE_URL "") - endif() - # other specific c4_project properties - c4_setg(PROJECT_AUTHOR "${_AUTHOR}") - c4_setg(${prefix}_AUTHOR "${_AUTHOR}") - - # CXX standard - if("${_CXX_STANDARD}" STREQUAL "latest") - _c4_find_latest_supported_cxx_standard(_CXX_STANDARD) - endif() - c4_log("using C++ standard: C++${_CXX_STANDARD}") - c4_set_proj_prop(CXX_STANDARD "${_CXX_STANDARD}") - c4_setg(${_c4_uprefix}CXX_STANDARD "${_CXX_STANDARD}") - if(${_CXX_STANDARD}) - c4_set_cxx(${_CXX_STANDARD}) - endif() - - # we are opinionated with respect to directory structure - c4_setg(${_c4_uprefix}SRC_DIR ${CMAKE_CURRENT_LIST_DIR}/src) - c4_setg(${_c4_uprefix}EXT_DIR ${CMAKE_CURRENT_LIST_DIR}/ext) - c4_setg(${_c4_uprefix}API_DIR ${CMAKE_CURRENT_LIST_DIR}/api) - # opionionated also for directory test - # opionionated also for directory bm (benchmarks) - - if("${C4_DEV}" STREQUAL "") - option(C4_DEV "enable development targets for all c4 projects" OFF) - endif() - option(${_c4_uprefix}DEV "enable development targets: tests, benchmarks, sanitize, static analysis, coverage" ${C4_DEV}) - - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/test") - cmake_dependent_option(${_c4_uprefix}BUILD_TESTS "build unit tests" ON ${_c4_uprefix}DEV OFF) - else() - c4_dbg("no tests: directory does not exist: ${CMAKE_CURRENT_LIST_DIR}/test") - endif() - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/bm") - cmake_dependent_option(${_c4_uprefix}BUILD_BENCHMARKS "build benchmarks" ON ${_c4_uprefix}DEV OFF) - else() - c4_dbg("no benchmarks: directory does not exist: ${CMAKE_CURRENT_LIST_DIR}/bm") - endif() - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/api") - cmake_dependent_option(${_c4_uprefix}BUILD_API "build API" OFF ${_c4_uprefix}DEV OFF) - else() - c4_dbg("no API generation: directory does not exist: ${CMAKE_CURRENT_LIST_DIR}/api") - endif() - if(_c4_is_root_proj) - c4_setup_coverage() - endif() - c4_setup_valgrind(${_c4_uprefix}DEV) - c4_setup_sanitize(${_c4_uprefix}DEV) - c4_setup_static_analysis(${_c4_uprefix}DEV) - c4_setup_doxygen(${_c4_uprefix}DEV) - - # option to use libc++ - option(${_c4_uprefix}USE_LIBCXX "use libc++ instead of the default standard library" OFF) - if(${_c4_uprefix}USE_LIBCXX) - if(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") - c4_log("using libc++") - list(APPEND CMAKE_CXX_FLAGS -stdlib=libc++) - list(APPEND CMAKE_EXE_LINKER_FLAGS -lc++) - list(APPEND CMAKE_MODULE_LINKER_FLAGS -lc++) - list(APPEND CMAKE_SHARED_LINKER_FLAGS -lc++) - list(APPEND CMAKE_STATIC_LINKER_FLAGS -lc++) - else() - c4_err("libc++ can only be used with clang") - endif() - endif() - - # default compilation flags - set(${_c4_uprefix}CXX_FLAGS "${${_c4_uprefix}CXX_FLAGS_FWD}" CACHE STRING "compilation flags for ${_c4_prefix} targets") - set(${_c4_uprefix}CXX_LINKER_FLAGS "${${_c4_uprefix}CXX_LINKER_FLAGS_FWD}" CACHE STRING "linker flags for ${_c4_prefix} targets") - c4_dbg_var_if(${_c4_uprefix}CXX_LINKER_FLAGS_FWD) - c4_dbg_var_if(${_c4_uprefix}CXX_FLAGS_FWD) - c4_dbg_var_if(${_c4_uprefix}CXX_LINKER_FLAGS) - c4_dbg_var_if(${_c4_uprefix}CXX_FLAGS) - - # Dev compilation flags, appended to the project's flags. They - # are enabled when in dev mode, but provided as a (default-disabled) - # option when not in dev mode - c4_dbg_var_if(${_c4_uprefix}CXX_FLAGS_OPT_FWD) - c4_setg(${_c4_uprefix}CXX_FLAGS_OPT "${${_c4_uprefix}CXX_FLAGS_OPT_FWD}") - c4_optional_compile_flags_dev(WERROR "Compile with warnings as errors" - GCC_CLANG -Werror -pedantic-errors - MSVC /WX - ) - c4_optional_compile_flags_dev(STRICT_ALIASING "Enable strict aliasing" - GCC_CLANG -fstrict-aliasing - MSVC # does it have this? - ) - c4_optional_compile_flags_dev(PEDANTIC "Compile in pedantic mode" - GCC ${_C4_PEDANTIC_FLAGS_GCC} - CLANG ${_C4_PEDANTIC_FLAGS_CLANG} - MSVC ${_C4_PEDANTIC_FLAGS_MSVC} - ) - c4_dbg_var_if(${_c4_uprefix}CXX_FLAGS_OPT) -endfunction(c4_project) - - -# cmake: VERSION argument in project() does not accept semantic versioning -# see: https://gitlab.kitware.com/cmake/cmake/-/issues/16716 -macro(_c4_handle_semantic_version version) - # https://stackoverflow.com/questions/18658233/split-string-to-3-variables-in-cmake - string(REPLACE "." ";" version_list ${version}) - list(GET version_list 0 _major) - list(GET version_list 1 _minor) - list(GET version_list 2 _patch) - if("${_patch}" STREQUAL "") - set(_patch 1) - set(_tweak) - else() - string(REGEX REPLACE "([0-9]+)[-_.]?(.*)" "\\2" _tweak ${_patch}) # do this first - string(REGEX REPLACE "([0-9]+)[-_.]?(.*)" "\\1" _patch ${_patch}) # ... because this replaces _patch - endif() - # because cmake handles only numeric tweak fields, make sure to skip our - # semantic tweak field if it is not numeric - if(${_tweak} MATCHES "^[0-9]+$") - set(_safe_tweak ${_tweak}) - set(_safe_version ${_major}.${_minor}.${_patch}.${_tweak}) - else() - set(_safe_tweak) - set(_safe_version ${_major}.${_minor}.${_patch}) - endif() - c4_setg(PROJECT_VERSION_FULL ${version}) - c4_setg(PROJECT_VERSION ${_safe_version}) - c4_setg(PROJECT_VERSION_MAJOR ${_major}) - c4_setg(PROJECT_VERSION_MINOR ${_minor}) - c4_setg(PROJECT_VERSION_PATCH ${_patch}) - c4_setg(PROJECT_VERSION_TWEAK "${_safe_tweak}") - c4_setg(PROJECT_VERSION_TWEAK_FULL "${_tweak}") - c4_setg(${prefix}_VERSION_FULL ${version}) - c4_setg(${prefix}_VERSION ${_safe_version}) - c4_setg(${prefix}_VERSION_MAJOR ${_major}) - c4_setg(${prefix}_VERSION_MINOR ${_minor}) - c4_setg(${prefix}_VERSION_PATCH ${_patch}) - c4_setg(${prefix}_VERSION_TWEAK "${_safe_tweak}") - c4_setg(${prefix}_VERSION_TWEAK_FULL "${_tweak}") -endmacro() - - -# Add targets for testing (dir=./test), benchmark (dir=./bm) and API (dir=./api). -# Call this macro towards the end of the project's main CMakeLists.txt. -# Experimental feature: docs. -function(c4_add_dev_targets) - if(NOT CMAKE_CURRENT_LIST_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) - c4_err("this macro needs to be called on the project's main CMakeLists.txt file") - endif() - # - if(${_c4_uprefix}BUILD_TESTS) - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/test") - c4_dbg("adding tests: ${CMAKE_CURRENT_LIST_DIR}/test") - enable_testing() # this must be done here (and not inside the - # test dir) so that the cmake-generated test - # targets are available at the top level - add_subdirectory(test) - endif() - endif() - # - if(${_c4_uprefix}BUILD_BENCHMARKS) - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/bm") - c4_dbg("adding benchmarks: ${CMAKE_CURRENT_LIST_DIR}/bm") - add_subdirectory(bm) - endif() - endif() - # - if(${_c4_uprefix}BUILD_API) - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/api") - c4_dbg("adding API: ${d}") - add_subdirectory(api) - endif() - endif() - # - # FIXME - c4_add_doxygen(doc DOXYFILE_IN ${_c4_project_dir}/Doxyfile.in - PROJ c4core - INPUT ${${_c4_uprefix}SRC_DIR} - EXCLUDE ${${_c4_uprefix}EXT_DIR} ${${_c4_uprefix}SRC_DIR}/c4/ext - STRIP_FROM_PATH ${${_c4_uprefix}SRC_DIR} - STRIP_FROM_INC_PATH ${${_c4_uprefix}SRC_DIR} - CLANG_DATABASE_PATH ${CMAKE_BINARY_DIR} - ) - c4_add_doxygen(doc-full DOXYFILE_IN ${_c4_project_dir}/Doxyfile.full.in - PROJ c4core - INPUT ${${_c4_uprefix}SRC_DIR} - EXCLUDE ${${_c4_uprefix}EXT_DIR} ${${_c4_uprefix}SRC_DIR}/c4/ext - STRIP_FROM_PATH ${${_c4_uprefix}SRC_DIR} - STRIP_FROM_INC_PATH ${${_c4_uprefix}SRC_DIR} - CLANG_DATABASE_PATH ${CMAKE_BINARY_DIR} - ) -endfunction() - - -function(_c4_get_san_targets target result) - _c4_get_tgt_prop(san_targets ${target} C4_SAN_TARGETS) - if(NOT san_targets) - #c4_err("${target} must have at least itself in its sanitized target list") - set(${result} ${target} PARENT_SCOPE) - else() - set(${result} ${san_targets} PARENT_SCOPE) - endif() -endfunction() - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -# utilities for compilation flags and defines - -# flags enabled only on dev mode -macro(c4_optional_compile_flags_dev tag desc) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 - _ARGS1 - _ARGSN - MSVC # flags for Visual Studio compilers - GCC # flags for gcc compilers - CLANG # flags for clang compilers - GCC_CLANG # flags common to gcc and clang - _DEPRECATE - ) - cmake_dependent_option(${_c4_uprefix}${tag} "${desc}" ON ${_c4_uprefix}DEV OFF) - set(optname ${_c4_uprefix}${tag}) - if(${optname}) - c4_dbg("${optname} is enabled. Adding flags...") - if(MSVC) - set(flags ${_MSVC}) - elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") - set(flags ${_GCC_CLANG};${_CLANG}) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(flags ${_GCC_CLANG};${_GCC}) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - set(flags ${_ALL};${_GCC_CLANG};${_GCC}) # FIXME - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - set(flags ${_ALL};${_GCC_CLANG};${_CLANG}) # FIXME - else() - c4_err("unknown compiler") - endif() - else() - c4_dbg("${optname} is disabled.") - endif() - if(flags) - c4_log("${tag} flags [${desc}]: ${flags}") - c4_setg(${_c4_uprefix}CXX_FLAGS_OPT "${${_c4_uprefix}CXX_FLAGS_OPT};${flags}") - endif() -endmacro() - - -function(c4_target_compile_flags target) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 - PUBLIC - PRIVATE - INTERFACE - AFTER # this is the default - BEFORE - _ARGS1 - _ARGSN - ALL # flags for all compilers - MSVC # flags for Visual Studio compilers - GCC # flags for gcc compilers - CLANG # flags for clang compilers - GCC_CLANG # flags common to gcc and clang - _DEPRECATE - ) - if(MSVC) - set(flags ${_ALL};${_MSVC}) - elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") - set(flags ${_ALL};${_GCC_CLANG};${_CLANG}) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(flags ${_ALL};${_GCC_CLANG};${_GCC}) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - set(flags ${_ALL};${_GCC_CLANG};${_GCC}) # FIXME - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") - set(flags ${_ALL};${_GCC_CLANG};${_CLANG}) # FIXME - else() - c4_err("unknown compiler") - endif() - if(NOT flags) - c4_dbg("no compile flags to be set") - return() - endif() - if(_AFTER OR (NOT _BEFORE)) - set(mode) - c4_log("${target}: adding compile flags AFTER: ${flags}") - elseif(_BEFORE) - set(mode BEFORE) - c4_log("${target}: adding compile flags BEFORE: ${flags}") - endif() - _c4_get_san_targets(${target} san_targets) - foreach(st ${san_targets}) - if(_PUBLIC) - target_compile_options(${st} ${mode} PUBLIC ${flags}) - elseif(_PRIVATE) - target_compile_options(${st} ${mode} PRIVATE ${flags}) - elseif(_INTERFACE) - target_compile_options(${st} ${mode} INTERFACE ${flags}) - else() - c4_err("${target}: must have one of PUBLIC, PRIVATE or INTERFACE") - endif() - endforeach() -endfunction() - - -function(c4_target_definitions target) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 - PUBLIC - PRIVATE - INTERFACE - _ARGS1 - _ARGSN - ALL # defines for all compilers - MSVC # defines for Visual Studio compilers - GCC # defines for gcc compilers - CLANG # defines for clang compilers - GCC_CLANG # defines common to gcc and clang - _DEPRECATE - ) - if(MSVC) - set(flags ${_ALL};${_MSVC}) - elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") - set(flags ${_ALL};${_GCC_CLANG};${_CLANG}) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(flags ${_ALL};${_GCC_CLANG};${_GCC}) - else() - c4_err("unknown compiler") - endif() - if(NOT flags) - c4_dbg("no compile flags to be set") - return() - endif() - if(_AFTER OR (NOT _BEFORE)) - set(mode) - c4_log("${target}: adding definitions AFTER: ${flags}") - elseif(_BEFORE) - set(mode BEFORE) - c4_log("${target}: adding definitions BEFORE: ${flags}") - endif() - _c4_get_san_targets(${target} san_targets) - foreach(st ${san_targets}) - if(_PUBLIC) - target_compile_definitions(${st} ${mode} PUBLIC ${flags}) - elseif(_PRIVATE) - target_compile_definitions(${st} ${mode} PRIVATE ${flags}) - elseif(_INTERFACE) - target_compile_definitions(${st} ${mode} INTERFACE ${flags}) - else() - c4_err("${target}: must have one of PUBLIC, PRIVATE or INTERFACE") - endif() - endforeach() -endfunction() - - -function(c4_target_remove_compile_flags target) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 - PUBLIC # remove only from public compile options - INTERFACE # remove only from interface compile options - _ARGS1 - _ARGSN - MSVC # flags for Visual Studio compilers - GCC # flags for gcc compilers - CLANG # flags for clang compilers - GCC_CLANG # flags common to gcc and clang - _DEPRECATE - ) - if(MSVC) - set(flags ${_MSVC}) - elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") - set(flags ${_GCC_CLANG};${_CLANG}) - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(flags ${_GCC_CLANG};${_GCC}) - else() - c4_err("unknown compiler") - endif() - if(NOT flags) - return() - endif() - _c4_get_san_targets(${target} san_targets) - foreach(st ${san_targets}) - if(_PUBLIC OR (NOT _INTERFACE)) - get_target_property(co ${st} COMPILE_OPTIONS) - if(co) - _c4_remove_entries_from_list("${flags}" co) - set_target_properties(${st} PROPERTIES COMPILE_OPTIONS "${co}") - endif() - endif() - if(_INTERFACE OR (NOT _PUBLIC)) - get_target_property(ico ${st} INTERFACE_COMPILE_OPTIONS) - if(ico) - _c4_remove_entries_from_list("${flags}" ico) - set_target_properties(${st} PROPERTIES INTERFACE_COMPILE_OPTIONS "${ico}") - endif() - endif() - endforeach() -endfunction() - - -function(_c4_remove_entries_from_list entries_to_remove list) - set(str ${${list}}) - string(REPLACE ";" "==?==" str "${str}") - foreach(entry ${entries_to_remove}) - string(REPLACE "${entry}" "" str "${str}") - endforeach() - string(REPLACE "==?==" ";" str "${str}") - string(REPLACE ";;" ";" str "${str}") - set(${list} "${str}" PARENT_SCOPE) -endfunction() - - - -# pedantic flags... -# default pedantic flags taken from: -# https://github.com/lefticus/cpp_starter_project/blob/master/cmake/CompilerWarnings.cmake -set(_C4_PEDANTIC_FLAGS_MSVC - /W4 # Baseline reasonable warnings - /w14242 # 'identifier': conversion from 'type1' to 'type1', possible loss of data - /w14254 # 'operator': conversion from 'type1:field_bits' to 'type2:field_bits', possible loss of data - /w14263 # 'function': member function does not override any base class virtual member function - /w14265 # 'classname': class has virtual functions, but destructor is not virtual instances of this class may not - # be destructed correctly - /w14287 # 'operator': unsigned/negative constant mismatch - /we4289 # nonstandard extension used: 'variable': loop control variable declared in the for-loop is used outside - # the for-loop scope - /w14296 # 'operator': expression is always 'boolean_value' - /w14311 # 'variable': pointer truncation from 'type1' to 'type2' - /w14545 # expression before comma evaluates to a function which is missing an argument list - /w14546 # function call before comma missing argument list - /w14547 # 'operator': operator before comma has no effect; expected operator with side-effect - /w14549 # 'operator': operator before comma has no effect; did you intend 'operator'? - /w14555 # expression has no effect; expected expression with side- effect - /w14619 # pragma warning: there is no warning number 'number' - /w14640 # Enable warning on thread un-safe static member initialization - /w14826 # Conversion from 'type1' to 'type_2' is sign-extended. This may cause unexpected runtime behavior. - /w14905 # wide string literal cast to 'LPSTR' - /w14906 # string literal cast to 'LPWSTR' - /w14928 # illegal copy-initialization; more than one user-defined conversion has been implicitly applied - $<$:/permissive-> # standards conformance mode for MSVC compiler (only vs2017+) - ) - -set(_C4_PEDANTIC_FLAGS_CLANG - -Wall - -Wextra - -pedantic - -Wshadow # warn the user if a variable declaration shadows one from a parent context - -Wnon-virtual-dtor # warn the user if a class with virtual functions has a non-virtual destructor. This helps - # catch hard to track down memory errors - #-Wold-style-cast # warn for c-style casts - -Wcast-align # warn for potential performance problem casts - -Wunused # warn on anything being unused - -Woverloaded-virtual # warn if you overload (not override) a virtual function - -Wpedantic # warn if non-standard C++ is used - -Wconversion # warn on type conversions that may lose data - -Wsign-conversion # warn on sign conversions - -Wdouble-promotion # warn if float is implicit promoted to double - -Wfloat-equal # warn if comparing floats - -Wformat=2 # warn on security issues around functions that format output (ie printf) - ) - -set(_C4_PEDANTIC_FLAGS_GCC ${_C4_PEDANTIC_FLAGS_CLANG} - -Wlogical-op # where logical operations are used where bitwise were probably wanted - -Wuseless-cast # where you perform a cast to the same type - ) - -if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 6.0)) - list(APPEND _C4_PEDANTIC_FLAGS_GCC - -Wnull-dereference # warn if a null dereference is detected - -Wmisleading-indentation # where indentation implies blocks where blocks do not exist - -Wduplicated-cond # where if-else chain has duplicated conditions - ) -endif() - -if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 7.0)) - list(APPEND _C4_PEDANTIC_FLAGS_GCC - -Wduplicated-branches # where if-else branches have duplicated code - ) -endif() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -function(c4_pack_project) - # if this is the top-level project... pack it. - if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) - c4_log("packing the project: ${ARGN}") - c4_set_default_pack_properties(${ARGN}) - include(CPack) - endif() -endfunction() - - -# [WIP] set convenient defaults for the properties used by CPack -function(c4_set_default_pack_properties) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 # zero-value macro arguments - _ARGS1 # one-value macro arguments - TYPE # one of LIBRARY, EXECUTABLE - _ARGSN # multi-value macro arguments - ) - set(pd "${PROJECT_SOURCE_DIR}") - _c4_handle_arg(TYPE EXECUTABLE) # default to EXECUTABLE - # - _c4_get_platform_tag(platform_tag) - if("${_TYPE}" STREQUAL "LIBRARY") - if(BUILD_SHARED_LIBS) - set(build_tag "-shared") - else() - set(build_tag "-static") - endif() - get_property(multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) - if(multi_config) - # doesn't work because generators are not evaluated: set(build_tag "${build_tag}-$") - # doesn't work because generators are not evaluated: set(build_tag "${build_tag}$<$:-Debug>$<$:-MinSizeRel>$<$:-Release>$<$:-RelWithDebInfo>") - # see also https://stackoverflow.com/questions/44153730/how-to-change-cpack-package-file-name-based-on-configuration - if(CMAKE_BUILD_TYPE) # in the off-chance it was explicitly set - set(build_tag "${build_tag}-${CMAKE_BUILD_TYPE}") - endif() - else() - set(build_tag "${build_tag}-${CMAKE_BUILD_TYPE}") - endif() - elseif("${_TYPE}" STREQUAL "EXECUTABLE") - set(build_tag) - elseif() - c4_err("unknown TYPE: ${_TYPE}") - endif() - # - c4_setg(CPACK_VERBATIM_VARIABLES true) - c4_setg(CPACK_PACKAGE_VENDOR "${${_c4_prefix}_HOMEPAGE_URL}") - c4_setg(CPACK_PACKAGE_CONTACT "${${_c4_prefix}_AUTHOR}") - c4_setg(CPACK_PACKAGE_DESCRIPTION_SUMMARY "${${_c4_prefix}_DESCRIPTION}") - if(EXISTS "${pd}/README.md") - c4_setg(CPACK_PACKAGE_DESCRIPTION_FILE "${pd}/README.md") - c4_setg(CPACK_PACKAGE_DESCRIPTION_README "${pd}/README.md") - c4_setg(CPACK_PACKAGE_DESCRIPTION_WELCOME "${pd}/README.md") - elseif(EXISTS "${pd}/README.txt") - c4_setg(CPACK_PACKAGE_DESCRIPTION_FILE "${pd}/README.txt") - c4_setg(CPACK_PACKAGE_DESCRIPTION_README "${pd}/README.txt") - c4_setg(CPACK_PACKAGE_DESCRIPTION_WELCOME "${pd}/README.txt") - endif() - if(EXISTS "${pd}/LICENSE.md") - c4_setg(CPACK_RESOURCE_FILE_LICENSE "${pd}/LICENSE.md") - elseif(EXISTS "${pd}/LICENSE.txt") - c4_setg(CPACK_RESOURCE_FILE_LICENSE "${pd}/LICENSE.txt") - endif() - c4_proj_get_version("${pd}" version_tag full major minor patch tweak) - c4_setg(CPACK_PACKAGE_VERSION "${full}") - c4_setg(CPACK_PACKAGE_VERSION_MAJOR "${major}") - c4_setg(CPACK_PACKAGE_VERSION_MINOR "${minor}") - c4_setg(CPACK_PACKAGE_VERSION_PATCH "${patch}") - c4_setg(CPACK_PACKAGE_VERSION_TWEAK "${tweak}") - c4_setg(CPACK_PACKAGE_INSTALL_DIRECTORY "${_c4_prefix}-${version_tag}") - c4_setg(CPACK_PACKAGE_FILE_NAME "${_c4_prefix}-${version_tag}-${platform_tag}${build_tag}") - if(WIN32 AND NOT UNIX) - # There is a bug in NSI that does not handle full UNIX paths properly. - # Make sure there is at least one set of four backlashes. - #c4_setg(CPACK_PACKAGE_ICON "${CMake_SOURCE_DIR}/Utilities/Release\\\\InstallIcon.bmp") - #c4_setg(CPACK_NSIS_INSTALLED_ICON_NAME "bin\\\\MyExecutable.exe") - c4_setg(CPACK_NSIS_DISPLAY_NAME "${_c4_prefix} ${version_tag}") - c4_setg(CPACK_NSIS_HELP_LINK "${${_c4_prefix}_HOMEPAGE_URL}") - c4_setg(CPACK_NSIS_URL_INFO_ABOUT "${${_c4_prefix}_HOMEPAGE_URL}") - c4_setg(CPACK_NSIS_CONTACT "${${_c4_prefix}_AUTHOR}") - c4_setg(CPACK_NSIS_MODIFY_PATH ON) - else() - #c4_setg(CPACK_STRIP_FILES "bin/MyExecutable") - #c4_setg(CPACK_SOURCE_STRIP_FILES "") - c4_setg(CPACK_DEBIAN_PACKAGE_MAINTAINER "${${_c4_prefix}_AUTHOR}") - endif() - #c4_setg(CPACK_PACKAGE_EXECUTABLES "MyExecutable" "My Executable") -endfunction() - - -function(_c4_get_platform_tag tag_) - if(WIN32 AND NOT UNIX) - set(tag win) - elseif(APPLE) - set(tag apple) - elseif(UNIX) - set(tag unix) - else() - set(tag ${CMAKE_SYSTEM_NAME}) - endif() - if(CMAKE_SIZEOF_VOID_P EQUAL 8) # 64 bits - set(tag ${tag}64) - elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) # 32 bits - set(tag ${tag}32) - else() - c4_err("not implemented") - endif() - set(${tag_} ${tag} PARENT_SCOPE) -endfunction() - - -function(_c4_extract_version_tag tag_) - # git describe --tags for unannotated tags - # git describe --contains -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -# set project-wide property -function(c4_set_proj_prop prop value) - c4_dbg("set ${prop}=${value}") - set(C4PROJ_${_c4_prefix}_${prop} ${value}) -endfunction() - -# set project-wide property -function(c4_get_proj_prop prop var) - c4_dbg("get ${prop}=${C4PROJ_${_c4_prefix}_${prop}}") - set(${var} ${C4PROJ_${_c4_prefix}_${prop}} PARENT_SCOPE) -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -# set target-wide c4 property -function(c4_set_target_prop target prop value) - _c4_set_tgt_prop(${target} C4_TGT_${prop} "${value}") -endfunction() -function(c4_append_target_prop target prop value) - _c4_append_tgt_prop(${target} C4_TGT_${prop} "${value}") -endfunction() - -# get target-wide c4 property -function(c4_get_target_prop target prop var) - _c4_get_tgt_prop(val ${target} C4_TGT_${prop}) - set(${var} ${val} PARENT_SCOPE) -endfunction() - - -# get target-wide property -function(_c4_get_tgt_prop out tgt prop) - get_target_property(target_type ${target} TYPE) - if(target_type STREQUAL "INTERFACE_LIBRARY") - get_property(val GLOBAL PROPERTY C4_TGT_${tgt}_${prop}) - else() - get_target_property(val ${tgt} ${prop}) - endif() - c4_dbg("target ${tgt}: get ${prop}=${val}") - set(${out} "${val}" PARENT_SCOPE) -endfunction() - -# set target-wide property -function(_c4_set_tgt_prop tgt prop propval) - c4_dbg("target ${tgt}: set ${prop}=${propval}") - get_target_property(target_type ${target} TYPE) - if(target_type STREQUAL "INTERFACE_LIBRARY") - set_property(GLOBAL PROPERTY C4_TGT_${tgt}_${prop} "${propval}") - else() - set_target_properties(${tgt} PROPERTIES ${prop} "${propval}") - endif() -endfunction() -function(_c4_append_tgt_prop tgt prop propval) - c4_dbg("target ${tgt}: appending ${prop}=${propval}") - _c4_get_tgt_prop(curr ${tgt} ${prop}) - if(curr) - list(APPEND curr "${propval}") - else() - set(curr "${propval}") - endif() - _c4_set_tgt_prop(${tgt} ${prop} "${curr}") -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -function(c4_set_var_tmp var value) - c4_dbg("tmp-setting ${var} to ${value} (was ${${value}})") - set(_c4_old_val_${var} ${${var}}) - set(${var} ${value} PARENT_SCOPE) -endfunction() - -function(c4_clean_var_tmp var) - c4_dbg("cleaning ${var} to ${_c4_old_val_${var}} (tmp was ${${var}})") - set(${var} ${_c4_old_val_${var}} PARENT_SCOPE) -endfunction() - -macro(c4_override opt val) - set(${opt} ${val} CACHE BOOL "" FORCE) -endmacro() - - -macro(c4_setg var val) - set(${var} ${val}) - set(${var} ${val} PARENT_SCOPE) -endmacro() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -function(c4_proj_get_version dir tag_o full_o major_o minor_o patch_o tweak_o) - if("${dir}" STREQUAL "") - set(dir ${CMAKE_CURRENT_LIST_DIR}) - endif() - find_program(GIT git REQUIRED) - function(_c4pgv_get_cmd outputvar) - execute_process(COMMAND ${ARGN} - WORKING_DIRECTORY ${dir} - ERROR_VARIABLE error - ERROR_STRIP_TRAILING_WHITESPACE - OUTPUT_VARIABLE output - OUTPUT_STRIP_TRAILING_WHITESPACE) - c4_dbg("output of ${ARGN}: ${outputvar}=${output} [@${dir}]") - set(${outputvar} ${output} PARENT_SCOPE) - endfunction() - # do we have any tags yet? - _c4pgv_get_cmd(head_desc ${GIT} describe HEAD) - _c4pgv_get_cmd(branch ${GIT} rev-parse --abbrev-ref HEAD) - if(NOT head_desc) - c4_dbg("the repo does not have any tags yet") - _c4pgv_get_cmd(commit_hash ${GIT} rev-parse --short HEAD) - set(otag "${commit_hash}-${branch}") - else() - c4_dbg("there are tags!") - # is the current commit tagged? - _c4pgv_get_cmd(commit_hash_full ${GIT} rev-parse HEAD) - _c4pgv_get_cmd(commit_desc ${GIT} describe --exact-match ${commit_hash_full}) - if(commit_desc) - c4_dbg("current commit is tagged") - # is the tag a version tag? - _c4_parse_version_tag(${commit_desc} is_version major minor patch tweak more) - if(is_version) - c4_dbg("current commit's tag is a version tag") - # is the tag the current version tag? - if("${is_version}" VERSION_EQUAL "${${_c4_prefix}_VERSION_FULL}") - c4_dbg("this is the official version commit") - else() - c4_dbg("this is a different version") - endif() - set(otag "${commit_desc}") - else() - c4_dbg("this is a non-version tag") - set(otag "${commit_desc}-${branch}") - endif() - else(commit_desc) - # is the latest tag in the head_desc a version tag? - string(REGEX REPLACE "(.*)-[0-9]+-[0-9a-f]+" "\\1" latest_tag "${head_desc}") - c4_dbg("current commit is NOT tagged. latest tag=${latest_tag}") - _c4_parse_version_tag(${latest_tag} latest_tag_is_a_version major minor patch tweak more) - if(latest_tag_is_a_version) - c4_dbg("latest tag is a version. stick to the head description") - set(otag "${head_desc}-${branch}") - set(full "${latest_tag_is_a_version}") - else() - c4_dbg("latest tag is NOT a version. Use the current project version from cmake + the output of git describe") - set(otag "v${full}-${head_desc}-${branch}") - set(full "${${_c4_prefix}_VERSION_FULL}") - set(major "${${_c4_prefix}_VERSION_MAJOR}") - set(minor "${${_c4_prefix}_VERSION_MINOR}") - set(patch "${${_c4_prefix}_VERSION_PATCH}") - set(tweak "${${_c4_prefix}_VERSION_TWEAK}") - endif() - endif(commit_desc) - endif(NOT head_desc) - c4_log("cpack tag: ${otag}") - set(${tag_o} "${otag}" PARENT_SCOPE) - set(${full_o} "${full}" PARENT_SCOPE) - set(${major_o} "${major}" PARENT_SCOPE) - set(${minor_o} "${minor}" PARENT_SCOPE) - set(${patch_o} "${patch}" PARENT_SCOPE) - set(${tweak_o} "${tweak}" PARENT_SCOPE) - # also: dirty index? - # https://stackoverflow.com/questions/2657935/checking-for-a-dirty-index-or-untracked-files-with-git -endfunction() - - -function(_c4_parse_version_tag tag is_version major minor patch tweak more) - # does the tag match a four-part version? - string(REGEX MATCH "v?([0-9]+)([\._][0-9]+)([\._][0-9]+)([\._][0-9]+)(.*)" match "${tag}") - function(_triml arg out) # trim the leading [\._] from the left - if("${arg}" STREQUAL "") - set(${out} "" PARENT_SCOPE) - else() - string(REGEX REPLACE "[\._](.*)" "\\1" ret "${arg}") - set("${out}" "${ret}" PARENT_SCOPE) - endif() - endfunction() - if(match) - set(${is_version} ${tag} PARENT_SCOPE) - _triml("${CMAKE_MATCH_1}" major_v) - _triml("${CMAKE_MATCH_2}" minor_v) - _triml("${CMAKE_MATCH_3}" patch_v) - _triml("${CMAKE_MATCH_4}" tweak_v) - _triml("${CMAKE_MATCH_5}" more_v) - else() - # does the tag match a three-part version? - string(REGEX MATCH "v?([0-9]+)([\._][0-9]+)([\._][0-9]+)(.*)" match "${tag}") - if(match) - set(${is_version} ${tag} PARENT_SCOPE) - _triml("${CMAKE_MATCH_1}" major_v) - _triml("${CMAKE_MATCH_2}" minor_v) - _triml("${CMAKE_MATCH_3}" patch_v) - _triml("${CMAKE_MATCH_4}" more_v) - else() - # does the tag match a two-part version? - string(REGEX MATCH "v?([0-9]+)([\._][0-9]+)(.*)" match "${tag}") - if(match) - set(${is_version} ${tag} PARENT_SCOPE) - _triml("${CMAKE_MATCH_1}" major_v) - _triml("${CMAKE_MATCH_2}" minor_v) - _triml("${CMAKE_MATCH_3}" more_v) - else() - # not a version! - set(${is_version} FALSE PARENT_SCOPE) - endif() - endif() - endif() - set(${major} "${major_v}" PARENT_SCOPE) - set(${minor} "${minor_v}" PARENT_SCOPE) - set(${patch} "${patch_v}" PARENT_SCOPE) - set(${tweak} "${tweak_v}" PARENT_SCOPE) - set(${more} "${more_v}" PARENT_SCOPE) -endfunction() - - -#function(testvtag) -# set(err FALSE) -# function(cmp value expected) -# if(NOT ("${${value}}" STREQUAL "${expected}")) -# c4_log("${tag}: error: expected ${value}=='${expected}': '${${value}}'=='${expected}'") -# set(err TRUE PARENT_SCOPE) -# else() -# c4_log("${tag}: ok: expected ${value}=='${expected}': '${${value}}'=='${expected}'") -# endif() -# endfunction() -# function(verify tag is_version_e major_e minor_e patch_e tweak_e more_e) -# _c4_parse_version_tag(${tag} is_version major minor patch tweak more) -# cmp(is_version ${is_version_e}) -# cmp(major "${major_e}") -# cmp(minor "${minor_e}") -# cmp(patch "${patch_e}") -# cmp(tweak "${tweak_e}") -# cmp(more "${more_e}") -# set(err ${err} PARENT_SCOPE) -# endfunction() -# verify(v12.34.567.89-rcfoo TRUE 12 34 567 89 -rcfoo) -# verify(v12_34_567_89-rcfoo TRUE 12 34 567 89 -rcfoo) -# verify(v12.34.567.89 TRUE 12 34 567 89 "") -# verify(v12_34_567_89 TRUE 12 34 567 89 "") -# verify(v12.34.567-rcfoo TRUE 12 34 567 "" -rcfoo) -# verify(v12_34_567-rcfoo TRUE 12 34 567 "" -rcfoo) -# verify(v12.34.567 TRUE 12 34 567 "" "") -# verify(v12_34_567 TRUE 12 34 567 "" "") -# verify(v12_34 TRUE 12 34 "" "" "") -# verify(v12.34 TRUE 12 34 "" "" "") -# if(err) -# c4_err("test failed") -# endif() -#endfunction() -#testvtag() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - - -macro(_c4_handle_cxx_standard_args) - # EXTENSIONS: - # enable compiler extensions eg, prefer gnu++11 to c++11 - if(EXTENSIONS IN_LIST ARGN) - set(_EXTENSIONS ON) - else() - c4_get_from_first_of(_EXTENSIONS - ENV - DEFAULT OFF - VARS ${_c4_uprefix}CXX_EXTENSIONS C4_CXX_EXTENSIONS CMAKE_CXX_EXTENSIONS) - endif() - # - # OPTIONAL - if(OPTIONAL IN_LIST ARGN) - set(_REQUIRED OFF) - else() - c4_get_from_first_of(_REQUIRED - ENV - DEFAULT ON - VARS ${_c4_uprefix}CXX_STANDARD_REQUIRED C4_CXX_STANDARD_REQUIRED CMAKE_CXX_STANDARD_REQUIRED) - endif() -endmacro() - - -# set the global cxx standard for the project. -# -# examples: -# c4_set_cxx(latest) # find the latest standard supported by the compiler, and use that -# c4_set_cxx(11) # required, no extensions (eg c++11) -# c4_set_cxx(14) # required, no extensions (eg c++14) -# c4_set_cxx(11 EXTENSIONS) # opt-in to extensions (eg, gnu++11) -# c4_set_cxx(14 EXTENSIONS) # opt-in to extensions (eg, gnu++14) -# c4_set_cxx(11 OPTIONAL) # not REQUIRED. no extensions -# c4_set_cxx(11 OPTIONAL EXTENSIONS) # not REQUIRED. with extensions. -macro(c4_set_cxx standard) - _c4_handle_cxx_standard_args(${ARGN}) - if(NOT DEFINED CMAKE_CXX_STANDARD) - c4_log("setting C++ standard: ${standard}") - c4_setg(CMAKE_CXX_STANDARD ${standard}) - endif() - if(NOT DEFINED CMAKE_CXX_STANDARD_REQUIRED) - c4_log("setting C++ standard required: ${_REQUIRED}") - c4_setg(CMAKE_CXX_STANDARD_REQUIRED ${_REQUIRED}) - endif() - if(NOT DEFINED CMAKE_CXX_STANDARD_REQUIRED) - c4_log("setting C++ standard extensions: ${_EXTENSIONS}") - c4_setg(CMAKE_CXX_EXTENSIONS ${_EXTENSIONS}) - endif() -endmacro() - - -# set the cxx standard for a target. -# -# examples: -# c4_target_set_cxx(target latest) # find the latest standard supported by the compiler, and use that -# c4_target_set_cxx(target 11) # required, no extensions (eg c++11) -# c4_target_set_cxx(target 14) # required, no extensions (eg c++14) -# c4_target_set_cxx(target 11 EXTENSIONS) # opt-in to extensions (eg, gnu++11) -# c4_target_set_cxx(target 14 EXTENSIONS) # opt-in to extensions (eg, gnu++14) -# c4_target_set_cxx(target 11 OPTIONAL) # not REQUIRED. no extensions -# c4_target_set_cxx(target 11 OPTIONAL EXTENSIONS) -function(c4_target_set_cxx target standard) - c4_dbg("setting C++ standard for target ${target}: ${standard}") - _c4_handle_cxx_standard_args(${ARGN}) - set_target_properties(${target} PROPERTIES - CXX_STANDARD ${standard} - CXX_STANDARD_REQUIRED ${_REQUIRED} - CXX_EXTENSIONS ${_EXTENSIONS}) - target_compile_features(${target} PUBLIC cxx_std_${standard}) -endfunction() - - -# set the cxx standard for a target based on the global project settings -function(c4_target_inherit_cxx_standard target) - c4_dbg("inheriting C++ standard for target ${target}: ${CMAKE_CXX_STANDARD}") - set_target_properties(${target} PROPERTIES - CXX_STANDARD "${CMAKE_CXX_STANDARD}" - CXX_STANDARD_REQUIRED "${CMAKE_CXX_STANDARD_REQUIRED}" - CXX_EXTENSIONS "${CMAKE_CXX_EXTENSIONS}") - target_compile_features(${target} PUBLIC cxx_std_${CMAKE_CXX_STANDARD}) -endfunction() - - -function(_c4_find_latest_supported_cxx_standard out) - if(NOT c4_latest_supported_cxx_standard) - include(CheckCXXCompilerFlag) - # make sure CMAKE_CXX_FLAGS is clean here - # see https://cmake.org/cmake/help/v3.16/module/CheckCXXCompilerFlag.html - # Note: since this is a function, we don't need to reset CMAKE_CXX_FLAGS - # back to its previous value - set(CMAKE_CXX_FLAGS) - set(standard 11) # default to C++11 if everything fails - foreach(s ${C4_CXX_STANDARDS}) - if(MSVC) - set(flag /std:c++${s}) - else() - # assume GNU-style compiler - set(flag -std=c++${s}) - endif() - c4_log("checking CXX standard: C++${s} flag=${flag}") - check_cxx_compiler_flag(${flag} has${s}) - if(has${s}) - c4_log("checking CXX standard: C++${s} is supported! flag=${flag}") - set(standard ${s}) - break() - else() - c4_log("checking CXX standard: C++${s}: no support for flag=${flag} no") - endif() - endforeach() - set(c4_latest_supported_cxx_standard ${standard} CACHE INTERNAL "") - endif() - set(${out} ${c4_latest_supported_cxx_standard} PARENT_SCOPE) -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -# examples: -# -# # require subproject c4core, as a subdirectory. c4core will be used -# # as a separate library -# c4_require_subproject(c4core SUBDIRECTORY ${C4OPT_EXT_DIR}/c4core) -# -# # require subproject c4core, as a remote proj -# c4_require_subproject(c4core REMOTE -# GIT_REPOSITORY https://github.com/biojppm/c4core -# GIT_TAG master -# ) -function(c4_require_subproject subproj) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 - INCORPORATE - EXCLUDE_FROM_ALL - _ARGS1 - SUBDIRECTORY # the subproject is located in the given directory name and - # will be added via add_subdirectory() - _ARGSN - REMOTE # the subproject is located in a remote repo/url - # and will be added via c4_import_remote_proj(), - # forwarding all the arguments in here. - OVERRIDE # a list of variable name+value pairs - # these variables will be set with c4_override() - # before calling add_subdirectory() - SET_FOLDER_TARGETS # Set the folder of the given targets using - # c4_set_folder_remote_project_targets(). - # The first expected argument is the folder, - # and the remaining arguments are the targets - # which we want to set the folder. - _DEPRECATE - INTERFACE - ) - list(APPEND _${_c4_uprefix}_deps ${subproj}) - c4_setg(_${_c4_uprefix}_deps ${_${_c4_uprefix}_deps}) - c4_dbg("-----------------------------------------------") - c4_dbg("requires subproject ${subproj}!") - if(_INCORPORATE) - c4_dbg("requires subproject ${subproj} in INCORPORATE mode!") - c4_dbg_var(${_c4_root_uproj}_STANDALONE) - if(${_c4_root_uproj}_STANDALONE) - c4_dbg("${_c4_root_uproj} is STANDALONE: honoring INCORPORATE mode...") - else() - c4_dbg("${_c4_root_uproj} is not STANDALONE: ignoring INCORPORATE mode...") - set(_INCORPORATE OFF) - endif() - endif() - # - _c4_get_subproject_property(${subproj} AVAILABLE _available) - if(_available) - c4_dbg("required subproject ${subproj} was already imported:") - c4_dbg_subproject(${subproj}) - # TODO check version compatibility - else() #elseif(NOT _${subproj}_available) - c4_dbg("required subproject ${subproj} is unknown. Importing...") - if(_EXCLUDE_FROM_ALL) - set(excl EXCLUDE_FROM_ALL) - endif() - # forward c4 compile flags - string(TOUPPER ${subproj} usubproj) - c4_setg(${usubproj}_CXX_FLAGS_FWD "${${_c4_uprefix}CXX_FLAGS}") - c4_setg(${usubproj}_CXX_FLAGS_OPT_FWD "${${_c4_uprefix}CXX_FLAGS_OPT}") - c4_setg(${usubproj}_CXX_LINKER_FLAGS_FWD "${${_c4_uprefix}CXX_LINKER_FLAGS}") - # root dir - set(_r ${CMAKE_CURRENT_BINARY_DIR}/subprojects/${subproj}) - if(_REMOTE) - c4_log("importing subproject ${subproj} (REMOTE)... ${_REMOTE}") - _c4_mark_subproject_imported(${subproj} ${_r}/src ${_r}/build ${_INCORPORATE}) - c4_import_remote_proj(${subproj} ${_r} REMOTE ${_REMOTE} OVERRIDE ${_OVERRIDE} ${excl}) - _c4_get_subproject_property(${subproj} SRC_DIR _srcdir) - c4_dbg("finished importing subproject ${subproj} (REMOTE, SRC_DIR=${_srcdir}).") - elseif(_SUBDIRECTORY) - c4_log("importing subproject ${subproj} (SUBDIRECTORY)... ${_SUBDIRECTORY}") - _c4_mark_subproject_imported(${subproj} ${_SUBDIRECTORY} ${_r}/build ${_INCORPORATE}) - c4_add_subproj(${subproj} ${_SUBDIRECTORY} ${_r}/build OVERRIDE ${_OVERRIDE} ${excl}) - set(_srcdir ${_SUBDIRECTORY}) - c4_dbg("finished importing subproject ${subproj} (SUBDIRECTORY=${_SUBDIRECTORY}).") - else() - c4_err("subproject type must be either REMOTE or SUBDIRECTORY") - endif() - endif() - # - if(_SET_FOLDER_TARGETS) - c4_set_folder_remote_project_targets(${_SET_FOLDER_TARGETS}) - endif() -endfunction(c4_require_subproject) - - -function(c4_add_subproj proj dir bindir) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 - EXCLUDE_FROM_ALL # forward to add_subdirectory() - _ARGS1 - _ARGSN - OVERRIDE # a list of variable name+value pairs - # these variables will be set with c4_override() - # before calling add_subdirectory() - ) - # push the subproj into the current path - set(prev_subproject ${_c4_curr_subproject}) - set(prev_path ${_c4_curr_path}) - set(_c4_curr_subproject ${proj}) - string(REGEX MATCH ".*/${proj}\$" pos "${_c4_curr_path}") - if(pos EQUAL -1) - string(REGEX MATCH "^${proj}\$" pos "${_c4_curr_path}") - if(pos EQUAL -1) - set(_c4_curr_path ${_c4_curr_path}/${proj}) - endif() - endif() - # - while(_OVERRIDE) - list(POP_FRONT _OVERRIDE varname) - list(POP_FRONT _OVERRIDE varvalue) - c4_override(${varname} ${varvalue}) - endwhile() - # - if(_EXCLUDE_FROM_ALL) - set(excl EXCLUDE_FROM_ALL) - endif() - # - c4_dbg("adding subproj: ${prev_subproject}->${_c4_curr_subproject}. path=${_c4_curr_path}") - add_subdirectory(${dir} ${bindir} ${excl}) - # pop the subproj from the current path - set(_c4_curr_subproject ${prev_subproject}) - set(_c4_curr_path ${prev_path}) -endfunction() - - -function(_c4_mark_subproject_imported subproject_name subproject_src_dir subproject_bin_dir incorporate) - c4_dbg("marking subproject imported: ${subproject_name} (imported by ${_c4_prefix}). src=${subproject_src_dir}") - _c4_append_subproject_property(${_c4_prefix} DEPENDENCIES ${subproject_name}) - _c4_get_folder(folder ${_c4_prefix} ${subproject_name}) - _c4_set_subproject_property(${subproject_name} AVAILABLE ON) - _c4_set_subproject_property(${subproject_name} IMPORTER "${_c4_prefix}") - _c4_set_subproject_property(${subproject_name} SRC_DIR "${subproject_src_dir}") - _c4_set_subproject_property(${subproject_name} BIN_DIR "${subproject_bin_dir}") - _c4_set_subproject_property(${subproject_name} FOLDER "${folder}") - _c4_set_subproject_property(${subproject_name} INCORPORATE "${incorporate}") -endfunction() - - -function(_c4_get_subproject_property subproject property var) - get_property(v GLOBAL PROPERTY _c4_subproject-${subproject}-${property}) - set(${var} "${v}" PARENT_SCOPE) -endfunction() - - -function(_c4_set_subproject_property subproject property value) - c4_dbg("setting subproj prop: ${subproject}: ${property}=${value}") - set_property(GLOBAL PROPERTY _c4_subproject-${subproject}-${property} "${value}") -endfunction() -function(_c4_append_subproject_property subproject property value) - _c4_get_subproject_property(${subproject} ${property} cval) - if(cval) - list(APPEND cval ${value}) - else() - set(cval ${value}) - endif() - _c4_set_subproject_property(${subproject} ${property} ${cval}) -endfunction() - - -function(_c4_is_incorporated subproj out) - if("${subproj}" STREQUAL "${_c4_root_proj}") - c4_dbg("${subproj} is incorporated? root proj, no") - set(${out} OFF PARENT_SCOPE) - else() - _c4_get_subproject_property(${subproj} INCORPORATE inc) - c4_dbg("${subproj} is incorporated? not root proj, incorporate=${inc}") - set(${out} ${inc} PARENT_SCOPE) - endif() -endfunction() - - -function(c4_dbg_subproject subproject) - set(props AVAILABLE IMPORTER SRC_DIR BIN_DIR DEPENDENCIES FOLDER INCORPORATE) - foreach(p ${props}) - _c4_get_subproject_property(${subproject} ${p} pv) - c4_dbg("${subproject}: ${p}=${pv}") - endforeach() -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -# -# -function(c4_import_remote_proj name dir) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 - EXCLUDE_FROM_ALL - _ARGS1 - SUBDIR # path to the subdirectory where the CMakeLists file is to be found. - _ARGSN - OVERRIDE # a list of variable name+value pairs - # these variables will be set with c4_override() - # before calling add_subdirectory() - REMOTE # to specify url, repo, tag, or branch, - # pass the needed arguments after dir. - # These arguments will be forwarded to ExternalProject_Add() - SET_FOLDER_TARGETS # Set the folder of the given targets using - # c4_set_folder_remote_project_targets(). - # The first expected argument is the folder, - # and the remaining arguments are the targets - # which we want to set the folder. - ) - set(srcdir_in_out "${dir}") - c4_download_remote_proj(${name} srcdir_in_out ${_REMOTE}) - if(_SUBDIR) - set(srcdir_in_out "${srcdir_in_out}/${_SUBDIR}") - endif() - _c4_set_subproject_property(${name} SRC_DIR "${srcdir_in_out}") - if(_EXCLUDE_FROM_ALL) - set(excl EXCLUDE_FROM_ALL) - endif() - c4_add_subproj(${name} "${srcdir_in_out}" "${dir}/build" OVERRIDE ${_OVERRIDE} ${excl}) - # - if(_SET_FOLDER_TARGETS) - c4_set_folder_remote_project_targets(${_SET_FOLDER_TARGETS}) - endif() -endfunction() - - -# download remote projects while running cmake -# to specify url, repo, tag, or branch, -# pass the needed arguments after dir. -# These arguments will be forwarded to ExternalProject_Add() -function(c4_download_remote_proj name candidate_dir) - # https://crascit.com/2015/07/25/cmake-gtest/ - # (via https://stackoverflow.com/questions/15175318/cmake-how-to-build-external-projects-and-include-their-targets) - set(dir ${${candidate_dir}}) - if("${dir}" STREQUAL "") - set(dir "${CMAKE_BINARY_DIR}/extern/${name}") - endif() - set(cvar _${_c4_uprefix}_DOWNLOAD_${name}_LOCATION) - set(cval ${${cvar}}) - # - # was it already downloaded in this project? - if(NOT ("${cval}" STREQUAL "")) - if(EXISTS "${cval}") - c4_log("${name} was previously imported into this project - found at \"${cval}\"!") - set(${candidate_dir} "${cval}" PARENT_SCOPE) - return() - else() - c4_log("${name} was previously imported into this project - but was NOT found at \"${cval}\"!") - endif() - endif() - # - # try to find an existing version (downloaded by some other project) - set(out "${dir}") - _c4_find_cached_proj(${name} out) - if(NOT ("${out}" STREQUAL "${dir}")) - c4_log("using ${name} from \"${out}\"...") - set(${cvar} "${out}" CACHE INTERNAL "") - set(${candidate_dir} "${out}" PARENT_SCOPE) - return() - endif() - # - # no version was found; need to download. - c4_log("downloading ${name}: not in cache...") - # check for a global place to download into - set(srcdir) - _c4_get_cached_srcdir_global_extern(${name} srcdir) - if("${srcdir}" STREQUAL "") - # none found; default to the given dir - set(srcdir "${dir}/src") - endif() - # - # do it - #if((EXISTS ${dir}/dl) AND (EXISTS ${dir}/dl/CMakeLists.txt)) - # return() - #endif() - c4_log("downloading remote project: ${name} -> \"${srcdir}\" (dir=${dir})...") - # - file(WRITE ${dir}/dl/CMakeLists.txt " -cmake_minimum_required(VERSION 2.8.2) -project(${_c4_lcprefix}-download-${name} NONE) - -# this project only downloads ${name} -# (ie, no configure, build or install step) -include(ExternalProject) - -ExternalProject_Add(${name}-dl - ${ARGN} - SOURCE_DIR \"${srcdir}\" - BINARY_DIR \"${dir}/build\" - CONFIGURE_COMMAND \"\" - BUILD_COMMAND \"\" - INSTALL_COMMAND \"\" - TEST_COMMAND \"\" -) -") - execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY ${dir}/dl) - execute_process(COMMAND ${CMAKE_COMMAND} --build . - WORKING_DIRECTORY ${dir}/dl) - # - set(${candidate_dir} "${srcdir}" PARENT_SCOPE) - set(_${_c4_uprefix}_DOWNLOAD_${name}_LOCATION "${srcdir}" CACHE INTERNAL "") -endfunction() - - -# checks if the project was already downloaded. If it was, then dir_in_out is -# changed to the directory where the project was found at. -function(_c4_find_cached_proj name dir_in_out) - c4_log("downloading ${name}: searching cached project...") - # - # 1. search in the per-import variable, eg RYML_CACHE_DOWNLOAD_GTEST - string(TOUPPER ${name} uname) - set(var ${_c4_uprefix}CACHE_DOWNLOAD_${uname}) - set(val "${${var}}") - if(NOT ("${val}" STREQUAL "")) - c4_log("downloading ${name}: searching in ${var}=${val}") - if(EXISTS "${val}") - c4_log("downloading ${name}: picked ${sav} instead of ${${dir_in_out}}") - set(${dir_in_out} ${sav} PARENT_SCOPE) - endif() - endif() - # - # 2. search in the global directory (if there is one) - _c4_get_cached_srcdir_global_extern(${name} sav) - if(NOT ("${sav}" STREQUAL "")) - c4_log("downloading ${name}: searching in C4_EXTERN_DIR: ${sav}") - if(EXISTS "${sav}") - c4_log("downloading ${name}: picked ${sav} instead of ${${dir_in_out}}") - set(${dir_in_out} ${sav} PARENT_SCOPE) - endif() - endif() -endfunction() - - -function(_c4_get_cached_srcdir_global_extern name out) - set(${out} "" PARENT_SCOPE) - if("${C4_EXTERN_DIR}" STREQUAL "") - set(C4_EXTERN_DIR "$ENV{C4_EXTERN_DIR}") - endif() - if(NOT ("${C4_EXTERN_DIR}" STREQUAL "")) - set(${out} "${C4_EXTERN_DIR}/${name}" PARENT_SCOPE) - endif() -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -function(_c4_get_folder output importer_subproject subproject_name) - _c4_get_subproject_property(${importer_subproject} FOLDER importer_folder) - if("${importer_folder}" STREQUAL "") - set(folder ${importer_subproject}) - else() - set(folder "${importer_folder}/deps/${subproject_name}") - endif() - set(${output} ${folder} PARENT_SCOPE) -endfunction() - - -function(_c4_set_target_folder target subfolder) - string(FIND "${subfolder}" "/" pos) - if(pos EQUAL 0) - if("${_c4_curr_path}" STREQUAL "") - string(SUBSTRING "${subfolder}" 1 -1 sf) - set_target_properties(${target} PROPERTIES - FOLDER "${sf}") - else() - set_target_properties(${target} PROPERTIES - FOLDER "${subfolder}") - endif() - elseif("${subfolder}" STREQUAL "") - set_target_properties(${target} PROPERTIES - FOLDER "${_c4_curr_path}") - else() - if("${_c4_curr_path}" STREQUAL "") - set_target_properties(${target} PROPERTIES - FOLDER "${subfolder}") - else() - set_target_properties(${target} PROPERTIES - FOLDER "${_c4_curr_path}/${subfolder}") - endif() - endif() -endfunction() - - -function(c4_set_folder_remote_project_targets subfolder) - foreach(target ${ARGN}) - if(TARGET ${target}) - _c4_set_target_folder(${target} "${subfolder}") - endif() - endforeach() -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -# a convenience alias to c4_add_target() -function(c4_add_executable target) - c4_add_target(${target} EXECUTABLE ${ARGN}) -endfunction(c4_add_executable) - - -# a convenience alias to c4_add_target() -function(c4_add_library target) - c4_add_target(${target} LIBRARY ${ARGN}) -endfunction(c4_add_library) - - -# example: c4_add_target(ryml LIBRARY SOURCES ${SRC}) -function(c4_add_target target) - c4_dbg("adding target: ${target}: ${ARGN}") - set(opt0arg - LIBRARY # the target is a library - EXECUTABLE # the target is an executable - WIN32 # the executable is WIN32 - SANITIZE # deprecated - ) - set(opt1arg - LIBRARY_TYPE # override global setting for C4_LIBRARY_TYPE - SHARED_MACRO # the name of the macro to turn on export/import symbols - # for compiling the library as a windows DLL. - # defaults to ${_c4_uprefix}SHARED. - SHARED_EXPORTS # the name of the macro to turn on export of symbols - # for compiling the library as a windows DLL. - # defaults to ${_c4_uprefix}EXPORTS. - SOURCE_ROOT # the directory where relative source paths - # should be resolved. when empty, - # use CMAKE_CURRENT_SOURCE_DIR - FOLDER # IDE folder to group the target in - SANITIZERS # outputs the list of sanitize targets in this var - SOURCE_TRANSFORM # WIP - ) - set(optnarg - INCORPORATE # incorporate these libraries into this target, - # subject to ${_c4_uprefix}STANDALONE and C4_STANDALONE - SOURCES PUBLIC_SOURCES INTERFACE_SOURCES PRIVATE_SOURCES - HEADERS PUBLIC_HEADERS INTERFACE_HEADERS PRIVATE_HEADERS - INC_DIRS PUBLIC_INC_DIRS INTERFACE_INC_DIRS PRIVATE_INC_DIRS - LIBS PUBLIC_LIBS INTERFACE_LIBS PRIVATE_LIBS - DEFS PUBLIC_DEFS INTERFACE_DEFS PRIVATE_DEFS # defines - CFLAGS PUBLIC_CFLAGS INTERFACE_CFLAGS PRIVATE_CFLAGS # compiler flags. TODO: linker flags - DLLS # DLLs required by this target - MORE_ARGS - ) - cmake_parse_arguments("" "${opt0arg}" "${opt1arg}" "${optnarg}" ${ARGN}) - # - if(_SANITIZE) - c4_err("SANITIZE is deprecated") - endif() - - if(${_LIBRARY}) - set(_what LIBRARY) - elseif(${_EXECUTABLE}) - set(_what EXECUTABLE) - else() - c4_err("must be either LIBRARY or EXECUTABLE") - endif() - - _c4_handle_arg(SHARED_MACRO ${_c4_uprefix}MACRO) - _c4_handle_arg(SHARED_EXPORTS ${_c4_uprefix}EXPORTS) - _c4_handle_arg_or_fallback(SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}") - function(_c4_transform_to_full_path list all) - set(l) - foreach(f ${${list}}) - if(NOT IS_ABSOLUTE "${f}") - set(f "${_SOURCE_ROOT}/${f}") - endif() - list(APPEND l "${f}") - endforeach() - set(${list} "${l}" PARENT_SCOPE) - set(cp ${${all}}) - list(APPEND cp ${l}) - set(${all} ${cp} PARENT_SCOPE) - endfunction() - _c4_transform_to_full_path( _SOURCES allsrc) - _c4_transform_to_full_path( _HEADERS allsrc) - _c4_transform_to_full_path( _PUBLIC_SOURCES allsrc) - _c4_transform_to_full_path(_INTERFACE_SOURCES allsrc) - _c4_transform_to_full_path( _PRIVATE_SOURCES allsrc) - _c4_transform_to_full_path( _PUBLIC_HEADERS allsrc) - _c4_transform_to_full_path(_INTERFACE_HEADERS allsrc) - _c4_transform_to_full_path( _PRIVATE_HEADERS allsrc) - create_source_group("" "${_SOURCE_ROOT}" "${allsrc}") - # is the target name prefixed with the project prefix? - string(REGEX MATCH "${_c4_prefix}::.*" target_is_prefixed "${target}") - if(NOT ${_c4_uprefix}SANITIZE_ONLY) - if(${_EXECUTABLE}) - c4_dbg("adding executable: ${target}") - if(WIN32) - if(${_WIN32}) - list(APPEND _MORE_ARGS WIN32) - endif() - endif() - add_executable(${target} ${_MORE_ARGS}) - if(NOT target_is_prefixed) - add_executable(${_c4_prefix}::${target} ALIAS ${target}) - endif() - set(src_mode PRIVATE) - set(tgt_type PUBLIC) - set(compiled_target ON) - set_target_properties(${target} PROPERTIES VERSION ${${_c4_prefix}_VERSION}) - elseif(${_LIBRARY}) - c4_dbg("adding library: ${target}") - set(_blt ${C4_LIBRARY_TYPE}) # build library type - if(NOT "${_LIBRARY_TYPE}" STREQUAL "") - set(_blt ${_LIBRARY_TYPE}) - endif() - if("${_blt}" STREQUAL "") - endif() - # - if("${_blt}" STREQUAL "INTERFACE") - c4_dbg("adding interface library ${target}") - add_library(${target} INTERFACE) - set(src_mode INTERFACE) - set(tgt_type INTERFACE) - set(compiled_target OFF) - else() - if("${_blt}" STREQUAL "") - # obey BUILD_SHARED_LIBS (ie, either static or shared library) - c4_dbg("adding library ${target} (defer to BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}) --- ${_MORE_ARGS}") - add_library(${target} ${_MORE_ARGS}) - if(BUILD_SHARED_LIBS) - set(_blt SHARED) - else() - set(_blt STATIC) - endif() - else() - c4_dbg("adding library ${target} with type ${_blt}") - add_library(${target} ${_blt} ${_MORE_ARGS}) - endif() - # libraries - set(src_mode PRIVATE) - set(tgt_type PUBLIC) - set(compiled_target ON) - set_target_properties(${target} PROPERTIES VERSION ${${_c4_prefix}_VERSION}) - if("${_blt}" STREQUAL SHARED) - set_target_properties(${target} PROPERTIES SOVERSION ${${_c4_prefix}_VERSION}) - endif() - # exports for shared libraries - if(WIN32) - if("${_blt}" STREQUAL SHARED) - set_target_properties(${target} PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS ON) - target_compile_definitions(${target} PUBLIC ${_SHARED_MACRO}) - target_compile_definitions(${target} PRIVATE $) - # save the name of the macro for later use when(if) incorporating this library - c4_set_target_prop(${target} SHARED_EXPORTS ${_SHARED_EXPORTS}) - endif() # shared lib - endif() # win32 - endif() # interface or lib - if(NOT target_is_prefixed) - add_library(${_c4_prefix}::${target} ALIAS ${target}) - endif() - endif(${_EXECUTABLE}) - - if(src_mode STREQUAL "PUBLIC") - c4_add_target_sources(${target} - PUBLIC "${_SOURCES};${_HEADERS};${_PUBLIC_SOURCES};${_PUBLIC_HEADERS}" - INTERFACE "${_INTERFACE_SOURCES};${_INTERFACE_HEADERS}" - PRIVATE "${_PRIVATE_SOURCES};${_PRIVATE_HEADERS}") - elseif(src_mode STREQUAL "INTERFACE") - c4_add_target_sources(${target} - PUBLIC "${_PUBLIC_SOURCES};${_PUBLIC_HEADERS}" - INTERFACE "${_SOURCES};${_HEADERS};${_INTERFACE_SOURCES};${_INTERFACE_HEADERS}" - PRIVATE "${_PRIVATE_SOURCES};${_PRIVATE_HEADERS}") - elseif(src_mode STREQUAL "PRIVATE") - c4_add_target_sources(${target} - PUBLIC "${_PUBLIC_SOURCES};${_PUBLIC_HEADERS}" - INTERFACE "${_INTERFACE_SOURCES};${_INTERFACE_HEADERS}" - PRIVATE "${_SOURCES};${_HEADERS};${_PRIVATE_SOURCES};${_PRIVATE_HEADERS}") - elseif() - c4_err("${target}: adding sources: invalid source mode") - endif() - _c4_set_tgt_prop(${target} C4_SOURCE_ROOT "${_SOURCE_ROOT}") - - if(_INC_DIRS) - c4_dbg("${target}: adding include dirs ${_INC_DIRS} [from target: ${tgt_type}]") - target_include_directories(${target} "${tgt_type}" ${_INC_DIRS}) - endif() - if(_PUBLIC_INC_DIRS) - c4_dbg("${target}: adding PUBLIC include dirs ${_PUBLIC_INC_DIRS}") - target_include_directories(${target} PUBLIC ${_PUBLIC_INC_DIRS}) - endif() - if(_INTERFACE_INC_DIRS) - c4_dbg("${target}: adding INTERFACE include dirs ${_INTERFACE_INC_DIRS}") - target_include_directories(${target} INTERFACE ${_INTERFACE_INC_DIRS}) - endif() - if(_PRIVATE_INC_DIRS) - c4_dbg("${target}: adding PRIVATE include dirs ${_PRIVATE_INC_DIRS}") - target_include_directories(${target} PRIVATE ${_PRIVATE_INC_DIRS}) - endif() - - if(_LIBS) - _c4_link_with_libs(${target} "${tgt_type}" "${_LIBS}" "${_INCORPORATE}") - endif() - if(_PUBLIC_LIBS) - _c4_link_with_libs(${target} PUBLIC "${_PUBLIC_LIBS}" "${_INCORPORATE}") - endif() - if(_INTERFACE_LIBS) - _c4_link_with_libs(${target} INTERFACE "${_INTERFACE_LIBS}" "${_INCORPORATE}") - endif() - if(_PRIVATE_LIBS) - _c4_link_with_libs(${target} PRIVATE "${_PRIVATE_LIBS}" "${_INCORPORATE}") - endif() - - if(compiled_target) - if(_FOLDER) - _c4_set_target_folder(${target} "${_FOLDER}") - else() - _c4_set_target_folder(${target} "") - endif() - # cxx standard - c4_target_inherit_cxx_standard(${target}) - # compile flags - set(_more_flags - ${${_c4_uprefix}CXX_FLAGS} - ${${_c4_uprefix}C_FLAGS} - ${${_c4_uprefix}CXX_FLAGS_OPT}) - if(_more_flags) - get_target_property(_flags ${target} COMPILE_OPTIONS) - if(_flags) - set(_more_flags ${_flags};${_more_flags}) - endif() - c4_dbg("${target}: COMPILE_FLAGS=${_more_flags}") - target_compile_options(${target} PRIVATE "${_more_flags}") - endif() - # linker flags - set(_link_flags ${${_c4_uprefix}CXX_LINKER_FLAGS}) - if(_link_flags) - get_target_property(_flags ${target} LINK_OPTIONS) - if(_flags) - set(_link_flags ${_flags};${_more_flags}) - endif() - c4_dbg("${target}: LINKER_FLAGS=${_link_flags}") - target_link_options(${target} PUBLIC "${_link_flags}") - endif() - # static analysis - if(${_c4_uprefix}LINT) - c4_static_analysis_target(${target} "${_FOLDER}" lint_targets) - endif() - endif(compiled_target) - - if(_DEFS) - target_compile_definitions(${target} "${tgt_type}" ${_DEFS}) - endif() - if(_PUBLIC_DEFS) - target_compile_definitions(${target} PUBLIC ${_PUBLIC_DEFS}) - endif() - if(_INTERFACE_DEFS) - target_compile_definitions(${target} INTERFACE ${_INTERFACE_DEFS}) - endif() - if(_PRIVATE_DEFS) - target_compile_definitions(${target} PRIVATE ${_PRIVATE_DEFS}) - endif() - - if(_CFLAGS) - target_compile_options(${target} "${tgt_type}" ${_CFLAGS}) - endif() - if(_PUBLIC_CFLAGS) - target_compile_options(${target} PUBLIC ${_PUBLIC_CFLAGS}) - endif() - if(_INTERFACE_CFLAGS) - target_compile_options(${target} INTERFACE ${_INTERFACE_CFLAGS}) - endif() - if(_PRIVATE_CFLAGS) - target_compile_options(${target} PRIVATE ${_PRIVATE_CFLAGS}) - endif() - - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND - (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) AND - (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)) - c4_dbg("${target}: adding compat include path") - target_include_directories(${target} PUBLIC $) - endif() - - endif(NOT ${_c4_uprefix}SANITIZE_ONLY) - - if(compiled_target) - if(${_c4_uprefix}SANITIZE) - c4_sanitize_target(${target} - ${_what} # LIBRARY or EXECUTABLE - SOURCES ${allsrc} - INC_DIRS ${_INC_DIRS} ${_PUBLIC_INC_DIRS} ${_INTERFACE_INC_DIRS} ${_PRIVATE_INC_DIRS} - LIBS ${_LIBS} ${_PUBLIC_LIBS} ${_INTERFACE_LIBS} ${_PRIVATE_LIBS} - DEFS ${_DEFS} ${_PUBLIC_DEFS} ${_INTERFACE_DEFS} ${_PRIVATE_DEFS} - CFLAGS ${_CFLAGS} ${_PUBLIC_CFLAGS} ${_INTERFACE_CFLAGS} ${_PRIVATE_CFLAGS} - OUTPUT_TARGET_NAMES san_targets - FOLDER "${_FOLDER}" - ) - endif() - - if(NOT ${_c4_uprefix}SANITIZE_ONLY) - list(INSERT san_targets 0 ${target}) - endif() - - if(_SANITIZERS) - set(${_SANITIZERS} ${san_targets} PARENT_SCOPE) - endif() - - _c4_set_tgt_prop(${target} C4_SAN_TARGETS "${san_targets}") - else() - _c4_set_tgt_prop(${target} C4_SAN_TARGETS "${target}") - endif() - - # gather dlls so that they can be automatically copied to the target directory - if(_DLLS) - c4_append_transitive_property(${target} _C4_DLLS "${_DLLS}") - endif() - - if(${_EXECUTABLE}) - if(WIN32) - c4_get_transitive_property(${target} _C4_DLLS transitive_dlls) - list(REMOVE_DUPLICATES transitive_dlls) - foreach(_dll ${transitive_dlls}) - if(_dll) - c4_dbg("enable copy of dll to target file dir: ${_dll} ---> $") - add_custom_command(TARGET ${target} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy_if_different "${_dll}" "$" - ) - else() - message(WARNING "dll required by ${_c4_prefix}/${target} was not found, so cannot copy: ${_dll}") - endif() - endforeach() - endif() - endif() -endfunction() # add_target - - -function(_c4_link_with_libs target link_type libs incorporate) - foreach(lib ${libs}) - # add targets that are DLLs - if(WIN32) - if(TARGET ${lib}) - get_target_property(lib_type ${lib} TYPE) - if(lib_type STREQUAL SHARED_LIBRARY) - c4_append_transitive_property(${target} _C4_DLLS "$") - endif() - endif() - endif() - _c4_lib_is_incorporated(${lib} isinc) - if(isinc OR (incorporate AND ${_c4_uprefix}STANDALONE)) - c4_log("-----> target ${target} ${link_type} incorporating lib ${lib}") - _c4_incorporate_lib(${target} ${link_type} ${lib}) - else() - c4_dbg("${target} ${link_type} linking with lib ${lib}") - target_link_libraries(${target} ${link_type} ${lib}) - endif() - endforeach() -endfunction() - - -function(_c4_lib_is_incorporated lib ret) - c4_dbg("${lib}: is incorporated?") - if(NOT TARGET ${lib}) - c4_dbg("${lib}: no, not a target") - set(${ret} OFF PARENT_SCOPE) - else() - c4_get_target_prop(${lib} INCORPORATING_TARGETS inc) - if(inc) - c4_dbg("${lib}: is incorporated!") - set(${ret} ON PARENT_SCOPE) - else() - c4_dbg("${lib}: is not incorporated!") - set(${ret} OFF PARENT_SCOPE) - endif() - endif() -endfunction() - - -function(_c4_incorporate_lib target link_type lib) - c4_dbg("target ${target}: incorporating lib ${lib} [${link_type}]") - _c4_get_tgt_prop(srcroot ${lib} C4_SOURCE_ROOT) - # - c4_append_target_prop(${lib} INCORPORATING_TARGETS ${target}) - c4_append_target_prop(${target} INCORPORATED_TARGETS ${lib}) - # - _c4_get_tgt_prop(lib_src ${lib} SOURCES) - if(lib_src) - create_source_group("${lib}" "${srcroot}" "${lib_src}") - c4_add_target_sources(${target} INCORPORATED_FROM ${lib} PRIVATE ${lib_src}) - endif() - # - _c4_get_tgt_prop(lib_isrc ${lib} INTERFACE_SOURCES) - if(lib_isrc) - create_source_group("${lib}" "${srcroot}" "${lib_isrc}") - c4_add_target_sources(${target} INCORPORATED_FROM ${lib} INTERFACE ${lib_isrc}) - endif() - # - _c4_get_tgt_prop(lib_psrc ${lib} PRIVATE_SOURCES) - if(lib_psrc) - create_source_group("${lib}" "${srcroot}" "${lib_psrc}") - c4_add_target_sources(${target} INCORPORATED_FROM ${lib} INTERFACE ${lib_psrc}) - endif() - # - # - _c4_get_tgt_prop(lib_incs ${lib} INCLUDE_DIRECTORIES) - if(lib_incs) - target_include_directories(${target} PUBLIC ${lib_incs}) - endif() - # - _c4_get_tgt_prop(lib_iincs ${lib} INTERFACE_INCLUDE_DIRECTORIES) - if(lib_iincs) - target_include_directories(${target} INTERFACE ${lib_iincs}) - endif() - # - # - _c4_get_tgt_prop(lib_lib ${lib} LINK_LIBRARIES) - if(lib_lib) - target_link_libraries(${target} PUBLIC ${lib_lib}) - endif() - _c4_get_tgt_prop(lib_ilib ${lib} INTERFACE_LIBRARY) - if(lib_ilib) - target_link_libraries(${target} INTERFACE ${lib_ilib}) - endif() - # - # - c4_get_target_prop(${lib} SHARED_EXPORTS lib_exports) - if(lib_exports) - target_compile_definitions(${target} PRIVATE $) - endif() -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -# -# -function(c4_add_target_sources target) - # https://steveire.wordpress.com/2016/08/09/opt-in-header-only-libraries-with-cmake/ - _c4_handle_args(_ARGS ${ARGN} - _ARGS1 # one-value macro arguments - INCORPORATED_FROM - TRANSFORM # Transform types: - # * NONE - do not transform the sources - # * UNITY - # * UNITY_HDR - # * SINGLE_HDR - # * SINGLE_UNIT - _ARGSN # multi-value macro arguments - PUBLIC - INTERFACE - PRIVATE - ) - if(("${_TRANSFORM}" STREQUAL "GLOBAL") OR ("${_TRANSFORM}" STREQUAL "")) - set(_TRANSFORM ${C4_SOURCE_TRANSFORM}) - endif() - if("${_TRANSFORM}" STREQUAL "") - set(_TRANSFORM NONE) - endif() - # - # is this target an interface? - set(_is_iface FALSE) - _c4_get_tgt_prop(target_type ${target} TYPE) - if("${target_type}" STREQUAL "INTERFACE_LIBRARY") - set(_is_iface TRUE) - elseif("${prop_name}" STREQUAL "LINK_LIBRARIES") - set(_is_iface FALSE) - endif() - # - set(out) - set(umbrella ${_c4_lprefix}transform-src) - # - if("${_TRANSFORM}" STREQUAL "NONE") - c4_dbg("target ${target}: source transform: NONE!") - # - # do not transform the sources - # - if(_PUBLIC) - c4_dbg("target=${target} PUBLIC sources: ${_PUBLIC}") - c4_append_target_prop(${target} PUBLIC_SRC "${_PUBLIC}") - if(_INCORPORATED_FROM) - c4_append_target_prop(${target} PUBLIC_SRC_${_INCORPORATED_FROM} "${_PUBLIC}") - else() - c4_append_target_prop(${target} PUBLIC_SRC_${target} "${_PUBLIC}") - endif() - target_sources(${target} PUBLIC "${_PUBLIC}") - endif() - if(_INTERFACE) - c4_dbg("target=${target} INTERFACE sources: ${_INTERFACE}") - c4_append_target_prop(${target} INTERFACE_SRC "${_INTERFACE}") - if(_INCORPORATED_FROM) - c4_append_target_prop(${target} INTERFACE_SRC_${_INCORPORATED_FROM} "${_INTERFACE}") - else() - c4_append_target_prop(${target} INTERFACE_SRC_${target} "${_INTERFACE}") - endif() - target_sources(${target} INTERFACE "${_INTERFACE}") - endif() - if(_PRIVATE) - c4_dbg("target=${target} PRIVATE sources: ${_PRIVATE}") - c4_append_target_prop(${target} PRIVATE_SRC "${_PRIVATE}") - if(_INCORPORATED_FROM) - c4_append_target_prop(${target} PRIVATE_SRC_${_INCORPORATED_FROM} "${_PRIVATE}") - else() - c4_append_target_prop(${target} PRIVATE_SRC_${target} "${_PRIVATE}") - endif() - target_sources(${target} PRIVATE "${_PRIVATE}") - endif() - # - elseif("${_TRANSFORM}" STREQUAL "UNITY") - c4_dbg("target ${target}: source transform: UNITY!") - c4_err("source transformation not implemented") - # - # concatenate all compilation unit files (excluding interface) - # into a single compilation unit - # - _c4cat_filter_srcs("${_PUBLIC}" cpublic) - _c4cat_filter_hdrs("${_PUBLIC}" hpublic) - _c4cat_filter_srcs("${_INTERFACE}" cinterface) - _c4cat_filter_hdrs("${_INTERFACE}" hinterface) - _c4cat_filter_srcs("${_PRIVATE}" cprivate) - _c4cat_filter_hdrs("${_PRIVATE}" hprivate) - if(cpublic OR cinterface OR cprivate) - _c4cat_get_outname(${target} "src" ${C4_GEN_SRC_EXT} out) - c4_dbg("${target}: output unit: ${out}") - c4_cat_sources("${cpublic};${cinterface};${cprivate}" "${out}" ${umbrella}) - add_dependencies(${target} ${out}) - endif() - if(_PUBLIC) - c4_append_target_prop(${target} PUBLIC_SRC - $ - $) - target_sources(${target} PUBLIC - $ - $) - endif() - if(_INTERFACE) - c4_append_target_prop(${target} INTERFACE_SRC - $ - $) - target_sources(${target} INTERFACE - $ - $) - endif() - if(_PRIVATE) - c4_append_target_prop(${target} PRIVATE_SRC - $ - $) - target_sources(${target} PRIVATE - $ - $) - endif() - elseif("${_TRANSFORM}" STREQUAL "UNITY_HDR") - c4_dbg("target ${target}: source transform: UNITY_HDR!") - c4_err("target ${target}: source transformation not implemented") - # - # like unity, but concatenate compilation units into - # a header file, leaving other header files untouched - # - _c4cat_filter_srcs("${_PUBLIC}" cpublic) - _c4cat_filter_hdrs("${_PUBLIC}" hpublic) - _c4cat_filter_srcs("${_INTERFACE}" cinterface) - _c4cat_filter_hdrs("${_INTERFACE}" hinterface) - _c4cat_filter_srcs("${_PRIVATE}" cprivate) - _c4cat_filter_hdrs("${_PRIVATE}" hprivate) - if(c) - _c4cat_get_outname(${target} "src" ${C4_GEN_HDR_EXT} out) - c4_dbg("${target}: output hdr: ${out}") - _c4cat_filter_srcs_hdrs("${_PUBLIC}" c_h) - c4_cat_sources("${c}" "${out}" ${umbrella}) - add_dependencies(${target} ${out}) - add_dependencies(${target} ${_c4_lprefix}cat) - endif() - set(${src} ${out} PARENT_SCOPE) - set(${hdr} ${h} PARENT_SCOPE) - # - elseif("${_TRANSFORM}" STREQUAL "SINGLE_HDR") - c4_dbg("target ${target}: source transform: SINGLE_HDR!") - c4_err("target ${target}: source transformation not implemented") - # - # concatenate everything into a single header file - # - _c4cat_get_outname(${target} "all" ${C4_GEN_HDR_EXT} out) - _c4cat_filter_srcs_hdrs("${_c4al_SOURCES}" ch) - c4_cat_sources("${ch}" "${out}" ${umbrella}) - # - elseif("${_TRANSFORM}" STREQUAL "SINGLE_UNIT") - c4_dbg("target ${target}: source transform: SINGLE_UNIT!") - c4_err("target ${target}: source transformation not implemented") - # - # concatenate: - # * all compilation units into a single compilation unit - # * all headers into a single header - # - _c4cat_get_outname(${target} "src" ${C4_GEN_SRC_EXT} out) - _c4cat_get_outname(${target} "hdr" ${C4_GEN_SRC_EXT} out) - _c4cat_filter_srcs_hdrs("${_c4al_SOURCES}" ch) - c4_cat_sources("${ch}" "${out}" ${umbrella}) - else() - c4_err("unknown transform type: ${transform_type}. Must be one of GLOBAL;NONE;UNITY;TO_HEADERS;SINGLE_HEADER") - endif() -endfunction() - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# WIP, under construction (still incomplete) -# see: https://github.com/pr0g/cmake-examples -# see: https://cliutils.gitlab.io/modern-cmake/ - - -function(c4_install_target target) - _c4_handle_args(_ARGS ${ARGN} - _ARGS1 # one-value macro arguments - EXPORT # the name of the export target. default: see below. - ) - _c4_handle_arg(EXPORT "${_c4_prefix}-export") - # - c4_dbg("installing target: ${target} ${ARGN}") - #_c4_is_incorporated(${_c4_prefix} inc) - #if(inc) - # c4_dbg("this project is INCORPORATEd. skipping install of targets") - # return() - #endif() - # - _c4_setup_install_vars() - install(TARGETS ${target} - EXPORT ${_EXPORT} - RUNTIME DESTINATION ${_RUNTIME_INSTALL_DIR} #COMPONENT runtime - BUNDLE DESTINATION ${_RUNTIME_INSTALL_DIR} #COMPONENT runtime - LIBRARY DESTINATION ${_LIBRARY_INSTALL_DIR} #COMPONENT runtime - ARCHIVE DESTINATION ${_ARCHIVE_INSTALL_DIR} #COMPONENT development - OBJECTS DESTINATION ${_OBJECTS_INSTALL_DIR} #COMPONENT development - INCLUDES DESTINATION ${_INCLUDE_INSTALL_DIR} #COMPONENT development - PUBLIC_HEADER DESTINATION ${_INCLUDE_INSTALL_DIR} #COMPONENT development - ) - c4_install_sources(${target} include) - # - # on windows, install also required DLLs - if(WIN32) - get_target_property(target_type ${target} TYPE) - if("${target_type}" STREQUAL "EXECUTABLE") - c4_get_transitive_property(${target} _C4_DLLS transitive_dlls) - if(transitive_dlls) - c4_dbg("${target}: installing dlls: ${transitive_dlls} to ${_RUNTIME_INSTALL_DIR}") - list(REMOVE_DUPLICATES transitive_dlls) - install(FILES ${transitive_dlls} - DESTINATION ${_RUNTIME_INSTALL_DIR} # shouldn't it be _LIBRARY_INSTALL_DIR? - #COMPONENT runtime - ) - endif() - endif() - endif() - # - set(l ${${_c4_prefix}_TARGETS}) - list(APPEND l ${target}) - set(${_c4_prefix}_TARGETS ${l} PARENT_SCOPE) - # -# # pkgconfig (WIP) -# set(pc ${CMAKE_CURRENT_BINARY_DIR}/pkgconfig/${target}.pc) -# file(WRITE ${pc} "# pkg-config: ${target} -# -#prefix=\"${CMAKE_INSTALL_PREFIX}\" -#exec_prefix=\"\${_c4_prefix}\" -#libdir=\"\${_c4_prefix}/${CMAKE_INSTALL_LIBDIR}\" -#includedir=\"\${_c4_prefix}/include\" -# -#Name: ${target} -#Description: A library for xyzzying frobnixes -#URL: https://github.com/me/mylibrary -#Version: 0.0.0 -#Requires: @PKGCONF_REQ_PUB@ -#Requires.private: @PKGCONF_REQ_PRIV@ -#Cflags: -I\"${includedir}\" -#Libs: -L\"${libdir}\" -lmylibrary -#Libs.private: -L\"${libdir}\" -lmylibrary @PKGCONF_LIBS_PRIV@ -#") -# _c4_setup_install_vars() -# install(FILES ${pc} DESTINATION "${_ARCHIVE_INSTALL_DIR}/pkgconfig/") -endfunction() - - -function(c4_install_sources target destination) - c4_dbg("target ${target}: installing sources to ${destination}") - # executables have no sources requiring install - _c4_get_tgt_prop(target_type ${target} TYPE) - if(target_type STREQUAL "EXECUTABLE") - c4_dbg("target ${target}: is executable, skipping source install") - return() - endif() - # install source from the target and incorporated targets - c4_get_target_prop(${target} INCORPORATED_TARGETS inctargets) - if(inctargets) - set(targets "${inctargets};${target}") - else() - set(targets "${target}") - endif() - foreach(t ${targets}) - _c4_get_tgt_prop(srcroot ${t} C4_SOURCE_ROOT) - # get the sources from the target - # - c4_get_target_prop(${t} PUBLIC_SRC_${t} src) - if(src) - _c4cat_filter_hdrs("${src}" srcf) - _c4cat_filter_additional_exts("${src}" add) - c4_install_files("${srcf}" "${destination}" "${srcroot}") - c4_install_files("${add}" "${destination}" "${srcroot}") - endif() - # - c4_get_target_prop(${t} PRIVATE_SRC_${t} psrc) - if(psrc) - _c4cat_filter_hdrs("${psrc}" psrcf) - _c4cat_filter_additional_exts("${psrc}" add) - c4_install_files("${psrcf}" "${destination}" "${srcroot}") - c4_install_files("${add}" "${destination}" "${srcroot}") - endif() - # - c4_get_target_prop(${t} INTERFACE_SRC_${t} isrc) - if(isrc) - _c4cat_filter_srcs_hdrs("${isrc}" isrcf) - _c4cat_filter_additional_exts("${isrc}" add) - c4_install_files("${isrcf}" "${destination}" "${srcroot}") - c4_install_files("${add}" "${destination}" "${srcroot}") - endif() - # - c4_get_target_prop(${t} ADDFILES addfiles) - if(addfiles) - foreach(af ${addfiles}) - string(REGEX REPLACE "(.*)!!(.*)!!(.*)" "\\1;\\2;\\3" li "${af}") - list(GET li 0 files) - list(GET li 1 dst) - list(GET li 2 relative_to) - string(REPLACE "%%%" ";" files "${files}") - c4_install_files("${files}" "${dst}" "${relative_to}") - endforeach() - endif() - # - c4_get_target_prop(${t} ADDDIRS adddirs) - if(adddirs) - foreach(af ${adddirs}) - string(REGEX REPLACE "(.*)!!(.*)!!(.*)" "\\1;\\2;\\3" li "${af}") - list(GET li 0 dirs) - list(GET li 1 dst) - list(GET li 2 relative_to) - string(REPLACE "%%%" ";" dirs "${files}") - c4_install_dirs("${dirs}" "${dst}" "${relative_to}") - endforeach() - endif() - endforeach() -endfunction() - - -function(c4_install_target_add_files target files destination relative_to) - c4_dbg("installing additional files for target ${target}, destination=${destination}: ${files}") - string(REPLACE ";" "%%%" rfiles "${files}") - c4_append_target_prop(${target} ADDFILES "${rfiles}!!${destination}!!${relative_to}") - # - _c4_is_incorporated(${_c4_prefix} inc) - if(inc) - c4_dbg("this project is INCORPORATEd. skipping install of targets") - return() - endif() - c4_install_files("${files}" "${destination}" "${relative_to}") -endfunction() - - -function(c4_install_target_add_dirs target dirs destination relative_to) - c4_dbg("installing additional dirs for target ${target}, destination=${destination}: ${dirs}") - string(REPLACE ";" "%%%" rdirs "${dirs}") - c4_append_target_prop(${target} ADDDIRS "${rdirs}!!${destination}!!${relative_to}") - # - _c4_is_incorporated(${_c4_prefix} inc) - if(inc) - c4_dbg("this project is INCORPORATEd. skipping install of targets") - return() - endif() - c4_install_dirs("${dirs}" "${destination}" "${relative_to}") -endfunction() - - -function(c4_install_files files destination relative_to) - c4_dbg("adding files to install list, destination ${destination}: ${files}") - foreach(f ${files}) - file(RELATIVE_PATH rf "${relative_to}" ${f}) - get_filename_component(rd "${rf}" DIRECTORY) - install(FILES ${f} DESTINATION "${destination}/${rd}" ${ARGN}) - endforeach() -endfunction() - - -function(c4_install_directories directories destination relative_to) - c4_dbg("adding directories to install list, destination ${destination}: ${directories}") - foreach(d ${directories}) - file(RELATIVE_PATH rf "${relative_to}" ${d}) - get_filename_component(rd "${rf}" DIRECTORY) - install(DIRECTORY ${d} DESTINATION "${destination}/${rd}" ${ARGN}) - endforeach() -endfunction() - - -function(c4_install_exports) - _c4_handle_args(_ARGS ${ARGN} - _ARGS1 # one-value macro arguments - PREFIX # override the c4 project-wide prefix. This will be used in the cmake - TARGET # the name of the exports target - NAMESPACE # the namespace for the targets - _ARGSN # multi-value macro arguments - DEPENDENCIES - ) - # - _c4_handle_arg(PREFIX "${_c4_prefix}") - _c4_handle_arg(TARGET "${_c4_prefix}-export") - _c4_handle_arg(NAMESPACE "${_c4_prefix}::") - # - c4_dbg("installing exports: ${ARGN}") - #_c4_is_incorporated(${_c4_prefix} inc) - #if(inc) - # c4_dbg("this project is INCORPORATEd. skipping install of exports") - # return() - #endif() - # - _c4_setup_install_vars() - # - list(GET ${_c4_prefix}_TARGETS 0 target) - set(exported_target "${_NAMESPACE}${target}") - set(targets_file "${_PREFIX}Targets.cmake") - # - set(deps) - if(_DEPENDENCIES) - set(deps "#----------------------------- -include(CMakeFindDependencyMacro) -") - foreach(d ${_DEPENDENCIES}) - _c4_is_incorporated(${d} inc) - if(inc) - c4_dbg("install: dependency ${d} is INCORPORATEd, skipping check") - continue() - endif() - c4_dbg("install: adding dependency check for ${d}") - set(deps "${deps}find_dependency(${d} REQUIRED) -") - endforeach() - set(deps "${deps}#-----------------------------") - endif() - # - # cfg_dst is the path relative to install root where the export - # should be installed; cfg_dst_rel is the path from there to - # the install root - macro(__c4_install_exports cfg_dst cfg_dst_rel) - # make sure that different exports are staged in different directories - set(case ${CMAKE_CURRENT_BINARY_DIR}/export_cases/${cfg_dst}) - file(MAKE_DIRECTORY ${case}) - # - install(EXPORT "${_TARGET}" - FILE "${targets_file}" - NAMESPACE "${_NAMESPACE}" - DESTINATION "${cfg_dst}") - export(EXPORT ${_TARGET} - FILE "${targets_file}" - NAMESPACE "${_NAMESPACE}") - # - # Config files - # the module below has nice docs in it; do read them - # to understand the macro calls below - include(CMakePackageConfigHelpers) - set(cfg ${case}/${_PREFIX}Config.cmake) - set(cfg_ver ${case}/${_PREFIX}ConfigVersion.cmake) - # - file(WRITE ${cfg}.in "${deps} -set(${_c4_uprefix}VERSION ${${_c4_uprefix}VERSION}) - -@PACKAGE_INIT@ - -if(NOT TARGET ${exported_target}) - include(\${PACKAGE_PREFIX_DIR}/${targets_file}) -endif() - -# HACK: PACKAGE_PREFIX_DIR is obtained from the PACKAGE_INIT macro above; -# When used below in the calls to set_and_check(), -# it points at the location of this file. So point it instead -# to the CMAKE_INSTALL_PREFIX, in relative terms -get_filename_component(PACKAGE_PREFIX_DIR - \"\${PACKAGE_PREFIX_DIR}/${cfg_dst_rel}\" ABSOLUTE) - -set_and_check(${_c4_uprefix}INCLUDE_DIR \"@PACKAGE__INCLUDE_INSTALL_DIR@\") -set_and_check(${_c4_uprefix}LIB_DIR \"@PACKAGE__LIBRARY_INSTALL_DIR@\") -#set_and_check(${_c4_uprefix}SYSCONFIG_DIR \"@PACKAGE__SYSCONFIG_INSTALL_DIR@\") - -check_required_components(${_c4_lcprefix}) -") - configure_package_config_file(${cfg}.in ${cfg} - INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}" # defaults to CMAKE_INSTALL_PREFIX - INSTALL_DESTINATION "${CMAKE_INSTALL_PREFIX}" - PATH_VARS - _INCLUDE_INSTALL_DIR - _LIBRARY_INSTALL_DIR - _SYSCONFIG_INSTALL_DIR - #NO_SET_AND_CHECK_MACRO - #NO_CHECK_REQUIRED_COMPONENTS_MACRO - ) - write_basic_package_version_file( - ${cfg_ver} - VERSION ${${_c4_uprefix}VERSION} - COMPATIBILITY AnyNewerVersion - ) - install(FILES ${cfg} ${cfg_ver} DESTINATION ${cfg_dst}) - endmacro(__c4_install_exports) - # - # To install the exports: - # - # Windows: - # / - # /(cmake|CMake)/ - # /*/ - # /*/(cmake|CMake)/ - # - # Unix: - # /(lib/|lib|share)/cmake/*/ - # /(lib/|lib|share)/*/ - # /(lib/|lib|share)/*/(cmake|CMake)/ - # - # Apple: - # /.framework/Resources/ - # /.framework/Resources/CMake/ - # /.framework/Versions/*/Resources/ - # /.framework/Versions/*/Resources/CMake/ - # /.app/Contents/Resources/ - # /.app/Contents/Resources/CMake/ - # - # (This was taken from the find_package() documentation) - if(WIN32) - __c4_install_exports(cmake/ "..") - elseif(APPLE) - __c4_install_exports(${_ARCHIVE_INSTALL_DIR}/cmake/${_c4_prefix} "../../..") - #__c4_install_exports(${_ARCHIVE_INSTALL_DIR}/${_c4_prefix}.framework/Resources/ "../../..") - elseif(UNIX OR (CMAKE_SYSTEM_NAME STREQUAL UNIX) OR (CMAKE_SYSTEM_NAME STREQUAL Linux) OR (CMAKE_SYSTEM_NAME STREQUAL Generic)) - __c4_install_exports(${_ARCHIVE_INSTALL_DIR}/cmake/${_c4_prefix} "../../..") - else() - c4_err("unknown platform. CMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME} CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}") - endif() -endfunction() - - -macro(_c4_setup_install_vars) - set(_RUNTIME_INSTALL_DIR bin/) - set(_ARCHIVE_INSTALL_DIR lib/) - set(_LIBRARY_INSTALL_DIR lib/) # TODO on Windows, ARCHIVE and LIBRARY dirs must be different to prevent name clashes - set(_INCLUDE_INSTALL_DIR include/) - set(_OBJECTS_INSTALL_DIR obj/) - set(_SYSCONFIG_INSTALL_DIR etc/${_c4_lcprefix}/) -endmacro() - - -function(c4_get_target_installed_headers target out) - c4_get_target_prop(${target} INCORPORATED_TARGETS inctargets) - if(inctargets) - set(targets "${inctargets};${target}") - else() - set(targets "${target}") - endif() - set(hdrs) - foreach(t ${targets}) - _c4_get_tgt_prop(srcroot ${t} C4_SOURCE_ROOT) - # - c4_get_target_prop(${t} PUBLIC_SRC_${t} src) - if(src) - _c4cat_filter_hdrs("${src}" srcf) - if(thdrs) - set(thdrs "${thdrs};${srcf}") - else() - set(thdrs "${srcf}") - endif() - endif() - # - c4_get_target_prop(${t} PRIVATE_SRC_${t} psrc) - if(src) - _c4cat_filter_hdrs("${psrc}" psrcf) - if(thdrs) - set(thdrs "${thdrs};${psrcf}") - else() - set(thdrs "${psrcf}") - endif() - endif() - # - c4_get_target_prop(${t} INTERFACE_SRC_${t} isrc) - if(src) - _c4cat_filter_hdrs("${isrc}" isrcf) - if(thdrs) - set(thdrs "${thdrs};${isrcf}") - else() - set(thdrs "${isrcf}") - endif() - endif() - # - foreach(h ${thdrs}) - file(RELATIVE_PATH rf "${srcroot}" "${h}") - list(APPEND hdrs "${rf}") - endforeach() - endforeach() - set(${out} ${hdrs} PARENT_SCOPE) -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -function(c4_setup_testing) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 - GTEST # download and import googletest - DOCTEST # download and import doctest - _ARGS1 - _ARGSN - ) - #include(GoogleTest) # this module requires at least cmake 3.9 - c4_dbg("enabling tests") - # umbrella target for building test binaries - add_custom_target(${_c4_lprefix}test-build) - _c4_set_target_folder(${_c4_lprefix}test-build test) - # umbrella targets for running tests - if(NOT TARGET test-build) - add_custom_target(test-build) - add_custom_target(test-verbose) - _c4_set_target_folder(test-build "/test") - _c4_set_target_folder(test-verbose "/test") - endif() - if(NOT TARGET test) - # add a test target. To prevent a warning, we need to set up a policy, - # and also suppress the resulting warning from suppressing the warning. - set(_depr_old_val ${CMAKE_WARN_DEPRECATED}) - set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE) # https://stackoverflow.com/questions/67432538/cannot-set-cmake-warn-deprecated-inside-the-cmakelists-txt - cmake_policy(PUSH) - cmake_policy(SET CMP0037 OLD) # target name "test" is reserved for CTesting - add_custom_target(test) - _c4_set_target_folder(test "/test") - cmake_policy(POP) - set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "${_depr_old_val}" FORCE) - unset(_depr_old_val) - endif() - function(_def_runner runner) - set(echo " -CWD=${CMAKE_CURRENT_BINARY_DIR} ----------------------------------- -${ARGN} ----------------------------------- -") - add_custom_target(${runner} - #${CMAKE_COMMAND} -E echo "${echo}" - COMMAND ${ARGN} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS ${_c4_lprefix}test-build - ) - _c4_set_target_folder(${runner} test) - endfunction() - _def_runner(${_c4_lprefix}test-run ${CMAKE_CTEST_COMMAND} --output-on-failure ${${_c4_uprefix}CTEST_OPTIONS} -C $) - _def_runner(${_c4_lprefix}test-run-verbose ${CMAKE_CTEST_COMMAND} -VV ${${_c4_uprefix}CTEST_OPTIONS} -C $) - add_dependencies(test ${_c4_lprefix}test-run) - add_dependencies(test-verbose ${_c4_lprefix}test-run-verbose) - add_dependencies(test-build ${_c4_lprefix}test-build) - # - # import required libraries - if(_GTEST) - c4_log("testing requires googletest") - if(NOT TARGET gtest) - c4_import_remote_proj(gtest ${CMAKE_CURRENT_BINARY_DIR}/ext/gtest - REMOTE - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG release-1.11.0 #GIT_SHALLOW ON - OVERRIDE - BUILD_GTEST ON - BUILD_GMOCK OFF - gtest_force_shared_crt ON - gtest_build_samples OFF - gtest_build_tests OFF - SET_FOLDER_TARGETS ext gtest gtest_main - EXCLUDE_FROM_ALL - ) - # old gcc-4.8 support - if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND - (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) AND - (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)) - _c4_get_subproject_property(gtest SRC_DIR _gtest_patch_src_dir) - apply_patch("${_c4_project_dir}/compat/gtest_gcc-4.8.patch" - "${_gtest_patch_src_dir}" - "${_gtest_patch_src_dir}/.gtest_gcc-4.8.patch") - unset(_gtest_patch_src_dir) - target_compile_options(gtest PUBLIC -include ${_c4_project_dir}/compat/c4/gcc-4.8.hpp) - endif() - endif() - endif() - if(_DOCTEST) - c4_log("testing requires doctest") - if(NOT TARGET doctest) - c4_import_remote_proj(doctest ${CMAKE_CURRENT_BINARY_DIR}/ext/doctest - REMOTE - GIT_REPOSITORY https://github.com/onqtam/doctest.git - GIT_TAG 2.4.6 #GIT_SHALLOW ON - OVERRIDE - DOCTEST_WITH_TESTS OFF - DOCTEST_WITH_MAIN_IN_STATIC_LIB ON - SET_FOLDER_TARGETS ext doctest_with_main - EXCLUDE_FROM_ALL - ) - endif() - endif() -endfunction(c4_setup_testing) - - -function(c4_add_test target) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 # zero-value macro arguments - _ARGS1 # one-value macro arguments - WORKING_DIRECTORY - _ARGSN # multi-value macro arguments - ARGS - ) - # - if(_WORKING_DIRECTORY) - set(_WORKING_DIRECTORY WORKING_DIRECTORY ${_WORKING_DIRECTORY}) - endif() - set(cmd_pfx) - if(CMAKE_CROSSCOMPILING) - set(cmd_pfx ${CMAKE_CROSSCOMPILING_EMULATOR}) - endif() - if(NOT ${uprefix}SANITIZE_ONLY) - if(${CMAKE_VERSION} VERSION_LESS "3.16.0") - add_test(NAME ${target} - COMMAND ${cmd_pfx} "$" ${_ARGS} - ${_WORKING_DIRECTORY}) - else() - add_test(NAME ${target} - COMMAND ${cmd_pfx} "$" ${_ARGS} - ${_WORKING_DIRECTORY} - COMMAND_EXPAND_LISTS) - endif() - endif() - # - if("${CMAKE_BUILD_TYPE}" STREQUAL "Coverage") - add_dependencies(${_c4_lprefix}test-build ${target}) - return() - endif() - # - set(sanitized_targets) - foreach(s asan msan tsan ubsan) - set(t ${target}-${s}) - if(TARGET ${t}) - list(APPEND sanitized_targets ${s}) - endif() - endforeach() - if(sanitized_targets) - add_custom_target(${target}-all) - add_dependencies(${target}-all ${target}) - add_dependencies(${_c4_lprefix}test-build ${target}-all) - _c4_set_target_folder(${target}-all test/${target}) - else() - add_dependencies(${_c4_lprefix}test-build ${target}) - endif() - if(sanitized_targets) - foreach(s asan msan tsan ubsan) - set(t ${target}-${s}) - if(TARGET ${t}) - add_dependencies(${target}-all ${t}) - c4_sanitize_get_target_command("${cmd_pfx};$" ${s} cmd) - #c4_log("adding test: ${t}") - add_test(NAME ${t} - COMMAND ${cmd} ${_ARGS} - ${_WORKING_DIRECTORY} - COMMAND_EXPAND_LISTS) - endif() - endforeach() - endif() - if(NOT CMAKE_CROSSCOMPILING) - if(NOT ${_c4_uprefix}SANITIZE_ONLY) - c4_add_valgrind(${target} ${ARGN}) - endif() - endif() - if(${_c4_uprefix}LINT) - c4_static_analysis_add_tests(${target}) # this will not actually run the executable - endif() -endfunction(c4_add_test) - - -# every excess argument is passed on to set_target_properties() -function(c4_add_test_fail_build name srccontent_or_srcfilename) - # - set(sdir ${CMAKE_CURRENT_BINARY_DIR}/test_fail_build) - set(src ${srccontent_or_srcfilename}) - if("${src}" STREQUAL "") - c4_err("must be given an existing source file name or a non-empty string") - endif() - # - if(EXISTS ${src}) - set(fn ${src}) - else() - if(NOT EXISTS ${sdir}) - file(MAKE_DIRECTORY ${sdir}) - endif() - set(fn ${sdir}/${name}.cpp) - file(WRITE ${fn} "${src}") - endif() - # - # https://stackoverflow.com/questions/30155619/expected-build-failure-tests-in-cmake - add_executable(${name} ${fn}) - # don't build this target - set_target_properties(${name} PROPERTIES - EXCLUDE_FROM_ALL TRUE - EXCLUDE_FROM_DEFAULT_BUILD TRUE - # and pass on further properties given by the caller - ${ARGN}) - add_test(NAME ${name} - COMMAND ${CMAKE_COMMAND} --build . --target ${name} --config $ - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - set_tests_properties(${name} PROPERTIES WILL_FAIL TRUE) -endfunction() - - -# add a test ensuring that a target linking and using code from a library -# successfully compiles and runs against the installed library -function(c4_add_install_link_test library namespace exe_source_code) - if(CMAKE_CROSSCOMPILING) - c4_log("cross-compiling: skip install link test") - return() - endif() - if("${library}" STREQUAL "${_c4_prefix}") - set(testname ${_c4_lprefix}test-install-link) - else() - set(testname ${_c4_lprefix}test-install-link-${library}) - endif() - _c4_add_library_client_test(${library} "${namespace}" "${testname}" "${exe_source_code}") -endfunction() - - -# add a test ensuring that a target consuming every header in a library -# successfully compiles and runs against the installed library -function(c4_add_install_include_test library namespace) - if(CMAKE_CROSSCOMPILING) - c4_log("cross-compiling: skip install include test") - return() - endif() - c4_get_target_installed_headers(${library} incfiles) - set(incblock) - foreach(i ${incfiles}) - set(incblock "${incblock} -#include <${i}>") - endforeach() - set(src "${incblock} - -int main() -{ - return 0; -} -") - if("${library}" STREQUAL "${_c4_prefix}") - set(testname ${_c4_lprefix}test-install-include) - else() - set(testname ${_c4_lprefix}test-install-include-${library}) - endif() - _c4_add_library_client_test(${library} "${namespace}" "${testname}" "${src}") -endfunction() - - -function(_c4_add_library_client_test library namespace pname source_code) - if("${CMAKE_BUILD_TYPE}" STREQUAL Coverage) - add_test(NAME ${pname} - COMMAND ${CMAKE_COMMAND} -E echo "skipping this test in coverage builds" - ) - return() - endif() - set(pdir "${CMAKE_CURRENT_BINARY_DIR}/${pname}") - set(bdir "${pdir}/build") - if(NOT EXISTS "${pdir}") - file(MAKE_DIRECTORY "${pdir}") - endif() - if(NOT EXISTS "${bdir}/build") - file(MAKE_DIRECTORY "${bdir}/build") - endif() - set(psrc "${pdir}/${pname}.cpp") - set(tsrc "${pdir}/${pname}-run.cmake") - set(tout "${pdir}/${pname}-run-out") - # generate the source file - file(WRITE "${psrc}" "${source_code}") - # generate the cmake project consuming this library - file(WRITE "${pdir}/CMakeLists.txt" " -cmake_minimum_required(VERSION 3.12) -project(${pname} LANGUAGES CXX) - -find_package(${library} REQUIRED) - -message(STATUS \" -found ${library}: - ${_c4_uprefix}INCLUDE_DIR=\${${_c4_uprefix}INCLUDE_DIR} - ${_c4_uprefix}LIB_DIR=\${${_c4_uprefix}LIB_DIR} -\") - -add_executable(${pname} ${pname}.cpp) -# this must be the only required setup to link with ${library} -target_link_libraries(${pname} PUBLIC ${namespace}${library}) - -get_target_property(lib_type ${namespace}${library} TYPE) -if(WIN32 AND (lib_type STREQUAL SHARED_LIBRARY)) - # add the directory containing the DLL to the path - get_target_property(imported_configs ${namespace}${library} IMPORTED_CONFIGURATIONS) - message(STATUS \"${namespace}${library}: it's a shared library. imported configs: \${imported_configs}\") - foreach(cfg \${imported_configs}) - get_target_property(implib ${namespace}${library} IMPORTED_IMPLIB_\${cfg}) - get_target_property(location ${namespace}${library} IMPORTED_LOCATION_\${cfg}) - message(STATUS \"${namespace}${library}: implib_\${cfg}=\${implib}\") - message(STATUS \"${namespace}${library}: location_\${cfg}=\${location}\") - break() - endforeach() - get_filename_component(dlldir \"\${location}\" DIRECTORY) - message(STATUS \"${namespace}${library}: dlldir=\${dlldir}\") - add_custom_target(${pname}-run - COMMAND \${CMAKE_COMMAND} -E echo \"cd \${dlldir} && \$\" - COMMAND \$ - DEPENDS ${pname} - WORKING_DIRECTORY \${dlldir}) -else() - add_custom_target(${pname}-run - COMMAND \$ - DEPENDS ${pname}) -endif() -") - # The test consists in running the script generated below. - # We force evaluation of the configuration generator expression - # by receiving its result via the command line. - add_test(NAME ${pname} - COMMAND ${CMAKE_COMMAND} -DCFG_IN=$ -P "${tsrc}" - ) - # NOTE: in the cmake configure command, be sure to NOT use quotes - # in -DCMAKE_PREFIX_PATH=\"${CMAKE_INSTALL_PREFIX}\". Use - # -DCMAKE_PREFIX_PATH=${CMAKE_INSTALL_PREFIX} instead. - # So here we add a check to make sure the install path has no spaces - string(FIND "${CMAKE_INSTALL_PREFIX}" " " has_spaces) - if(NOT (has_spaces EQUAL -1)) - c4_err("install tests will fail if the install path has spaces: '${CMAKE_INSTALL_PREFIX}' : ... ${has_spaces}") - endif() - # make sure the test project uses the same architecture - # CMAKE_VS_PLATFORM_NAME is available only since cmake 3.9 - # see https://cmake.org/cmake/help/v3.9/variable/CMAKE_GENERATOR_PLATFORM.html - if(WIN32) - set(cfg_opt "--config \${cfg}") - if(CMAKE_GENERATOR_PLATFORM OR CMAKE_VS_PLATFORM_NAME) - set(arch "-DCMAKE_GENERATOR_PLATFORM=${CMAKE_GENERATOR_PLATFORM}" "-DCMAKE_VS_PLATFORM_NAME=${CMAKE_VS_PLATFORM_NAME}") - else() - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(arch -A x64) - elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) - set(arch -A Win32) - else() - c4_err("not implemented") - endif() - endif() - elseif(ANDROID OR IOS OR WINCE OR WINDOWS_PHONE) - c4_err("not implemented") - elseif(IOS) - c4_err("not implemented") - elseif(UNIX) - if(CMAKE_GENERATOR_PLATFORM OR CMAKE_VS_PLATFORM_NAME) - set(arch "-DCMAKE_GENERATOR_PLATFORM=${CMAKE_GENERATOR_PLATFORM}" "-DCMAKE_VS_PLATFORM_NAME=${CMAKE_VS_PLATFORM_NAME}") - else() - if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64) - else() - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - set(arch "-DCMAKE_CXX_FLAGS=-m64") - elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) - set(arch "-DCMAKE_CXX_FLAGS=-m32") - else() - c4_err("not implemented") - endif() - endif() - endif() - endif() - # generate the cmake script with the test content - file(WRITE "${tsrc}" " -# run a command and check its return status -function(runcmd id) - set(cmdout \"${tout}-\${id}.log\") - message(STATUS \"Running command: \${ARGN}\") - message(STATUS \"Running command: output goes to \${cmdout}\") - execute_process( - COMMAND \${ARGN} - RESULT_VARIABLE retval - OUTPUT_FILE \"\${cmdout}\" - ERROR_FILE \"\${cmdout}\" - # COMMAND_ECHO STDOUT # only available from cmake-3.15 - ) - message(STATUS \"Running command: exit status was \${retval}\") - file(READ \"\${cmdout}\" output) - if(\"\${cmdout}\" STREQUAL \"\") - message(STATUS \"Running command: no output\") - else() - message(STATUS \"Running command: output: --------------------- -\${output}--------------------\") - endif() - if(NOT (\${retval} EQUAL 0)) - message(FATAL_ERROR \"Command failed with exit status \${retval}: \${ARGN}\") - endif() -endfunction() - -set(cmk \"${CMAKE_COMMAND}\") -set(pfx \"${CMAKE_INSTALL_PREFIX}\") -set(idir \"${CMAKE_BINARY_DIR}\") -set(pdir \"${pdir}\") -set(bdir \"${bdir}\") - -# force evaluation of the configuration generator expression -# by receiving its result via the command line -set(cfg \${CFG_IN}) - -# remove any existing library install -if(EXISTS \"\${pfx}\") - runcmd(0_rmdir \"\${cmk}\" -E remove_directory \"\${pfx}\") -else() - message(STATUS \"does not exist: \${pfx}\") -endif() - -# install the library -#runcmd(1_install_lib \"\${cmk}\" --install \"\${idir}\" ${cfg_opt}) # requires cmake>3.13 (at least) -runcmd(1_install_lib \"\${cmk}\" --build \"\${idir}\" ${cfg_opt} --target install) - -# configure the client project -runcmd(2_config \"\${cmk}\" -S \"\${pdir}\" -B \"\${bdir}\" \"-DCMAKE_PREFIX_PATH=\${pfx}\" \"-DCMAKE_GENERATOR=${CMAKE_GENERATOR}\" ${arch} \"-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}\" \"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}\") - -# build the client project -runcmd(3_build \"\${cmk}\" --build \"\${bdir}\" ${cfg_opt}) - -# run the client executable -runcmd(4_install \"\${cmk}\" --build \"\${bdir}\" --target \"${pname}-run\" ${cfg_opt}) -") -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -function(c4_setup_valgrind umbrella_option) - if(UNIX AND (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "Coverage")) - if("${C4_VALGRIND}" STREQUAL "") - option(C4_VALGRIND "enable valgrind tests (all subprojects)" ON) - endif() - if("${C4_VALGRIND_SGCHECK}" STREQUAL "") - option(C4_VALGRIND_SGCHECK "enable valgrind tests with the exp-sgcheck tool (all subprojects)" OFF) - endif() - cmake_dependent_option(${_c4_uprefix}VALGRIND "enable valgrind tests" ${C4_VALGRIND} ${umbrella_option} OFF) - cmake_dependent_option(${_c4_uprefix}VALGRIND_SGCHECK "enable valgrind tests with the exp-sgcheck tool" ${C4_VALGRIND_SGCHECK} ${umbrella_option} OFF) - set(${_c4_uprefix}VALGRIND_OPTIONS "--gen-suppressions=all --error-exitcode=10101" CACHE STRING "options for valgrind tests") - endif() -endfunction(c4_setup_valgrind) - - -function(c4_add_valgrind target_name) - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 # zero-value macro arguments - _ARGS1 # one-value macro arguments - WORKING_DIRECTORY - _ARGSN # multi-value macro arguments - ARGS - ) - # - if(_WORKING_DIRECTORY) - set(_WORKING_DIRECTORY WORKING_DIRECTORY ${_WORKING_DIRECTORY}) - endif() - # @todo: consider doing this for valgrind: - # http://stackoverflow.com/questions/40325957/how-do-i-add-valgrind-tests-to-my-cmake-test-target - # for now we explicitly run it: - if(${_c4_uprefix}VALGRIND) - separate_arguments(_vg_opts UNIX_COMMAND "${${_c4_uprefix}VALGRIND_OPTIONS}") - add_test(NAME ${target_name}-valgrind - COMMAND valgrind ${_vg_opts} $ ${_ARGS} - ${_WORKING_DIRECTORY} - COMMAND_EXPAND_LISTS) - endif() - if(${_c4_uprefix}VALGRIND_SGCHECK) - # stack and global array overrun detector - # http://valgrind.org/docs/manual/sg-manual.html - separate_arguments(_sg_opts UNIX_COMMAND "--tool=exp-sgcheck ${${_c4_uprefix}VALGRIND_OPTIONS}") - add_test(NAME ${target_name}-sgcheck - COMMAND valgrind ${_sg_opts} $ ${_ARGS} - ${_WORKING_DIRECTORY} - COMMAND_EXPAND_LISTS) - endif() -endfunction(c4_add_valgrind) - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -function(c4_setup_coverage) - if(NOT ("${CMAKE_BUILD_TYPE}" STREQUAL "Coverage")) - return() - endif() - # - _c4_handle_args(_ARGS ${ARGN} - _ARGS0 # zero-value macro arguments - _ARGS1 # one-value macro arguments - _ARGSN # multi-value macro arguments - COVFLAGS # coverage compilation flags - INCLUDE # patterns to include in the coverage, relative to CMAKE_SOURCE_DIR - EXCLUDE # patterns to exclude in the coverage, relative to CMAKE_SOURCE_DIR - EXCLUDE_ABS # absolute paths to exclude in the coverage - GENHTML_ARGS # options to pass to genhtml - ) - # defaults for the macro arguments - set(_genhtml_args "--title ${_c4_lcprefix} --demangle-cpp --sort --function-coverage --branch-coverage --prefix '${CMAKE_SOURCE_DIR}' --prefix '${CMAKE_BINARY_DIR}'") - set(covflags "-g -O0 --coverage") #set(covflags "-g -O0 -fprofile-arcs -ftest-coverage") - if(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(covflags "${covflags} -fprofile-arcs -ftest-coverage -fno-inline -fno-inline-small-functions -fno-default-inline") - endif() - set(${_c4_uprefix}COVERAGE_FLAGS "${covflags}" CACHE STRING "coverage compilation flags") - set(${_c4_uprefix}COVERAGE_GENHTML_ARGS "${_genhtml_args}" CACHE STRING "arguments to pass to genhtml") - set(${_c4_uprefix}COVERAGE_INCLUDE src CACHE STRING "relative paths to include in the coverage, relative to CMAKE_SOURCE_DIR") - set(${_c4_uprefix}COVERAGE_EXCLUDE bm;build;extern;ext;src/c4/ext;test CACHE STRING "relative paths to exclude from the coverage, relative to CMAKE_SOURCE_DIR") - set(${_c4_uprefix}COVERAGE_EXCLUDE_ABS /usr CACHE STRING "absolute paths to exclude from the coverage") - _c4_handle_arg(COVFLAGS ${${_c4_uprefix}COVERAGE_FLAGS}) - _c4_handle_arg(INCLUDE ${${_c4_uprefix}COVERAGE_INCLUDE}) - _c4_handle_arg(EXCLUDE ${${_c4_uprefix}COVERAGE_EXCLUDE}) - _c4_handle_arg(EXCLUDE_ABS ${${_c4_uprefix}COVERAGE_EXCLUDE_ABS} "${CMAKE_BINARY_DIR}") - _c4_handle_arg(GENHTML_ARGS ${${_c4_uprefix}COVERAGE_GENHTML_ARGS}) - # - function(_c4cov_transform_filters var reldir) - set(_filters) - foreach(pat ${${var}}) - list(APPEND _filters "'${reldir}${pat}/*'") - endforeach() - set(${var} ${_filters} PARENT_SCOPE) - endfunction() - _c4cov_transform_filters(_INCLUDE "${CMAKE_SOURCE_DIR}/") - _c4cov_transform_filters(_EXCLUDE "${CMAKE_SOURCE_DIR}/") - _c4cov_transform_filters(_EXCLUDE_ABS "") - # - if("${CMAKE_CXX_COMPILER_ID}" MATCHES "(Apple)?[Cc]lang") - if("${CMAKE_CXX_COMPILER_VERSION}" VERSION_LESS 3) - c4_err("coverage: clang version must be 3.0.0 or greater") - endif() - elseif(NOT CMAKE_COMPILER_IS_GNUCXX) - c4_err("coverage: compiler is not GNUCXX") - endif() - # - find_program(GCOV gcov) - find_program(LCOV lcov) - find_program(GENHTML genhtml) - find_program(CTEST ctest) - if(NOT (GCOV AND LCOV AND GENHTML AND CTEST)) - c4_err("Coverage tools not available: - gcov: ${GCOV} - lcov: ${LCOV} - genhtml: ${GENHTML} - ctest: ${CTEST} - --coverage flags: ${_COVFLAGS}") - endif() - # - add_configuration_type(Coverage - DEFAULT_FROM DEBUG - C_FLAGS ${_COVFLAGS} - CXX_FLAGS ${_COVFLAGS} - ) - # - option(${_c4_uprefix}COVERAGE_CODECOV "enable target to submit coverage to codecov.io" OFF) - option(${_c4_uprefix}COVERAGE_COVERALLS "enable target to submit coverage to coveralls.io" OFF) - # - c4_dbg("adding coverage targets") - # - set(sd "${CMAKE_SOURCE_DIR}") - set(bd "${CMAKE_BINARY_DIR}") - set(coverage_result ${bd}/lcov/index.html) - set(lcov_result ${bd}/coverage3-final_filtered.lcov) - separate_arguments(_GENHTML_ARGS NATIVE_COMMAND ${_GENHTML_ARGS}) - add_custom_command(OUTPUT ${coverage_result} ${lcov_result} - COMMAND echo "cd ${CMAKE_BINARY_DIR}" - COMMAND ${LCOV} -q --zerocounters --directory . - COMMAND ${LCOV} -q --no-external --capture --base-directory "${sd}" --directory . --output-file ${bd}/coverage0-before.lcov --initial - COMMAND ${CMAKE_COMMAND} --build . --target ${_c4_lprefix}test-run || echo "Failed running the tests. Proceeding with coverage, but results may be affected or even empty." - COMMAND ${LCOV} -q --no-external --capture --base-directory "${sd}" --directory . --output-file ${bd}/coverage1-after.lcov - COMMAND ${LCOV} -q --add-tracefile ${bd}/coverage0-before.lcov --add-tracefile ${bd}/coverage1-after.lcov --output-file ${bd}/coverage2-final.lcov - COMMAND ${LCOV} -q --remove ${bd}/coverage2-final.lcov ${_EXCLUDE} ${EXCLUDE_ABS} --output-file ${bd}/coverage3-final_filtered.lcov - COMMAND ${GENHTML} ${bd}/coverage3-final_filtered.lcov -o ${bd}/lcov ${_GENHTML_ARGS} - COMMAND echo "Coverage report: ${coverage_result}" - DEPENDS ${_c4_lprefix}test-build - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMENT "${_c4_prefix} coverage: LCOV report at ${coverage_result}" - #VERBATIM - ) - add_custom_target(${_c4_lprefix}coverage SOURCES ${coverage_result} ${lcov_result}) - # - if(${_c4_uprefix}COVERAGE_CODECOV) - set(_subm ${_c4_lprefix}coverage-submit-codecov) - _c4cov_get_service_token(codecov _token) - if(NOT ("${_token}" STREQUAL "")) - set(_token -t "${_token}") - endif() - set(_silent_codecov) - if(${_c4_uprefix}COVERAGE_CODECOV_SILENT) - set(_silent_codecov >${CMAKE_BINARY_DIR}/codecov.log 2>&1) - endif() - # - c4_log("coverage: enabling submission of results to https://codecov.io: ${_subm}") - set(submitcc "${CMAKE_BINARY_DIR}/submit_codecov.sh") - c4_download_file("https://codecov.io/bash" "${submitcc}") - set(submit_cmd bash ${submitcc} -Z ${_token} -X gcov -X gcovout -p ${CMAKE_SOURCE_DIR} -f ${lcov_result} ${_silent_codecov}) - string(REPLACE ";" " " submit_cmd_str "${submit_cmd}") - add_custom_target(${_subm} - SOURCES ${lcov_result} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMAND echo "cd ${CMAKE_BINARY_DIR} && ${submit_cmd_str}" - COMMAND ${submit_cmd} - VERBATIM - COMMENT "${_c4_lcprefix} coverage: submit to codecov" - ) - c4_add_umbrella_target(coverage-submit-codecov coverage-submit) # uses the current prefix - endif() - # - if(${_c4_uprefix}COVERAGE_COVERALLS) - set(_subm ${_c4_lprefix}coverage-submit-coveralls) - _c4cov_get_service_token(coveralls _token) - if(NOT ("${_token}" STREQUAL "")) - set(_token --repo-token "${_token}") - endif() - set(_silent_coveralls) - if(${_c4_uprefix}COVERAGE_COVERALLS_SILENT) - set(_silent_coveralls >${CMAKE_BINARY_DIR}/coveralls.log 2>&1) - endif() - # - c4_log("coverage: enabling submission of results to https://coveralls.io: ${_subm}") - set(submit_cmd coveralls ${_token} --build-root ${CMAKE_BINARY_DIR} --root ${CMAKE_SOURCE_DIR} --no-gcov --lcov-file ${lcov_result} ${_silent_coveralls}) - string(REPLACE ";" " " submit_cmd_str "${submit_cmd}") - add_custom_target(${_subm} - SOURCES ${lcov_result} - WORKING_DIRECTORY ${CMAKE_BINARY_DIR} - COMMAND echo "cd ${CMAKE_BINARY_DIR} && ${submit_cmd_str}" - COMMAND ${submit_cmd} - VERBATIM - COMMENT "${_c4_lcprefix} coverage: submit to coveralls" - ) - c4_add_umbrella_target(coverage-submit-coveralls coverage-submit) # uses the current prefix - endif() -endfunction(c4_setup_coverage) - - -# 1. try cmake or environment variables -# 2. try local file -function(_c4cov_get_service_token service out) - # try cmake var - string(TOUPPER ${service} uservice) - c4_get_from_first_of(token COVERAGE_${uservice}_TOKEN ENV) - if(NOT ("${token}" STREQUAL "")) - c4_dbg("${service}: found token from variable: ${token}") - else() - # try local file - set(service_token_file ${CMAKE_SOURCE_DIR}/.ci/${service}.token) - if(EXISTS ${service_token_file}) - file(READ ${service_token_file} token) - c4_dbg("found token file for ${service} coverage report: ${service_token_file}") - else() - c4_dbg("could not find token for ${service} coverage report") - endif() - endif() - set(${out} ${token} PARENT_SCOPE) -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -function(c4_add_umbrella_target target umbrella_target) - _c4_handle_args(_ARGS ${ARGN} - # zero-value macro arguments - _ARGS0 - ALWAYS # Add the umbrella target even if this is the only one under it. - # The default behavior is to add the umbrella target only if - # there is more than one target under it. - # one-value macro arguments - _ARGS1 - PREFIX # The project prefix. Defaults to ${_c4_lprefix} - # multi-value macro arguments - _ARGSN - ARGS # more args to add_custom_target() - ) - if(NOT _PREFIX) - set(_PREFIX "${_c4_lprefix}") - endif() - set(t ${_PREFIX}${target}) - set(ut ${_PREFIX}${umbrella_target}) - # if the umbrella target already exists, just add the dependency - if(TARGET ${ut}) - add_dependencies(${ut} ${t}) - else() - if(_ALWAYS) - add_custom_target(${ut} ${_ARGS}) - add_dependencies(${ut} ${t}) - else() - # check if there is more than one under the same umbrella - c4_get_proj_prop(${ut}_subtargets sub) - if(sub) - add_custom_target(${ut} ${_ARGS}) - add_dependencies(${ut} ${sub}) - add_dependencies(${ut} ${t}) - else() - c4_set_proj_prop(${ut}_subtargets ${t}) - endif() - endif() - endif() -endfunction() - - - -function(c4_download_file url dstpath) - c4_dbg("downloading file: ${url} ---> ${dstpath}") - get_filename_component(abspath ${dstpath} ABSOLUTE) - if(NOT EXISTS ${abspath}) - c4_dbg("downloading file: does not exist: ${dstpath}") - file(DOWNLOAD ${url} ${abspath} LOG dl_log STATUS status ${ARGN}) - if((NOT (status EQUAL 0)) OR (NOT EXISTS ${abspath})) - c4_err("error downloading file: ${url} -> ${abspath}:\n${dl_log}") - endif() - endif() -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -function(c4_setup_benchmarking) - c4_log("enabling benchmarks: to build, ${_c4_lprefix}bm-build") - c4_log("enabling benchmarks: to run, ${_c4_lprefix}bm-run") - # umbrella target for building test binaries - add_custom_target(${_c4_lprefix}bm-build) - # umbrella target for running benchmarks - add_custom_target(${_c4_lprefix}bm-run - ${CMAKE_COMMAND} -E echo CWD=${CMAKE_CURRENT_BINARY_DIR} - DEPENDS ${_c4_lprefix}bm-build - ) - if(NOT TARGET bm-run) - add_custom_target(bm-run) - add_custom_target(bm-build) - endif() - add_dependencies(bm-run ${_c4_lprefix}bm-run) - add_dependencies(bm-build ${_c4_lprefix}bm-build) - _c4_set_target_folder(${_c4_lprefix}bm-run bm) - _c4_set_target_folder(${_c4_lprefix}bm-build bm) - _c4_set_target_folder(bm-build "/bm") - _c4_set_target_folder(bm-run "/bm") - # download google benchmark - if(NOT TARGET benchmark) - c4_import_remote_proj(googlebenchmark ${CMAKE_CURRENT_BINARY_DIR}/ext/googlebenchmark - REMOTE - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG main GIT_SHALLOW ON - OVERRIDE - BENCHMARK_ENABLE_TESTING OFF - BENCHMARK_ENABLE_EXCEPTIONS OFF - BENCHMARK_ENABLE_LTO OFF - SET_FOLDER_TARGETS ext benchmark benchmark_main - EXCLUDE_FROM_ALL - ) - # - if((CMAKE_CXX_COMPILER_ID STREQUAL GNU) OR (CMAKE_COMPILER_IS_GNUCC)) - target_compile_options(benchmark PRIVATE -Wno-deprecated-declarations) - target_compile_options(benchmark PRIVATE -Wno-restrict) - endif() - # - if(NOT WIN32) - option(${_c4_uprefix}BENCHMARK_CPUPOWER - "set the cpu mode to performance before / powersave after the benchmark" OFF) - if(${_c4_uprefix}BENCHMARK_CPUPOWER) - find_program(C4_SUDO sudo) - find_program(C4_CPUPOWER cpupower) - endif() - endif() - endif() -endfunction() - - -function(c4_add_benchmark_cmd casename) - add_custom_target(${casename} - COMMAND ${ARGN} - VERBATIM - COMMENT "${_c4_prefix}: running benchmark ${casename}: ${ARGN}") - add_dependencies(${_c4_lprefix}bm-build ${casename}) - _c4_set_target_folder(${casename} bm) -endfunction() - - -# assumes this is a googlebenchmark target, and that multiple -# benchmarks are defined from it -function(c4_add_target_benchmark target casename) - set(opt0arg - ) - set(opt1arg - WORKDIR # working directory - FILTER # benchmark patterns to filter - UMBRELLA_TARGET - RESULTS_FILE - ) - set(optnarg - ARGS - ) - cmake_parse_arguments("" "${opt0arg}" "${opt1arg}" "${optnarg}" ${ARGN}) - # - set(name "${target}-${casename}") - set(rdir "${CMAKE_CURRENT_BINARY_DIR}/bm-results") - set(rfile "${rdir}/${name}.json") - if(_RESULTS_FILE) - set(${_RESULTS_FILE} "${rfile}" PARENT_SCOPE) - endif() - if(NOT EXISTS "${rdir}") - file(MAKE_DIRECTORY "${rdir}") - endif() - set(filter) - if(NOT ("${_FILTER}" STREQUAL "")) - set(filter "--benchmark_filter=${_FILTER}") - endif() - set(args_fwd ${filter} --benchmark_out_format=json --benchmark_out=${rfile} ${_ARGS}) - c4_add_benchmark(${target} - "${name}" - "${_WORKDIR}" - "saving results in ${rfile}" - ${args_fwd} - OUTPUT_FILE ${rfile}) - if(_UMBRELLA_TARGET) - add_dependencies(${_UMBRELLA_TARGET} "${name}") - endif() -endfunction() - - -function(c4_add_benchmark target casename work_dir comment) - set(opt0arg - ) - set(opt1arg - OUTPUT_FILE - ) - set(optnarg - ) - cmake_parse_arguments("" "${opt0arg}" "${opt1arg}" "${optnarg}" ${ARGN}) - if(NOT TARGET ${target}) - c4_err("target ${target} does not exist...") - endif() - if(NOT ("${work_dir}" STREQUAL "")) - if(NOT EXISTS "${work_dir}") - file(MAKE_DIRECTORY "${work_dir}") - endif() - endif() - set(exe $) - if(${_c4_uprefix}BENCHMARK_CPUPOWER) - if(C4_BM_SUDO AND C4_BM_CPUPOWER) - set(c ${C4_SUDO} ${C4_CPUPOWER} frequency-set --governor performance) - set(cpupow_before - COMMAND echo ${c} - COMMAND ${c}) - set(c ${C4_SUDO} ${C4_CPUPOWER} frequency-set --governor powersave) - set(cpupow_after - COMMAND echo ${c} - COMMAND ${c}) - endif() - endif() - if(_OUTPUT_FILE) - set(_OUTPUT_FILE BYPRODUCTS ${_OUTPUT_FILE}) - set(_OUTPUT_FILE) # otherwise the benchmarks run everytime when building depending targets - endif() - add_custom_target(${casename} - ${cpupow_before} - # this is useful to show the target file (you cannot echo generator variables) - #COMMAND ${CMAKE_COMMAND} -E echo "target file = $" - COMMAND ${CMAKE_COMMAND} -E echo "${exe} ${ARGN}" - COMMAND "${exe}" ${ARGN} - ${cpupow_after} - VERBATIM - ${_OUTPUT_FILE} - WORKING_DIRECTORY "${work_dir}" - DEPENDS ${target} - COMMENT "${_c4_lcprefix}: running benchmark ${target}, case ${casename}: ${comment}" - ) - add_dependencies(${_c4_lprefix}bm-build ${target}) - add_dependencies(${_c4_lprefix}bm-run ${casename}) - _c4_set_target_folder(${casename} bm/run) -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -function(_c4cat_get_outname target id ext out) - if("${_c4_lcprefix}" STREQUAL "${target}") - set(p "${target}") - else() - set(p "${_c4_lcprefix}.${target}") - endif() - set(${out} "${CMAKE_CURRENT_BINARY_DIR}/${p}.${id}.${ext}" PARENT_SCOPE) -endfunction() - -function(_c4cat_filter_srcs in out) - _c4cat_filter_extensions("${in}" "${C4_SRC_EXTS}" l) - set(${out} ${l} PARENT_SCOPE) -endfunction() - -function(_c4cat_filter_hdrs in out) - _c4cat_filter_extensions("${in}" "${C4_HDR_EXTS}" l) - set(${out} ${l} PARENT_SCOPE) -endfunction() - -function(_c4cat_filter_srcs_hdrs in out) - _c4cat_filter_extensions("${in}" "${C4_HDR_EXTS};${C4_SRC_EXTS}" l) - set(${out} ${l} PARENT_SCOPE) -endfunction() - -function(_c4cat_filter_additional_exts in out) - _c4cat_filter_extensions("${in}" "${C4_ADD_EXTS}" l) - set(${out} ${l} PARENT_SCOPE) -endfunction() - -function(_c4cat_filter_extensions in filter out) - set(l) - foreach(fn ${in}) # don't quote the list here - _c4cat_get_file_ext("${fn}" ext) - _c4cat_one_of("${ext}" "${filter}" yes) - if(${yes}) - list(APPEND l "${fn}") - endif() - endforeach() - set(${out} "${l}" PARENT_SCOPE) -endfunction() - -function(_c4cat_get_file_ext in out) - # https://stackoverflow.com/questions/30049180/strip-filename-shortest-extension-by-cmake-get-filename-removing-the-last-ext - string(REGEX MATCH "^.*\\.([^.]*)$" dummy ${in}) - set(${out} ${CMAKE_MATCH_1} PARENT_SCOPE) -endfunction() - -function(_c4cat_one_of ext candidates out) - foreach(e ${candidates}) - if("${ext}" STREQUAL "${e}") - set(${out} TRUE PARENT_SCOPE) - return() - endif() - endforeach() - set(${out} FALSE PARENT_SCOPE) -endfunction() - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ - -# given a list of source files, return a list with full paths -function(c4_to_full_path source_list source_list_with_full_paths) - set(l) - foreach(f ${source_list}) - if(IS_ABSOLUTE "${f}") - list(APPEND l "${f}") - else() - list(APPEND l "${CMAKE_CURRENT_SOURCE_DIR}/${f}") - endif() - endforeach() - set(${source_list_with_full_paths} ${l} PARENT_SCOPE) -endfunction() - - -# convert a list to a string separated with spaces -function(c4_separate_list input_list output_string) - set(s) - foreach(e ${input_list}) - set(s "${s} ${e}") - endforeach() - set(${output_string} ${s} PARENT_SCOPE) -endfunction() - - - -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -#------------------------------------------------------------------------------ -endif(NOT _c4_project_included) diff --git a/thirdparty/ryml/ext/c4core/cmake/c4SanitizeTarget.cmake b/thirdparty/ryml/ext/c4core/cmake/c4SanitizeTarget.cmake deleted file mode 100644 index a064f91ee..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/c4SanitizeTarget.cmake +++ /dev/null @@ -1,292 +0,0 @@ -# (C) 2017 Joao Paulo Magalhaes -if(NOT _c4_sanitize_target_included) -set(_c4_sanitize_target_included ON) - -include(CMakeDependentOption) -include(PrintVar) - -function(_c4_default_if_not_set var dft) - if("${${var}}" STREQUAL "") - option(${var} "" ${dft}) - endif() -endfunction() - - -#------------------------------------------------------------------------------ -function(c4_setup_sanitize umbrella_option) - if("${CMAKE_BUILD_TYPE}" STREQUAL "Coverage") - return() - endif() - if(NOT ((CMAKE_CXX_COMPILER_ID MATCHES ".*Clang") OR (CMAKE_CXX_COMPILER_ID STREQUAL "GNU"))) - return() - endif() - - _c4_default_if_not_set(C4_SANITIZE ON) - _c4_default_if_not_set(C4_SANITIZE_ONLY OFF) - _c4_default_if_not_set(C4_ASAN ON) - _c4_default_if_not_set(C4_TSAN ON) - _c4_default_if_not_set(C4_MSAN ON) - _c4_default_if_not_set(C4_UBSAN ON) - - cmake_dependent_option(${_c4_uprefix}SANITIZE "turn on clang sanitizer targets" ${C4_SANITIZE} ${umbrella_option} OFF) - cmake_dependent_option(${_c4_uprefix}SANITIZE_ONLY "compile only sanitize targets (not the regular unsanitized targets)" ${C4_SANITIZE_ONLY} ${umbrella_option} OFF) - - # options for individual sanitizers - contingent on sanitize on/off - cmake_dependent_option(${_c4_uprefix}ASAN "" ${C4_ASAN} "${_c4_uprefix}SANITIZE" OFF) - cmake_dependent_option(${_c4_uprefix}TSAN "" ${C4_TSAN} "${_c4_uprefix}SANITIZE" OFF) - cmake_dependent_option(${_c4_uprefix}MSAN "" ${C4_MSAN} "${_c4_uprefix}SANITIZE" OFF) - cmake_dependent_option(${_c4_uprefix}UBSAN "" ${C4_UBSAN} "${_c4_uprefix}SANITIZE" OFF) - - if(${_c4_uprefix}SANITIZE) - string(REGEX REPLACE "([0-9]+\\.[0-9]+).*" "\\1" LLVM_VERSION "${CMAKE_CXX_COMPILER_VERSION}") - find_program(LLVM_SYMBOLIZER llvm-symbolizer - NAMES llvm-symbolizer-${LLVM_VERSION} llvm-symbolizer - DOC "symbolizer to use in sanitize tools") - if(NOT LLVM_SYMBOLIZER) - string(REGEX REPLACE "([0-9]+)\\.[0-9]+.*" "\\1" LLVM_VERSION "${CMAKE_CXX_COMPILER_VERSION}") - find_program(LLVM_SYMBOLIZER llvm-symbolizer - NAMES llvm-symbolizer-${LLVM_VERSION} llvm-symbolizer - DOC "symbolizer to use in sanitize tools") - if(NOT LLVM_SYMBOLIZER) - message(FATAL_ERROR "could not find symbolizer. LLVM_VERSION=${LLVM_VERSION}") - endif() - endif() - - set(ss) # string to report enabled sanitizers - - if(${_c4_uprefix}ASAN) - set(ss "asan") - set(${_c4_uprefix}ASAN_CFLAGS "-O1 -g -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls" CACHE STRING "compile flags for clang address sanitizer: https://clang.llvm.org/docs/AddressSanitizer.html") - set(${_c4_uprefix}ASAN_LFLAGS "-g -fsanitize=address" CACHE STRING "linker flags for clang address sanitizer: https://clang.llvm.org/docs/AddressSanitizer.html") - set(${_c4_uprefix}ASAN_RENV "env ASAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER} ASAN_OPTIONS=symbolize=1" CACHE STRING "run environment for clang address sanitizer: https://clang.llvm.org/docs/AddressSanitizer.html") - # the flags are strings; we need to separate them into a list - # to prevent cmake from quoting them when passing to the targets - separate_arguments(${_c4_uprefix}ASAN_CFLAGS_SEP UNIX_COMMAND ${${_c4_uprefix}ASAN_CFLAGS}) - separate_arguments(${_c4_uprefix}ASAN_LFLAGS_SEP UNIX_COMMAND ${${_c4_uprefix}ASAN_LFLAGS}) - endif() - - if(${_c4_uprefix}TSAN) - set(ss "${ss} tsan") - set(${_c4_uprefix}TSAN_CFLAGS "-O1 -g -fsanitize=thread -fno-omit-frame-pointer" CACHE STRING "compile flags for clang thread sanitizer: https://clang.llvm.org/docs/ThreadSanitizer.html") - set(${_c4_uprefix}TSAN_LFLAGS "-g -fsanitize=thread" CACHE STRING "linker flags for clang thread sanitizer: https://clang.llvm.org/docs/ThreadSanitizer.html") - set(${_c4_uprefix}TSAN_RENV "env TSAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER} TSAN_OPTIONS=symbolize=1" CACHE STRING "run environment for clang thread sanitizer: https://clang.llvm.org/docs/ThreadSanitizer.html") - separate_arguments(${_c4_uprefix}TSAN_CFLAGS_SEP UNIX_COMMAND ${${_c4_uprefix}TSAN_CFLAGS}) - separate_arguments(${_c4_uprefix}TSAN_LFLAGS_SEP UNIX_COMMAND ${${_c4_uprefix}TSAN_LFLAGS}) - endif() - - if(${_c4_uprefix}MSAN) - set(ss "${ss} msan") - set(${_c4_uprefix}MSAN_CFLAGS "-O1 -g -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -fno-optimize-sibling-calls" CACHE STRING "compile flags for clang memory sanitizer: https://clang.llvm.org/docs/MemorySanitizer.html") - set(${_c4_uprefix}MSAN_LFLAGS "-g -fsanitize=memory" CACHE STRING "linker flags for clang memory sanitizer: https://clang.llvm.org/docs/MemorySanitizer.html") - set(${_c4_uprefix}MSAN_RENV "env MSAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER} MSAN_OPTIONS=symbolize=1" CACHE STRING "run environment for clang memory sanitizer: https://clang.llvm.org/docs/MemorySanitizer.html") - separate_arguments(${_c4_uprefix}MSAN_CFLAGS_SEP UNIX_COMMAND ${${_c4_uprefix}MSAN_CFLAGS}) - separate_arguments(${_c4_uprefix}MSAN_LFLAGS_SEP UNIX_COMMAND ${${_c4_uprefix}MSAN_LFLAGS}) - endif() - - if(${_c4_uprefix}UBSAN) - set(ss "${ss} ubsan") - set(${_c4_uprefix}UBSAN_CFLAGS "-g -fsanitize=undefined" CACHE STRING "compile flags for clang undefined behaviour sanitizer: https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html") - set(${_c4_uprefix}UBSAN_LFLAGS "-g -fsanitize=undefined" CACHE STRING "linker flags for clang undefined behaviour sanitizer: https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html") - set(${_c4_uprefix}UBSAN_RENV "env UBSAN_SYMBOLIZER_PATH=${LLVM_SYMBOLIZER} UBSAN_OPTIONS='symbolize=1 print_stacktrace=1'" CACHE STRING "run environment for clang undefined behaviour sanitizer: https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html") - separate_arguments(${_c4_uprefix}UBSAN_CFLAGS_SEP UNIX_COMMAND ${${_c4_uprefix}UBSAN_CFLAGS}) - separate_arguments(${_c4_uprefix}UBSAN_LFLAGS_SEP UNIX_COMMAND ${${_c4_uprefix}UBSAN_LFLAGS}) - endif() - - c4_dbg("enabled clang sanitizers: ${ss}") - endif() # ${_c4_uprefix}SANITIZE - -endfunction() - - -#------------------------------------------------------------------------------ -function(c4_sanitize_get_target_command name which_sanitizer_ output) - string(TOUPPER ${which_sanitizer_} which_sanitizer) - if("${which_sanitizer}" STREQUAL ASAN) - elseif("${which_sanitizer}" STREQUAL TSAN) - elseif("${which_sanitizer}" STREQUAL MSAN) - elseif("${which_sanitizer}" STREQUAL UBSAN) - else() - message(FATAL_ERROR "the sanitizer must be one of: ASAN, TSAN, MSAN, UBSAN") - endif() - separate_arguments(cmd UNIX_COMMAND "${${_c4_uprefix}${which_sanitizer}_RENV} ${name}") - set(${output} ${cmd} PARENT_SCOPE) -endfunction() - -function(_sanitize_set_target_folder tgt folder) - if(folder) - set_target_properties(${tgt} PROPERTIES FOLDER "${folder}") - endif() -endfunction() - - -#------------------------------------------------------------------------------ -function(c4_sanitize_target name) - set(opt0arg - LIBRARY - EXECUTABLE - ) - set(opt1arg - OUTPUT_TARGET_NAMES - FOLDER - ) - set(optnarg - SOURCES - INC_DIRS # TODO public, interface, private - LIBS # TODO public, interface, private - LIB_DIRS # TODO public, interface, private - DEFS # TODO public, interface, private - CFLAGS # TODO public, interface, private - ) - cmake_parse_arguments("" "${opt0arg}" "${opt1arg}" "${optnarg}" ${ARGN}) - - if((NOT _LIBRARY) AND (NOT _EXECUTABLE)) - c4_err("either LIBRARY or EXECUTABLE must be specified") - endif() - - if(${_c4_uprefix}SANITIZE AND NOT TARGET ${_c4_lprefix}sanitize) - add_custom_target(${_c4_lprefix}sanitize) - _sanitize_set_target_folder(${_c4_lprefix}sanitize "${_FOLDER}") - endif() - if(${_c4_uprefix}ASAN AND NOT TARGET ${_c4_lprefix}asan-all) - add_custom_target(${_c4_lprefix}asan-all) - add_dependencies(${_c4_lprefix}sanitize ${_c4_lprefix}asan-all) - _sanitize_set_target_folder(${_c4_lprefix}asan-all "${_FOLDER}") - endif() - if(${_c4_uprefix}MSAN AND NOT TARGET ${_c4_lprefix}msan-all) - add_custom_target(${_c4_lprefix}msan-all) - add_dependencies(${_c4_lprefix}sanitize ${_c4_lprefix}msan-all) - _sanitize_set_target_folder(${_c4_lprefix}msan-all "${_FOLDER}") - endif() - if(${_c4_uprefix}TSAN AND NOT TARGET ${_c4_lprefix}tsan-all) - add_custom_target(${_c4_lprefix}tsan-all) - add_dependencies(${_c4_lprefix}sanitize ${_c4_lprefix}tsan-all) - _sanitize_set_target_folder(${_c4_lprefix}tsan-all "${_FOLDER}") - endif() - if(${_c4_uprefix}UBSAN AND NOT TARGET ${_c4_lprefix}ubsan-all) - add_custom_target(${_c4_lprefix}ubsan-all) - add_dependencies(${_c4_lprefix}sanitize ${_c4_lprefix}ubsan-all) - _sanitize_set_target_folder(${_c4_lprefix}ubsan-all "${_FOLDER}") - endif() - - if(${_c4_uprefix}ASAN OR ${_c4_uprefix}MSAN OR ${_c4_uprefix}TSAN OR ${_c4_uprefix}UBSAN) - add_custom_target(${name}-sanitize-all) - _sanitize_set_target_folder(${name}-sanitize-all "${_FOLDER}") - endif() - - set(targets) - - # https://clang.llvm.org/docs/AddressSanitizer.html - if(${_c4_uprefix}ASAN) - if(${_LIBRARY}) - add_library(${name}-asan EXCLUDE_FROM_ALL ${_SOURCES}) - elseif(${_EXECUTABLE}) - add_executable(${name}-asan EXCLUDE_FROM_ALL ${_SOURCES}) - endif() - _sanitize_set_target_folder(${name}-asan "${_FOLDER}") - list(APPEND targets ${name}-asan) - target_include_directories(${name}-asan PUBLIC ${_INC_DIRS}) - set(_real_libs) - foreach(_l ${_LIBS}) - if(TARGET ${_l}-asan) - list(APPEND _real_libs ${_l}-asan) - else() - list(APPEND _real_libs ${_l}) - endif() - endforeach() - target_link_libraries(${name}-asan PUBLIC ${_real_libs}) - target_compile_definitions(${name}-asan PUBLIC ${_DEFS}) - target_compile_options(${name}-asan PUBLIC ${_CFLAGS} ${${_c4_uprefix}ASAN_CFLAGS_SEP}) - # http://stackoverflow.com/questions/25043458/does-cmake-have-something-like-target-link-options - target_link_libraries(${name}-asan PUBLIC ${${_c4_uprefix}ASAN_LFLAGS_SEP}) - add_dependencies(${_c4_lprefix}asan-all ${name}-asan) - add_dependencies(${name}-sanitize-all ${name}-asan) - endif() - - # https://clang.llvm.org/docs/ThreadSanitizer.html - if(${_c4_uprefix}TSAN) - if(${_LIBRARY}) - add_library(${name}-tsan EXCLUDE_FROM_ALL ${_SOURCES}) - elseif(${_EXECUTABLE}) - add_executable(${name}-tsan EXCLUDE_FROM_ALL ${_SOURCES}) - endif() - _sanitize_set_target_folder(${name}-tsan "${_FOLDER}") - list(APPEND targets ${name}-tsan) - target_include_directories(${name}-tsan PUBLIC ${_INC_DIRS}) - set(_real_libs) - foreach(_l ${_LIBS}) - if(TARGET ${_l}-tsan) - list(APPEND _real_libs ${_l}-tsan) - else() - list(APPEND _real_libs ${_l}) - endif() - endforeach() - target_link_libraries(${name}-tsan PUBLIC ${_real_libs}) - target_compile_definitions(${name}-tsan PUBLIC ${_DEFS}) - target_compile_options(${name}-tsan PUBLIC ${_CFLAGS} ${${_c4_uprefix}TSAN_CFLAGS_SEP}) - # http://stackoverflow.com/questions/25043458/does-cmake-have-something-like-target-link-options - target_link_libraries(${name}-tsan PUBLIC ${${_c4_uprefix}TSAN_LFLAGS_SEP}) - add_dependencies(${_c4_lprefix}tsan-all ${name}-tsan) - add_dependencies(${name}-sanitize-all ${name}-tsan) - endif() - - # https://clang.llvm.org/docs/MemorySanitizer.html - if(${_c4_uprefix}MSAN) - if(${_LIBRARY}) - add_library(${name}-msan EXCLUDE_FROM_ALL ${_SOURCES}) - elseif(${_EXECUTABLE}) - add_executable(${name}-msan EXCLUDE_FROM_ALL ${_SOURCES}) - endif() - _sanitize_set_target_folder(${name}-msan "${_FOLDER}") - list(APPEND targets ${name}-msan) - target_include_directories(${name}-msan PUBLIC ${_INC_DIRS}) - set(_real_libs) - foreach(_l ${_LIBS}) - if(TARGET ${_l}-msan) - list(APPEND _real_libs ${_l}-msan) - else() - list(APPEND _real_libs ${_l}) - endif() - endforeach() - target_link_libraries(${name}-msan PUBLIC ${_real_libs}) - target_compile_definitions(${name}-msan PUBLIC ${_DEFS}) - target_compile_options(${name}-msan PUBLIC ${_CFLAGS} ${${_c4_uprefix}MSAN_CFLAGS_SEP}) - # http://stackoverflow.com/questions/25043458/does-cmake-have-something-like-target-link-options - target_link_libraries(${name}-msan PUBLIC ${${_c4_uprefix}MSAN_LFLAGS_SEP}) - add_dependencies(${_c4_lprefix}msan-all ${name}-msan) - add_dependencies(${name}-sanitize-all ${name}-msan) - endif() - - # https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html - if(${_c4_uprefix}UBSAN) - if(${_LIBRARY}) - add_library(${name}-ubsan EXCLUDE_FROM_ALL ${_SOURCES}) - elseif(${_EXECUTABLE}) - add_executable(${name}-ubsan EXCLUDE_FROM_ALL ${_SOURCES}) - endif() - _sanitize_set_target_folder(${name}-ubsan "${_FOLDER}") - list(APPEND targets ${name}-ubsan) - target_include_directories(${name}-ubsan PUBLIC ${_INC_DIRS}) - set(_real_libs) - foreach(_l ${_LIBS}) - if(TARGET ${_l}-ubsan) - list(APPEND _real_libs ${_l}-ubsan) - else() - list(APPEND _real_libs ${_l}) - endif() - endforeach() - target_link_libraries(${name}-ubsan PUBLIC ${_real_libs}) - target_compile_definitions(${name}-ubsan PUBLIC ${_DEFS}) - target_compile_options(${name}-ubsan PUBLIC ${_CFLAGS} ${${_c4_uprefix}UBSAN_CFLAGS_SEP}) - # http://stackoverflow.com/questions/25043458/does-cmake-have-something-like-target-link-options - target_link_libraries(${name}-ubsan PUBLIC ${${_c4_uprefix}UBSAN_LFLAGS_SEP}) - add_dependencies(${_c4_lprefix}ubsan-all ${name}-ubsan) - add_dependencies(${name}-sanitize-all ${name}-ubsan) - endif() - - if(_OUTPUT_TARGET_NAMES) - set(${_OUTPUT_TARGET_NAMES} ${targets} PARENT_SCOPE) - endif() -endfunction() - - -endif(NOT _c4_sanitize_target_included) diff --git a/thirdparty/ryml/ext/c4core/cmake/c4StaticAnalysis.cmake b/thirdparty/ryml/ext/c4core/cmake/c4StaticAnalysis.cmake deleted file mode 100644 index 06de5ca2f..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/c4StaticAnalysis.cmake +++ /dev/null @@ -1,154 +0,0 @@ -include(PVS-Studio) -include(GetFlags) -include(c4GetTargetPropertyRecursive) - - -function(_c4sta_default_if_not_set var dft) - if("${${var}}" STREQUAL "") - set(${var} "${dft}" PARENT_SCOPE) - endif() -endfunction() - - -function(c4_setup_static_analysis umbrella_option) - if(WIN32) - c4_dbg("no static analyzer available in WIN32") - return() - endif() - if("${CMAKE_BUILD_TYPE}" STREQUAL "Coverage") - c4_dbg("Coverage build: disabling static analyzers") - return() - endif() - _c4sta_default_if_not_set(C4_LINT ${umbrella_option}) - _c4sta_default_if_not_set(C4_LINT_TESTS ${umbrella_option}) - _c4sta_default_if_not_set(C4_LINT_CLANG_TIDY ${umbrella_option}) - _c4sta_default_if_not_set(C4_LINT_PVS_STUDIO OFF) - # option to turn lints on/off - cmake_dependent_option(${_c4_uprefix}LINT "add static analyzer targets" ${C4_LINT} ${umbrella_option} OFF) - cmake_dependent_option(${_c4_uprefix}LINT_TESTS "add tests to run static analyzer targets" ${C4_LINT_TESTS} ${umbrella_option} OFF) - # options for individual lints - contingent on linting on/off - cmake_dependent_option(${_c4_uprefix}LINT_CLANG_TIDY "use the clang-tidy static analyzer" ${C4_LINT_CLANG_TIDY} "${_c4_uprefix}LINT" ON) - cmake_dependent_option(${_c4_uprefix}LINT_PVS_STUDIO "use the PVS-Studio static analyzer https://www.viva64.com/en/b/0457/" ${C4_LINT_PVS_STUDIO} "${_c4_uprefix}LINT" OFF) - if(${_c4_uprefix}LINT_CLANG_TIDY) - find_program(CLANG_TIDY clang-tidy) - endif() - if(${_c4_uprefix}LINT_PVS_STUDIO) - set(${_c4_uprefix}LINT_PVS_STUDIO_FORMAT "errorfile" CACHE STRING "PVS-Studio output format. Choices: xml,csv,errorfile(like gcc/clang),tasklist(qtcreator)") - endif() - # - set(sa) - if(${_c4_uprefix}LINT_CLANG_TIDY) - set(sa "clang_tidy") - endif() - if(${_c4_uprefix}LINT_PVS_STUDIO) - set(sa "${sa} PVS-Studio") - endif() - if(sa) - c4_dbg("enabled static analyzers: ${sa}") - endif() -endfunction() - - -function(c4_static_analysis_target target_name folder generated_targets) - set(any_linter OFF) - if(${_c4_uprefix}LINT_CLANG_TIDY OR ${_c4_uprefix}LINT_PVS_STUDIO) - set(any_linter ON) - endif() - if(${_c4_uprefix}LINT AND any_linter) - # umbrella target for running all linters for this particular target - if(any_linter AND NOT TARGET ${_c4_lprefix}lint-all) - add_custom_target(${_c4_lprefix}lint-all) - if(folder) - #message(STATUS "${target_name}: folder=${folder}") - set_target_properties(${_c4_lprefix}lint-all PROPERTIES FOLDER "${folder}") - endif() - endif() - if(${_c4_uprefix}LINT_CLANG_TIDY) - c4_static_analysis_clang_tidy(${target_name} - ${target_name}-lint-clang_tidy - ${_c4_lprefix}lint-all-clang_tidy - "${folder}") - list(APPEND ${generated_targets} ${_c4_lprefix}lint-clang_tidy) - add_dependencies(${_c4_lprefix}lint-all ${_c4_lprefix}lint-all-clang_tidy) - endif() - if(${_c4_uprefix}LINT_PVS_STUDIO) - c4_static_analysis_pvs_studio(${target_name} - ${target_name}-lint-pvs_studio - ${_c4_lprefix}lint-all-pvs_studio - "${folder}") - list(APPEND ${generated_targets} ${_c4_lprefix}lint-pvs_studio) - add_dependencies(${_c4_lprefix}lint-all ${_c4_lprefix}lint-all-pvs_studio) - endif() - endif() -endfunction() - - -function(c4_static_analysis_add_tests target_name) - if(${_c4_uprefix}LINT_CLANG_TIDY AND ${_c4_uprefix}LINT_TESTS) - add_test(NAME ${target_name}-lint-clang_tidy-run - COMMAND - ${CMAKE_COMMAND} --build ${CMAKE_CURRENT_BINARY_DIR} --target ${target_name}-lint-clang_tidy) - endif() - if(${_c4_uprefix}LINT_PVS_STUDIO AND ${_c4_uprefix}LINT_TESTS) - add_test(NAME ${target_name}-lint-pvs_studio-run - COMMAND - ${CMAKE_COMMAND} --build ${CMAKE_CURRENT_BINARY_DIR} --target ${target_name}-lint-pvs_studio) - endif() -endfunction() - - -#------------------------------------------------------------------------------ -function(c4_static_analysis_clang_tidy subj_target lint_target umbrella_target folder) - c4_static_analysis_clang_tidy_get_cmd(${subj_target} ${lint_target} cmd) - string(REPLACE ";" " " cmd_str "${cmd}") - add_custom_target(${lint_target} - COMMAND ${CMAKE_COMMAND} -E echo "cd ${CMAKE_CURRENT_SOURCE_DIR} ; ${cmd_str}" - COMMAND ${cmd} - VERBATIM - COMMENT "clang-tidy: analyzing sources of ${subj_target}" - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - if(folder) - set_target_properties(${lint_target} PROPERTIES FOLDER "${folder}") - endif() - if(NOT TARGET ${umbrella_target}) - add_custom_target(${umbrella_target}) - endif() - add_dependencies(${umbrella_target} ${lint_target}) -endfunction() - -function(c4_static_analysis_clang_tidy_get_cmd subj_target lint_target cmd) - get_target_property(_clt_all_srcs ${subj_target} SOURCES) - _c4cat_filter_srcs_hdrs("${_clt_all_srcs}" _clt_srcs) - set(result "${CLANG_TIDY}" -p ${CMAKE_BINARY_DIR} --header-filter=.* ${_clt_srcs}) - set(${cmd} ${result} PARENT_SCOPE) -endfunction() - - -#------------------------------------------------------------------------------ -function(c4_static_analysis_pvs_studio subj_target lint_target umbrella_target folder) - c4_get_target_property_recursive(_c4al_pvs_incs ${subj_target} INCLUDE_DIRECTORIES) - c4_get_include_flags(_c4al_pvs_incs ${_c4al_pvs_incs}) - separate_arguments(_c4al_cxx_flags_sep UNIX_COMMAND "${CMAKE_CXX_FLAGS} ${_c4al_pvs_incs}") - separate_arguments(_c4al_c_flags_sep UNIX_COMMAND "${CMAKE_C_FLAGS} ${_c4al_pvs_incs}") - pvs_studio_add_target(TARGET ${lint_target} - ALL # indicates that the analysis starts when you build the project - #PREPROCESSOR ${_c4al_preproc} - FORMAT tasklist - LOG "${CMAKE_CURRENT_BINARY_DIR}/${subj_target}.pvs-analysis.tasks" - ANALYZE ${name} #main_target subtarget:path/to/subtarget - CXX_FLAGS ${_c4al_cxx_flags_sep} - C_FLAGS ${_c4al_c_flags_sep} - #CONFIG "/path/to/PVS-Studio.cfg" - ) - if(folder) - set_target_properties(${lint_target} PROPERTIES FOLDER "${folder}") - endif() - if(NOT TARGET ${umbrella_target}) - add_custom_target(${umbrella_target}) - endif() - add_dependencies(${umbrella_target} ${lint_target}) -endfunction() - -function(c4_static_analysis_pvs_studio_get_cmd subj_target lint_target cmd) - set(${cmd} $ PARENT_SCOPE) -endfunction() diff --git a/thirdparty/ryml/ext/c4core/cmake/c4stlAddTarget.cmake b/thirdparty/ryml/ext/c4core/cmake/c4stlAddTarget.cmake deleted file mode 100644 index 07835977b..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/c4stlAddTarget.cmake +++ /dev/null @@ -1,5 +0,0 @@ -include(c4project) - -function(c4stl_add_target name) - c4_add_target(c4stl ${name} ${ARGN}) -endfunction() # c4stl_add_target diff --git a/thirdparty/ryml/ext/c4core/cmake/compat/c4/gcc-4.8.hpp b/thirdparty/ryml/ext/c4core/cmake/compat/c4/gcc-4.8.hpp deleted file mode 100644 index 83cad6256..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/compat/c4/gcc-4.8.hpp +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef _C4_COMPAT_GCC_4_8_HPP_ -#define _C4_COMPAT_GCC_4_8_HPP_ - -#if __GNUC__ == 4 && __GNUC_MINOR__ >= 8 -/* STL polyfills for old GNU compilers */ - -_Pragma("GCC diagnostic ignored \"-Wshadow\"") -_Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") - -#if __cplusplus -#include -#include - -namespace std { - -template -struct is_trivially_copyable : public integral_constant::value && __has_trivial_destructor(_Tp) && - (__has_trivial_constructor(_Tp) || __has_trivial_copy(_Tp) || __has_trivial_assign(_Tp))> -{ }; - -template -using is_trivially_copy_constructible = has_trivial_copy_constructor<_Tp>; - -template -using is_trivially_default_constructible = has_trivial_default_constructor<_Tp>; - -template -using is_trivially_copy_assignable = has_trivial_copy_assign<_Tp>; - -/* not supported */ -template -struct is_trivially_move_constructible : false_type -{ }; - -/* not supported */ -template -struct is_trivially_move_assignable : false_type -{ }; - -inline void *align(size_t __align, size_t __size, void*& __ptr, size_t& __space) noexcept -{ - if (__space < __size) - return nullptr; - const auto __intptr = reinterpret_cast(__ptr); - const auto __aligned = (__intptr - 1u + __align) & -__align; - const auto __diff = __aligned - __intptr; - if (__diff > (__space - __size)) - return nullptr; - else - { - __space -= __diff; - return __ptr = reinterpret_cast(__aligned); - } -} -typedef long double max_align_t ; - -} -#else // __cplusplus - -#include -// see https://sourceware.org/bugzilla/show_bug.cgi?id=25399 (ubuntu gcc-4.8) -#define memset(s, c, count) __builtin_memset(s, c, count) - -#endif // __cplusplus - -#endif // __GNUC__ == 4 && __GNUC_MINOR__ >= 8 - -#endif // _C4_COMPAT_GCC_4_8_HPP_ diff --git a/thirdparty/ryml/ext/c4core/cmake/compat/gtest_gcc-4.8.patch b/thirdparty/ryml/ext/c4core/cmake/compat/gtest_gcc-4.8.patch deleted file mode 100644 index 07f0ca577..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/compat/gtest_gcc-4.8.patch +++ /dev/null @@ -1,97 +0,0 @@ -Multi-line macros support is not guaranteed with gcc-4.8. - -This uses temporary objects to work-arround this limitation, main drawback is -that compared code is not displayed in message anymore (only "val" placeholders). - ---- googletest/include/gtest/gtest.h -+++ googletest/include/gtest/gtest.h -@@ -2040,6 +2040,80 @@ class TestWithParam : public Test, publi - // ASSERT_LT(i, array_size); - // ASSERT_GT(records.size(), 0) << "There is no record left."; - -+#if __GNUC__ == 4 && __GNUC_MINOR__ >= 8 -+/* -+ * multi-line macros support is not guaranteed with gcc-4.8. -+ * This uses temporary objects to work-arround this limitation, main drawback is -+ * that compared code is not displayed in message anymore (only "val" placeholders) -+ */ -+ -+enum class CompatExpectSelector { EQ, NE, LE, LT, GE, GT }; -+ -+template -+struct CompatExpect -+{ -+ const char *file; -+ int line; -+ ::testing::AssertionResult gtest_ar = AssertionSuccess(); -+ ::testing::Message msg; -+ -+ CompatExpect(const char *file, int line) : file(file), line(line) {} -+ ~CompatExpect() { -+ if (!gtest_ar) -+ GTEST_MESSAGE_AT_(file, line, gtest_ar.failure_message(), ::testing::TestPartResult::kNonFatalFailure) << msg; -+ } -+ -+ template ::type = 0> -+ CompatExpect &operator ()(const T1 &val1, const T2 &val2) { -+ gtest_ar = ::testing::internal::EqHelper::Compare("val1", "val2", val1, val2); -+ return *this; -+ } -+ -+ template ::type = 0> -+ CompatExpect &operator ()(const T1 &val1, const T2 &val2) { -+ gtest_ar = ::testing::internal::CmpHelperNE("val1", "val2", val1, val2); -+ return *this; -+ } -+ -+ template ::type = 0> -+ CompatExpect &operator ()(const T1 &val1, const T2 &val2) { -+ gtest_ar = ::testing::internal::CmpHelperLE("val1", "val2", val1, val2); -+ return *this; -+ } -+ -+ template ::type = 0> -+ CompatExpect &operator ()(const T1 &val1, const T2 &val2) { -+ gtest_ar = ::testing::internal::CmpHelperLT("val1", "val2", val1, val2); -+ return *this; -+ } -+ -+ template ::type = 0> -+ CompatExpect &operator ()(const T1 &val1, const T2 &val2) { -+ gtest_ar = ::testing::internal::CmpHelperGE("val1", "val2", val1, val2); -+ return *this; -+ } -+ -+ template ::type = 0> -+ CompatExpect &operator ()(const T1 &val1, const T2 &val2) { -+ gtest_ar = ::testing::internal::CmpHelperGT("val1", "val2", val1, val2); -+ return *this; -+ } -+ -+ template -+ CompatExpect &operator << (const T1 &t) { -+ msg << t; -+ return *this; -+ } -+}; -+#define EXPECT_EQ ::testing::CompatExpect<::testing::CompatExpectSelector::EQ>{__FILE__,__LINE__} -+#define EXPECT_NE ::testing::CompatExpect<::testing::CompatExpectSelector::NE>{__FILE__,__LINE__} -+#define EXPECT_LE ::testing::CompatExpect<::testing::CompatExpectSelector::LE>{__FILE__,__LINE__} -+#define EXPECT_LT ::testing::CompatExpect<::testing::CompatExpectSelector::LT>{__FILE__,__LINE__} -+#define EXPECT_GE ::testing::CompatExpect<::testing::CompatExpectSelector::GE>{__FILE__,__LINE__} -+#define EXPECT_GT ::testing::CompatExpect<::testing::CompatExpectSelector::GT>{__FILE__,__LINE__} -+ -+#else -+ - #define EXPECT_EQ(val1, val2) \ - EXPECT_PRED_FORMAT2(::testing::internal::EqHelper::Compare, val1, val2) - #define EXPECT_NE(val1, val2) \ -@@ -2053,6 +2127,8 @@ class TestWithParam : public Test, publi - #define EXPECT_GT(val1, val2) \ - EXPECT_PRED_FORMAT2(::testing::internal::CmpHelperGT, val1, val2) - -+#endif -+ - #define GTEST_ASSERT_EQ(val1, val2) \ - ASSERT_PRED_FORMAT2(::testing::internal::EqHelper::Compare, val1, val2) - #define GTEST_ASSERT_NE(val1, val2) \ diff --git a/thirdparty/ryml/ext/c4core/cmake/requirements_doc.txt b/thirdparty/ryml/ext/c4core/cmake/requirements_doc.txt deleted file mode 100644 index baeb2ce4f..000000000 --- a/thirdparty/ryml/ext/c4core/cmake/requirements_doc.txt +++ /dev/null @@ -1,3 +0,0 @@ -sphinx -sphinx_rtd_theme -breathe diff --git a/thirdparty/ryml/ext/c4core/compat.cmake b/thirdparty/ryml/ext/c4core/compat.cmake deleted file mode 100644 index 186b84e0b..000000000 --- a/thirdparty/ryml/ext/c4core/compat.cmake +++ /dev/null @@ -1,16 +0,0 @@ - -# old gcc-4.8 support -if((CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND - (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) AND - (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)) - - c4_install_files( - "${CMAKE_CURRENT_LIST_DIR}/cmake/compat/c4/gcc-4.8.hpp" - "include" - "${CMAKE_CURRENT_LIST_DIR}/cmake/compat") - - # c++17 compiler required - set(C4CORE_BUILD_BENCHMARKS OFF CACHE BOOL "" FORCE) - # LLVM required - set(C4CORE_SANITIZE OFF CACHE BOOL "" FORCE) -endif() diff --git a/thirdparty/ryml/ext/c4core/src/c4/allocator.hpp b/thirdparty/ryml/ext/c4core/src/c4/allocator.hpp deleted file mode 100644 index d221341c5..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/allocator.hpp +++ /dev/null @@ -1,405 +0,0 @@ -#ifndef _C4_ALLOCATOR_HPP_ -#define _C4_ALLOCATOR_HPP_ - -#include "c4/memory_resource.hpp" -#include "c4/ctor_dtor.hpp" - -#include // std::allocator_traits -#include - -/** @file allocator.hpp Contains classes to make typeful allocations (note - * that memory resources are typeless) */ - -/** @defgroup mem_res_providers Memory resource providers - * @brief Policy classes which provide a memory resource for - * use in an allocator. - * @ingroup memory - */ - -/** @defgroup allocators Allocators - * @brief Lightweight classes that act as handles to specific memory - * resources and provide typeful memory. - * @ingroup memory - */ - -namespace c4 { - -namespace detail { -template inline size_t size_for (size_t num_objs) noexcept { return num_objs * sizeof(T); } -template< > inline size_t size_for(size_t num_objs) noexcept { return num_objs; } -} // namespace detail - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** provides a per-allocator memory resource - * @ingroup mem_res_providers */ -class MemRes -{ -public: - - MemRes() : m_resource(get_memory_resource()) {} - MemRes(MemoryResource* r) noexcept : m_resource(r ? r : get_memory_resource()) {} - - inline MemoryResource* resource() const { return m_resource; } - -private: - - MemoryResource* m_resource; - -}; - - -/** the allocators using this will default to the global memory resource - * @ingroup mem_res_providers */ -class MemResGlobal -{ -public: - - MemResGlobal() {} - MemResGlobal(MemoryResource* r) noexcept { C4_UNUSED(r); C4_ASSERT(r == get_memory_resource()); } - - inline MemoryResource* resource() const { return get_memory_resource(); } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace detail { -template -struct _AllocatorUtil; - -template -struct has_no_alloc - : public std::integral_constant::value) - && std::is_constructible::value> {}; - -// std::uses_allocator_v && std::is_constructible -// ie can construct(std::allocator_arg_t, MemoryResource*, Args...) -template -struct has_alloc_arg - : public std::integral_constant::value - && std::is_constructible::value> {}; -// std::uses_allocator && std::is_constructible -// ie, can construct(Args..., MemoryResource*) -template -struct has_alloc - : public std::integral_constant::value - && std::is_constructible::value> {}; - -} // namespace detail - - -template -struct detail::_AllocatorUtil : public MemRes -{ - using MemRes::MemRes; - - /** for construct: - * @see http://en.cppreference.com/w/cpp/experimental/polymorphic_allocator/construct */ - - // 1. types with no allocators - template - C4_ALWAYS_INLINE typename std::enable_if::value, void>::type - construct(U *ptr, Args &&...args) - { - c4::construct(ptr, std::forward(args)...); - } - template - C4_ALWAYS_INLINE typename std::enable_if::value, void>::type - construct_n(U* ptr, I n, Args&&... args) - { - c4::construct_n(ptr, n, std::forward(args)...); - } - - // 2. types using allocators (ie, containers) - - // 2.1. can construct(std::allocator_arg_t, MemoryResource*, Args...) - template - C4_ALWAYS_INLINE typename std::enable_if::value, void>::type - construct(U* ptr, Args&&... args) - { - c4::construct(ptr, std::allocator_arg, this->resource(), std::forward(args)...); - } - template - C4_ALWAYS_INLINE typename std::enable_if::value, void>::type - construct_n(U* ptr, I n, Args&&... args) - { - c4::construct_n(ptr, n, std::allocator_arg, this->resource(), std::forward(args)...); - } - - // 2.2. can construct(Args..., MemoryResource*) - template - C4_ALWAYS_INLINE typename std::enable_if::value, void>::type - construct(U* ptr, Args&&... args) - { - c4::construct(ptr, std::forward(args)..., this->resource()); - } - template - C4_ALWAYS_INLINE typename std::enable_if::value, void>::type - construct_n(U* ptr, I n, Args&&... args) - { - c4::construct_n(ptr, n, std::forward(args)..., this->resource()); - } - - template - static C4_ALWAYS_INLINE void destroy(U* ptr) - { - c4::destroy(ptr); - } - template - static C4_ALWAYS_INLINE void destroy_n(U* ptr, I n) - { - c4::destroy_n(ptr, n); - } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** An allocator is simply a proxy to a memory resource. - * @param T - * @param MemResProvider - * @ingroup allocators */ -template -class Allocator : public detail::_AllocatorUtil -{ -public: - - using impl_type = detail::_AllocatorUtil; - - using value_type = T; - using pointer = T*; - using const_pointer = T const*; - using reference = T&; - using const_reference = T const&; - using size_type = size_t; - using difference_type = std::ptrdiff_t; - using propagate_on_container_move_assigment = std::true_type; - -public: - - template - bool operator== (Allocator const& that) const - { - return this->resource() == that.resource(); - } - template - bool operator!= (Allocator const& that) const - { - return this->resource() != that.resource(); - } - -public: - - template friend class Allocator; - template - struct rebind - { - using other = Allocator; - }; - template - typename rebind::other rebound() - { - return typename rebind::other(*this); - } - -public: - - using impl_type::impl_type; - Allocator() : impl_type() {} // VS demands this - - template Allocator(Allocator const& that) : impl_type(that.resource()) {} - - Allocator(Allocator const&) = default; - Allocator(Allocator &&) = default; - - Allocator& operator= (Allocator const&) = default; // WTF? why? @see http://en.cppreference.com/w/cpp/memory/polymorphic_allocator - Allocator& operator= (Allocator &&) = default; - - /** returns a default-constructed polymorphic allocator object - * @see http://en.cppreference.com/w/cpp/memory/polymorphic_allocator/select_on_container_copy_construction */ - Allocator select_on_container_copy_construct() const { return Allocator(*this); } - - T* allocate(size_t num_objs, size_t alignment=alignof(T)) - { - C4_ASSERT(this->resource() != nullptr); - C4_ASSERT(alignment >= alignof(T)); - void* vmem = this->resource()->allocate(detail::size_for(num_objs), alignment); - T* mem = static_cast(vmem); - return mem; - } - - void deallocate(T * ptr, size_t num_objs, size_t alignment=alignof(T)) - { - C4_ASSERT(this->resource() != nullptr); - C4_ASSERT(alignment>= alignof(T)); - this->resource()->deallocate(ptr, detail::size_for(num_objs), alignment); - } - - T* reallocate(T* ptr, size_t oldnum, size_t newnum, size_t alignment=alignof(T)) - { - C4_ASSERT(this->resource() != nullptr); - C4_ASSERT(alignment >= alignof(T)); - void* vmem = this->resource()->reallocate(ptr, detail::size_for(oldnum), detail::size_for(newnum), alignment); - T* mem = static_cast(vmem); - return mem; - } - -}; - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** @ingroup allocators */ -template -class SmallAllocator : public detail::_AllocatorUtil -{ - static_assert(Alignment >= alignof(T), "invalid alignment"); - - using impl_type = detail::_AllocatorUtil; - - alignas(Alignment) char m_arr[N * sizeof(T)]; - size_t m_num{0}; - -public: - - using value_type = T; - using pointer = T*; - using const_pointer = T const*; - using reference = T&; - using const_reference = T const&; - using size_type = size_t; - using difference_type = std::ptrdiff_t; - using propagate_on_container_move_assigment = std::true_type; - - template - bool operator== (SmallAllocator const&) const - { - return false; - } - template - bool operator!= (SmallAllocator const&) const - { - return true; - } - -public: - - template friend class SmallAllocator; - template - struct rebind - { - using other = SmallAllocator; - }; - template - typename rebind::other rebound() - { - return typename rebind::other(*this); - } - -public: - - using impl_type::impl_type; - SmallAllocator() : impl_type() {} // VS demands this - - template - SmallAllocator(SmallAllocator const& that) : impl_type(that.resource()) - { - C4_ASSERT(that.m_num == 0); - } - - SmallAllocator(SmallAllocator const&) = default; - SmallAllocator(SmallAllocator &&) = default; - - SmallAllocator& operator= (SmallAllocator const&) = default; // WTF? why? @see http://en.cppreference.com/w/cpp/memory/polymorphic_allocator - SmallAllocator& operator= (SmallAllocator &&) = default; - - /** returns a default-constructed polymorphic allocator object - * @see http://en.cppreference.com/w/cpp/memory/polymorphic_allocator/select_on_container_copy_construction */ - SmallAllocator select_on_container_copy_construct() const { return SmallAllocator(*this); } - - T* allocate(size_t num_objs, size_t alignment=Alignment) - { - C4_ASSERT(this->resource() != nullptr); - C4_ASSERT(alignment >= alignof(T)); - void *vmem; - if(m_num + num_objs <= N) - { - vmem = (m_arr + m_num * sizeof(T)); - } - else - { - vmem = this->resource()->allocate(num_objs * sizeof(T), alignment); - } - m_num += num_objs; - T *mem = static_cast(vmem); - return mem; - } - - void deallocate(T * ptr, size_t num_objs, size_t alignment=Alignment) - { - C4_ASSERT(m_num >= num_objs); - m_num -= num_objs; - if((char*)ptr >= m_arr && (char*)ptr < m_arr + (N * sizeof(T))) - { - return; - } - C4_ASSERT(this->resource() != nullptr); - C4_ASSERT(alignment >= alignof(T)); - this->resource()->deallocate(ptr, num_objs * sizeof(T), alignment); - } - - T* reallocate(T * ptr, size_t oldnum, size_t newnum, size_t alignment=Alignment) - { - C4_ASSERT(this->resource() != nullptr); - C4_ASSERT(alignment >= alignof(T)); - if(oldnum <= N && newnum <= N) - { - return m_arr; - } - else if(oldnum <= N && newnum > N) - { - return allocate(newnum, alignment); - } - else if(oldnum > N && newnum <= N) - { - deallocate(ptr, oldnum, alignment); - return m_arr; - } - void* vmem = this->resource()->reallocate(ptr, oldnum * sizeof(T), newnum * sizeof(T), alignment); - T* mem = static_cast(vmem); - return mem; - } - -}; - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** An allocator making use of the global memory resource. - * @ingroup allocators */ -template using allocator = Allocator; -/** An allocator with a per-instance memory resource - * @ingroup allocators */ -template using allocator_mr = Allocator; - -/** @ingroup allocators */ -template using small_allocator = SmallAllocator; -/** @ingroup allocators */ -template using small_allocator_mr = SmallAllocator; - -} // namespace c4 - -#endif /* _C4_ALLOCATOR_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/base64.cpp b/thirdparty/ryml/ext/c4core/src/c4/base64.cpp deleted file mode 100644 index bc75d3e0e..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/base64.cpp +++ /dev/null @@ -1,220 +0,0 @@ -#include "c4/base64.hpp" - -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wchar-subscripts" // array subscript is of type 'char' -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wchar-subscripts" -# pragma GCC diagnostic ignored "-Wtype-limits" -#endif - -namespace c4 { - -namespace detail { - -constexpr static const char base64_sextet_to_char_[64] = { - /* 0/ 65*/ 'A', /* 1/ 66*/ 'B', /* 2/ 67*/ 'C', /* 3/ 68*/ 'D', - /* 4/ 69*/ 'E', /* 5/ 70*/ 'F', /* 6/ 71*/ 'G', /* 7/ 72*/ 'H', - /* 8/ 73*/ 'I', /* 9/ 74*/ 'J', /*10/ 75*/ 'K', /*11/ 74*/ 'L', - /*12/ 77*/ 'M', /*13/ 78*/ 'N', /*14/ 79*/ 'O', /*15/ 78*/ 'P', - /*16/ 81*/ 'Q', /*17/ 82*/ 'R', /*18/ 83*/ 'S', /*19/ 82*/ 'T', - /*20/ 85*/ 'U', /*21/ 86*/ 'V', /*22/ 87*/ 'W', /*23/ 88*/ 'X', - /*24/ 89*/ 'Y', /*25/ 90*/ 'Z', /*26/ 97*/ 'a', /*27/ 98*/ 'b', - /*28/ 99*/ 'c', /*29/100*/ 'd', /*30/101*/ 'e', /*31/102*/ 'f', - /*32/103*/ 'g', /*33/104*/ 'h', /*34/105*/ 'i', /*35/106*/ 'j', - /*36/107*/ 'k', /*37/108*/ 'l', /*38/109*/ 'm', /*39/110*/ 'n', - /*40/111*/ 'o', /*41/112*/ 'p', /*42/113*/ 'q', /*43/114*/ 'r', - /*44/115*/ 's', /*45/116*/ 't', /*46/117*/ 'u', /*47/118*/ 'v', - /*48/119*/ 'w', /*49/120*/ 'x', /*50/121*/ 'y', /*51/122*/ 'z', - /*52/ 48*/ '0', /*53/ 49*/ '1', /*54/ 50*/ '2', /*55/ 51*/ '3', - /*56/ 52*/ '4', /*57/ 53*/ '5', /*58/ 54*/ '6', /*59/ 55*/ '7', - /*60/ 56*/ '8', /*61/ 57*/ '9', /*62/ 43*/ '+', /*63/ 47*/ '/', -}; - -// https://www.cs.cmu.edu/~pattis/15-1XX/common/handouts/ascii.html -constexpr static const char base64_char_to_sextet_[128] = { - #define __ char(-1) // undefined below - /* 0 NUL*/ __, /* 1 SOH*/ __, /* 2 STX*/ __, /* 3 ETX*/ __, - /* 4 EOT*/ __, /* 5 ENQ*/ __, /* 6 ACK*/ __, /* 7 BEL*/ __, - /* 8 BS */ __, /* 9 TAB*/ __, /* 10 LF */ __, /* 11 VT */ __, - /* 12 FF */ __, /* 13 CR */ __, /* 14 SO */ __, /* 15 SI */ __, - /* 16 DLE*/ __, /* 17 DC1*/ __, /* 18 DC2*/ __, /* 19 DC3*/ __, - /* 20 DC4*/ __, /* 21 NAK*/ __, /* 22 SYN*/ __, /* 23 ETB*/ __, - /* 24 CAN*/ __, /* 25 EM */ __, /* 26 SUB*/ __, /* 27 ESC*/ __, - /* 28 FS */ __, /* 29 GS */ __, /* 30 RS */ __, /* 31 US */ __, - /* 32 SPC*/ __, /* 33 ! */ __, /* 34 " */ __, /* 35 # */ __, - /* 36 $ */ __, /* 37 % */ __, /* 38 & */ __, /* 39 ' */ __, - /* 40 ( */ __, /* 41 ) */ __, /* 42 * */ __, /* 43 + */ 62, - /* 44 , */ __, /* 45 - */ __, /* 46 . */ __, /* 47 / */ 63, - /* 48 0 */ 52, /* 49 1 */ 53, /* 50 2 */ 54, /* 51 3 */ 55, - /* 52 4 */ 56, /* 53 5 */ 57, /* 54 6 */ 58, /* 55 7 */ 59, - /* 56 8 */ 60, /* 57 9 */ 61, /* 58 : */ __, /* 59 ; */ __, - /* 60 < */ __, /* 61 = */ __, /* 62 > */ __, /* 63 ? */ __, - /* 64 @ */ __, /* 65 A */ 0, /* 66 B */ 1, /* 67 C */ 2, - /* 68 D */ 3, /* 69 E */ 4, /* 70 F */ 5, /* 71 G */ 6, - /* 72 H */ 7, /* 73 I */ 8, /* 74 J */ 9, /* 75 K */ 10, - /* 76 L */ 11, /* 77 M */ 12, /* 78 N */ 13, /* 79 O */ 14, - /* 80 P */ 15, /* 81 Q */ 16, /* 82 R */ 17, /* 83 S */ 18, - /* 84 T */ 19, /* 85 U */ 20, /* 86 V */ 21, /* 87 W */ 22, - /* 88 X */ 23, /* 89 Y */ 24, /* 90 Z */ 25, /* 91 [ */ __, - /* 92 \ */ __, /* 93 ] */ __, /* 94 ^ */ __, /* 95 _ */ __, - /* 96 ` */ __, /* 97 a */ 26, /* 98 b */ 27, /* 99 c */ 28, - /*100 d */ 29, /*101 e */ 30, /*102 f */ 31, /*103 g */ 32, - /*104 h */ 33, /*105 i */ 34, /*106 j */ 35, /*107 k */ 36, - /*108 l */ 37, /*109 m */ 38, /*110 n */ 39, /*111 o */ 40, - /*112 p */ 41, /*113 q */ 42, /*114 r */ 43, /*115 s */ 44, - /*116 t */ 45, /*117 u */ 46, /*118 v */ 47, /*119 w */ 48, - /*120 x */ 49, /*121 y */ 50, /*122 z */ 51, /*123 { */ __, - /*124 | */ __, /*125 } */ __, /*126 ~ */ __, /*127 DEL*/ __, - #undef __ -}; - -#ifndef NDEBUG -void base64_test_tables() -{ - for(size_t i = 0; i < C4_COUNTOF(detail::base64_sextet_to_char_); ++i) - { - char s2c = base64_sextet_to_char_[i]; - char c2s = base64_char_to_sextet_[(int)s2c]; - C4_CHECK((size_t)c2s == i); - } - for(size_t i = 0; i < C4_COUNTOF(detail::base64_char_to_sextet_); ++i) - { - char c2s = base64_char_to_sextet_[i]; - if(c2s == char(-1)) - continue; - char s2c = base64_sextet_to_char_[(int)c2s]; - C4_CHECK((size_t)s2c == i); - } -} -#endif -} // namespace detail - - -bool base64_valid(csubstr encoded) -{ - if(encoded.len % 4) return false; - for(const char c : encoded) - { - if(c < 0/* || c >= 128*/) - return false; - if(c == '=') - continue; - if(detail::base64_char_to_sextet_[c] == char(-1)) - return false; - } - return true; -} - - -size_t base64_encode(substr buf, cblob data) -{ - #define c4append_(c) { if(pos < buf.len) { buf.str[pos] = (c); } ++pos; } - #define c4append_idx_(char_idx) \ - {\ - C4_XASSERT((char_idx) < sizeof(detail::base64_sextet_to_char_));\ - c4append_(detail::base64_sextet_to_char_[(char_idx)]);\ - } - - size_t rem, pos = 0; - constexpr const uint32_t sextet_mask = uint32_t(1 << 6) - 1; - const unsigned char *C4_RESTRICT d = (unsigned char *) data.buf; // cast to unsigned to avoid wrapping high-bits - for(rem = data.len; rem >= 3; rem -= 3, d += 3) - { - const uint32_t val = ((uint32_t(d[0]) << 16) | (uint32_t(d[1]) << 8) | (uint32_t(d[2]))); - c4append_idx_((val >> 18) & sextet_mask); - c4append_idx_((val >> 12) & sextet_mask); - c4append_idx_((val >> 6) & sextet_mask); - c4append_idx_((val ) & sextet_mask); - } - C4_ASSERT(rem < 3); - if(rem == 2) - { - const uint32_t val = ((uint32_t(d[0]) << 16) | (uint32_t(d[1]) << 8)); - c4append_idx_((val >> 18) & sextet_mask); - c4append_idx_((val >> 12) & sextet_mask); - c4append_idx_((val >> 6) & sextet_mask); - c4append_('='); - } - else if(rem == 1) - { - const uint32_t val = ((uint32_t(d[0]) << 16)); - c4append_idx_((val >> 18) & sextet_mask); - c4append_idx_((val >> 12) & sextet_mask); - c4append_('='); - c4append_('='); - } - return pos; - - #undef c4append_ - #undef c4append_idx_ -} - - -size_t base64_decode(csubstr encoded, blob data) -{ - #define c4append_(c) { if(wpos < data.len) { data.buf[wpos] = static_cast(c); } ++wpos; } - #define c4appendval_(c, shift)\ - {\ - C4_XASSERT(c >= 0);\ - C4_XASSERT(size_t(c) < sizeof(detail::base64_char_to_sextet_));\ - val |= static_cast(detail::base64_char_to_sextet_[(c)]) << ((shift) * 6);\ - } - - C4_ASSERT(base64_valid(encoded)); - C4_CHECK(encoded.len % 4 == 0); - size_t wpos = 0; // the write position - const char *C4_RESTRICT d = encoded.str; - constexpr const uint32_t full_byte = 0xff; - // process every quartet of input 6 bits --> triplet of output bytes - for(size_t rpos = 0; rpos < encoded.len; rpos += 4, d += 4) - { - if(d[2] == '=' || d[3] == '=') // skip the last quartet if it is padded - { - C4_ASSERT(d + 4 == encoded.str + encoded.len); - break; - } - uint32_t val = 0; - c4appendval_(d[3], 0); - c4appendval_(d[2], 1); - c4appendval_(d[1], 2); - c4appendval_(d[0], 3); - c4append_((val >> (2 * 8)) & full_byte); - c4append_((val >> (1 * 8)) & full_byte); - c4append_((val ) & full_byte); - } - // deal with the last quartet when it is padded - if(d == encoded.str + encoded.len) - return wpos; - if(d[2] == '=') // 2 padding chars - { - C4_ASSERT(d + 4 == encoded.str + encoded.len); - C4_ASSERT(d[3] == '='); - uint32_t val = 0; - c4appendval_(d[1], 2); - c4appendval_(d[0], 3); - c4append_((val >> (2 * 8)) & full_byte); - } - else if(d[3] == '=') // 1 padding char - { - C4_ASSERT(d + 4 == encoded.str + encoded.len); - uint32_t val = 0; - c4appendval_(d[2], 1); - c4appendval_(d[1], 2); - c4appendval_(d[0], 3); - c4append_((val >> (2 * 8)) & full_byte); - c4append_((val >> (1 * 8)) & full_byte); - } - return wpos; - #undef c4append_ - #undef c4appendval_ -} - -} // namespace c4 - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/base64.hpp b/thirdparty/ryml/ext/c4core/src/c4/base64.hpp deleted file mode 100644 index 673f4d77e..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/base64.hpp +++ /dev/null @@ -1,98 +0,0 @@ -#ifndef _C4_BASE64_HPP_ -#define _C4_BASE64_HPP_ - -/** @file base64.hpp encoding/decoding for base64. - * @see https://en.wikipedia.org/wiki/Base64 - * @see https://www.base64encode.org/ - * */ - -#include "c4/charconv.hpp" -#include "c4/blob.hpp" - -namespace c4 { - -/** check that the given buffer is a valid base64 encoding - * @see https://en.wikipedia.org/wiki/Base64 */ -bool base64_valid(csubstr encoded); - -/** base64-encode binary data. - * @param encoded [out] output buffer for encoded data - * @param data [in] the input buffer with the binary data - * @return the number of bytes needed to return the output. No writes occur beyond the end of the output buffer. - * @see https://en.wikipedia.org/wiki/Base64 */ -size_t base64_encode(substr encoded, cblob data); - -/** decode the base64 encoding in the given buffer - * @param encoded [in] the encoded base64 - * @param data [out] the output buffer - * @return the number of bytes needed to return the output.. No writes occur beyond the end of the output buffer. - * @see https://en.wikipedia.org/wiki/Base64 */ -size_t base64_decode(csubstr encoded, blob data); - - -namespace fmt { - -template -struct base64_wrapper_ -{ - blob_ data; - base64_wrapper_() : data() {} - base64_wrapper_(blob_ blob) : data(blob) {} -}; -using const_base64_wrapper = base64_wrapper_; -using base64_wrapper = base64_wrapper_; - - -/** mark a variable to be written in base64 format */ -template -C4_ALWAYS_INLINE const_base64_wrapper cbase64(Args const& C4_RESTRICT ...args) -{ - return const_base64_wrapper(cblob(args...)); -} -/** mark a csubstr to be written in base64 format */ -C4_ALWAYS_INLINE const_base64_wrapper cbase64(csubstr s) -{ - return const_base64_wrapper(cblob(s.str, s.len)); -} -/** mark a variable to be written in base64 format */ -template -C4_ALWAYS_INLINE const_base64_wrapper base64(Args const& C4_RESTRICT ...args) -{ - return const_base64_wrapper(cblob(args...)); -} -/** mark a csubstr to be written in base64 format */ -C4_ALWAYS_INLINE const_base64_wrapper base64(csubstr s) -{ - return const_base64_wrapper(cblob(s.str, s.len)); -} - -/** mark a variable to be read in base64 format */ -template -C4_ALWAYS_INLINE base64_wrapper base64(Args &... args) -{ - return base64_wrapper(blob(args...)); -} -/** mark a variable to be read in base64 format */ -C4_ALWAYS_INLINE base64_wrapper base64(substr s) -{ - return base64_wrapper(blob(s.str, s.len)); -} - -} // namespace fmt - - -/** write a variable in base64 format */ -inline size_t to_chars(substr buf, fmt::const_base64_wrapper b) -{ - return base64_encode(buf, b.data); -} - -/** read a variable in base64 format */ -inline size_t from_chars(csubstr buf, fmt::base64_wrapper *b) -{ - return base64_decode(buf, b->data); -} - -} // namespace c4 - -#endif /* _C4_BASE64_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/bitmask.hpp b/thirdparty/ryml/ext/c4core/src/c4/bitmask.hpp deleted file mode 100644 index 8c239da30..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/bitmask.hpp +++ /dev/null @@ -1,330 +0,0 @@ -#ifndef _C4_BITMASK_HPP_ -#define _C4_BITMASK_HPP_ - -/** @file bitmask.hpp bitmask utilities */ - -#include -#include - -#include "c4/enum.hpp" -#include "c4/format.hpp" - -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable : 4996) // 'strncpy', fopen, etc: This function or variable may be unsafe -#elif defined(__clang__) -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# if __GNUC__ >= 8 -# pragma GCC diagnostic ignored "-Wstringop-truncation" -# pragma GCC diagnostic ignored "-Wstringop-overflow" -# endif -#endif - -namespace c4 { - -//----------------------------------------------------------------------------- -/** write a bitmask to a stream, formatted as a string */ - -template -Stream& bm2stream(Stream &s, typename std::underlying_type::type bits, EnumOffsetType offst=EOFFS_PFX) -{ - using I = typename std::underlying_type::type; - bool written = false; - - auto const& pairs = esyms(); - - // write non null value - if(bits) - { - // do reverse iteration to give preference to composite enum symbols, - // which are likely to appear at the end of the enum sequence - for(size_t i = pairs.size() - 1; i != size_t(-1); --i) - { - auto p = pairs[i]; - I b(static_cast(p.value)); - if(b && (bits & b) == b) - { - if(written) s << '|'; // append bit-or character - written = true; - s << p.name_offs(offst); // append bit string - bits &= ~b; - } - } - return s; - } - else - { - // write a null value - for(size_t i = pairs.size() - 1; i != size_t(-1); --i) - { - auto p = pairs[i]; - I b(static_cast(p.value)); - if(b == 0) - { - s << p.name_offs(offst); - written = true; - break; - } - } - } - if(!written) - { - s << '0'; - } - return s; -} - -template -typename std::enable_if::value, Stream&>::type -bm2stream(Stream &s, Enum value, EnumOffsetType offst=EOFFS_PFX) -{ - using I = typename std::underlying_type::type; - return bm2stream(s, static_cast(value), offst); -} - - -//----------------------------------------------------------------------------- - -// some utility macros, undefed below - -/// @cond dev - -/* Execute `code` if the `num` of characters is available in the str - * buffer. This macro simplifies the code for bm2str(). - * @todo improve performance by writing from the end and moving only once. */ -#define _c4prependchars(code, num) \ - if(str && (pos + num <= sz)) \ - { \ - /* move the current string to the right */ \ - memmove(str + num, str, pos); \ - /* now write in the beginning of the string */ \ - code; \ - } \ - else if(str && sz) \ - { \ - C4_ERROR("cannot write to string pos=%d num=%d sz=%d", \ - (int)pos, (int)num, (int)sz); \ - } \ - pos += num - -/* Execute `code` if the `num` of characters is available in the str - * buffer. This macro simplifies the code for bm2str(). */ -#define _c4appendchars(code, num) \ - if(str && (pos + num <= sz)) \ - { \ - code; \ - } \ - else if(str && sz) \ - { \ - C4_ERROR("cannot write to string pos=%d num=%d sz=%d", \ - (int)pos, (int)num, (int)sz); \ - } \ - pos += num - -/// @endcond - - -/** convert a bitmask to string. - * return the number of characters written. To find the needed size, - * call first with str=nullptr and sz=0 */ -template -size_t bm2str -( - typename std::underlying_type::type bits, - char *str=nullptr, - size_t sz=0, - EnumOffsetType offst=EOFFS_PFX -) -{ - using I = typename std::underlying_type::type; - C4_ASSERT((str == nullptr) == (sz == 0)); - - auto syms = esyms(); - size_t pos = 0; - typename EnumSymbols::Sym const* C4_RESTRICT zero = nullptr; - - // do reverse iteration to give preference to composite enum symbols, - // which are likely to appear later in the enum sequence - for(size_t i = syms.size()-1; i != size_t(-1); --i) - { - auto const &C4_RESTRICT p = syms[i]; // do not copy, we are assigning to `zero` - I b = static_cast(p.value); - if(b == 0) - { - zero = &p; // save this symbol for later - } - else if((bits & b) == b) - { - bits &= ~b; - // append bit-or character - if(pos > 0) - { - _c4prependchars(*str = '|', 1); - } - // append bit string - const char *pname = p.name_offs(offst); - size_t len = strlen(pname); - _c4prependchars(strncpy(str, pname, len), len); - } - } - - C4_CHECK_MSG(bits == 0, "could not find all bits"); - if(pos == 0) // make sure at least something is written - { - if(zero) // if we have a zero symbol, use that - { - const char *pname = zero->name_offs(offst); - size_t len = strlen(pname); - _c4prependchars(strncpy(str, pname, len), len); - } - else // otherwise just write an integer zero - { - _c4prependchars(*str = '0', 1); - } - } - _c4appendchars(str[pos] = '\0', 1); - - return pos; -} - - -// cleanup! -#undef _c4appendchars -#undef _c4prependchars - - -/** scoped enums do not convert automatically to their underlying type, - * so this SFINAE overload will accept scoped enum symbols and cast them - * to the underlying type */ -template -typename std::enable_if::value, size_t>::type -bm2str -( - Enum bits, - char *str=nullptr, - size_t sz=0, - EnumOffsetType offst=EOFFS_PFX -) -{ - using I = typename std::underlying_type::type; - return bm2str(static_cast(bits), str, sz, offst); -} - - -//----------------------------------------------------------------------------- - -namespace detail { - -#ifdef __clang__ -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# if __GNUC__ >= 6 -# pragma GCC diagnostic ignored "-Wnull-dereference" -# endif -#endif - -template -typename std::underlying_type::type str2bm_read_one(const char *str, size_t sz, bool alnum) -{ - using I = typename std::underlying_type::type; - auto pairs = esyms(); - if(alnum) - { - auto *p = pairs.find(str, sz); - C4_CHECK_MSG(p != nullptr, "no valid enum pair name for '%.*s'", (int)sz, str); - return static_cast(p->value); - } - I tmp; - size_t len = uncat(csubstr(str, sz), tmp); - C4_CHECK_MSG(len != csubstr::npos, "could not read string as an integral type: '%.*s'", (int)sz, str); - return tmp; -} - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif -} // namespace detail - -/** convert a string to a bitmask */ -template -typename std::underlying_type::type str2bm(const char *str, size_t sz) -{ - using I = typename std::underlying_type::type; - - I val = 0; - bool started = false; - bool alnum = false, num = false; - const char *f = nullptr, *pc = str; - for( ; pc < str+sz; ++pc) - { - const char c = *pc; - if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_') - { - C4_CHECK(( ! num) || ((pc - f) == 1 && (c == 'x' || c == 'X'))); // accept hexadecimal numbers - if( ! started) - { - f = pc; - alnum = started = true; - } - } - else if(c >= '0' && c <= '9') - { - C4_CHECK( ! alnum); - if(!started) - { - f = pc; - num = started = true; - } - } - else if(c == ':' || c == ' ') - { - // skip this char - } - else if(c == '|' || c == '\0') - { - C4_ASSERT(num != alnum); - C4_ASSERT(pc >= f); - val |= detail::str2bm_read_one(f, static_cast(pc-f), alnum); - started = num = alnum = false; - if(c == '\0') - { - return val; - } - } - else - { - C4_ERROR("bad character '%c' in bitmask string", c); - } - } - - if(f) - { - C4_ASSERT(num != alnum); - C4_ASSERT(pc >= f); - val |= detail::str2bm_read_one(f, static_cast(pc-f), alnum); - } - - return val; -} - -/** convert a string to a bitmask */ -template -typename std::underlying_type::type str2bm(const char *str) -{ - return str2bm(str, strlen(str)); -} - -} // namespace c4 - -#ifdef _MSC_VER -# pragma warning(pop) -#elif defined(__clang__) -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -#endif // _C4_BITMASK_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/blob.hpp b/thirdparty/ryml/ext/c4core/src/c4/blob.hpp deleted file mode 100644 index 9c233d9f3..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/blob.hpp +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef _C4_BLOB_HPP_ -#define _C4_BLOB_HPP_ - -#include "c4/types.hpp" -#include "c4/error.hpp" - -/** @file blob.hpp Mutable and immutable binary data blobs. -*/ - -namespace c4 { - -template -struct blob_ -{ - T * buf; - size_t len; - - C4_ALWAYS_INLINE blob_() noexcept : buf(), len() {} - - C4_ALWAYS_INLINE blob_(blob_ const& that) noexcept = default; - C4_ALWAYS_INLINE blob_(blob_ && that) noexcept = default; - C4_ALWAYS_INLINE blob_& operator=(blob_ && that) noexcept = default; - C4_ALWAYS_INLINE blob_& operator=(blob_ const& that) noexcept = default; - - // need to sfinae out copy constructors! (why? isn't the above sufficient?) - #define _C4_REQUIRE_NOT_SAME class=typename std::enable_if<( ! std::is_same::value) && ( ! std::is_pointer::value), T>::type - template C4_ALWAYS_INLINE blob_(U &var) noexcept : buf(reinterpret_cast(&var)), len(sizeof(U)) {} - template C4_ALWAYS_INLINE blob_& operator= (U &var) noexcept { buf = reinterpret_cast(&var); len = sizeof(U); return *this; } - #undef _C4_REQUIRE_NOT_SAME - - template C4_ALWAYS_INLINE blob_(U (&arr)[N]) noexcept : buf(reinterpret_cast(arr)), len(sizeof(U) * N) {} - template C4_ALWAYS_INLINE blob_& operator= (U (&arr)[N]) noexcept { buf = reinterpret_cast(arr); len = sizeof(U) * N; return *this; } - - template - C4_ALWAYS_INLINE blob_(U *ptr, size_t n) noexcept : buf(reinterpret_cast(ptr)), len(sizeof(U) * n) { C4_ASSERT(is_aligned(ptr)); } - C4_ALWAYS_INLINE blob_(void *ptr, size_t n) noexcept : buf(reinterpret_cast(ptr)), len(n) {} - C4_ALWAYS_INLINE blob_(void const *ptr, size_t n) noexcept : buf(reinterpret_cast(ptr)), len(n) {} -}; - -/** an immutable binary blob */ -using cblob = blob_; -/** a mutable binary blob */ -using blob = blob_< byte>; - -C4_MUST_BE_TRIVIAL_COPY(blob); -C4_MUST_BE_TRIVIAL_COPY(cblob); - -} // namespace c4 - -#endif // _C4_BLOB_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/c4_pop.hpp b/thirdparty/ryml/ext/c4core/src/c4/c4_pop.hpp deleted file mode 100644 index 3aa213352..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/c4_pop.hpp +++ /dev/null @@ -1,19 +0,0 @@ -#ifdef _C4_PUSH_HPP_ // this must match the include guard from c4_push - -/** @file c4_pop.hpp disables the macros and control directives - * enabled in c4_push.hpp. - * @see c4_push.hpp */ - -#include "c4/unrestrict.hpp" - -#ifdef C4_WIN -# include "c4/windows_pop.hpp" -#endif - -#ifdef _MSC_VER -# pragma warning(pop) -#endif - -#undef _C4_PUSH_HPP_ - -#endif /* _C4_PUSH_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/c4_push.hpp b/thirdparty/ryml/ext/c4core/src/c4/c4_push.hpp deleted file mode 100644 index 5498bdb28..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/c4_push.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef _C4_PUSH_HPP_ -#define _C4_PUSH_HPP_ - - -/** @file c4_push.hpp enables macros and warning control directives - * needed by c4core. This is implemented in a push/pop way. - * @see c4_pop.hpp */ - - -#ifndef _C4_CONFIG_HPP_ -#include "c4/config.hpp" -#endif - -#include "c4/restrict.hpp" - -#ifdef C4_WIN -# include "c4/windows_push.hpp" -#endif - -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable : 4068) // unknown pragma -# pragma warning(disable : 4100) // unreferenced formal parameter -# pragma warning(disable : 4127) // conditional expression is constant -- eg do {} while(1); -# pragma warning(disable : 4201) // nonstandard extension used : nameless struct/union -//# pragma warning(disable : 4238) // nonstandard extension used: class rvalue used as lvalue -# pragma warning(disable : 4244) -# pragma warning(disable : 4503) // decorated name length exceeded, name was truncated -# pragma warning(disable : 4702) // unreachable code -# pragma warning(disable : 4714) // function marked as __forceinline not inlined -# pragma warning(disable : 4996) // 'strncpy', fopen, etc: This function or variable may be unsafe -# if C4_MSVC_VERSION != C4_MSVC_VERSION_2017 -# pragma warning(disable : 4800) // forcing value to bool 'true' or 'false' (performance warning) -# endif -#endif - -#endif /* _C4_PUSH_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/c4core.natvis b/thirdparty/ryml/ext/c4core/src/c4/c4core.natvis deleted file mode 100644 index 7cd138fe6..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/c4core.natvis +++ /dev/null @@ -1,168 +0,0 @@ - - - - - - - - {str,[len]} (sz={len}) - str,[len] - - len - - len - str - - - - - - {m_ptr,[m_size]} (sz={m_size}) - - m_size - - m_size - m_ptr - - - - - {m_ptr,[m_size]} (sz={m_size}, cap={m_capacity}) - - m_size - m_capacity - - m_size - m_ptr - - - - - - {m_ptr,[m_size]} (sz={m_size}) - m_ptr,[m_size] - - m_size - - m_size - m_ptr - - - - - {m_ptr,[m_size]} (sz={m_size}) - m_ptr,[m_size] - - m_size - - m_size - m_ptr - - - - - - {m_ptr,[m_size]} (sz={m_size}, cap={m_capacity}) - m_ptr,[m_size] - - m_size - m_capacity - - m_size - m_ptr - - - - - {m_ptr,[m_size]} (sz={m_size}, cap={m_capacity}) - m_ptr,[m_size] - - m_size - m_capacity - - m_size - m_ptr - - - - - - - {(($T3*)this)->m_str,[(($T3*)this)->m_size]} (sz={(($T3*)this)->m_size}) - (($T3*)this)->m_str,[(($T3*)this)->m_size] - - - {(($T3*)this)->m_str,[(($T3*)this)->m_size]} - (($T3*)this)->m_str,[(($T3*)this)->m_size] - - - {(($T3*)this)->m_size} - - - - - {m_str,[m_size]} (sz={m_size}) - m_str,[m_size] - - - {m_size} - - - - - {m_str,[m_size]} (sz={m_size},cap={m_capacity}) - m_str,[m_size] - - - {m_size} - - - {m_capacity} - - - {m_str,[m_capacity]} - m_str,[m_capacity] - - - - - {m_str,[m_size]} (sz={m_size},cap={m_capacity}) - m_str,[m_size] - - - {m_size} - - - {m_str,[m_capacity]} - m_str,[m_capacity] - - - - - - - {value} - {name} - - value - name - - - - {m_symbols,[m_num]} (sz={m_num}) - - m_num - - m_num - m_symbols - - - - - diff --git a/thirdparty/ryml/ext/c4core/src/c4/char_traits.cpp b/thirdparty/ryml/ext/c4core/src/c4/char_traits.cpp deleted file mode 100644 index 053a7c3b1..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/char_traits.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "c4/char_traits.hpp" - -namespace c4 { - -constexpr const char char_traits< char >::whitespace_chars[]; -constexpr const size_t char_traits< char >::num_whitespace_chars; -constexpr const wchar_t char_traits< wchar_t >::whitespace_chars[]; -constexpr const size_t char_traits< wchar_t >::num_whitespace_chars; - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/src/c4/char_traits.hpp b/thirdparty/ryml/ext/c4core/src/c4/char_traits.hpp deleted file mode 100644 index b080659fb..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/char_traits.hpp +++ /dev/null @@ -1,98 +0,0 @@ -#ifndef _C4_CHAR_TRAITS_HPP_ -#define _C4_CHAR_TRAITS_HPP_ - -#include "c4/config.hpp" - -#include // needed because of std::char_traits -#include -#include - -namespace c4 { - -C4_ALWAYS_INLINE bool isspace(char c) { return std::isspace(c) != 0; } -C4_ALWAYS_INLINE bool isspace(wchar_t c) { return std::iswspace(static_cast(c)) != 0; } - -//----------------------------------------------------------------------------- -template -struct char_traits; - -template<> -struct char_traits : public std::char_traits -{ - constexpr static const char whitespace_chars[] = " \f\n\r\t\v"; - constexpr static const size_t num_whitespace_chars = sizeof(whitespace_chars) - 1; -}; - -template<> -struct char_traits : public std::char_traits -{ - constexpr static const wchar_t whitespace_chars[] = L" \f\n\r\t\v"; - constexpr static const size_t num_whitespace_chars = sizeof(whitespace_chars) - 1; -}; - - -//----------------------------------------------------------------------------- -namespace detail { -template -struct needed_chars; -template<> -struct needed_chars -{ - template - C4_ALWAYS_INLINE constexpr static SizeType for_bytes(SizeType num_bytes) - { - return num_bytes; - } -}; -template<> -struct needed_chars -{ - template - C4_ALWAYS_INLINE constexpr static SizeType for_bytes(SizeType num_bytes) - { - // wchar_t is not necessarily 2 bytes. - return (num_bytes / static_cast(sizeof(wchar_t))) + ((num_bytes & static_cast(SizeType(sizeof(wchar_t)) - SizeType(1))) != 0); - } -}; -} // namespace detail - -/** get the number of C characters needed to store a number of bytes */ -template -C4_ALWAYS_INLINE constexpr SizeType num_needed_chars(SizeType num_bytes) -{ - return detail::needed_chars::for_bytes(num_bytes); -} - - -//----------------------------------------------------------------------------- - -/** get the given text string as either char or wchar_t according to the given type */ -#define C4_TXTTY(txt, type) \ - /* is there a smarter way to do this? */\ - c4::detail::literal_as::get(txt, C4_WIDEN(txt)) - -namespace detail { -template -struct literal_as; - -template<> -struct literal_as -{ - C4_ALWAYS_INLINE static constexpr const char* get(const char* str, const wchar_t *) - { - return str; - } -}; -template<> -struct literal_as -{ - C4_ALWAYS_INLINE static constexpr const wchar_t* get(const char*, const wchar_t *wstr) - { - return wstr; - } -}; -} // namespace detail - -} // namespace c4 - -#endif /* _C4_CHAR_TRAITS_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/charconv.hpp b/thirdparty/ryml/ext/c4core/src/c4/charconv.hpp deleted file mode 100644 index af44f136c..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/charconv.hpp +++ /dev/null @@ -1,2407 +0,0 @@ -#ifndef _C4_CHARCONV_HPP_ -#define _C4_CHARCONV_HPP_ - -/** @file charconv.hpp Lightweight generic type-safe wrappers for - * converting individual values to/from strings. - * - * These are the main functions: - * - * @code{.cpp} - * // Convert the given value, writing into the string. - * // The resulting string will NOT be null-terminated. - * // Return the number of characters needed. - * // This function is safe to call when the string is too small - - * // no writes will occur beyond the string's last character. - * template size_t c4::to_chars(substr buf, T const& C4_RESTRICT val); - * - * - * // Convert the given value to a string using to_chars(), and - * // return the resulting string, up to and including the last - * // written character. - * template substr c4::to_chars_sub(substr buf, T const& C4_RESTRICT val); - * - * - * // Read a value from the string, which must be - * // trimmed to the value (ie, no leading/trailing whitespace). - * // return true if the conversion succeeded. - * // There is no check for overflow; the value wraps around in a way similar - * // to the standard C/C++ overflow behavior. For example, - * // from_chars("128", &val) returns true and val will be - * // set tot 0. - * template bool c4::from_chars(csubstr buf, T * C4_RESTRICT val); - * - * - * // Read the first valid sequence of characters from the string, - * // skipping leading whitespace, and convert it using from_chars(). - * // Return the number of characters read for converting. - * template size_t c4::from_chars_first(csubstr buf, T * C4_RESTRICT val); - * @endcode - */ - -#include "c4/language.hpp" -#include -#include -#include -#include -#include - -#include "c4/config.hpp" -#include "c4/substr.hpp" -#include "c4/std/std_fwd.hpp" -#include "c4/memory_util.hpp" -#include "c4/szconv.hpp" - -#ifndef C4CORE_NO_FAST_FLOAT -# if (C4_CPP >= 17) -# if defined(_MSC_VER) -# if (C4_MSVC_VERSION >= C4_MSVC_VERSION_2019) // VS2017 and lower do not have these macros -# include -# define C4CORE_HAVE_STD_TOCHARS 1 -# define C4CORE_HAVE_STD_FROMCHARS 0 // prefer fast_float with MSVC -# define C4CORE_HAVE_FAST_FLOAT 1 -# else -# define C4CORE_HAVE_STD_TOCHARS 0 -# define C4CORE_HAVE_STD_FROMCHARS 0 -# define C4CORE_HAVE_FAST_FLOAT 1 -# endif -# else -# if __has_include() -# include -# if defined(__cpp_lib_to_chars) -# define C4CORE_HAVE_STD_TOCHARS 1 -# define C4CORE_HAVE_STD_FROMCHARS 0 // glibc uses fast_float internally -# define C4CORE_HAVE_FAST_FLOAT 1 -# else -# define C4CORE_HAVE_STD_TOCHARS 0 -# define C4CORE_HAVE_STD_FROMCHARS 0 -# define C4CORE_HAVE_FAST_FLOAT 1 -# endif -# else -# define C4CORE_HAVE_STD_TOCHARS 0 -# define C4CORE_HAVE_STD_FROMCHARS 0 -# define C4CORE_HAVE_FAST_FLOAT 1 -# endif -# endif -# else -# define C4CORE_HAVE_STD_TOCHARS 0 -# define C4CORE_HAVE_STD_FROMCHARS 0 -# define C4CORE_HAVE_FAST_FLOAT 1 -# endif -# if C4CORE_HAVE_FAST_FLOAT - C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wsign-conversion") - C4_SUPPRESS_WARNING_GCC("-Warray-bounds") -# if __GNUC__ >= 5 - C4_SUPPRESS_WARNING_GCC("-Wshift-count-overflow") -# endif -# include "c4/ext/fast_float.hpp" - C4_SUPPRESS_WARNING_GCC_POP -# endif -#elif (C4_CPP >= 17) -# define C4CORE_HAVE_FAST_FLOAT 0 -# if defined(_MSC_VER) -# if (C4_MSVC_VERSION >= C4_MSVC_VERSION_2019) // VS2017 and lower do not have these macros -# include -# define C4CORE_HAVE_STD_TOCHARS 1 -# define C4CORE_HAVE_STD_FROMCHARS 1 -# else -# define C4CORE_HAVE_STD_TOCHARS 0 -# define C4CORE_HAVE_STD_FROMCHARS 0 -# endif -# else -# if __has_include() -# include -# if defined(__cpp_lib_to_chars) -# define C4CORE_HAVE_STD_TOCHARS 1 -# define C4CORE_HAVE_STD_FROMCHARS 1 // glibc uses fast_float internally -# else -# define C4CORE_HAVE_STD_TOCHARS 0 -# define C4CORE_HAVE_STD_FROMCHARS 0 -# endif -# else -# define C4CORE_HAVE_STD_TOCHARS 0 -# define C4CORE_HAVE_STD_FROMCHARS 0 -# endif -# endif -#else -# define C4CORE_HAVE_STD_TOCHARS 0 -# define C4CORE_HAVE_STD_FROMCHARS 0 -# define C4CORE_HAVE_FAST_FLOAT 0 -#endif - - -#if !C4CORE_HAVE_STD_FROMCHARS -#include -#endif - - -#ifdef _MSC_VER -# pragma warning(push) -# if C4_MSVC_VERSION != C4_MSVC_VERSION_2017 -# pragma warning(disable: 4800) //'int': forcing value to bool 'true' or 'false' (performance warning) -# endif -# pragma warning(disable: 4996) // snprintf/scanf: this function or variable may be unsafe -#elif defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wtautological-constant-out-of-range-compare" -# pragma clang diagnostic ignored "-Wformat-nonliteral" -# pragma clang diagnostic ignored "-Wdouble-promotion" // implicit conversion increases floating-point precision -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wformat-nonliteral" -# pragma GCC diagnostic ignored "-Wdouble-promotion" // implicit conversion increases floating-point precision -# pragma GCC diagnostic ignored "-Wuseless-cast" -#endif - - -namespace c4 { - -#if C4CORE_HAVE_STD_TOCHARS -/** @warning Use only the symbol. Do not rely on the type or naked value of this enum. */ -typedef enum : std::underlying_type::type { - /** print the real number in floating point format (like %f) */ - FTOA_FLOAT = static_cast::type>(std::chars_format::fixed), - /** print the real number in scientific format (like %e) */ - FTOA_SCIENT = static_cast::type>(std::chars_format::scientific), - /** print the real number in flexible format (like %g) */ - FTOA_FLEX = static_cast::type>(std::chars_format::general), - /** print the real number in hexadecimal format (like %a) */ - FTOA_HEXA = static_cast::type>(std::chars_format::hex), -} RealFormat_e; -#else -/** @warning Use only the symbol. Do not rely on the type or naked value of this enum. */ -typedef enum : char { - /** print the real number in floating point format (like %f) */ - FTOA_FLOAT = 'f', - /** print the real number in scientific format (like %e) */ - FTOA_SCIENT = 'e', - /** print the real number in flexible format (like %g) */ - FTOA_FLEX = 'g', - /** print the real number in hexadecimal format (like %a) */ - FTOA_HEXA = 'a', -} RealFormat_e; -#endif - - -/** in some platforms, int,unsigned int - * are not any of int8_t...int64_t and - * long,unsigned long are not any of uint8_t...uint64_t */ -template -struct is_fixed_length -{ - enum : bool { - /** true if T is one of the fixed length signed types */ - value_i = (std::is_integral::value - && (std::is_same::value - || std::is_same::value - || std::is_same::value - || std::is_same::value)), - /** true if T is one of the fixed length unsigned types */ - value_u = (std::is_integral::value - && (std::is_same::value - || std::is_same::value - || std::is_same::value - || std::is_same::value)), - /** true if T is one of the fixed length signed or unsigned types */ - value = value_i || value_u - }; -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -#ifdef _MSC_VER -# pragma warning(push) -#elif defined(__clang__) -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wconversion" -# if __GNUC__ >= 6 -# pragma GCC diagnostic ignored "-Wnull-dereference" -# endif -#endif - -namespace detail { - -/* python command to get the values below: -def dec(v): - return str(v) -for bits in (8, 16, 32, 64): - imin, imax, umax = (-(1 << (bits - 1))), (1 << (bits - 1)) - 1, (1 << bits) - 1 - for vname, v in (("imin", imin), ("imax", imax), ("umax", umax)): - for f in (bin, oct, dec, hex): - print(f"{bits}b: {vname}={v} {f.__name__}: len={len(f(v)):2d}: {v} {f(v)}") -*/ - -// do not use the type as the template argument because in some -// platforms long!=int32 and long!=int64. Just use the numbytes -// which is more generic and spares lengthy SFINAE code. -template struct charconv_digits_; -template using charconv_digits = charconv_digits_::value>; - -template<> struct charconv_digits_<1u, true> // int8_t -{ - enum : size_t { - maxdigits_bin = 1 + 2 + 8, // -128==-0b10000000 - maxdigits_oct = 1 + 2 + 3, // -128==-0o200 - maxdigits_dec = 1 + 3, // -128 - maxdigits_hex = 1 + 2 + 2, // -128==-0x80 - maxdigits_bin_nopfx = 8, // -128==-0b10000000 - maxdigits_oct_nopfx = 3, // -128==-0o200 - maxdigits_dec_nopfx = 3, // -128 - maxdigits_hex_nopfx = 2, // -128==-0x80 - }; - // min values without sign! - static constexpr csubstr min_value_dec() noexcept { return csubstr("128"); } - static constexpr csubstr min_value_hex() noexcept { return csubstr("80"); } - static constexpr csubstr min_value_oct() noexcept { return csubstr("200"); } - static constexpr csubstr min_value_bin() noexcept { return csubstr("10000000"); } - static constexpr csubstr max_value_dec() noexcept { return csubstr("127"); } - static constexpr bool is_oct_overflow(csubstr str) noexcept { return !((str.len < 3) || (str.len == 3 && str[0] <= '1')); } -}; -template<> struct charconv_digits_<1u, false> // uint8_t -{ - enum : size_t { - maxdigits_bin = 2 + 8, // 255 0b11111111 - maxdigits_oct = 2 + 3, // 255 0o377 - maxdigits_dec = 3, // 255 - maxdigits_hex = 2 + 2, // 255 0xff - maxdigits_bin_nopfx = 8, // 255 0b11111111 - maxdigits_oct_nopfx = 3, // 255 0o377 - maxdigits_dec_nopfx = 3, // 255 - maxdigits_hex_nopfx = 2, // 255 0xff - }; - static constexpr csubstr max_value_dec() noexcept { return csubstr("255"); } - static constexpr bool is_oct_overflow(csubstr str) noexcept { return !((str.len < 3) || (str.len == 3 && str[0] <= '3')); } -}; -template<> struct charconv_digits_<2u, true> // int16_t -{ - enum : size_t { - maxdigits_bin = 1 + 2 + 16, // -32768 -0b1000000000000000 - maxdigits_oct = 1 + 2 + 6, // -32768 -0o100000 - maxdigits_dec = 1 + 5, // -32768 -32768 - maxdigits_hex = 1 + 2 + 4, // -32768 -0x8000 - maxdigits_bin_nopfx = 16, // -32768 -0b1000000000000000 - maxdigits_oct_nopfx = 6, // -32768 -0o100000 - maxdigits_dec_nopfx = 5, // -32768 -32768 - maxdigits_hex_nopfx = 4, // -32768 -0x8000 - }; - // min values without sign! - static constexpr csubstr min_value_dec() noexcept { return csubstr("32768"); } - static constexpr csubstr min_value_hex() noexcept { return csubstr("8000"); } - static constexpr csubstr min_value_oct() noexcept { return csubstr("100000"); } - static constexpr csubstr min_value_bin() noexcept { return csubstr("1000000000000000"); } - static constexpr csubstr max_value_dec() noexcept { return csubstr("32767"); } - static constexpr bool is_oct_overflow(csubstr str) noexcept { return !((str.len < 6)); } -}; -template<> struct charconv_digits_<2u, false> // uint16_t -{ - enum : size_t { - maxdigits_bin = 2 + 16, // 65535 0b1111111111111111 - maxdigits_oct = 2 + 6, // 65535 0o177777 - maxdigits_dec = 6, // 65535 65535 - maxdigits_hex = 2 + 4, // 65535 0xffff - maxdigits_bin_nopfx = 16, // 65535 0b1111111111111111 - maxdigits_oct_nopfx = 6, // 65535 0o177777 - maxdigits_dec_nopfx = 6, // 65535 65535 - maxdigits_hex_nopfx = 4, // 65535 0xffff - }; - static constexpr csubstr max_value_dec() noexcept { return csubstr("65535"); } - static constexpr bool is_oct_overflow(csubstr str) noexcept { return !((str.len < 6) || (str.len == 6 && str[0] <= '1')); } -}; -template<> struct charconv_digits_<4u, true> // int32_t -{ - enum : size_t { - maxdigits_bin = 1 + 2 + 32, // len=35: -2147483648 -0b10000000000000000000000000000000 - maxdigits_oct = 1 + 2 + 11, // len=14: -2147483648 -0o20000000000 - maxdigits_dec = 1 + 10, // len=11: -2147483648 -2147483648 - maxdigits_hex = 1 + 2 + 8, // len=11: -2147483648 -0x80000000 - maxdigits_bin_nopfx = 32, // len=35: -2147483648 -0b10000000000000000000000000000000 - maxdigits_oct_nopfx = 11, // len=14: -2147483648 -0o20000000000 - maxdigits_dec_nopfx = 10, // len=11: -2147483648 -2147483648 - maxdigits_hex_nopfx = 8, // len=11: -2147483648 -0x80000000 - }; - // min values without sign! - static constexpr csubstr min_value_dec() noexcept { return csubstr("2147483648"); } - static constexpr csubstr min_value_hex() noexcept { return csubstr("80000000"); } - static constexpr csubstr min_value_oct() noexcept { return csubstr("20000000000"); } - static constexpr csubstr min_value_bin() noexcept { return csubstr("10000000000000000000000000000000"); } - static constexpr csubstr max_value_dec() noexcept { return csubstr("2147483647"); } - static constexpr bool is_oct_overflow(csubstr str) noexcept { return !((str.len < 11) || (str.len == 11 && str[0] <= '1')); } -}; -template<> struct charconv_digits_<4u, false> // uint32_t -{ - enum : size_t { - maxdigits_bin = 2 + 32, // len=34: 4294967295 0b11111111111111111111111111111111 - maxdigits_oct = 2 + 11, // len=13: 4294967295 0o37777777777 - maxdigits_dec = 10, // len=10: 4294967295 4294967295 - maxdigits_hex = 2 + 8, // len=10: 4294967295 0xffffffff - maxdigits_bin_nopfx = 32, // len=34: 4294967295 0b11111111111111111111111111111111 - maxdigits_oct_nopfx = 11, // len=13: 4294967295 0o37777777777 - maxdigits_dec_nopfx = 10, // len=10: 4294967295 4294967295 - maxdigits_hex_nopfx = 8, // len=10: 4294967295 0xffffffff - }; - static constexpr csubstr max_value_dec() noexcept { return csubstr("4294967295"); } - static constexpr bool is_oct_overflow(csubstr str) noexcept { return !((str.len < 11) || (str.len == 11 && str[0] <= '3')); } -}; -template<> struct charconv_digits_<8u, true> // int32_t -{ - enum : size_t { - maxdigits_bin = 1 + 2 + 64, // len=67: -9223372036854775808 -0b1000000000000000000000000000000000000000000000000000000000000000 - maxdigits_oct = 1 + 2 + 22, // len=25: -9223372036854775808 -0o1000000000000000000000 - maxdigits_dec = 1 + 19, // len=20: -9223372036854775808 -9223372036854775808 - maxdigits_hex = 1 + 2 + 16, // len=19: -9223372036854775808 -0x8000000000000000 - maxdigits_bin_nopfx = 64, // len=67: -9223372036854775808 -0b1000000000000000000000000000000000000000000000000000000000000000 - maxdigits_oct_nopfx = 22, // len=25: -9223372036854775808 -0o1000000000000000000000 - maxdigits_dec_nopfx = 19, // len=20: -9223372036854775808 -9223372036854775808 - maxdigits_hex_nopfx = 16, // len=19: -9223372036854775808 -0x8000000000000000 - }; - static constexpr csubstr min_value_dec() noexcept { return csubstr("9223372036854775808"); } - static constexpr csubstr min_value_hex() noexcept { return csubstr("8000000000000000"); } - static constexpr csubstr min_value_oct() noexcept { return csubstr("1000000000000000000000"); } - static constexpr csubstr min_value_bin() noexcept { return csubstr("1000000000000000000000000000000000000000000000000000000000000000"); } - static constexpr csubstr max_value_dec() noexcept { return csubstr("9223372036854775807"); } - static constexpr bool is_oct_overflow(csubstr str) noexcept { return !((str.len < 22)); } -}; -template<> struct charconv_digits_<8u, false> -{ - enum : size_t { - maxdigits_bin = 2 + 64, // len=66: 18446744073709551615 0b1111111111111111111111111111111111111111111111111111111111111111 - maxdigits_oct = 2 + 22, // len=24: 18446744073709551615 0o1777777777777777777777 - maxdigits_dec = 20, // len=20: 18446744073709551615 18446744073709551615 - maxdigits_hex = 2 + 16, // len=18: 18446744073709551615 0xffffffffffffffff - maxdigits_bin_nopfx = 64, // len=66: 18446744073709551615 0b1111111111111111111111111111111111111111111111111111111111111111 - maxdigits_oct_nopfx = 22, // len=24: 18446744073709551615 0o1777777777777777777777 - maxdigits_dec_nopfx = 20, // len=20: 18446744073709551615 18446744073709551615 - maxdigits_hex_nopfx = 16, // len=18: 18446744073709551615 0xffffffffffffffff - }; - static constexpr csubstr max_value_dec() noexcept { return csubstr("18446744073709551615"); } - static constexpr bool is_oct_overflow(csubstr str) noexcept { return !((str.len < 22) || (str.len == 22 && str[0] <= '1')); } -}; -} // namespace detail - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// Helper macros, undefined below -#define _c4append(c) { if(C4_LIKELY(pos < buf.len)) { buf.str[pos++] = static_cast(c); } else { ++pos; } } -#define _c4appendhex(i) { if(C4_LIKELY(pos < buf.len)) { buf.str[pos++] = hexchars[i]; } else { ++pos; } } - -/** @name digits_dec return the number of digits required to encode a - * decimal number. - * - * @note At first sight this code may look heavily branchy and - * therefore inefficient. However, measurements revealed this to be - * the fastest among the alternatives. - * - * @see https://github.com/biojppm/c4core/pull/77 */ -/** @{ */ - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto digits_dec(T v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - return ((v >= 100) ? 3u : ((v >= 10) ? 2u : 1u)); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto digits_dec(T v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - return ((v >= 10000) ? 5u : (v >= 1000) ? 4u : (v >= 100) ? 3u : (v >= 10) ? 2u : 1u); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto digits_dec(T v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - return ((v >= 1000000000) ? 10u : (v >= 100000000) ? 9u : (v >= 10000000) ? 8u : - (v >= 1000000) ? 7u : (v >= 100000) ? 6u : (v >= 10000) ? 5u : - (v >= 1000) ? 4u : (v >= 100) ? 3u : (v >= 10) ? 2u : 1u); -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -auto digits_dec(T v) noexcept - -> typename std::enable_if::type -{ - // thanks @fargies!!! - // https://github.com/biojppm/c4core/pull/77#issuecomment-1063753568 - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - if(v >= 1000000000) // 10 - { - if(v >= 100000000000000) // 15 [15-20] range - { - if(v >= 100000000000000000) // 18 (15 + (20 - 15) / 2) - { - if((typename std::make_unsigned::type)v >= 10000000000000000000u) // 20 - return 20u; - else - return (v >= 1000000000000000000) ? 19u : 18u; - } - else if(v >= 10000000000000000) // 17 - return 17u; - else - return(v >= 1000000000000000) ? 16u : 15u; - } - else if(v >= 1000000000000) // 13 - return (v >= 10000000000000) ? 14u : 13u; - else if(v >= 100000000000) // 12 - return 12; - else - return(v >= 10000000000) ? 11u : 10u; - } - else if(v >= 10000) // 5 [5-9] range - { - if(v >= 10000000) // 8 - return (v >= 100000000) ? 9u : 8u; - else if(v >= 1000000) // 7 - return 7; - else - return (v >= 100000) ? 6u : 5u; - } - else if(v >= 100) - return (v >= 1000) ? 4u : 3u; - else - return (v >= 10) ? 2u : 1u; -} - -/** @} */ - - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned digits_hex(T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - return v ? 1u + (msb((typename std::make_unsigned::type)v) >> 2u) : 1u; -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned digits_bin(T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - return v ? 1u + msb((typename std::make_unsigned::type)v) : 1u; -} - -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE unsigned digits_oct(T v_) noexcept -{ - // TODO: is there a better way? - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v_ >= 0); - using U = typename - std::conditional::type>::type; - U v = (U) v_; // safe because we require v_ >= 0 - unsigned __n = 1; - const unsigned __b2 = 64u; - const unsigned __b3 = __b2 * 8u; - const unsigned long __b4 = __b3 * 8u; - while(true) - { - if(v < 8u) - return __n; - if(v < __b2) - return __n + 1; - if(v < __b3) - return __n + 2; - if(v < __b4) - return __n + 3; - v /= (U) __b4; - __n += 4; - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace detail { -C4_INLINE_CONSTEXPR const char hexchars[] = "0123456789abcdef"; -C4_INLINE_CONSTEXPR const char digits0099[] = - "0001020304050607080910111213141516171819" - "2021222324252627282930313233343536373839" - "4041424344454647484950515253545556575859" - "6061626364656667686970717273747576777879" - "8081828384858687888990919293949596979899"; -} // namespace detail - -C4_SUPPRESS_WARNING_GCC_PUSH -C4_SUPPRESS_WARNING_GCC("-Warray-bounds") // gcc has false positives here -#if (defined(__GNUC__) && (__GNUC__ >= 7)) -C4_SUPPRESS_WARNING_GCC("-Wstringop-overflow") // gcc has false positives here -#endif - -template -C4_HOT C4_ALWAYS_INLINE -void write_dec_unchecked(substr buf, T v, unsigned digits_v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - C4_ASSERT(buf.len >= digits_v); - C4_XASSERT(digits_v == digits_dec(v)); - // in bm_xtoa: checkoncelog_singlediv_write2 - while(v >= T(100)) - { - const T quo = v / T(100); - const auto num = (v - quo * T(100)) << 1u; - v = quo; - buf.str[--digits_v] = detail::digits0099[num + 1]; - buf.str[--digits_v] = detail::digits0099[num]; - } - if(v >= T(10)) - { - C4_ASSERT(digits_v == 2); - const auto num = v << 1u; - buf.str[1] = detail::digits0099[num + 1]; - buf.str[0] = detail::digits0099[num]; - } - else - { - C4_ASSERT(digits_v == 1); - buf.str[0] = (char)('0' + v); - } -} - - -template -C4_HOT C4_ALWAYS_INLINE -void write_hex_unchecked(substr buf, T v, unsigned digits_v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - C4_ASSERT(buf.len >= digits_v); - C4_XASSERT(digits_v == digits_hex(v)); - do { - buf.str[--digits_v] = detail::hexchars[v & T(15)]; - v >>= 4; - } while(v); - C4_ASSERT(digits_v == 0); -} - - -template -C4_HOT C4_ALWAYS_INLINE -void write_oct_unchecked(substr buf, T v, unsigned digits_v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - C4_ASSERT(buf.len >= digits_v); - C4_XASSERT(digits_v == digits_oct(v)); - do { - buf.str[--digits_v] = (char)('0' + (v & T(7))); - v >>= 3; - } while(v); - C4_ASSERT(digits_v == 0); -} - - -template -C4_HOT C4_ALWAYS_INLINE -void write_bin_unchecked(substr buf, T v, unsigned digits_v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - C4_ASSERT(buf.len >= digits_v); - C4_XASSERT(digits_v == digits_bin(v)); - do { - buf.str[--digits_v] = (char)('0' + (v & T(1))); - v >>= 1; - } while(v); - C4_ASSERT(digits_v == 0); -} - - -/** write an integer to a string in decimal format. This is the - * lowest level (and the fastest) function to do this task. - * @note does not accept negative numbers - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the required size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t write_dec(substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - unsigned digits = digits_dec(v); - if(C4_LIKELY(buf.len >= digits)) - write_dec_unchecked(buf, v, digits); - return digits; -} - -/** write an integer to a string in hexadecimal format. This is the - * lowest level (and the fastest) function to do this task. - * @note does not accept negative numbers - * @note does not prefix with 0x - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the required size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t write_hex(substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - unsigned digits = digits_hex(v); - if(C4_LIKELY(buf.len >= digits)) - write_hex_unchecked(buf, v, digits); - return digits; -} - -/** write an integer to a string in octal format. This is the - * lowest level (and the fastest) function to do this task. - * @note does not accept negative numbers - * @note does not prefix with 0o - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the required size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t write_oct(substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - unsigned digits = digits_oct(v); - if(C4_LIKELY(buf.len >= digits)) - write_oct_unchecked(buf, v, digits); - return digits; -} - -/** write an integer to a string in binary format. This is the - * lowest level (and the fastest) function to do this task. - * @note does not accept negative numbers - * @note does not prefix with 0b - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the required size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t write_bin(substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(v >= 0); - unsigned digits = digits_bin(v); - C4_ASSERT(digits > 0); - if(C4_LIKELY(buf.len >= digits)) - write_bin_unchecked(buf, v, digits); - return digits; -} - - -namespace detail { -template using NumberWriter = size_t (*)(substr, U); -template writer> -size_t write_num_digits(substr buf, T v, size_t num_digits) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - size_t ret = writer(buf, v); - if(ret >= num_digits) - return ret; - else if(ret >= buf.len || num_digits > buf.len) - return num_digits; - C4_ASSERT(num_digits >= ret); - size_t delta = static_cast(num_digits - ret); - memmove(buf.str + delta, buf.str, ret); - memset(buf.str, '0', delta); - return num_digits; -} -} // namespace detail - - -/** same as c4::write_dec(), but pad with zeroes on the left - * such that the resulting string is @p num_digits wide. - * If the given number is requires more than num_digits, then the number prevails. */ -template -C4_ALWAYS_INLINE size_t write_dec(substr buf, T val, size_t num_digits) noexcept -{ - return detail::write_num_digits>(buf, val, num_digits); -} - -/** same as c4::write_hex(), but pad with zeroes on the left - * such that the resulting string is @p num_digits wide. - * If the given number is requires more than num_digits, then the number prevails. */ -template -C4_ALWAYS_INLINE size_t write_hex(substr buf, T val, size_t num_digits) noexcept -{ - return detail::write_num_digits>(buf, val, num_digits); -} - -/** same as c4::write_bin(), but pad with zeroes on the left - * such that the resulting string is @p num_digits wide. - * If the given number is requires more than num_digits, then the number prevails. */ -template -C4_ALWAYS_INLINE size_t write_bin(substr buf, T val, size_t num_digits) noexcept -{ - return detail::write_num_digits>(buf, val, num_digits); -} - -/** same as c4::write_oct(), but pad with zeroes on the left - * such that the resulting string is @p num_digits wide. - * If the given number is requires more than num_digits, then the number prevails. */ -template -C4_ALWAYS_INLINE size_t write_oct(substr buf, T val, size_t num_digits) noexcept -{ - return detail::write_num_digits>(buf, val, num_digits); -} - -C4_SUPPRESS_WARNING_GCC_POP - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** read a decimal integer from a string. This is the - * lowest level (and the fastest) function to do this task. - * @note does not accept negative numbers - * @note The string must be trimmed. Whitespace is not accepted. - * @note the string must not be empty - * @note there is no check for overflow; the value wraps around - * in a way similar to the standard C/C++ overflow behavior. - * For example, `read_dec("128", &val)` returns true - * and val will be set to 0 because 127 is the max i8 value. - * @see overflows() to find out if a number string overflows a type range - * @return true if the conversion was successful (no overflow check) */ -template -C4_ALWAYS_INLINE bool read_dec(csubstr s, I *C4_RESTRICT v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(!s.empty()); - *v = 0; - for(char c : s) - { - if(C4_UNLIKELY(c < '0' || c > '9')) - return false; - *v = (*v) * I(10) + (I(c) - I('0')); - } - return true; -} - -/** read an hexadecimal integer from a string. This is the - * lowest level (and the fastest) function to do this task. - * @note does not accept negative numbers - * @note does not accept leading 0x or 0X - * @note the string must not be empty - * @note the string must be trimmed. Whitespace is not accepted. - * @note there is no check for overflow; the value wraps around - * in a way similar to the standard C/C++ overflow behavior. - * For example, `read_hex("80", &val)` returns true - * and val will be set to 0 because 7f is the max i8 value. - * @see overflows() to find out if a number string overflows a type range - * @return true if the conversion was successful (no overflow check) */ -template -C4_ALWAYS_INLINE bool read_hex(csubstr s, I *C4_RESTRICT v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(!s.empty()); - *v = 0; - for(char c : s) - { - I cv; - if(c >= '0' && c <= '9') - cv = I(c) - I('0'); - else if(c >= 'a' && c <= 'f') - cv = I(10) + (I(c) - I('a')); - else if(c >= 'A' && c <= 'F') - cv = I(10) + (I(c) - I('A')); - else - return false; - *v = (*v) * I(16) + cv; - } - return true; -} - -/** read a binary integer from a string. This is the - * lowest level (and the fastest) function to do this task. - * @note does not accept negative numbers - * @note does not accept leading 0b or 0B - * @note the string must not be empty - * @note the string must be trimmed. Whitespace is not accepted. - * @note there is no check for overflow; the value wraps around - * in a way similar to the standard C/C++ overflow behavior. - * For example, `read_bin("10000000", &val)` returns true - * and val will be set to 0 because 1111111 is the max i8 value. - * @see overflows() to find out if a number string overflows a type range - * @return true if the conversion was successful (no overflow check) */ -template -C4_ALWAYS_INLINE bool read_bin(csubstr s, I *C4_RESTRICT v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(!s.empty()); - *v = 0; - for(char c : s) - { - *v <<= 1; - if(c == '1') - *v |= 1; - else if(c != '0') - return false; - } - return true; -} - -/** read an octal integer from a string. This is the - * lowest level (and the fastest) function to do this task. - * @note does not accept negative numbers - * @note does not accept leading 0o or 0O - * @note the string must not be empty - * @note the string must be trimmed. Whitespace is not accepted. - * @note there is no check for overflow; the value wraps around - * in a way similar to the standard C/C++ overflow behavior. - * For example, `read_oct("200", &val)` returns true - * and val will be set to 0 because 177 is the max i8 value. - * @see overflows() to find out if a number string overflows a type range - * @return true if the conversion was successful (no overflow check) */ -template -C4_ALWAYS_INLINE bool read_oct(csubstr s, I *C4_RESTRICT v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_ASSERT(!s.empty()); - *v = 0; - for(char c : s) - { - if(C4_UNLIKELY(c < '0' || c > '7')) - return false; - *v = (*v) * I(8) + (I(c) - I('0')); - } - return true; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace detail { -inline size_t _itoa2buf(substr buf, size_t pos, csubstr val) noexcept -{ - C4_ASSERT(pos + val.len <= buf.len); - memcpy(buf.str + pos, val.str, val.len); - return pos + val.len; -} -inline size_t _itoa2bufwithdigits(substr buf, size_t pos, size_t num_digits, csubstr val) noexcept -{ - num_digits = num_digits > val.len ? num_digits - val.len : 0; - C4_ASSERT(num_digits + val.len <= buf.len); - for(size_t i = 0; i < num_digits; ++i) - _c4append('0'); - return detail::_itoa2buf(buf, pos, val); -} -template -C4_NO_INLINE size_t _itoadec2buf(substr buf) noexcept -{ - using digits_type = detail::charconv_digits; - if(C4_UNLIKELY(buf.len < digits_type::maxdigits_dec)) - return digits_type::maxdigits_dec; - buf.str[0] = '-'; - return detail::_itoa2buf(buf, 1, digits_type::min_value_dec()); -} -template -C4_NO_INLINE size_t _itoa2buf(substr buf, I radix) noexcept -{ - using digits_type = detail::charconv_digits; - size_t pos = 0; - if(C4_LIKELY(buf.len > 0)) - buf.str[pos++] = '-'; - switch(radix) - { - case I(10): - if(C4_UNLIKELY(buf.len < digits_type::maxdigits_dec)) - return digits_type::maxdigits_dec; - pos =_itoa2buf(buf, pos, digits_type::min_value_dec()); - break; - case I(16): - if(C4_UNLIKELY(buf.len < digits_type::maxdigits_hex)) - return digits_type::maxdigits_hex; - buf.str[pos++] = '0'; - buf.str[pos++] = 'x'; - pos = _itoa2buf(buf, pos, digits_type::min_value_hex()); - break; - case I( 2): - if(C4_UNLIKELY(buf.len < digits_type::maxdigits_bin)) - return digits_type::maxdigits_bin; - buf.str[pos++] = '0'; - buf.str[pos++] = 'b'; - pos = _itoa2buf(buf, pos, digits_type::min_value_bin()); - break; - case I( 8): - if(C4_UNLIKELY(buf.len < digits_type::maxdigits_oct)) - return digits_type::maxdigits_oct; - buf.str[pos++] = '0'; - buf.str[pos++] = 'o'; - pos = _itoa2buf(buf, pos, digits_type::min_value_oct()); - break; - } - return pos; -} -template -C4_NO_INLINE size_t _itoa2buf(substr buf, I radix, size_t num_digits) noexcept -{ - using digits_type = detail::charconv_digits; - size_t pos = 0; - size_t needed_digits = 0; - if(C4_LIKELY(buf.len > 0)) - buf.str[pos++] = '-'; - switch(radix) - { - case I(10): - // add 1 to account for - - needed_digits = num_digits+1 > digits_type::maxdigits_dec ? num_digits+1 : digits_type::maxdigits_dec; - if(C4_UNLIKELY(buf.len < needed_digits)) - return needed_digits; - pos = _itoa2bufwithdigits(buf, pos, num_digits, digits_type::min_value_dec()); - break; - case I(16): - // add 3 to account for -0x - needed_digits = num_digits+3 > digits_type::maxdigits_hex ? num_digits+3 : digits_type::maxdigits_hex; - if(C4_UNLIKELY(buf.len < needed_digits)) - return needed_digits; - buf.str[pos++] = '0'; - buf.str[pos++] = 'x'; - pos = _itoa2bufwithdigits(buf, pos, num_digits, digits_type::min_value_hex()); - break; - case I( 2): - // add 3 to account for -0b - needed_digits = num_digits+3 > digits_type::maxdigits_bin ? num_digits+3 : digits_type::maxdigits_bin; - if(C4_UNLIKELY(buf.len < needed_digits)) - return needed_digits; - C4_ASSERT(buf.len >= digits_type::maxdigits_bin); - buf.str[pos++] = '0'; - buf.str[pos++] = 'b'; - pos = _itoa2bufwithdigits(buf, pos, num_digits, digits_type::min_value_bin()); - break; - case I( 8): - // add 3 to account for -0o - needed_digits = num_digits+3 > digits_type::maxdigits_oct ? num_digits+3 : digits_type::maxdigits_oct; - if(C4_UNLIKELY(buf.len < needed_digits)) - return needed_digits; - C4_ASSERT(buf.len >= digits_type::maxdigits_oct); - buf.str[pos++] = '0'; - buf.str[pos++] = 'o'; - pos = _itoa2bufwithdigits(buf, pos, num_digits, digits_type::min_value_oct()); - break; - } - return pos; -} -} // namespace detail - - -/** convert an integral signed decimal to a string. - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the needed size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t itoa(substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_signed::value); - if(v >= T(0)) - { - // write_dec() checks the buffer size, so no need to check here - return write_dec(buf, v); - } - // when T is the min value (eg i8: -128), negating it - // will overflow, so treat the min as a special case - else if(C4_LIKELY(v != std::numeric_limits::min())) - { - v = -v; - unsigned digits = digits_dec(v); - if(C4_LIKELY(buf.len >= digits + 1u)) - { - buf.str[0] = '-'; - write_dec_unchecked(buf.sub(1), v, digits); - } - return digits + 1u; - } - return detail::_itoadec2buf(buf); -} - -/** convert an integral signed integer to a string, using a specific - * radix. The radix must be 2, 8, 10 or 16. - * - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the needed size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t itoa(substr buf, T v, T radix) noexcept -{ - C4_STATIC_ASSERT(std::is_signed::value); - C4_ASSERT(radix == 2 || radix == 8 || radix == 10 || radix == 16); - C4_SUPPRESS_WARNING_GCC_PUSH - #if (defined(__GNUC__) && (__GNUC__ >= 7)) - C4_SUPPRESS_WARNING_GCC("-Wstringop-overflow") // gcc has a false positive here - #endif - // when T is the min value (eg i8: -128), negating it - // will overflow, so treat the min as a special case - if(C4_LIKELY(v != std::numeric_limits::min())) - { - unsigned pos = 0; - if(v < 0) - { - v = -v; - if(C4_LIKELY(buf.len > 0)) - buf.str[pos] = '-'; - ++pos; - } - unsigned digits = 0; - switch(radix) - { - case T(10): - digits = digits_dec(v); - if(C4_LIKELY(buf.len >= pos + digits)) - write_dec_unchecked(buf.sub(pos), v, digits); - break; - case T(16): - digits = digits_hex(v); - if(C4_LIKELY(buf.len >= pos + 2u + digits)) - { - buf.str[pos + 0] = '0'; - buf.str[pos + 1] = 'x'; - write_hex_unchecked(buf.sub(pos + 2), v, digits); - } - digits += 2u; - break; - case T(2): - digits = digits_bin(v); - if(C4_LIKELY(buf.len >= pos + 2u + digits)) - { - buf.str[pos + 0] = '0'; - buf.str[pos + 1] = 'b'; - write_bin_unchecked(buf.sub(pos + 2), v, digits); - } - digits += 2u; - break; - case T(8): - digits = digits_oct(v); - if(C4_LIKELY(buf.len >= pos + 2u + digits)) - { - buf.str[pos + 0] = '0'; - buf.str[pos + 1] = 'o'; - write_oct_unchecked(buf.sub(pos + 2), v, digits); - } - digits += 2u; - break; - } - return pos + digits; - } - C4_SUPPRESS_WARNING_GCC_POP - // when T is the min value (eg i8: -128), negating it - // will overflow - return detail::_itoa2buf(buf, radix); -} - - -/** same as c4::itoa(), but pad with zeroes on the left such that the - * resulting string is @p num_digits wide, not accounting for radix - * prefix (0x,0o,0b). The @p radix must be 2, 8, 10 or 16. - * - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the needed size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t itoa(substr buf, T v, T radix, size_t num_digits) noexcept -{ - C4_STATIC_ASSERT(std::is_signed::value); - C4_ASSERT(radix == 2 || radix == 8 || radix == 10 || radix == 16); - C4_SUPPRESS_WARNING_GCC_PUSH - #if (defined(__GNUC__) && (__GNUC__ >= 7)) - C4_SUPPRESS_WARNING_GCC("-Wstringop-overflow") // gcc has a false positive here - #endif - // when T is the min value (eg i8: -128), negating it - // will overflow, so treat the min as a special case - if(C4_LIKELY(v != std::numeric_limits::min())) - { - unsigned pos = 0; - if(v < 0) - { - v = -v; - if(C4_LIKELY(buf.len > 0)) - buf.str[pos] = '-'; - ++pos; - } - unsigned total_digits = 0; - switch(radix) - { - case T(10): - total_digits = digits_dec(v); - total_digits = pos + (unsigned)(num_digits > total_digits ? num_digits : total_digits); - if(C4_LIKELY(buf.len >= total_digits)) - write_dec(buf.sub(pos), v, num_digits); - break; - case T(16): - total_digits = digits_hex(v); - total_digits = pos + 2u + (unsigned)(num_digits > total_digits ? num_digits : total_digits); - if(C4_LIKELY(buf.len >= total_digits)) - { - buf.str[pos + 0] = '0'; - buf.str[pos + 1] = 'x'; - write_hex(buf.sub(pos + 2), v, num_digits); - } - break; - case T(2): - total_digits = digits_bin(v); - total_digits = pos + 2u + (unsigned)(num_digits > total_digits ? num_digits : total_digits); - if(C4_LIKELY(buf.len >= total_digits)) - { - buf.str[pos + 0] = '0'; - buf.str[pos + 1] = 'b'; - write_bin(buf.sub(pos + 2), v, num_digits); - } - break; - case T(8): - total_digits = digits_oct(v); - total_digits = pos + 2u + (unsigned)(num_digits > total_digits ? num_digits : total_digits); - if(C4_LIKELY(buf.len >= total_digits)) - { - buf.str[pos + 0] = '0'; - buf.str[pos + 1] = 'o'; - write_oct(buf.sub(pos + 2), v, num_digits); - } - break; - } - return total_digits; - } - C4_SUPPRESS_WARNING_GCC_POP - // when T is the min value (eg i8: -128), negating it - // will overflow - return detail::_itoa2buf(buf, radix, num_digits); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** convert an integral unsigned decimal to a string. - * - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the needed size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t utoa(substr buf, T v) noexcept -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - // write_dec() does the buffer length check, so no need to check here - return write_dec(buf, v); -} - -/** convert an integral unsigned integer to a string, using a specific - * radix. The radix must be 2, 8, 10 or 16. - * - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the needed size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t utoa(substr buf, T v, T radix) noexcept -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(radix == 10 || radix == 16 || radix == 2 || radix == 8); - unsigned digits = 0; - switch(radix) - { - case T(10): - digits = digits_dec(v); - if(C4_LIKELY(buf.len >= digits)) - write_dec_unchecked(buf, v, digits); - break; - case T(16): - digits = digits_hex(v); - if(C4_LIKELY(buf.len >= digits+2u)) - { - buf.str[0] = '0'; - buf.str[1] = 'x'; - write_hex_unchecked(buf.sub(2), v, digits); - } - digits += 2u; - break; - case T(2): - digits = digits_bin(v); - if(C4_LIKELY(buf.len >= digits+2u)) - { - buf.str[0] = '0'; - buf.str[1] = 'b'; - write_bin_unchecked(buf.sub(2), v, digits); - } - digits += 2u; - break; - case T(8): - digits = digits_oct(v); - if(C4_LIKELY(buf.len >= digits+2u)) - { - buf.str[0] = '0'; - buf.str[1] = 'o'; - write_oct_unchecked(buf.sub(2), v, digits); - } - digits += 2u; - break; - } - return digits; -} - -/** same as c4::utoa(), but pad with zeroes on the left such that the - * resulting string is @p num_digits wide. The @p radix must be 2, - * 8, 10 or 16. - * - * @note the resulting string is NOT zero-terminated. - * @note it is ok to call this with an empty or too-small buffer; - * no writes will occur, and the needed size will be returned - * @return the number of characters required for the buffer. */ -template -C4_ALWAYS_INLINE size_t utoa(substr buf, T v, T radix, size_t num_digits) noexcept -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(radix == 10 || radix == 16 || radix == 2 || radix == 8); - unsigned total_digits = 0; - switch(radix) - { - case T(10): - total_digits = digits_dec(v); - total_digits = (unsigned)(num_digits > total_digits ? num_digits : total_digits); - if(C4_LIKELY(buf.len >= total_digits)) - write_dec(buf, v, num_digits); - break; - case T(16): - total_digits = digits_hex(v); - total_digits = 2u + (unsigned)(num_digits > total_digits ? num_digits : total_digits); - if(C4_LIKELY(buf.len >= total_digits)) - { - buf.str[0] = '0'; - buf.str[1] = 'x'; - write_hex(buf.sub(2), v, num_digits); - } - break; - case T(2): - total_digits = digits_bin(v); - total_digits = 2u + (unsigned)(num_digits > total_digits ? num_digits : total_digits); - if(C4_LIKELY(buf.len >= total_digits)) - { - buf.str[0] = '0'; - buf.str[1] = 'b'; - write_bin(buf.sub(2), v, num_digits); - } - break; - case T(8): - total_digits = digits_oct(v); - total_digits = 2u + (unsigned)(num_digits > total_digits ? num_digits : total_digits); - if(C4_LIKELY(buf.len >= total_digits)) - { - buf.str[0] = '0'; - buf.str[1] = 'o'; - write_oct(buf.sub(2), v, num_digits); - } - break; - } - return total_digits; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** Convert a trimmed string to a signed integral value. The input - * string can be formatted as decimal, binary (prefix 0b or 0B), octal - * (prefix 0o or 0O) or hexadecimal (prefix 0x or 0X). Strings with - * leading zeroes are considered as decimal and not octal (unlike the - * C/C++ convention). Every character in the input string is read for - * the conversion; the input string must not contain any leading or - * trailing whitespace. - * - * @return true if the conversion was successful. - * - * @note overflow is not detected: the return status is true even if - * the conversion would return a value outside of the type's range, in - * which case the result will wrap around the type's range. - * This is similar to native behavior. - * - * @note a positive sign is not accepted. ie, the string must not - * start with '+' - * - * @see atoi_first() if the string is not trimmed to the value to read. */ -template -C4_ALWAYS_INLINE bool atoi(csubstr str, T * C4_RESTRICT v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_STATIC_ASSERT(std::is_signed::value); - - if(C4_UNLIKELY(str.len == 0)) - return false; - - C4_ASSERT(str.str[0] != '+'); - - T sign = 1; - size_t start = 0; - if(str.str[0] == '-') - { - if(C4_UNLIKELY(str.len == ++start)) - return false; - sign = -1; - } - - bool parsed_ok = true; - if(str.str[start] != '0') // this should be the common case, so put it first - { - parsed_ok = read_dec(str.sub(start), v); - } - else if(str.len > start + 1) - { - // starts with 0: is it 0x, 0o, 0b? - const char pfx = str.str[start + 1]; - if(pfx == 'x' || pfx == 'X') - parsed_ok = str.len > start + 2 && read_hex(str.sub(start + 2), v); - else if(pfx == 'b' || pfx == 'B') - parsed_ok = str.len > start + 2 && read_bin(str.sub(start + 2), v); - else if(pfx == 'o' || pfx == 'O') - parsed_ok = str.len > start + 2 && read_oct(str.sub(start + 2), v); - else - parsed_ok = read_dec(str.sub(start + 1), v); - } - else - { - parsed_ok = read_dec(str.sub(start), v); - } - if(C4_LIKELY(parsed_ok)) - *v *= sign; - return parsed_ok; -} - - -/** Select the next range of characters in the string that can be parsed - * as a signed integral value, and convert it using atoi(). Leading - * whitespace (space, newline, tabs) is skipped. - * @return the number of characters read for conversion, or csubstr::npos if the conversion failed - * @see atoi() if the string is already trimmed to the value to read. - * @see csubstr::first_int_span() */ -template -C4_ALWAYS_INLINE size_t atoi_first(csubstr str, T * C4_RESTRICT v) -{ - csubstr trimmed = str.first_int_span(); - if(trimmed.len == 0) - return csubstr::npos; - if(atoi(trimmed, v)) - return static_cast(trimmed.end() - str.begin()); - return csubstr::npos; -} - - -//----------------------------------------------------------------------------- - -/** Convert a trimmed string to an unsigned integral value. The string can be - * formatted as decimal, binary (prefix 0b or 0B), octal (prefix 0o or 0O) - * or hexadecimal (prefix 0x or 0X). Every character in the input string is read - * for the conversion; it must not contain any leading or trailing whitespace. - * - * @return true if the conversion was successful. - * - * @note overflow is not detected: the return status is true even if - * the conversion would return a value outside of the type's range, in - * which case the result will wrap around the type's range. - * - * @note If the string has a minus character, the return status - * will be false. - * - * @see atou_first() if the string is not trimmed to the value to read. */ -template -bool atou(csubstr str, T * C4_RESTRICT v) noexcept -{ - C4_STATIC_ASSERT(std::is_integral::value); - - if(C4_UNLIKELY(str.len == 0 || str.front() == '-')) - return false; - - bool parsed_ok = true; - if(str.str[0] != '0') - { - parsed_ok = read_dec(str, v); - } - else - { - if(str.len > 1) - { - const char pfx = str.str[1]; - if(pfx == 'x' || pfx == 'X') - parsed_ok = str.len > 2 && read_hex(str.sub(2), v); - else if(pfx == 'b' || pfx == 'B') - parsed_ok = str.len > 2 && read_bin(str.sub(2), v); - else if(pfx == 'o' || pfx == 'O') - parsed_ok = str.len > 2 && read_oct(str.sub(2), v); - else - parsed_ok = read_dec(str, v); - } - else - { - *v = 0; // we know the first character is 0 - } - } - return parsed_ok; -} - - -/** Select the next range of characters in the string that can be parsed - * as an unsigned integral value, and convert it using atou(). Leading - * whitespace (space, newline, tabs) is skipped. - * @return the number of characters read for conversion, or csubstr::npos if the conversion faileds - * @see atou() if the string is already trimmed to the value to read. - * @see csubstr::first_uint_span() */ -template -C4_ALWAYS_INLINE size_t atou_first(csubstr str, T *v) -{ - csubstr trimmed = str.first_uint_span(); - if(trimmed.len == 0) - return csubstr::npos; - if(atou(trimmed, v)) - return static_cast(trimmed.end() - str.begin()); - return csubstr::npos; -} - - -#ifdef _MSC_VER -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -namespace detail { -inline bool check_overflow(csubstr str, csubstr limit) noexcept -{ - if(str.len == limit.len) - { - for(size_t i = 0; i < limit.len; ++i) - { - if(str[i] < limit[i]) - return false; - else if(str[i] > limit[i]) - return true; - } - return false; - } - else - return str.len > limit.len; -} -} // namespace detail - - -/** Test if the following string would overflow when converted to associated - * types. - * @return true if number will overflow, false if it fits (or doesn't parse) - */ -template -auto overflows(csubstr str) noexcept - -> typename std::enable_if::value, bool>::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - - if(C4_UNLIKELY(str.len == 0)) - { - return false; - } - else if(str.str[0] == '0') - { - if (str.len == 1) - return false; - switch (str.str[1]) - { - case 'x': - case 'X': - { - size_t fno = str.first_not_of('0', 2); - if (fno == csubstr::npos) - return false; - return !(str.len <= fno + (sizeof(T) * 2)); - } - case 'b': - case 'B': - { - size_t fno = str.first_not_of('0', 2); - if (fno == csubstr::npos) - return false; - return !(str.len <= fno +(sizeof(T) * 8)); - } - case 'o': - case 'O': - { - size_t fno = str.first_not_of('0', 2); - if(fno == csubstr::npos) - return false; - return detail::charconv_digits::is_oct_overflow(str.sub(fno)); - } - default: - { - size_t fno = str.first_not_of('0', 1); - if(fno == csubstr::npos) - return false; - return detail::check_overflow(str.sub(fno), detail::charconv_digits::max_value_dec()); - } - } - } - else if(C4_UNLIKELY(str[0] == '-')) - { - return true; - } - else - { - return detail::check_overflow(str, detail::charconv_digits::max_value_dec()); - } -} - - -/** Test if the following string would overflow when converted to associated - * types. - * @return true if number will overflow, false if it fits (or doesn't parse) - */ -template -auto overflows(csubstr str) - -> typename std::enable_if::value, bool>::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - if(C4_UNLIKELY(str.len == 0)) - return false; - if(str.str[0] == '-') - { - if(str.str[1] == '0') - { - if(str.len == 2) - return false; - switch(str.str[2]) - { - case 'x': - case 'X': - { - size_t fno = str.first_not_of('0', 3); - if (fno == csubstr::npos) - return false; - return detail::check_overflow(str.sub(fno), detail::charconv_digits::min_value_hex()); - } - case 'b': - case 'B': - { - size_t fno = str.first_not_of('0', 3); - if (fno == csubstr::npos) - return false; - return detail::check_overflow(str.sub(fno), detail::charconv_digits::min_value_bin()); - } - case 'o': - case 'O': - { - size_t fno = str.first_not_of('0', 3); - if(fno == csubstr::npos) - return false; - return detail::check_overflow(str.sub(fno), detail::charconv_digits::min_value_oct()); - } - default: - { - size_t fno = str.first_not_of('0', 2); - if(fno == csubstr::npos) - return false; - return detail::check_overflow(str.sub(fno), detail::charconv_digits::min_value_dec()); - } - } - } - else - return detail::check_overflow(str.sub(1), detail::charconv_digits::min_value_dec()); - } - else if(str.str[0] == '0') - { - if (str.len == 1) - return false; - switch(str.str[1]) - { - case 'x': - case 'X': - { - size_t fno = str.first_not_of('0', 2); - if (fno == csubstr::npos) - return false; - const size_t len = str.len - fno; - return !((len < sizeof (T) * 2) || (len == sizeof(T) * 2 && str[fno] <= '7')); - } - case 'b': - case 'B': - { - size_t fno = str.first_not_of('0', 2); - if (fno == csubstr::npos) - return false; - return !(str.len <= fno + (sizeof(T) * 8 - 1)); - } - case 'o': - case 'O': - { - size_t fno = str.first_not_of('0', 2); - if(fno == csubstr::npos) - return false; - return detail::charconv_digits::is_oct_overflow(str.sub(fno)); - } - default: - { - size_t fno = str.first_not_of('0', 1); - if(fno == csubstr::npos) - return false; - return detail::check_overflow(str.sub(fno), detail::charconv_digits::max_value_dec()); - } - } - } - else - return detail::check_overflow(str, detail::charconv_digits::max_value_dec()); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace detail { - - -#if (!C4CORE_HAVE_STD_FROMCHARS) -/** @see http://www.exploringbinary.com/ for many good examples on float-str conversion */ -template -void get_real_format_str(char (& C4_RESTRICT fmt)[N], int precision, RealFormat_e formatting, const char* length_modifier="") -{ - int iret; - if(precision == -1) - iret = snprintf(fmt, sizeof(fmt), "%%%s%c", length_modifier, formatting); - else if(precision == 0) - iret = snprintf(fmt, sizeof(fmt), "%%.%s%c", length_modifier, formatting); - else - iret = snprintf(fmt, sizeof(fmt), "%%.%d%s%c", precision, length_modifier, formatting); - C4_ASSERT(iret >= 2 && size_t(iret) < sizeof(fmt)); - C4_UNUSED(iret); -} - - -/** @todo we're depending on snprintf()/sscanf() for converting to/from - * floating point numbers. Apparently, this increases the binary size - * by a considerable amount. There are some lightweight printf - * implementations: - * - * @see http://www.sparetimelabs.com/tinyprintf/tinyprintf.php (BSD) - * @see https://github.com/weiss/c99-snprintf - * @see https://github.com/nothings/stb/blob/master/stb_sprintf.h - * @see http://www.exploringbinary.com/ - * @see https://blog.benoitblanchon.fr/lightweight-float-to-string/ - * @see http://www.ryanjuckett.com/programming/printing-floating-point-numbers/ - */ -template -size_t print_one(substr str, const char* full_fmt, T v) -{ -#ifdef _MSC_VER - /** use _snprintf() to prevent early termination of the output - * for writing the null character at the last position - * @see https://msdn.microsoft.com/en-us/library/2ts7cx93.aspx */ - int iret = _snprintf(str.str, str.len, full_fmt, v); - if(iret < 0) - { - /* when buf.len is not enough, VS returns a negative value. - * so call it again with a negative value for getting an - * actual length of the string */ - iret = snprintf(nullptr, 0, full_fmt, v); - C4_ASSERT(iret > 0); - } - size_t ret = (size_t) iret; - return ret; -#else - int iret = snprintf(str.str, str.len, full_fmt, v); - C4_ASSERT(iret >= 0); - size_t ret = (size_t) iret; - if(ret >= str.len) - ++ret; /* snprintf() reserves the last character to write \0 */ - return ret; -#endif -} -#endif // (!C4CORE_HAVE_STD_FROMCHARS) - - -#if (!C4CORE_HAVE_STD_FROMCHARS) && (!C4CORE_HAVE_FAST_FLOAT) -/** scans a string using the given type format, while at the same time - * allowing non-null-terminated strings AND guaranteeing that the given - * string length is strictly respected, so that no buffer overflows - * might occur. */ -template -inline size_t scan_one(csubstr str, const char *type_fmt, T *v) -{ - /* snscanf() is absolutely needed here as we must be sure that - * str.len is strictly respected, because substr is - * generally not null-terminated. - * - * Alas, there is no snscanf(). - * - * So we fake it by using a dynamic format with an explicit - * field size set to the length of the given span. - * This trick is taken from: - * https://stackoverflow.com/a/18368910/5875572 */ - - /* this is the actual format we'll use for scanning */ - char fmt[16]; - - /* write the length into it. Eg "%12f". - * Also, get the number of characters read from the string. - * So the final format ends up as "%12f%n"*/ - int iret = std::snprintf(fmt, sizeof(fmt), "%%" "%zu" "%s" "%%n", str.len, type_fmt); - /* no nasty surprises, please! */ - C4_ASSERT(iret >= 0 && size_t(iret) < C4_COUNTOF(fmt)); - - /* now we scan with confidence that the span length is respected */ - int num_chars; - iret = std::sscanf(str.str, fmt, v, &num_chars); - /* scanf returns the number of successful conversions */ - if(iret != 1) return csubstr::npos; - C4_ASSERT(num_chars >= 0); - return (size_t)(num_chars); -} -#endif // (!C4CORE_HAVE_STD_FROMCHARS) && (!C4CORE_HAVE_FAST_FLOAT) - - -#if C4CORE_HAVE_STD_TOCHARS -template -C4_ALWAYS_INLINE size_t rtoa(substr buf, T v, int precision=-1, RealFormat_e formatting=FTOA_FLEX) noexcept -{ - std::to_chars_result result; - size_t pos = 0; - if(formatting == FTOA_HEXA) - { - if(buf.len > size_t(2)) - { - buf.str[0] = '0'; - buf.str[1] = 'x'; - } - pos += size_t(2); - } - if(precision == -1) - result = std::to_chars(buf.str + pos, buf.str + buf.len, v, (std::chars_format)formatting); - else - result = std::to_chars(buf.str + pos, buf.str + buf.len, v, (std::chars_format)formatting, precision); - if(result.ec == std::errc()) - { - // all good, no errors. - C4_ASSERT(result.ptr >= buf.str); - ptrdiff_t delta = result.ptr - buf.str; - return static_cast(delta); - } - C4_ASSERT(result.ec == std::errc::value_too_large); - // This is unfortunate. - // - // When the result can't fit in the given buffer, - // std::to_chars() returns the end pointer it was originally - // given, which is useless because here we would like to know - // _exactly_ how many characters the buffer must have to fit - // the result. - // - // So we take the pessimistic view, and assume as many digits - // as could ever be required: - size_t ret = static_cast(std::numeric_limits::max_digits10); - return ret > buf.len ? ret : buf.len + 1; -} -#endif // C4CORE_HAVE_STD_TOCHARS - - -#if C4CORE_HAVE_FAST_FLOAT -template -C4_ALWAYS_INLINE bool scan_rhex(csubstr s, T *C4_RESTRICT val) noexcept -{ - C4_ASSERT(s.len > 0); - C4_ASSERT(s.str[0] != '-'); - C4_ASSERT(s.str[0] != '+'); - C4_ASSERT(!s.begins_with("0x")); - C4_ASSERT(!s.begins_with("0X")); - size_t pos = 0; - // integer part - for( ; pos < s.len; ++pos) - { - const char c = s.str[pos]; - if(c >= '0' && c <= '9') - *val = *val * T(16) + T(c - '0'); - else if(c >= 'a' && c <= 'f') - *val = *val * T(16) + T(c - 'a'); - else if(c >= 'A' && c <= 'F') - *val = *val * T(16) + T(c - 'A'); - else if(c == '.') - { - ++pos; - break; // follow on to mantissa - } - else if(c == 'p' || c == 'P') - { - ++pos; - goto power; // no mantissa given, jump to power - } - else - { - return false; - } - } - // mantissa - { - // 0.0625 == 1/16 == value of first digit after the comma - for(T digit = T(0.0625); pos < s.len; ++pos, digit /= T(16)) - { - const char c = s.str[pos]; - if(c >= '0' && c <= '9') - *val += digit * T(c - '0'); - else if(c >= 'a' && c <= 'f') - *val += digit * T(c - 'a'); - else if(c >= 'A' && c <= 'F') - *val += digit * T(c - 'A'); - else if(c == 'p' || c == 'P') - { - ++pos; - goto power; // mantissa finished, jump to power - } - else - { - return false; - } - } - } - return true; -power: - if(C4_LIKELY(pos < s.len)) - { - if(s.str[pos] == '+') // atoi() cannot handle a leading '+' - ++pos; - if(C4_LIKELY(pos < s.len)) - { - int16_t powval = {}; - if(C4_LIKELY(atoi(s.sub(pos), &powval))) - { - *val *= ipow(powval); - return true; - } - } - } - return false; -} -#endif - -} // namespace detail - - -#undef _c4appendhex -#undef _c4append - - -/** Convert a single-precision real number to string. The string will - * in general be NOT null-terminated. For FTOA_FLEX, \p precision is - * the number of significand digits. Otherwise \p precision is the - * number of decimals. It is safe to call this function with an empty - * or too-small buffer. - * - * @return the size of the buffer needed to write the number - */ -C4_ALWAYS_INLINE size_t ftoa(substr str, float v, int precision=-1, RealFormat_e formatting=FTOA_FLEX) noexcept -{ -#if C4CORE_HAVE_STD_TOCHARS - return detail::rtoa(str, v, precision, formatting); -#else - char fmt[16]; - detail::get_real_format_str(fmt, precision, formatting, /*length_modifier*/""); - return detail::print_one(str, fmt, v); -#endif -} - - -/** Convert a double-precision real number to string. The string will - * in general be NOT null-terminated. For FTOA_FLEX, \p precision is - * the number of significand digits. Otherwise \p precision is the - * number of decimals. It is safe to call this function with an empty - * or too-small buffer. - * - * @return the size of the buffer needed to write the number - */ -C4_ALWAYS_INLINE size_t dtoa(substr str, double v, int precision=-1, RealFormat_e formatting=FTOA_FLEX) noexcept -{ -#if C4CORE_HAVE_STD_TOCHARS - return detail::rtoa(str, v, precision, formatting); -#else - char fmt[16]; - detail::get_real_format_str(fmt, precision, formatting, /*length_modifier*/"l"); - return detail::print_one(str, fmt, v); -#endif -} - - -/** Convert a string to a single precision real number. - * The input string must be trimmed to the value, ie - * no leading or trailing whitespace can be present. - * @return true iff the conversion succeeded - * @see atof_first() if the string is not trimmed - */ -C4_ALWAYS_INLINE bool atof(csubstr str, float * C4_RESTRICT v) noexcept -{ - C4_ASSERT(str.len > 0); - C4_ASSERT(str.triml(" \r\t\n").len == str.len); -#if C4CORE_HAVE_FAST_FLOAT - // fastfloat cannot parse hexadecimal floats - bool isneg = (str.str[0] == '-'); - csubstr rem = str.sub(isneg || str.str[0] == '+'); - if(!(rem.len >= 2 && (rem.str[0] == '0' && (rem.str[1] == 'x' || rem.str[1] == 'X')))) - { - fast_float::from_chars_result result; - result = fast_float::from_chars(str.str, str.str + str.len, *v); - return result.ec == std::errc(); - } - else if(detail::scan_rhex(rem.sub(2), v)) - { - *v *= isneg ? -1.f : 1.f; - return true; - } - return false; -#elif C4CORE_HAVE_STD_FROMCHARS - std::from_chars_result result; - result = std::from_chars(str.str, str.str + str.len, *v); - return result.ec == std::errc(); -#else - csubstr rem = str.sub(str.str[0] == '-' || str.str[0] == '+'); - if(!(rem.len >= 2 && (rem.str[0] == '0' && (rem.str[1] == 'x' || rem.str[1] == 'X')))) - return detail::scan_one(str, "f", v) != csubstr::npos; - else - return detail::scan_one(str, "a", v) != csubstr::npos; -#endif -} - - -/** Convert a string to a double precision real number. - * The input string must be trimmed to the value, ie - * no leading or trailing whitespace can be present. - * @return true iff the conversion succeeded - * @see atod_first() if the string is not trimmed - */ -C4_ALWAYS_INLINE bool atod(csubstr str, double * C4_RESTRICT v) noexcept -{ - C4_ASSERT(str.triml(" \r\t\n").len == str.len); -#if C4CORE_HAVE_FAST_FLOAT - // fastfloat cannot parse hexadecimal floats - bool isneg = (str.str[0] == '-'); - csubstr rem = str.sub(isneg || str.str[0] == '+'); - if(!(rem.len >= 2 && (rem.str[0] == '0' && (rem.str[1] == 'x' || rem.str[1] == 'X')))) - { - fast_float::from_chars_result result; - result = fast_float::from_chars(str.str, str.str + str.len, *v); - return result.ec == std::errc(); - } - else if(detail::scan_rhex(rem.sub(2), v)) - { - *v *= isneg ? -1. : 1.; - return true; - } - return false; -#elif C4CORE_HAVE_STD_FROMCHARS - std::from_chars_result result; - result = std::from_chars(str.str, str.str + str.len, *v); - return result.ec == std::errc(); -#else - csubstr rem = str.sub(str.str[0] == '-' || str.str[0] == '+'); - if(!(rem.len >= 2 && (rem.str[0] == '0' && (rem.str[1] == 'x' || rem.str[1] == 'X')))) - return detail::scan_one(str, "lf", v) != csubstr::npos; - else - return detail::scan_one(str, "la", v) != csubstr::npos; -#endif -} - - -/** Convert a string to a single precision real number. - * Leading whitespace is skipped until valid characters are found. - * @return the number of characters read from the string, or npos if - * conversion was not successful or if the string was empty */ -inline size_t atof_first(csubstr str, float * C4_RESTRICT v) noexcept -{ - csubstr trimmed = str.first_real_span(); - if(trimmed.len == 0) - return csubstr::npos; - if(atof(trimmed, v)) - return static_cast(trimmed.end() - str.begin()); - return csubstr::npos; -} - - -/** Convert a string to a double precision real number. - * Leading whitespace is skipped until valid characters are found. - * @return the number of characters read from the string, or npos if - * conversion was not successful or if the string was empty */ -inline size_t atod_first(csubstr str, double * C4_RESTRICT v) noexcept -{ - csubstr trimmed = str.first_real_span(); - if(trimmed.len == 0) - return csubstr::npos; - if(atod(trimmed, v)) - return static_cast(trimmed.end() - str.begin()); - return csubstr::npos; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// generic versions - -C4_ALWAYS_INLINE size_t xtoa(substr s, uint8_t v) noexcept { return write_dec(s, v); } -C4_ALWAYS_INLINE size_t xtoa(substr s, uint16_t v) noexcept { return write_dec(s, v); } -C4_ALWAYS_INLINE size_t xtoa(substr s, uint32_t v) noexcept { return write_dec(s, v); } -C4_ALWAYS_INLINE size_t xtoa(substr s, uint64_t v) noexcept { return write_dec(s, v); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int8_t v) noexcept { return itoa(s, v); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int16_t v) noexcept { return itoa(s, v); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int32_t v) noexcept { return itoa(s, v); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int64_t v) noexcept { return itoa(s, v); } -C4_ALWAYS_INLINE size_t xtoa(substr s, float v) noexcept { return ftoa(s, v); } -C4_ALWAYS_INLINE size_t xtoa(substr s, double v) noexcept { return dtoa(s, v); } - -C4_ALWAYS_INLINE size_t xtoa(substr s, uint8_t v, uint8_t radix) noexcept { return utoa(s, v, radix); } -C4_ALWAYS_INLINE size_t xtoa(substr s, uint16_t v, uint16_t radix) noexcept { return utoa(s, v, radix); } -C4_ALWAYS_INLINE size_t xtoa(substr s, uint32_t v, uint32_t radix) noexcept { return utoa(s, v, radix); } -C4_ALWAYS_INLINE size_t xtoa(substr s, uint64_t v, uint64_t radix) noexcept { return utoa(s, v, radix); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int8_t v, int8_t radix) noexcept { return itoa(s, v, radix); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int16_t v, int16_t radix) noexcept { return itoa(s, v, radix); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int32_t v, int32_t radix) noexcept { return itoa(s, v, radix); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int64_t v, int64_t radix) noexcept { return itoa(s, v, radix); } - -C4_ALWAYS_INLINE size_t xtoa(substr s, uint8_t v, uint8_t radix, size_t num_digits) noexcept { return utoa(s, v, radix, num_digits); } -C4_ALWAYS_INLINE size_t xtoa(substr s, uint16_t v, uint16_t radix, size_t num_digits) noexcept { return utoa(s, v, radix, num_digits); } -C4_ALWAYS_INLINE size_t xtoa(substr s, uint32_t v, uint32_t radix, size_t num_digits) noexcept { return utoa(s, v, radix, num_digits); } -C4_ALWAYS_INLINE size_t xtoa(substr s, uint64_t v, uint64_t radix, size_t num_digits) noexcept { return utoa(s, v, radix, num_digits); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int8_t v, int8_t radix, size_t num_digits) noexcept { return itoa(s, v, radix, num_digits); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int16_t v, int16_t radix, size_t num_digits) noexcept { return itoa(s, v, radix, num_digits); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int32_t v, int32_t radix, size_t num_digits) noexcept { return itoa(s, v, radix, num_digits); } -C4_ALWAYS_INLINE size_t xtoa(substr s, int64_t v, int64_t radix, size_t num_digits) noexcept { return itoa(s, v, radix, num_digits); } - -C4_ALWAYS_INLINE size_t xtoa(substr s, float v, int precision, RealFormat_e formatting=FTOA_FLEX) noexcept { return ftoa(s, v, precision, formatting); } -C4_ALWAYS_INLINE size_t xtoa(substr s, double v, int precision, RealFormat_e formatting=FTOA_FLEX) noexcept { return dtoa(s, v, precision, formatting); } - -C4_ALWAYS_INLINE bool atox(csubstr s, uint8_t *C4_RESTRICT v) noexcept { return atou(s, v); } -C4_ALWAYS_INLINE bool atox(csubstr s, uint16_t *C4_RESTRICT v) noexcept { return atou(s, v); } -C4_ALWAYS_INLINE bool atox(csubstr s, uint32_t *C4_RESTRICT v) noexcept { return atou(s, v); } -C4_ALWAYS_INLINE bool atox(csubstr s, uint64_t *C4_RESTRICT v) noexcept { return atou(s, v); } -C4_ALWAYS_INLINE bool atox(csubstr s, int8_t *C4_RESTRICT v) noexcept { return atoi(s, v); } -C4_ALWAYS_INLINE bool atox(csubstr s, int16_t *C4_RESTRICT v) noexcept { return atoi(s, v); } -C4_ALWAYS_INLINE bool atox(csubstr s, int32_t *C4_RESTRICT v) noexcept { return atoi(s, v); } -C4_ALWAYS_INLINE bool atox(csubstr s, int64_t *C4_RESTRICT v) noexcept { return atoi(s, v); } -C4_ALWAYS_INLINE bool atox(csubstr s, float *C4_RESTRICT v) noexcept { return atof(s, v); } -C4_ALWAYS_INLINE bool atox(csubstr s, double *C4_RESTRICT v) noexcept { return atod(s, v); } - -C4_ALWAYS_INLINE size_t to_chars(substr buf, uint8_t v) noexcept { return write_dec(buf, v); } -C4_ALWAYS_INLINE size_t to_chars(substr buf, uint16_t v) noexcept { return write_dec(buf, v); } -C4_ALWAYS_INLINE size_t to_chars(substr buf, uint32_t v) noexcept { return write_dec(buf, v); } -C4_ALWAYS_INLINE size_t to_chars(substr buf, uint64_t v) noexcept { return write_dec(buf, v); } -C4_ALWAYS_INLINE size_t to_chars(substr buf, int8_t v) noexcept { return itoa(buf, v); } -C4_ALWAYS_INLINE size_t to_chars(substr buf, int16_t v) noexcept { return itoa(buf, v); } -C4_ALWAYS_INLINE size_t to_chars(substr buf, int32_t v) noexcept { return itoa(buf, v); } -C4_ALWAYS_INLINE size_t to_chars(substr buf, int64_t v) noexcept { return itoa(buf, v); } -C4_ALWAYS_INLINE size_t to_chars(substr buf, float v) noexcept { return ftoa(buf, v); } -C4_ALWAYS_INLINE size_t to_chars(substr buf, double v) noexcept { return dtoa(buf, v); } - -C4_ALWAYS_INLINE bool from_chars(csubstr buf, uint8_t *C4_RESTRICT v) noexcept { return atou(buf, v); } -C4_ALWAYS_INLINE bool from_chars(csubstr buf, uint16_t *C4_RESTRICT v) noexcept { return atou(buf, v); } -C4_ALWAYS_INLINE bool from_chars(csubstr buf, uint32_t *C4_RESTRICT v) noexcept { return atou(buf, v); } -C4_ALWAYS_INLINE bool from_chars(csubstr buf, uint64_t *C4_RESTRICT v) noexcept { return atou(buf, v); } -C4_ALWAYS_INLINE bool from_chars(csubstr buf, int8_t *C4_RESTRICT v) noexcept { return atoi(buf, v); } -C4_ALWAYS_INLINE bool from_chars(csubstr buf, int16_t *C4_RESTRICT v) noexcept { return atoi(buf, v); } -C4_ALWAYS_INLINE bool from_chars(csubstr buf, int32_t *C4_RESTRICT v) noexcept { return atoi(buf, v); } -C4_ALWAYS_INLINE bool from_chars(csubstr buf, int64_t *C4_RESTRICT v) noexcept { return atoi(buf, v); } -C4_ALWAYS_INLINE bool from_chars(csubstr buf, float *C4_RESTRICT v) noexcept { return atof(buf, v); } -C4_ALWAYS_INLINE bool from_chars(csubstr buf, double *C4_RESTRICT v) noexcept { return atod(buf, v); } - -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, uint8_t *C4_RESTRICT v) noexcept { return atou_first(buf, v); } -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, uint16_t *C4_RESTRICT v) noexcept { return atou_first(buf, v); } -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, uint32_t *C4_RESTRICT v) noexcept { return atou_first(buf, v); } -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, uint64_t *C4_RESTRICT v) noexcept { return atou_first(buf, v); } -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, int8_t *C4_RESTRICT v) noexcept { return atoi_first(buf, v); } -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, int16_t *C4_RESTRICT v) noexcept { return atoi_first(buf, v); } -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, int32_t *C4_RESTRICT v) noexcept { return atoi_first(buf, v); } -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, int64_t *C4_RESTRICT v) noexcept { return atoi_first(buf, v); } -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, float *C4_RESTRICT v) noexcept { return atof_first(buf, v); } -C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, double *C4_RESTRICT v) noexcept { return atod_first(buf, v); } - - -//----------------------------------------------------------------------------- -// on some platforms, (unsigned) int and (unsigned) long -// are not any of the fixed length types above - -#define _C4_IF_NOT_FIXED_LENGTH_I(T, ty) C4_ALWAYS_INLINE typename std::enable_if::value && !is_fixed_length::value_i, ty> -#define _C4_IF_NOT_FIXED_LENGTH_U(T, ty) C4_ALWAYS_INLINE typename std::enable_if::value && !is_fixed_length::value_u, ty> - -template _C4_IF_NOT_FIXED_LENGTH_I(T, size_t)::type xtoa(substr buf, T v) noexcept { return itoa(buf, v); } -template _C4_IF_NOT_FIXED_LENGTH_U(T, size_t)::type xtoa(substr buf, T v) noexcept { return write_dec(buf, v); } - -template _C4_IF_NOT_FIXED_LENGTH_I(T, bool )::type atox(csubstr buf, T *C4_RESTRICT v) noexcept { return atoi(buf, v); } -template _C4_IF_NOT_FIXED_LENGTH_U(T, bool )::type atox(csubstr buf, T *C4_RESTRICT v) noexcept { return atou(buf, v); } - -template _C4_IF_NOT_FIXED_LENGTH_I(T, size_t)::type to_chars(substr buf, T v) noexcept { return itoa(buf, v); } -template _C4_IF_NOT_FIXED_LENGTH_U(T, size_t)::type to_chars(substr buf, T v) noexcept { return write_dec(buf, v); } - -template _C4_IF_NOT_FIXED_LENGTH_I(T, bool )::type from_chars(csubstr buf, T *C4_RESTRICT v) noexcept { return atoi(buf, v); } -template _C4_IF_NOT_FIXED_LENGTH_U(T, bool )::type from_chars(csubstr buf, T *C4_RESTRICT v) noexcept { return atou(buf, v); } - -template _C4_IF_NOT_FIXED_LENGTH_I(T, size_t)::type from_chars_first(csubstr buf, T *C4_RESTRICT v) noexcept { return atoi_first(buf, v); } -template _C4_IF_NOT_FIXED_LENGTH_U(T, size_t)::type from_chars_first(csubstr buf, T *C4_RESTRICT v) noexcept { return atou_first(buf, v); } - -#undef _C4_IF_NOT_FIXED_LENGTH_I -#undef _C4_IF_NOT_FIXED_LENGTH_U - - -//----------------------------------------------------------------------------- -// for pointers - -template C4_ALWAYS_INLINE size_t xtoa(substr s, T *v) noexcept { return itoa(s, (intptr_t)v, (intptr_t)16); } -template C4_ALWAYS_INLINE bool atox(csubstr s, T **v) noexcept { intptr_t tmp; bool ret = atox(s, &tmp); if(ret) { *v = (T*)tmp; } return ret; } -template C4_ALWAYS_INLINE size_t to_chars(substr s, T *v) noexcept { return itoa(s, (intptr_t)v, (intptr_t)16); } -template C4_ALWAYS_INLINE bool from_chars(csubstr buf, T **v) noexcept { intptr_t tmp; bool ret = from_chars(buf, &tmp); if(ret) { *v = (T*)tmp; } return ret; } -template C4_ALWAYS_INLINE size_t from_chars_first(csubstr buf, T **v) noexcept { intptr_t tmp; bool ret = from_chars_first(buf, &tmp); if(ret) { *v = (T*)tmp; } return ret; } - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** call to_chars() and return a substr consisting of the - * written portion of the input buffer. Ie, same as to_chars(), - * but return a substr instead of a size_t. - * - * @see to_chars() */ -template -C4_ALWAYS_INLINE substr to_chars_sub(substr buf, T const& C4_RESTRICT v) noexcept -{ - size_t sz = to_chars(buf, v); - return buf.left_of(sz <= buf.len ? sz : buf.len); -} - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// bool implementation - -C4_ALWAYS_INLINE size_t to_chars(substr buf, bool v) noexcept -{ - int val = v; - return to_chars(buf, val); -} - -inline bool from_chars(csubstr buf, bool * C4_RESTRICT v) noexcept -{ - if(buf == '0') - { - *v = false; return true; - } - else if(buf == '1') - { - *v = true; return true; - } - else if(buf == "false") - { - *v = false; return true; - } - else if(buf == "true") - { - *v = true; return true; - } - else if(buf == "False") - { - *v = false; return true; - } - else if(buf == "True") - { - *v = true; return true; - } - else if(buf == "FALSE") - { - *v = false; return true; - } - else if(buf == "TRUE") - { - *v = true; return true; - } - // fallback to c-style int bools - int val = 0; - bool ret = from_chars(buf, &val); - if(C4_LIKELY(ret)) - { - *v = (val != 0); - } - return ret; -} - -inline size_t from_chars_first(csubstr buf, bool * C4_RESTRICT v) noexcept -{ - csubstr trimmed = buf.first_non_empty_span(); - if(trimmed.len == 0 || !from_chars(buf, v)) - return csubstr::npos; - return trimmed.len; -} - - -//----------------------------------------------------------------------------- -// single-char implementation - -inline size_t to_chars(substr buf, char v) noexcept -{ - if(buf.len > 0) - buf[0] = v; - return 1; -} - -/** extract a single character from a substring - * @note to extract a string instead and not just a single character, use the csubstr overload */ -inline bool from_chars(csubstr buf, char * C4_RESTRICT v) noexcept -{ - if(buf.len != 1) - return false; - *v = buf[0]; - return true; -} - -inline size_t from_chars_first(csubstr buf, char * C4_RESTRICT v) noexcept -{ - if(buf.len < 1) - return csubstr::npos; - *v = buf[0]; - return 1; -} - - -//----------------------------------------------------------------------------- -// csubstr implementation - -inline size_t to_chars(substr buf, csubstr v) noexcept -{ - C4_ASSERT(!buf.overlaps(v)); - size_t len = buf.len < v.len ? buf.len : v.len; - // calling memcpy with null strings is undefined behavior - // and will wreak havoc in calling code's branches. - // see https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637 - if(len) - { - C4_ASSERT(buf.str != nullptr); - C4_ASSERT(v.str != nullptr); - memcpy(buf.str, v.str, len); - } - return v.len; -} - -inline bool from_chars(csubstr buf, csubstr *C4_RESTRICT v) noexcept -{ - *v = buf; - return true; -} - -inline size_t from_chars_first(substr buf, csubstr * C4_RESTRICT v) noexcept -{ - csubstr trimmed = buf.first_non_empty_span(); - if(trimmed.len == 0) - return csubstr::npos; - *v = trimmed; - return static_cast(trimmed.end() - buf.begin()); -} - - -//----------------------------------------------------------------------------- -// substr - -inline size_t to_chars(substr buf, substr v) noexcept -{ - C4_ASSERT(!buf.overlaps(v)); - size_t len = buf.len < v.len ? buf.len : v.len; - // calling memcpy with null strings is undefined behavior - // and will wreak havoc in calling code's branches. - // see https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637 - if(len) - { - C4_ASSERT(buf.str != nullptr); - C4_ASSERT(v.str != nullptr); - memcpy(buf.str, v.str, len); - } - return v.len; -} - -inline bool from_chars(csubstr buf, substr * C4_RESTRICT v) noexcept -{ - C4_ASSERT(!buf.overlaps(*v)); - // is the destination buffer wide enough? - if(v->len >= buf.len) - { - // calling memcpy with null strings is undefined behavior - // and will wreak havoc in calling code's branches. - // see https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637 - if(buf.len) - { - C4_ASSERT(buf.str != nullptr); - C4_ASSERT(v->str != nullptr); - memcpy(v->str, buf.str, buf.len); - } - v->len = buf.len; - return true; - } - return false; -} - -inline size_t from_chars_first(csubstr buf, substr * C4_RESTRICT v) noexcept -{ - csubstr trimmed = buf.first_non_empty_span(); - C4_ASSERT(!trimmed.overlaps(*v)); - if(C4_UNLIKELY(trimmed.len == 0)) - return csubstr::npos; - size_t len = trimmed.len > v->len ? v->len : trimmed.len; - // calling memcpy with null strings is undefined behavior - // and will wreak havoc in calling code's branches. - // see https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637 - if(len) - { - C4_ASSERT(buf.str != nullptr); - C4_ASSERT(v->str != nullptr); - memcpy(v->str, trimmed.str, len); - } - if(C4_UNLIKELY(trimmed.len > v->len)) - return csubstr::npos; - return static_cast(trimmed.end() - buf.begin()); -} - - -//----------------------------------------------------------------------------- - -template -inline size_t to_chars(substr buf, const char (& C4_RESTRICT v)[N]) noexcept -{ - csubstr sp(v); - return to_chars(buf, sp); -} - -inline size_t to_chars(substr buf, const char * C4_RESTRICT v) noexcept -{ - return to_chars(buf, to_csubstr(v)); -} - -} // namespace c4 - -#ifdef _MSC_VER -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -#endif /* _C4_CHARCONV_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/common.hpp b/thirdparty/ryml/ext/c4core/src/c4/common.hpp deleted file mode 100644 index e3063ffc0..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/common.hpp +++ /dev/null @@ -1,15 +0,0 @@ -#ifndef _C4_COMMON_HPP_ -#define _C4_COMMON_HPP_ - -#include "c4/config.hpp" - -#include "c4/preprocessor.hpp" -#include "c4/platform.hpp" -#include "c4/cpu.hpp" -#include "c4/compiler.hpp" -#include "c4/language.hpp" -#include "c4/error.hpp" -#include "c4/time.hpp" -#include "c4/types.hpp" - -#endif /* _C4_COMMON_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/compiler.hpp b/thirdparty/ryml/ext/c4core/src/c4/compiler.hpp deleted file mode 100644 index ba8e5b07c..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/compiler.hpp +++ /dev/null @@ -1,117 +0,0 @@ -#ifndef _C4_COMPILER_HPP_ -#define _C4_COMPILER_HPP_ - -/** @file compiler.hpp Provides compiler information macros - * @ingroup basic_headers */ - -#include "c4/platform.hpp" - -// Compilers: -// C4_MSVC -// Visual Studio 2022: MSVC++ 17, 1930 -// Visual Studio 2019: MSVC++ 16, 1920 -// Visual Studio 2017: MSVC++ 15 -// Visual Studio 2015: MSVC++ 14 -// Visual Studio 2013: MSVC++ 13 -// Visual Studio 2013: MSVC++ 12 -// Visual Studio 2012: MSVC++ 11 -// Visual Studio 2010: MSVC++ 10 -// Visual Studio 2008: MSVC++ 09 -// Visual Studio 2005: MSVC++ 08 -// C4_CLANG -// C4_GCC -// C4_ICC (intel compiler) -/** @see http://sourceforge.net/p/predef/wiki/Compilers/ for a list of compiler identifier macros */ -/** @see https://msdn.microsoft.com/en-us/library/b0084kay.aspx for VS2013 predefined macros */ - -#if defined(_MSC_VER)// && (defined(C4_WIN) || defined(C4_XBOX) || defined(C4_UE4)) -# define C4_MSVC -# define C4_MSVC_VERSION_2022 17 -# define C4_MSVC_VERSION_2019 16 -# define C4_MSVC_VERSION_2017 15 -# define C4_MSVC_VERSION_2015 14 -# define C4_MSVC_VERSION_2013 12 -# define C4_MSVC_VERSION_2012 11 -# if _MSC_VER >= 1930 -# define C4_MSVC_VERSION C4_MSVC_VERSION_2022 // visual studio 2022 -# define C4_MSVC_2022 -# elif _MSC_VER >= 1920 -# define C4_MSVC_VERSION C_4MSVC_VERSION_2019 // visual studio 2019 -# define C4_MSVC_2019 -# elif _MSC_VER >= 1910 -# define C4_MSVC_VERSION C4_MSVC_VERSION_2017 // visual studio 2017 -# define C4_MSVC_2017 -# elif _MSC_VER == 1900 -# define C4_MSVC_VERSION C4_MSVC_VERSION_2015 // visual studio 2015 -# define C4_MSVC_2015 -# elif _MSC_VER == 1800 -# error "MSVC version not supported" -# define C4_MSVC_VERSION C4_MSVC_VERSION_2013 // visual studio 2013 -# define C4_MSVC_2013 -# elif _MSC_VER == 1700 -# error "MSVC version not supported" -# define C4_MSVC_VERSION C4_MSVC_VERSION_2012 // visual studio 2012 -# define C4_MSVC_2012 -# elif _MSC_VER == 1600 -# error "MSVC version not supported" -# define C4_MSVC_VERSION 10 // visual studio 2010 -# define C4_MSVC_2010 -# elif _MSC_VER == 1500 -# error "MSVC version not supported" -# define C4_MSVC_VERSION 09 // visual studio 2008 -# define C4_MSVC_2008 -# elif _MSC_VER == 1400 -# error "MSVC version not supported" -# define C4_MSVC_VERSION 08 // visual studio 2005 -# define C4_MSVC_2005 -# else -# error "MSVC version not supported" -# endif // _MSC_VER -#else -# define C4_MSVC_VERSION 0 // visual studio not present -# define C4_GCC_LIKE -# ifdef __INTEL_COMPILER // check ICC before checking GCC, as ICC defines __GNUC__ too -# define C4_ICC -# define C4_ICC_VERSION __INTEL_COMPILER -# elif defined(__APPLE_CC__) -# define C4_XCODE -# if defined(__clang__) -# define C4_CLANG -# ifndef __apple_build_version__ -# define C4_CLANG_VERSION C4_VERSION_ENCODED(__clang_major__, __clang_minor__, __clang_patchlevel__) -# else -# define C4_CLANG_VERSION __apple_build_version__ -# endif -# else -# define C4_XCODE_VERSION __APPLE_CC__ -# endif -# elif defined(__clang__) -# define C4_CLANG -# ifndef __apple_build_version__ -# define C4_CLANG_VERSION C4_VERSION_ENCODED(__clang_major__, __clang_minor__, __clang_patchlevel__) -# else -# define C4_CLANG_VERSION __apple_build_version__ -# endif -# elif defined(__GNUC__) -# define C4_GCC -# if defined(__GNUC_PATCHLEVEL__) -# define C4_GCC_VERSION C4_VERSION_ENCODED(__GNUC__, __GNUC_MINOR__, __GNUC_PATCHLEVEL__) -# else -# define C4_GCC_VERSION C4_VERSION_ENCODED(__GNUC__, __GNUC_MINOR__, 0) -# endif -# if __GNUC__ < 5 -# if __GNUC__ == 4 && __GNUC_MINOR__ >= 8 -// provided by cmake sub-project -# include "c4/gcc-4.8.hpp" -# else -// we do not support GCC < 4.8: -// * misses std::is_trivially_copyable -// * misses std::align -// * -Wshadow has false positives when a local function parameter has the same name as a method -# error "GCC < 4.8 is not supported" -# endif -# endif -# endif -#endif // defined(C4_WIN) && defined(_MSC_VER) - -#endif /* _C4_COMPILER_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/config.hpp b/thirdparty/ryml/ext/c4core/src/c4/config.hpp deleted file mode 100644 index bda8033b3..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/config.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#ifndef _C4_CONFIG_HPP_ -#define _C4_CONFIG_HPP_ - -/** @defgroup basic_headers Basic headers - * @brief Headers providing basic macros, platform+cpu+compiler information, - * C++ facilities and basic typedefs. */ - -/** @file config.hpp Contains configuration defines and includes the basic_headers. - * @ingroup basic_headers */ - -//#define C4_DEBUG - -#define C4_ERROR_SHOWS_FILELINE -//#define C4_ERROR_SHOWS_FUNC -//#define C4_ERROR_THROWS_EXCEPTION -//#define C4_NO_ALLOC_DEFAULTS -//#define C4_REDEFINE_CPPNEW - -#ifndef C4_SIZE_TYPE -# define C4_SIZE_TYPE size_t -#endif - -#ifndef C4_STR_SIZE_TYPE -# define C4_STR_SIZE_TYPE C4_SIZE_TYPE -#endif - -#ifndef C4_TIME_TYPE -# define C4_TIME_TYPE double -#endif - -#include "c4/export.hpp" -#include "c4/preprocessor.hpp" -#include "c4/platform.hpp" -#include "c4/cpu.hpp" -#include "c4/compiler.hpp" -#include "c4/language.hpp" -#include "c4/types.hpp" - -#endif // _C4_CONFIG_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/cpu.hpp b/thirdparty/ryml/ext/c4core/src/c4/cpu.hpp deleted file mode 100644 index f8877d115..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/cpu.hpp +++ /dev/null @@ -1,139 +0,0 @@ -#ifndef _C4_CPU_HPP_ -#define _C4_CPU_HPP_ - -/** @file cpu.hpp Provides processor information macros - * @ingroup basic_headers */ - -// see also https://sourceforge.net/p/predef/wiki/Architectures/ -// see also https://sourceforge.net/p/predef/wiki/Endianness/ -// see also https://github.com/googlesamples/android-ndk/blob/android-mk/hello-jni/jni/hello-jni.c -// see http://code.qt.io/cgit/qt/qtbase.git/tree/src/corelib/global/qprocessordetection.h - -#ifdef __ORDER_LITTLE_ENDIAN__ - #define _C4EL __ORDER_LITTLE_ENDIAN__ -#else - #define _C4EL 1234 -#endif - -#ifdef __ORDER_BIG_ENDIAN__ - #define _C4EB __ORDER_BIG_ENDIAN__ -#else - #define _C4EB 4321 -#endif - -// mixed byte order (eg, PowerPC or ia64) -#define _C4EM 1111 - -#if defined(__x86_64) || defined(__x86_64__) || defined(__amd64) || defined(_M_X64) - #define C4_CPU_X86_64 - #define C4_WORDSIZE 8 - #define C4_BYTE_ORDER _C4EL - -#elif defined(__i386) || defined(__i386__) || defined(_M_IX86) - #define C4_CPU_X86 - #define C4_WORDSIZE 4 - #define C4_BYTE_ORDER _C4EL - -#elif defined(__arm__) || defined(_M_ARM) \ - || defined(__TARGET_ARCH_ARM) || defined(__aarch64__) || defined(_M_ARM64) - #if defined(__aarch64__) || defined(_M_ARM64) - #define C4_CPU_ARM64 - #define C4_CPU_ARMV8 - #define C4_WORDSIZE 8 - #else - #define C4_CPU_ARM - #define C4_WORDSIZE 4 - #if defined(__ARM_ARCH_8__) || defined(__ARM_ARCH_8A__) \ - || (defined(__ARCH_ARM) && __ARCH_ARM >= 8) - || (defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM >= 8) \ - #define C4_CPU_ARMV8 - #elif defined(__ARM_ARCH_7__) || defined(_ARM_ARCH_7) \ - || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) \ - || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) \ - || defined(__ARM_ARCH_7EM__) \ - || (defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM >= 7) \ - || (defined(_M_ARM) && _M_ARM >= 7) - #define C4_CPU_ARMV7 - #elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) \ - || defined(__ARM_ARCH_6T2__) || defined(__ARM_ARCH_6Z__) \ - || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6ZK__) \ - || defined(__ARM_ARCH_6M__) || defined(__ARM_ARCH_6KZ__) \ - || (defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM >= 6) - #define C4_CPU_ARMV6 - #elif defined(__ARM_ARCH_5TEJ__) \ - || defined(__ARM_ARCH_5TE__) \ - || (defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM >= 5) - #define C4_CPU_ARMV5 - #elif defined(__ARM_ARCH_4T__) \ - || (defined(__TARGET_ARCH_ARM) && __TARGET_ARCH_ARM >= 4) - #define C4_CPU_ARMV4 - #else - #error "unknown CPU architecture: ARM" - #endif - #endif - #if defined(__ARMEL__) || defined(__LITTLE_ENDIAN__) || defined(__AARCH64EL__) \ - || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) \ - || defined(_MSC_VER) // winarm64 does not provide any of the above macros, - // but advises little-endianess: - // https://docs.microsoft.com/en-us/cpp/build/overview-of-arm-abi-conventions?view=msvc-170 - // So if it is visual studio compiling, we'll assume little endian. - #define C4_BYTE_ORDER _C4EL - #elif defined(__ARMEB__) || defined(__BIG_ENDIAN__) || defined(__AARCH64EB__) \ - || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) - #define C4_BYTE_ORDER _C4EB - #elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_PDP_ENDIAN__) - #define C4_BYTE_ORDER _C4EM - #else - #error "unknown endianness" - #endif - -#elif defined(__ia64) || defined(__ia64__) || defined(_M_IA64) - #define C4_CPU_IA64 - #define C4_WORDSIZE 8 - #define C4_BYTE_ORDER _C4EM - // itanium is bi-endian - check byte order below - -#elif defined(__ppc__) || defined(__ppc) || defined(__powerpc__) \ - || defined(_ARCH_COM) || defined(_ARCH_PWR) || defined(_ARCH_PPC) \ - || defined(_M_MPPC) || defined(_M_PPC) - #if defined(__ppc64__) || defined(__powerpc64__) || defined(__64BIT__) - #define C4_CPU_PPC64 - #define C4_WORDSIZE 8 - #else - #define C4_CPU_PPC - #define C4_WORDSIZE 4 - #endif - #define C4_BYTE_ORDER _C4EM - // ppc is bi-endian - check byte order below - -#elif defined(__s390x__) || defined(__zarch__) || defined(__SYSC_ZARCH_) -# define C4_CPU_S390_X -# define C4_WORDSIZE 8 -# define C4_BYTE_ORDER _C4EB - -#elif defined(__riscv) - #if __riscv_xlen == 64 - #define C4_CPU_RISCV64 - #define C4_WORDSIZE 8 - #else - #define C4_CPU_RISCV32 - #define C4_WORDSIZE 4 - #endif - #define C4_BYTE_ORDER _C4EL - -#elif defined(__EMSCRIPTEN__) -# define C4_BYTE_ORDER _C4EL -# define C4_WORDSIZE 4 - -#elif defined(SWIG) - #error "please define CPU architecture macros when compiling with swig" - -#else - #error "unknown CPU architecture" -#endif - -#define C4_LITTLE_ENDIAN (C4_BYTE_ORDER == _C4EL) -#define C4_BIG_ENDIAN (C4_BYTE_ORDER == _C4EB) -#define C4_MIXED_ENDIAN (C4_BYTE_ORDER == _C4EM) - -#endif /* _C4_CPU_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/ctor_dtor.hpp b/thirdparty/ryml/ext/c4core/src/c4/ctor_dtor.hpp deleted file mode 100644 index 8624df7b5..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ctor_dtor.hpp +++ /dev/null @@ -1,462 +0,0 @@ -#ifndef _C4_CTOR_DTOR_HPP_ -#define _C4_CTOR_DTOR_HPP_ - -#include "c4/preprocessor.hpp" -#include "c4/language.hpp" -#include "c4/memory_util.hpp" -#include "c4/error.hpp" - -#include -#include // std::forward - -/** @file ctor_dtor.hpp object construction and destruction facilities. - * Some of these are not yet available in C++11. */ - -namespace c4 { - -/** default-construct an object, trivial version */ -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -construct(U *ptr) noexcept -{ - memset(ptr, 0, sizeof(U)); -} -/** default-construct an object, non-trivial version */ -template C4_ALWAYS_INLINE typename std ::enable_if< ! std::is_trivially_default_constructible::value, void>::type -construct(U* ptr) noexcept -{ - new ((void*)ptr) U(); -} - -/** default-construct n objects, trivial version */ -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -construct_n(U* ptr, I n) noexcept -{ - memset(ptr, 0, n * sizeof(U)); -} -/** default-construct n objects, non-trivial version */ -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_default_constructible::value, void>::type -construct_n(U* ptr, I n) noexcept -{ - for(I i = 0; i < n; ++i) - { - new ((void*)(ptr + i)) U(); - } -} - -#ifdef __clang__ -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# if __GNUC__ >= 6 -# pragma GCC diagnostic ignored "-Wnull-dereference" -# endif -#endif - -template -inline void construct(U* ptr, Args&&... args) -{ - new ((void*)ptr) U(std::forward(args)...); -} -template -inline void construct_n(U* ptr, I n, Args&&... args) -{ - for(I i = 0; i < n; ++i) - { - new ((void*)(ptr + i)) U(args...); - } -} - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - - -//----------------------------------------------------------------------------- -// copy-construct - -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -copy_construct(U* dst, U const* src) noexcept -{ - C4_ASSERT(dst != src); - memcpy(dst, src, sizeof(U)); -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_copy_constructible::value, void>::type -copy_construct(U* dst, U const* src) -{ - C4_ASSERT(dst != src); - new ((void*)dst) U(*src); -} -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -copy_construct_n(U* dst, U const* src, I n) noexcept -{ - C4_ASSERT(dst != src); - memcpy(dst, src, n * sizeof(U)); -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_copy_constructible::value, void>::type -copy_construct_n(U* dst, U const* src, I n) -{ - C4_ASSERT(dst != src); - for(I i = 0; i < n; ++i) - { - new ((void*)(dst + i)) U(*(src + i)); - } -} - -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -copy_construct(U* dst, U src) noexcept // pass by value for scalar types -{ - *dst = src; -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_scalar::value, void>::type -copy_construct(U* dst, U const& src) // pass by reference for non-scalar types -{ - C4_ASSERT(dst != &src); - new ((void*)dst) U(src); -} -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -copy_construct_n(U* dst, U src, I n) noexcept // pass by value for scalar types -{ - for(I i = 0; i < n; ++i) - { - dst[i] = src; - } -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_scalar::value, void>::type -copy_construct_n(U* dst, U const& src, I n) // pass by reference for non-scalar types -{ - C4_ASSERT(dst != &src); - for(I i = 0; i < n; ++i) - { - new ((void*)(dst + i)) U(src); - } -} - -template -C4_ALWAYS_INLINE void copy_construct(U (&dst)[N], U const (&src)[N]) noexcept -{ - copy_construct_n(dst, src, N); -} - -//----------------------------------------------------------------------------- -// copy-assign - -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -copy_assign(U* dst, U const* src) noexcept -{ - C4_ASSERT(dst != src); - memcpy(dst, src, sizeof(U)); -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_copy_assignable::value, void>::type -copy_assign(U* dst, U const* src) noexcept -{ - C4_ASSERT(dst != src); - *dst = *src; -} -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -copy_assign_n(U* dst, U const* src, I n) noexcept -{ - C4_ASSERT(dst != src); - memcpy(dst, src, n * sizeof(U)); -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_copy_assignable::value, void>::type -copy_assign_n(U* dst, U const* src, I n) noexcept -{ - C4_ASSERT(dst != src); - for(I i = 0; i < n; ++i) - { - dst[i] = src[i]; - } -} - -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -copy_assign(U* dst, U src) noexcept // pass by value for scalar types -{ - *dst = src; -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_scalar::value, void>::type -copy_assign(U* dst, U const& src) noexcept // pass by reference for non-scalar types -{ - C4_ASSERT(dst != &src); - *dst = src; -} -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -copy_assign_n(U* dst, U src, I n) noexcept // pass by value for scalar types -{ - for(I i = 0; i < n; ++i) - { - dst[i] = src; - } -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_scalar::value, void>::type -copy_assign_n(U* dst, U const& src, I n) noexcept // pass by reference for non-scalar types -{ - C4_ASSERT(dst != &src); - for(I i = 0; i < n; ++i) - { - dst[i] = src; - } -} - -template -C4_ALWAYS_INLINE void copy_assign(U (&dst)[N], U const (&src)[N]) noexcept -{ - copy_assign_n(dst, src, N); -} - -//----------------------------------------------------------------------------- -// move-construct - -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -move_construct(U* dst, U* src) noexcept -{ - C4_ASSERT(dst != src); - memcpy(dst, src, sizeof(U)); -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_move_constructible::value, void>::type -move_construct(U* dst, U* src) noexcept -{ - C4_ASSERT(dst != src); - new ((void*)dst) U(std::move(*src)); -} -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -move_construct_n(U* dst, U* src, I n) noexcept -{ - C4_ASSERT(dst != src); - memcpy(dst, src, n * sizeof(U)); -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_move_constructible::value, void>::type -move_construct_n(U* dst, U* src, I n) noexcept -{ - C4_ASSERT(dst != src); - for(I i = 0; i < n; ++i) - { - new ((void*)(dst + i)) U(std::move(src[i])); - } -} - -//----------------------------------------------------------------------------- -// move-assign - -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -move_assign(U* dst, U* src) noexcept -{ - C4_ASSERT(dst != src); - memcpy(dst, src, sizeof(U)); -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_move_assignable::value, void>::type -move_assign(U* dst, U* src) noexcept -{ - C4_ASSERT(dst != src); - *dst = std::move(*src); -} -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -move_assign_n(U* dst, U* src, I n) noexcept -{ - C4_ASSERT(dst != src); - memcpy(dst, src, n * sizeof(U)); -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_move_assignable::value, void>::type -move_assign_n(U* dst, U* src, I n) noexcept -{ - C4_ASSERT(dst != src); - for(I i = 0; i < n; ++i) - { - *(dst + i) = std::move(*(src + i)); - } -} - -//----------------------------------------------------------------------------- -// destroy - -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -destroy(U* ptr) noexcept -{ - C4_UNUSED(ptr); // nothing to do -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_destructible::value, void>::type -destroy(U* ptr) noexcept -{ - ptr->~U(); -} -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -destroy_n(U* ptr, I n) noexcept -{ - C4_UNUSED(ptr); - C4_UNUSED(n); // nothing to do -} -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_destructible::value, void>::type -destroy_n(U* ptr, I n) noexcept -{ - for(I i = 0; i C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -make_room(U *buf, I bufsz, I room) C4_NOEXCEPT_A -{ - C4_ASSERT(bufsz >= 0 && room >= 0); - if(room >= bufsz) - { - memcpy (buf + room, buf, bufsz * sizeof(U)); - } - else - { - memmove(buf + room, buf, bufsz * sizeof(U)); - } -} -/** makes room at the beginning of buf, which has a current size of bufsz */ -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_move_constructible::value, void>::type -make_room(U *buf, I bufsz, I room) C4_NOEXCEPT_A -{ - C4_ASSERT(bufsz >= 0 && room >= 0); - if(room >= bufsz) - { - for(I i = 0; i < bufsz; ++i) - { - new ((void*)(buf + (i + room))) U(std::move(buf[i])); - } - } - else - { - for(I i = 0; i < bufsz; ++i) - { - I w = bufsz-1 - i; // do a backwards loop - new ((void*)(buf + (w + room))) U(std::move(buf[w])); - } - } -} - -/** make room to the right of pos */ -template -C4_ALWAYS_INLINE void make_room(U *buf, I bufsz, I currsz, I pos, I room) -{ - C4_ASSERT(pos >= 0 && pos <= currsz); - C4_ASSERT(currsz <= bufsz); - C4_ASSERT(room + currsz <= bufsz); - C4_UNUSED(bufsz); - make_room(buf + pos, currsz - pos, room); -} - - -/** make room to the right of pos, copying to the beginning of a different buffer */ -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -make_room(U *dst, U const* src, I srcsz, I room, I pos) C4_NOEXCEPT_A -{ - C4_ASSERT(srcsz >= 0 && room >= 0 && pos >= 0); - C4_ASSERT(pos < srcsz || (pos == 0 && srcsz == 0)); - memcpy(dst , src , pos * sizeof(U)); - memcpy(dst + room + pos, src + pos, (srcsz - pos) * sizeof(U)); -} -/** make room to the right of pos, copying to the beginning of a different buffer */ -template C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_move_constructible::value, void>::type -make_room(U *dst, U const* src, I srcsz, I room, I pos) -{ - C4_ASSERT(srcsz >= 0 && room >= 0 && pos >= 0); - C4_ASSERT(pos < srcsz || (pos == 0 && srcsz == 0)); - for(I i = 0; i < pos; ++i) - { - new ((void*)(dst + i)) U(std::move(src[i])); - } - src += pos; - dst += room + pos; - for(I i = 0, e = srcsz - pos; i < e; ++i) - { - new ((void*)(dst + i)) U(std::move(src[i])); - } -} - -template -C4_ALWAYS_INLINE void make_room -( - U * dst, I dstsz, - U const* src, I srcsz, - I room, I pos -) -{ - C4_ASSERT(pos >= 0 && pos < srcsz || (srcsz == 0 && pos == 0)); - C4_ASSERT(pos >= 0 && pos < dstsz || (dstsz == 0 && pos == 0)); - C4_ASSERT(srcsz+room <= dstsz); - C4_UNUSED(dstsz); - make_room(dst, src, srcsz, room, pos); -} - - -//----------------------------------------------------------------------------- -/** destroy room at the beginning of buf, which has a current size of n */ -template C4_ALWAYS_INLINE typename std::enable_if::value || (std::is_standard_layout::value && std::is_trivial::value), void>::type -destroy_room(U *buf, I n, I room) C4_NOEXCEPT_A -{ - C4_ASSERT(n >= 0 && room >= 0); - C4_ASSERT(room <= n); - if(room < n) - { - memmove(buf, buf + room, (n - room) * sizeof(U)); - } - else - { - // nothing to do - no need to destroy scalar types - } -} -/** destroy room at the beginning of buf, which has a current size of n */ -template C4_ALWAYS_INLINE typename std::enable_if< ! (std::is_scalar::value || (std::is_standard_layout::value && std::is_trivial::value)), void>::type -destroy_room(U *buf, I n, I room) -{ - C4_ASSERT(n >= 0 && room >= 0); - C4_ASSERT(room <= n); - if(room < n) - { - for(I i = 0, e = n - room; i < e; ++i) - { - buf[i] = std::move(buf[i + room]); - } - } - else - { - for(I i = 0; i < n; ++i) - { - buf[i].~U(); - } - } -} - -/** destroy room to the right of pos, copying to a different buffer */ -template C4_ALWAYS_INLINE typename std::enable_if::value, void>::type -destroy_room(U *dst, U const* src, I n, I room, I pos) C4_NOEXCEPT_A -{ - C4_ASSERT(n >= 0 && room >= 0 && pos >= 0); - C4_ASSERT(pos C4_ALWAYS_INLINE typename std::enable_if< ! std::is_trivially_move_constructible::value, void>::type -destroy_room(U *dst, U const* src, I n, I room, I pos) -{ - C4_ASSERT(n >= 0 && room >= 0 && pos >= 0); - C4_ASSERT(pos < n); - C4_ASSERT(pos + room <= n); - for(I i = 0; i < pos; ++i) - { - new ((void*)(dst + i)) U(std::move(src[i])); - } - src += room + pos; - dst += pos; - for(I i = 0, e = n - pos - room; i < e; ++i) - { - new ((void*)(dst + i)) U(std::move(src[i])); - } -} - -} // namespace c4 - -#undef _C4REQUIRE - -#endif /* _C4_CTOR_DTOR_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/dump.hpp b/thirdparty/ryml/ext/c4core/src/c4/dump.hpp deleted file mode 100644 index 483acf982..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/dump.hpp +++ /dev/null @@ -1,579 +0,0 @@ -#ifndef C4_DUMP_HPP_ -#define C4_DUMP_HPP_ - -#include - -namespace c4 { - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** type of the function to dump characters */ -using DumperPfn = void (*)(csubstr buf); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -inline size_t dump(substr buf, Arg const& a) -{ - size_t sz = to_chars(buf, a); // need to serialize to the buffer - if(C4_LIKELY(sz <= buf.len)) - dumpfn(buf.first(sz)); - return sz; -} - -template -inline size_t dump(DumperFn &&dumpfn, substr buf, Arg const& a) -{ - size_t sz = to_chars(buf, a); // need to serialize to the buffer - if(C4_LIKELY(sz <= buf.len)) - dumpfn(buf.first(sz)); - return sz; -} - -template -inline size_t dump(substr buf, csubstr a) -{ - if(buf.len) - dumpfn(a); // dump directly, no need to serialize to the buffer - return 0; // no space was used in the buffer -} - -template -inline size_t dump(DumperFn &&dumpfn, substr buf, csubstr a) -{ - if(buf.len) - dumpfn(a); // dump directly, no need to serialize to the buffer - return 0; // no space was used in the buffer -} - -template -inline size_t dump(substr buf, const char (&a)[N]) -{ - if(buf.len) - dumpfn(csubstr(a)); // dump directly, no need to serialize to the buffer - return 0; // no space was used in the buffer -} - -template -inline size_t dump(DumperFn &&dumpfn, substr buf, const char (&a)[N]) -{ - if(buf.len) - dumpfn(csubstr(a)); // dump directly, no need to serialize to the buffer - return 0; // no space was used in the buffer -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** */ -struct DumpResults -{ - enum : size_t { noarg = (size_t)-1 }; - size_t bufsize = 0; - size_t lastok = noarg; - bool success_until(size_t expected) const { return lastok == noarg ? false : lastok >= expected; } - bool write_arg(size_t arg) const { return lastok == noarg || arg > lastok; } - size_t argfail() const { return lastok + 1; } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/// @cond dev -// terminates the variadic recursion -template -size_t cat_dump(DumperFn &&, substr) -{ - return 0; -} - -// terminates the variadic recursion -template -size_t cat_dump(substr) -{ - return 0; -} -/// @endcond - -/** take the function pointer as a function argument */ -template -size_t cat_dump(DumperFn &&dumpfn, substr buf, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - size_t size_for_a = dump(dumpfn, buf, a); - if(C4_UNLIKELY(size_for_a > buf.len)) - buf = buf.first(0); // ensure no more calls - size_t size_for_more = cat_dump(dumpfn, buf, more...); - return size_for_more > size_for_a ? size_for_more : size_for_a; -} - -/** take the function pointer as a template argument */ -template -size_t cat_dump(substr buf, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - size_t size_for_a = dump(buf, a); - if(C4_LIKELY(size_for_a > buf.len)) - buf = buf.first(0); // ensure no more calls - size_t size_for_more = cat_dump(buf, more...); - return size_for_more > size_for_a ? size_for_more : size_for_a; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/// @cond dev -namespace detail { - -// terminates the variadic recursion -template -DumpResults cat_dump_resume(size_t currarg, DumpResults results, substr buf, Arg const& C4_RESTRICT a) -{ - if(C4_LIKELY(results.write_arg(currarg))) - { - size_t sz = dump(buf, a); // yield to the specialized function - if(currarg == results.lastok + 1 && sz <= buf.len) - results.lastok = currarg; - results.bufsize = sz > results.bufsize ? sz : results.bufsize; - } - return results; -} - -// terminates the variadic recursion -template -DumpResults cat_dump_resume(size_t currarg, DumperFn &&dumpfn, DumpResults results, substr buf, Arg const& C4_RESTRICT a) -{ - if(C4_LIKELY(results.write_arg(currarg))) - { - size_t sz = dump(dumpfn, buf, a); // yield to the specialized function - if(currarg == results.lastok + 1 && sz <= buf.len) - results.lastok = currarg; - results.bufsize = sz > results.bufsize ? sz : results.bufsize; - } - return results; -} - -template -DumpResults cat_dump_resume(size_t currarg, DumpResults results, substr buf, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - results = detail::cat_dump_resume(currarg, results, buf, a); - return detail::cat_dump_resume(currarg + 1u, results, buf, more...); -} - -template -DumpResults cat_dump_resume(size_t currarg, DumperFn &&dumpfn, DumpResults results, substr buf, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - results = detail::cat_dump_resume(currarg, dumpfn, results, buf, a); - return detail::cat_dump_resume(currarg + 1u, dumpfn, results, buf, more...); -} -} // namespace detail -/// @endcond - - -template -C4_ALWAYS_INLINE DumpResults cat_dump_resume(DumpResults results, substr buf, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - if(results.bufsize > buf.len) - return results; - return detail::cat_dump_resume(0u, results, buf, a, more...); -} - -template -C4_ALWAYS_INLINE DumpResults cat_dump_resume(DumperFn &&dumpfn, DumpResults results, substr buf, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - if(results.bufsize > buf.len) - return results; - return detail::cat_dump_resume(0u, dumpfn, results, buf, a, more...); -} - -template -C4_ALWAYS_INLINE DumpResults cat_dump_resume(substr buf, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - return detail::cat_dump_resume(0u, DumpResults{}, buf, a, more...); -} - -template -C4_ALWAYS_INLINE DumpResults cat_dump_resume(DumperFn &&dumpfn, substr buf, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - return detail::cat_dump_resume(0u, dumpfn, DumpResults{}, buf, a, more...); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/// @cond dev -// terminate the recursion -template -size_t catsep_dump(DumperFn &&, substr, Sep const& C4_RESTRICT) -{ - return 0; -} - -// terminate the recursion -template -size_t catsep_dump(substr, Sep const& C4_RESTRICT) -{ - return 0; -} -/// @endcond - -/** take the function pointer as a function argument */ -template -size_t catsep_dump(DumperFn &&dumpfn, substr buf, Sep const& C4_RESTRICT sep, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - size_t sz = dump(dumpfn, buf, a); - if(C4_UNLIKELY(sz > buf.len)) - buf = buf.first(0); // ensure no more calls - if C4_IF_CONSTEXPR (sizeof...(more) > 0) - { - size_t szsep = dump(dumpfn, buf, sep); - if(C4_UNLIKELY(szsep > buf.len)) - buf = buf.first(0); // ensure no more calls - sz = sz > szsep ? sz : szsep; - } - size_t size_for_more = catsep_dump(dumpfn, buf, sep, more...); - return size_for_more > sz ? size_for_more : sz; -} - -/** take the function pointer as a template argument */ -template -size_t catsep_dump(substr buf, Sep const& C4_RESTRICT sep, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - size_t sz = dump(buf, a); - if(C4_UNLIKELY(sz > buf.len)) - buf = buf.first(0); // ensure no more calls - if C4_IF_CONSTEXPR (sizeof...(more) > 0) - { - size_t szsep = dump(buf, sep); - if(C4_UNLIKELY(szsep > buf.len)) - buf = buf.first(0); // ensure no more calls - sz = sz > szsep ? sz : szsep; - } - size_t size_for_more = catsep_dump(buf, sep, more...); - return size_for_more > sz ? size_for_more : sz; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/// @cond dev -namespace detail { -template -void catsep_dump_resume_(size_t currarg, DumpResults *C4_RESTRICT results, substr *C4_RESTRICT buf, Arg const& C4_RESTRICT a) -{ - if(C4_LIKELY(results->write_arg(currarg))) - { - size_t sz = dump(*buf, a); - results->bufsize = sz > results->bufsize ? sz : results->bufsize; - if(C4_LIKELY(sz <= buf->len)) - results->lastok = currarg; - else - buf->len = 0; - } -} - -template -void catsep_dump_resume_(size_t currarg, DumperFn &&dumpfn, DumpResults *C4_RESTRICT results, substr *C4_RESTRICT buf, Arg const& C4_RESTRICT a) -{ - if(C4_LIKELY(results->write_arg(currarg))) - { - size_t sz = dump(dumpfn, *buf, a); - results->bufsize = sz > results->bufsize ? sz : results->bufsize; - if(C4_LIKELY(sz <= buf->len)) - results->lastok = currarg; - else - buf->len = 0; - } -} - -template -C4_ALWAYS_INLINE void catsep_dump_resume(size_t currarg, DumpResults *C4_RESTRICT results, substr *C4_RESTRICT buf, Sep const& C4_RESTRICT, Arg const& C4_RESTRICT a) -{ - detail::catsep_dump_resume_(currarg, results, buf, a); -} - -template -C4_ALWAYS_INLINE void catsep_dump_resume(size_t currarg, DumperFn &&dumpfn, DumpResults *C4_RESTRICT results, substr *C4_RESTRICT buf, Sep const& C4_RESTRICT, Arg const& C4_RESTRICT a) -{ - detail::catsep_dump_resume_(currarg, dumpfn, results, buf, a); -} - -template -C4_ALWAYS_INLINE void catsep_dump_resume(size_t currarg, DumpResults *C4_RESTRICT results, substr *C4_RESTRICT buf, Sep const& C4_RESTRICT sep, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - detail::catsep_dump_resume_(currarg , results, buf, a); - detail::catsep_dump_resume_(currarg + 1u, results, buf, sep); - detail::catsep_dump_resume (currarg + 2u, results, buf, sep, more...); -} - -template -C4_ALWAYS_INLINE void catsep_dump_resume(size_t currarg, DumperFn &&dumpfn, DumpResults *C4_RESTRICT results, substr *C4_RESTRICT buf, Sep const& C4_RESTRICT sep, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - detail::catsep_dump_resume_(currarg , dumpfn, results, buf, a); - detail::catsep_dump_resume_(currarg + 1u, dumpfn, results, buf, sep); - detail::catsep_dump_resume (currarg + 2u, dumpfn, results, buf, sep, more...); -} -} // namespace detail -/// @endcond - - -template -C4_ALWAYS_INLINE DumpResults catsep_dump_resume(DumpResults results, substr buf, Sep const& C4_RESTRICT sep, Args const& C4_RESTRICT ...more) -{ - detail::catsep_dump_resume(0u, &results, &buf, sep, more...); - return results; -} - -template -C4_ALWAYS_INLINE DumpResults catsep_dump_resume(DumperFn &&dumpfn, DumpResults results, substr buf, Sep const& C4_RESTRICT sep, Args const& C4_RESTRICT ...more) -{ - detail::catsep_dump_resume(0u, dumpfn, &results, &buf, sep, more...); - return results; -} - -template -C4_ALWAYS_INLINE DumpResults catsep_dump_resume(substr buf, Sep const& C4_RESTRICT sep, Args const& C4_RESTRICT ...more) -{ - DumpResults results; - detail::catsep_dump_resume(0u, &results, &buf, sep, more...); - return results; -} - -template -C4_ALWAYS_INLINE DumpResults catsep_dump_resume(DumperFn &&dumpfn, substr buf, Sep const& C4_RESTRICT sep, Args const& C4_RESTRICT ...more) -{ - DumpResults results; - detail::catsep_dump_resume(0u, dumpfn, &results, &buf, sep, more...); - return results; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** take the function pointer as a function argument */ -template -C4_ALWAYS_INLINE size_t format_dump(DumperFn &&dumpfn, substr buf, csubstr fmt) -{ - // we can dump without using buf - // but we'll only dump if the buffer is ok - if(C4_LIKELY(buf.len > 0 && fmt.len)) - dumpfn(fmt); - return 0u; -} - -/** take the function pointer as a function argument */ -template -C4_ALWAYS_INLINE size_t format_dump(substr buf, csubstr fmt) -{ - // we can dump without using buf - // but we'll only dump if the buffer is ok - if(C4_LIKELY(buf.len > 0 && fmt.len > 0)) - dumpfn(fmt); - return 0u; -} - -/** take the function pointer as a function argument */ -template -size_t format_dump(DumperFn &&dumpfn, substr buf, csubstr fmt, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - // we can dump without using buf - // but we'll only dump if the buffer is ok - size_t pos = fmt.find("{}"); // @todo use _find_fmt() - if(C4_UNLIKELY(pos == csubstr::npos)) - { - if(C4_LIKELY(buf.len > 0 && fmt.len > 0)) - dumpfn(fmt); - return 0u; - } - if(C4_LIKELY(buf.len > 0 && pos > 0)) - dumpfn(fmt.first(pos)); // we can dump without using buf - fmt = fmt.sub(pos + 2); // skip {} do this before assigning to pos again - pos = dump(dumpfn, buf, a); - if(C4_UNLIKELY(pos > buf.len)) - buf.len = 0; // ensure no more calls to dump - size_t size_for_more = format_dump(dumpfn, buf, fmt, more...); - return size_for_more > pos ? size_for_more : pos; -} - -/** take the function pointer as a template argument */ -template -size_t format_dump(substr buf, csubstr fmt, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - // we can dump without using buf - // but we'll only dump if the buffer is ok - size_t pos = fmt.find("{}"); // @todo use _find_fmt() - if(C4_UNLIKELY(pos == csubstr::npos)) - { - if(C4_LIKELY(buf.len > 0 && fmt.len > 0)) - dumpfn(fmt); - return 0u; - } - if(C4_LIKELY(buf.len > 0 && pos > 0)) - dumpfn(fmt.first(pos)); // we can dump without using buf - fmt = fmt.sub(pos + 2); // skip {} do this before assigning to pos again - pos = dump(buf, a); - if(C4_UNLIKELY(pos > buf.len)) - buf.len = 0; // ensure no more calls to dump - size_t size_for_more = format_dump(buf, fmt, more...); - return size_for_more > pos ? size_for_more : pos; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/// @cond dev -namespace detail { - -template -DumpResults format_dump_resume(size_t currarg, DumpResults results, substr buf, csubstr fmt) -{ - // we can dump without using buf - // but we'll only dump if the buffer is ok - if(C4_LIKELY(buf.len > 0)) - { - dumpfn(fmt); - results.lastok = currarg; - } - return results; -} - -template -DumpResults format_dump_resume(size_t currarg, DumperFn &&dumpfn, DumpResults results, substr buf, csubstr fmt) -{ - // we can dump without using buf - // but we'll only dump if the buffer is ok - if(C4_LIKELY(buf.len > 0)) - { - dumpfn(fmt); - results.lastok = currarg; - } - return results; -} - -template -DumpResults format_dump_resume(size_t currarg, DumpResults results, substr buf, csubstr fmt, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - // we need to process the format even if we're not - // going to print the first arguments because we're resuming - size_t pos = fmt.find("{}"); // @todo use _find_fmt() - // we can dump without using buf - // but we'll only dump if the buffer is ok - if(C4_LIKELY(results.write_arg(currarg))) - { - if(C4_UNLIKELY(pos == csubstr::npos)) - { - if(C4_LIKELY(buf.len > 0)) - { - results.lastok = currarg; - dumpfn(fmt); - } - return results; - } - if(C4_LIKELY(buf.len > 0)) - { - results.lastok = currarg; - dumpfn(fmt.first(pos)); - } - } - fmt = fmt.sub(pos + 2); - if(C4_LIKELY(results.write_arg(currarg + 1))) - { - pos = dump(buf, a); - results.bufsize = pos > results.bufsize ? pos : results.bufsize; - if(C4_LIKELY(pos <= buf.len)) - results.lastok = currarg + 1; - else - buf.len = 0; - } - return detail::format_dump_resume(currarg + 2u, results, buf, fmt, more...); -} -/// @endcond - - -template -DumpResults format_dump_resume(size_t currarg, DumperFn &&dumpfn, DumpResults results, substr buf, csubstr fmt, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - // we need to process the format even if we're not - // going to print the first arguments because we're resuming - size_t pos = fmt.find("{}"); // @todo use _find_fmt() - // we can dump without using buf - // but we'll only dump if the buffer is ok - if(C4_LIKELY(results.write_arg(currarg))) - { - if(C4_UNLIKELY(pos == csubstr::npos)) - { - if(C4_LIKELY(buf.len > 0)) - { - results.lastok = currarg; - dumpfn(fmt); - } - return results; - } - if(C4_LIKELY(buf.len > 0)) - { - results.lastok = currarg; - dumpfn(fmt.first(pos)); - } - } - fmt = fmt.sub(pos + 2); - if(C4_LIKELY(results.write_arg(currarg + 1))) - { - pos = dump(dumpfn, buf, a); - results.bufsize = pos > results.bufsize ? pos : results.bufsize; - if(C4_LIKELY(pos <= buf.len)) - results.lastok = currarg + 1; - else - buf.len = 0; - } - return detail::format_dump_resume(currarg + 2u, dumpfn, results, buf, fmt, more...); -} -} // namespace detail - - -template -C4_ALWAYS_INLINE DumpResults format_dump_resume(DumpResults results, substr buf, csubstr fmt, Args const& C4_RESTRICT ...more) -{ - return detail::format_dump_resume(0u, results, buf, fmt, more...); -} - -template -C4_ALWAYS_INLINE DumpResults format_dump_resume(DumperFn &&dumpfn, DumpResults results, substr buf, csubstr fmt, Args const& C4_RESTRICT ...more) -{ - return detail::format_dump_resume(0u, dumpfn, results, buf, fmt, more...); -} - - -template -C4_ALWAYS_INLINE DumpResults format_dump_resume(substr buf, csubstr fmt, Args const& C4_RESTRICT ...more) -{ - return detail::format_dump_resume(0u, DumpResults{}, buf, fmt, more...); -} - -template -C4_ALWAYS_INLINE DumpResults format_dump_resume(DumperFn &&dumpfn, substr buf, csubstr fmt, Args const& C4_RESTRICT ...more) -{ - return detail::format_dump_resume(0u, dumpfn, DumpResults{}, buf, fmt, more...); -} - - -} // namespace c4 - - -#endif /* C4_DUMP_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/enum.hpp b/thirdparty/ryml/ext/c4core/src/c4/enum.hpp deleted file mode 100644 index 785cf5b28..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/enum.hpp +++ /dev/null @@ -1,276 +0,0 @@ -#ifndef _C4_ENUM_HPP_ -#define _C4_ENUM_HPP_ - -#include "c4/error.hpp" -#include - -/** @file enum.hpp utilities for enums: convert to/from string - */ - - -namespace c4 { - -//! taken from http://stackoverflow.com/questions/15586163/c11-type-trait-to-differentiate-between-enum-class-and-regular-enum -template -using is_scoped_enum = std::integral_constant::value && !std::is_convertible::value>; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -typedef enum { - EOFFS_NONE = 0, ///< no offset - EOFFS_CLS = 1, ///< get the enum offset for the class name. @see eoffs_cls() - EOFFS_PFX = 2, ///< get the enum offset for the enum prefix. @see eoffs_pfx() - _EOFFS_LAST ///< reserved -} EnumOffsetType; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** A simple (proxy) container for the value-name pairs of an enum type. - * Uses linear search for finds; this could be improved for time-critical - * code. */ -template -class EnumSymbols -{ -public: - - struct Sym - { - Enum value; - const char *name; - - bool cmp(const char *s) const; - bool cmp(const char *s, size_t len) const; - - const char *name_offs(EnumOffsetType t) const; - }; - - using const_iterator = Sym const*; - -public: - - template - EnumSymbols(Sym const (&p)[N]) : m_symbols(p), m_num(N) {} - - size_t size() const { return m_num; } - bool empty() const { return m_num == 0; } - - Sym const* get(Enum v) const { auto p = find(v); C4_CHECK_MSG(p != nullptr, "could not find symbol=%zd", (std::ptrdiff_t)v); return p; } - Sym const* get(const char *s) const { auto p = find(s); C4_CHECK_MSG(p != nullptr, "could not find symbol \"%s\"", s); return p; } - Sym const* get(const char *s, size_t len) const { auto p = find(s, len); C4_CHECK_MSG(p != nullptr, "could not find symbol \"%.*s\"", len, s); return p; } - - Sym const* find(Enum v) const; - Sym const* find(const char *s) const; - Sym const* find(const char *s, size_t len) const; - - Sym const& operator[] (size_t i) const { C4_CHECK(i < m_num); return m_symbols[i]; } - - Sym const* begin() const { return m_symbols; } - Sym const* end () const { return m_symbols + m_num; } - -private: - - Sym const* m_symbols; - size_t const m_num; - -}; - -//----------------------------------------------------------------------------- -/** return an EnumSymbols object for the enum type T - * - * @warning SPECIALIZE! This needs to be specialized for each enum - * type. Failure to provide a specialization will cause a linker - * error. */ -template -EnumSymbols const esyms(); - - -/** return the offset for an enum symbol class. For example, - * eoffs_cls() would be 13=strlen("MyEnumClass::"). - * - * With this function you can announce that the full prefix (including - * an eventual enclosing class or C++11 enum class) is of a certain - * length. - * - * @warning Needs to be specialized for each enum class type that - * wants to use this. When no specialization is given, will return - * 0. */ -template -size_t eoffs_cls() -{ - return 0; -} - - -/** return the offset for an enum symbol prefix. This includes - * eoffs_cls(). With this function you can announce that the full - * prefix (including an eventual enclosing class or C++11 enum class - * plus the string prefix) is of a certain length. - * - * @warning Needs to be specialized for each enum class type that - * wants to use this. When no specialization is given, will return - * 0. */ -template -size_t eoffs_pfx() -{ - return 0; -} - - -template -size_t eoffs(EnumOffsetType which) -{ - switch(which) - { - case EOFFS_NONE: - return 0; - case EOFFS_CLS: - return eoffs_cls(); - case EOFFS_PFX: - { - size_t pfx = eoffs_pfx(); - return pfx > 0 ? pfx : eoffs_cls(); - } - default: - C4_ERROR("unknown offset type %d", (int)which); - return 0; - } -} - - -//----------------------------------------------------------------------------- -/** get the enum value corresponding to a c-string */ - -#ifdef __clang__ -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# if __GNUC__ >= 6 -# pragma GCC diagnostic ignored "-Wnull-dereference" -# endif -#endif - -template -Enum str2e(const char* str) -{ - auto pairs = esyms(); - auto *p = pairs.get(str); - C4_CHECK_MSG(p != nullptr, "no valid enum pair name for '%s'", str); - return p->value; -} - -/** get the c-string corresponding to an enum value */ -template -const char* e2str(Enum e) -{ - auto es = esyms(); - auto *p = es.get(e); - C4_CHECK_MSG(p != nullptr, "no valid enum pair name"); - return p->name; -} - -/** like e2str(), but add an offset. */ -template -const char* e2stroffs(Enum e, EnumOffsetType ot=EOFFS_PFX) -{ - const char *s = e2str(e) + eoffs(ot); - return s; -} - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -//----------------------------------------------------------------------------- -/** Find a symbol by value. Returns nullptr when none is found */ -template -typename EnumSymbols::Sym const* EnumSymbols::find(Enum v) const -{ - for(Sym const* p = this->m_symbols, *e = p+this->m_num; p < e; ++p) - if(p->value == v) - return p; - return nullptr; -} - -/** Find a symbol by name. Returns nullptr when none is found */ -template -typename EnumSymbols::Sym const* EnumSymbols::find(const char *s) const -{ - for(Sym const* p = this->m_symbols, *e = p+this->m_num; p < e; ++p) - if(p->cmp(s)) - return p; - return nullptr; -} - -/** Find a symbol by name. Returns nullptr when none is found */ -template -typename EnumSymbols::Sym const* EnumSymbols::find(const char *s, size_t len) const -{ - for(Sym const* p = this->m_symbols, *e = p+this->m_num; p < e; ++p) - if(p->cmp(s, len)) - return p; - return nullptr; -} - -//----------------------------------------------------------------------------- -template -bool EnumSymbols::Sym::cmp(const char *s) const -{ - if(strcmp(name, s) == 0) - return true; - - for(int i = 1; i < _EOFFS_LAST; ++i) - { - auto o = eoffs((EnumOffsetType)i); - if(o > 0) - if(strcmp(name + o, s) == 0) - return true; - } - - return false; -} - -template -bool EnumSymbols::Sym::cmp(const char *s, size_t len) const -{ - if(strncmp(name, s, len) == 0) - return true; - - size_t nlen = 0; - for(int i = 1; i <_EOFFS_LAST; ++i) - { - auto o = eoffs((EnumOffsetType)i); - if(o > 0) - { - if(!nlen) - { - nlen = strlen(name); - } - C4_ASSERT(o < nlen); - size_t rem = nlen - o; - auto m = len > rem ? len : rem; - if(len >= m && strncmp(name + o, s, m) == 0) - return true; - } - } - - return false; -} - -//----------------------------------------------------------------------------- -template -const char* EnumSymbols::Sym::name_offs(EnumOffsetType t) const -{ - C4_ASSERT(eoffs(t) < strlen(name)); - return name + eoffs(t); -} - -} // namespace c4 - -#endif // _C4_ENUM_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/error.cpp b/thirdparty/ryml/ext/c4core/src/c4/error.cpp deleted file mode 100644 index ec2d2f81e..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/error.cpp +++ /dev/null @@ -1,227 +0,0 @@ -#include "c4/error.hpp" - -#include -#include -#include - -#define C4_LOGF_ERR(...) fprintf(stderr, __VA_ARGS__); fflush(stderr) -#define C4_LOGF_WARN(...) fprintf(stderr, __VA_ARGS__); fflush(stderr) -#define C4_LOGP(msg, ...) printf(msg) - -#if defined(C4_XBOX) || (defined(C4_WIN) && defined(C4_MSVC)) -# include "c4/windows.hpp" -#elif defined(C4_PS4) -# include -#elif defined(C4_UNIX) || defined(C4_LINUX) -# include -# include -# include -#elif defined(C4_MACOS) || defined(C4_IOS) -# include -# include -# include -# include -#endif -// the amalgamation tool is dumb and was omitting this include under MACOS. -// So do it only once: -#if defined(C4_UNIX) || defined(C4_LINUX) || defined(C4_MACOS) || defined(C4_IOS) -# include -#endif - -#if defined(C4_EXCEPTIONS_ENABLED) && defined(C4_ERROR_THROWS_EXCEPTION) -# include -#endif - -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wformat-nonliteral" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wformat-nonliteral" -#endif - - -//----------------------------------------------------------------------------- -namespace c4 { - -static error_flags s_error_flags = ON_ERROR_DEFAULTS; -static error_callback_type s_error_callback = nullptr; - -//----------------------------------------------------------------------------- - -error_flags get_error_flags() -{ - return s_error_flags; -} -void set_error_flags(error_flags flags) -{ - s_error_flags = flags; -} - -error_callback_type get_error_callback() -{ - return s_error_callback; -} -/** Set the function which is called when an error occurs. */ -void set_error_callback(error_callback_type cb) -{ - s_error_callback = cb; -} - -//----------------------------------------------------------------------------- - -void handle_error(srcloc where, const char *fmt, ...) -{ - char buf[1024]; - size_t msglen = 0; - if(s_error_flags & (ON_ERROR_LOG|ON_ERROR_CALLBACK)) - { - va_list args; - va_start(args, fmt); - int ilen = vsnprintf(buf, sizeof(buf), fmt, args); // ss.vprintf(fmt, args); - va_end(args); - msglen = ilen >= 0 && ilen < (int)sizeof(buf) ? static_cast(ilen) : sizeof(buf)-1; - } - - if(s_error_flags & ON_ERROR_LOG) - { - C4_LOGF_ERR("\n"); -#if defined(C4_ERROR_SHOWS_FILELINE) && defined(C4_ERROR_SHOWS_FUNC) - C4_LOGF_ERR("%s:%d: ERROR: %s\n", where.file, where.line, buf); - C4_LOGF_ERR("%s:%d: ERROR here: %s\n", where.file, where.line, where.func); -#elif defined(C4_ERROR_SHOWS_FILELINE) - C4_LOGF_ERR("%s:%d: ERROR: %s\n", where.file, where.line, buf); -#elif ! defined(C4_ERROR_SHOWS_FUNC) - C4_LOGF_ERR("ERROR: %s\n", buf); -#endif - } - - if(s_error_flags & ON_ERROR_CALLBACK) - { - if(s_error_callback) - { - s_error_callback(buf, msglen/*ss.c_strp(), ss.tellp()*/); - } - } - - if(s_error_flags & ON_ERROR_ABORT) - { - abort(); - } - - if(s_error_flags & ON_ERROR_THROW) - { -#if defined(C4_EXCEPTIONS_ENABLED) && defined(C4_ERROR_THROWS_EXCEPTION) - throw Exception(buf); -#else - abort(); -#endif - } -} - -//----------------------------------------------------------------------------- - -void handle_warning(srcloc where, const char *fmt, ...) -{ - va_list args; - char buf[1024]; //sstream ss; - va_start(args, fmt); - vsnprintf(buf, sizeof(buf), fmt, args); - va_end(args); - C4_LOGF_WARN("\n"); -#if defined(C4_ERROR_SHOWS_FILELINE) && defined(C4_ERROR_SHOWS_FUNC) - C4_LOGF_WARN("%s:%d: WARNING: %s\n", where.file, where.line, buf/*ss.c_strp()*/); - C4_LOGF_WARN("%s:%d: WARNING: here: %s\n", where.file, where.line, where.func); -#elif defined(C4_ERROR_SHOWS_FILELINE) - C4_LOGF_WARN("%s:%d: WARNING: %s\n", where.file, where.line, buf/*ss.c_strp()*/); -#elif ! defined(C4_ERROR_SHOWS_FUNC) - C4_LOGF_WARN("WARNING: %s\n", buf/*ss.c_strp()*/); -#endif - //c4::log.flush(); -} - -//----------------------------------------------------------------------------- -bool is_debugger_attached() -{ -#if defined(C4_UNIX) || defined(C4_LINUX) - static bool first_call = true; - static bool first_call_result = false; - if(first_call) - { - first_call = false; - //! @see http://stackoverflow.com/questions/3596781/how-to-detect-if-the-current-process-is-being-run-by-gdb - //! (this answer: http://stackoverflow.com/a/24969863/3968589 ) - char buf[1024] = ""; - - int status_fd = open("/proc/self/status", O_RDONLY); - if (status_fd == -1) - { - return 0; - } - - ssize_t num_read = ::read(status_fd, buf, sizeof(buf)); - - if (num_read > 0) - { - static const char TracerPid[] = "TracerPid:"; - char *tracer_pid; - - if(num_read < 1024) - { - buf[num_read] = 0; - } - tracer_pid = strstr(buf, TracerPid); - if (tracer_pid) - { - first_call_result = !!::atoi(tracer_pid + sizeof(TracerPid) - 1); - } - } - } - return first_call_result; -#elif defined(C4_PS4) - return (sceDbgIsDebuggerAttached() != 0); -#elif defined(C4_XBOX) || (defined(C4_WIN) && defined(C4_MSVC)) - return IsDebuggerPresent() != 0; -#elif defined(C4_MACOS) || defined(C4_IOS) - // https://stackoverflow.com/questions/2200277/detecting-debugger-on-mac-os-x - // Returns true if the current process is being debugged (either - // running under the debugger or has a debugger attached post facto). - int junk; - int mib[4]; - struct kinfo_proc info; - size_t size; - - // Initialize the flags so that, if sysctl fails for some bizarre - // reason, we get a predictable result. - - info.kp_proc.p_flag = 0; - - // Initialize mib, which tells sysctl the info we want, in this case - // we're looking for information about a specific process ID. - - mib[0] = CTL_KERN; - mib[1] = KERN_PROC; - mib[2] = KERN_PROC_PID; - mib[3] = getpid(); - - // Call sysctl. - - size = sizeof(info); - junk = sysctl(mib, sizeof(mib) / sizeof(*mib), &info, &size, NULL, 0); - assert(junk == 0); - - // We're being debugged if the P_TRACED flag is set. - return ((info.kp_proc.p_flag & P_TRACED) != 0); -#else - return false; -#endif -} // is_debugger_attached() - -} // namespace c4 - - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/error.hpp b/thirdparty/ryml/ext/c4core/src/c4/error.hpp deleted file mode 100644 index 26e457bfa..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/error.hpp +++ /dev/null @@ -1,432 +0,0 @@ -#ifndef _C4_ERROR_HPP_ -#define _C4_ERROR_HPP_ - -/** @file error.hpp Facilities for error reporting and runtime assertions. */ - -/** @defgroup error_checking Error checking */ - -#include "c4/config.hpp" - -#ifdef _DOXYGEN_ - /** if this is defined and exceptions are enabled, then calls to C4_ERROR() - * will throw an exception - * @ingroup error_checking */ -# define C4_EXCEPTIONS_ENABLED - /** if this is defined and exceptions are enabled, then calls to C4_ERROR() - * will throw an exception - * @see C4_EXCEPTIONS_ENABLED - * @ingroup error_checking */ -# define C4_ERROR_THROWS_EXCEPTION - /** evaluates to noexcept when C4_ERROR might be called and - * exceptions are disabled. Otherwise, defaults to nothing. - * @ingroup error_checking */ -# define C4_NOEXCEPT -#endif // _DOXYGEN_ - -#if defined(C4_EXCEPTIONS_ENABLED) && defined(C4_ERROR_THROWS_EXCEPTION) -# define C4_NOEXCEPT -#else -# define C4_NOEXCEPT noexcept -#endif - - -namespace c4 { -namespace detail { -struct fail_type__ {}; -} // detail -} // c4 -#define C4_STATIC_ERROR(dummy_type, errmsg) \ - static_assert(std::is_same::value, errmsg) - - -//----------------------------------------------------------------------------- - -#define C4_ASSERT_SAME_TYPE(ty1, ty2) \ - C4_STATIC_ASSERT(std::is_same::value) - -#define C4_ASSERT_DIFF_TYPE(ty1, ty2) \ - C4_STATIC_ASSERT( ! std::is_same::value) - - -//----------------------------------------------------------------------------- - -#ifdef _DOXYGEN_ -/** utility macro that triggers a breakpoint when - * the debugger is attached and NDEBUG is not defined. - * @ingroup error_checking */ -# define C4_DEBUG_BREAK() -#endif // _DOXYGEN_ - - -#if defined(NDEBUG) || defined(C4_NO_DEBUG_BREAK) -# define C4_DEBUG_BREAK() -#else -# ifdef __clang__ -# pragma clang diagnostic push -# if !defined(__APPLE_CC__) -# if __clang_major__ >= 10 -# pragma clang diagnostic ignored "-Wgnu-inline-cpp-without-extern" // debugbreak/debugbreak.h:50:16: error: 'gnu_inline' attribute without 'extern' in C++ treated as externally available, this changed in Clang 10 [-Werror,-Wgnu-inline-cpp-without-extern] -# endif -# else -# if __clang_major__ >= 13 -# pragma clang diagnostic ignored "-Wgnu-inline-cpp-without-extern" // debugbreak/debugbreak.h:50:16: error: 'gnu_inline' attribute without 'extern' in C++ treated as externally available, this changed in Clang 10 [-Werror,-Wgnu-inline-cpp-without-extern] -# endif -# endif -# elif defined(__GNUC__) -# endif -# include -# define C4_DEBUG_BREAK() if(c4::is_debugger_attached()) { ::debug_break(); } -# ifdef __clang__ -# pragma clang diagnostic pop -# elif defined(__GNUC__) -# endif -#endif - -namespace c4 { -C4CORE_EXPORT bool is_debugger_attached(); -} // namespace c4 - - -//----------------------------------------------------------------------------- - -#ifdef __clang__ - /* NOTE: using , ## __VA_ARGS__ to deal with zero-args calls to - * variadic macros is not portable, but works in clang, gcc, msvc, icc. - * clang requires switching off compiler warnings for pedantic mode. - * @see http://stackoverflow.com/questions/32047685/variadic-macro-without-arguments */ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" // warning: token pasting of ',' and __VA_ARGS__ is a GNU extension -#elif defined(__GNUC__) - /* GCC also issues a warning for zero-args calls to variadic macros. - * This warning is switched on with -pedantic and apparently there is no - * easy way to turn it off as with clang. But marking this as a system - * header works. - * @see https://gcc.gnu.org/onlinedocs/cpp/System-Headers.html - * @see http://stackoverflow.com/questions/35587137/ */ -# pragma GCC system_header -#endif - - -//----------------------------------------------------------------------------- - -namespace c4 { - -typedef enum : uint32_t { - /** when an error happens and the debugger is attached, call C4_DEBUG_BREAK(). - * Without effect otherwise. */ - ON_ERROR_DEBUGBREAK = 0x01 << 0, - /** when an error happens log a message. */ - ON_ERROR_LOG = 0x01 << 1, - /** when an error happens invoke a callback if it was set with - * set_error_callback(). */ - ON_ERROR_CALLBACK = 0x01 << 2, - /** when an error happens call std::terminate(). */ - ON_ERROR_ABORT = 0x01 << 3, - /** when an error happens and exceptions are enabled throw an exception. - * Without effect otherwise. */ - ON_ERROR_THROW = 0x01 << 4, - /** the default flags. */ - ON_ERROR_DEFAULTS = ON_ERROR_DEBUGBREAK|ON_ERROR_LOG|ON_ERROR_CALLBACK|ON_ERROR_ABORT -} ErrorFlags_e; -using error_flags = uint32_t; -C4CORE_EXPORT void set_error_flags(error_flags f); -C4CORE_EXPORT error_flags get_error_flags(); - - -using error_callback_type = void (*)(const char* msg, size_t msg_size); -C4CORE_EXPORT void set_error_callback(error_callback_type cb); -C4CORE_EXPORT error_callback_type get_error_callback(); - - -//----------------------------------------------------------------------------- -/** RAII class controling the error settings inside a scope. */ -struct ScopedErrorSettings -{ - error_flags m_flags; - error_callback_type m_callback; - - explicit ScopedErrorSettings(error_callback_type cb) - : m_flags(get_error_flags()), - m_callback(get_error_callback()) - { - set_error_callback(cb); - } - explicit ScopedErrorSettings(error_flags flags) - : m_flags(get_error_flags()), - m_callback(get_error_callback()) - { - set_error_flags(flags); - } - explicit ScopedErrorSettings(error_flags flags, error_callback_type cb) - : m_flags(get_error_flags()), - m_callback(get_error_callback()) - { - set_error_flags(flags); - set_error_callback(cb); - } - ~ScopedErrorSettings() - { - set_error_flags(m_flags); - set_error_callback(m_callback); - } -}; - - -//----------------------------------------------------------------------------- - -/** source location */ -struct srcloc; - -C4CORE_EXPORT void handle_error(srcloc s, const char *fmt, ...); -C4CORE_EXPORT void handle_warning(srcloc s, const char *fmt, ...); - - -# define C4_ERROR(msg, ...) \ - do { \ - if(c4::get_error_flags() & c4::ON_ERROR_DEBUGBREAK) \ - { \ - C4_DEBUG_BREAK() \ - } \ - c4::handle_error(C4_SRCLOC(), msg, ## __VA_ARGS__); \ - } while(0) - - -# define C4_WARNING(msg, ...) \ - c4::handle_warning(C4_SRCLOC(), msg, ## __VA_ARGS__) - - -#if defined(C4_ERROR_SHOWS_FILELINE) && defined(C4_ERROR_SHOWS_FUNC) - -struct srcloc -{ - const char *file = ""; - const char *func = ""; - int line = 0; -}; -#define C4_SRCLOC() c4::srcloc{__FILE__, C4_PRETTY_FUNC, __LINE__} - -#elif defined(C4_ERROR_SHOWS_FILELINE) - -struct srcloc -{ - const char *file; - int line; -}; -#define C4_SRCLOC() c4::srcloc{__FILE__, __LINE__} - -#elif ! defined(C4_ERROR_SHOWS_FUNC) - -struct srcloc -{ -}; -#define C4_SRCLOC() c4::srcloc() - -#else -# error not implemented -#endif - - -//----------------------------------------------------------------------------- -// assertions - -// Doxygen needs this so that only one definition counts -#ifdef _DOXYGEN_ - /** Explicitly enables assertions, independently of NDEBUG status. - * This is meant to allow enabling assertions even when NDEBUG is defined. - * Defaults to undefined. - * @ingroup error_checking */ -# define C4_USE_ASSERT - /** assert that a condition is true; this is turned off when NDEBUG - * is defined and C4_USE_ASSERT is not true. - * @ingroup error_checking */ -# define C4_ASSERT - /** same as C4_ASSERT(), additionally prints a printf-formatted message - * @ingroup error_checking */ -# define C4_ASSERT_MSG - /** evaluates to C4_NOEXCEPT when C4_XASSERT is disabled; otherwise, defaults - * to noexcept - * @ingroup error_checking */ -# define C4_NOEXCEPT_A -#endif // _DOXYGEN_ - -#ifndef C4_USE_ASSERT -# ifdef NDEBUG -# define C4_USE_ASSERT 0 -# else -# define C4_USE_ASSERT 1 -# endif -#endif - -#if C4_USE_ASSERT -# define C4_ASSERT(cond) C4_CHECK(cond) -# define C4_ASSERT_MSG(cond, /*fmt, */...) C4_CHECK_MSG(cond, ## __VA_ARGS__) -# define C4_ASSERT_IF(predicate, cond) if(predicate) { C4_ASSERT(cond); } -# define C4_NOEXCEPT_A C4_NOEXCEPT -#else -# define C4_ASSERT(cond) -# define C4_ASSERT_MSG(cond, /*fmt, */...) -# define C4_ASSERT_IF(predicate, cond) -# define C4_NOEXCEPT_A noexcept -#endif - - -//----------------------------------------------------------------------------- -// extreme assertions - -// Doxygen needs this so that only one definition counts -#ifdef _DOXYGEN_ - /** Explicitly enables extreme assertions; this is meant to allow enabling - * assertions even when NDEBUG is defined. Defaults to undefined. - * @ingroup error_checking */ -# define C4_USE_XASSERT - /** extreme assertion: can be switched off independently of - * the regular assertion; use for example for bounds checking in hot code. - * Turned on only when C4_USE_XASSERT is defined - * @ingroup error_checking */ -# define C4_XASSERT - /** same as C4_XASSERT(), and additionally prints a printf-formatted message - * @ingroup error_checking */ -# define C4_XASSERT_MSG - /** evaluates to C4_NOEXCEPT when C4_XASSERT is disabled; otherwise, defaults to noexcept - * @ingroup error_checking */ -# define C4_NOEXCEPT_X -#endif // _DOXYGEN_ - -#ifndef C4_USE_XASSERT -# define C4_USE_XASSERT C4_USE_ASSERT -#endif - -#if C4_USE_XASSERT -# define C4_XASSERT(cond) C4_CHECK(cond) -# define C4_XASSERT_MSG(cond, /*fmt, */...) C4_CHECK_MSG(cond, ## __VA_ARGS__) -# define C4_XASSERT_IF(predicate, cond) if(predicate) { C4_XASSERT(cond); } -# define C4_NOEXCEPT_X C4_NOEXCEPT -#else -# define C4_XASSERT(cond) -# define C4_XASSERT_MSG(cond, /*fmt, */...) -# define C4_XASSERT_IF(predicate, cond) -# define C4_NOEXCEPT_X noexcept -#endif - - -//----------------------------------------------------------------------------- -// checks: never switched-off - -/** Check that a condition is true, or raise an error when not - * true. Unlike C4_ASSERT(), this check is not disabled in non-debug - * builds. - * @see C4_ASSERT - * @ingroup error_checking - * - * @todo add constexpr-compatible compile-time assert: - * https://akrzemi1.wordpress.com/2017/05/18/asserts-in-constexpr-functions/ - */ -#define C4_CHECK(cond) \ - do { \ - if(C4_UNLIKELY(!(cond))) \ - { \ - C4_ERROR("check failed: %s", #cond); \ - } \ - } while(0) - - -/** like C4_CHECK(), and additionally log a printf-style message. - * @see C4_CHECK - * @ingroup error_checking */ -#define C4_CHECK_MSG(cond, fmt, ...) \ - do { \ - if(C4_UNLIKELY(!(cond))) \ - { \ - C4_ERROR("check failed: " #cond "\n" fmt, ## __VA_ARGS__); \ - } \ - } while(0) - - -//----------------------------------------------------------------------------- -// Common error conditions - -#define C4_NOT_IMPLEMENTED() C4_ERROR("NOT IMPLEMENTED") -#define C4_NOT_IMPLEMENTED_MSG(/*msg, */...) C4_ERROR("NOT IMPLEMENTED: " ## __VA_ARGS__) -#define C4_NOT_IMPLEMENTED_IF(condition) do { if(C4_UNLIKELY(condition)) { C4_ERROR("NOT IMPLEMENTED"); } } while(0) -#define C4_NOT_IMPLEMENTED_IF_MSG(condition, /*msg, */...) do { if(C4_UNLIKELY(condition)) { C4_ERROR("NOT IMPLEMENTED: " ## __VA_ARGS__); } } while(0) - -#define C4_NEVER_REACH() do { C4_ERROR("never reach this point"); C4_UNREACHABLE(); } while(0) -#define C4_NEVER_REACH_MSG(/*msg, */...) do { C4_ERROR("never reach this point: " ## __VA_ARGS__); C4_UNREACHABLE(); } while(0) - - - -//----------------------------------------------------------------------------- -// helpers for warning suppression -// idea adapted from https://github.com/onqtam/doctest/ - - -#ifdef C4_MSVC -#define C4_SUPPRESS_WARNING_MSVC_PUSH __pragma(warning(push)) -#define C4_SUPPRESS_WARNING_MSVC(w) __pragma(warning(disable : w)) -#define C4_SUPPRESS_WARNING_MSVC_POP __pragma(warning(pop)) -#define C4_SUPPRESS_WARNING_MSVC_WITH_PUSH(w) \ - C4_SUPPRESS_WARNING_MSVC_PUSH \ - C4_SUPPRESS_WARNING_MSVC(w) -#else // C4_MSVC -#define C4_SUPPRESS_WARNING_MSVC_PUSH -#define C4_SUPPRESS_WARNING_MSVC(w) -#define C4_SUPPRESS_WARNING_MSVC_POP -#define C4_SUPPRESS_WARNING_MSVC_WITH_PUSH(w) -#endif // C4_MSVC - - -#ifdef C4_CLANG -#define C4_PRAGMA_TO_STR(x) _Pragma(#x) -#define C4_SUPPRESS_WARNING_CLANG_PUSH _Pragma("clang diagnostic push") -#define C4_SUPPRESS_WARNING_CLANG(w) C4_PRAGMA_TO_STR(clang diagnostic ignored w) -#define C4_SUPPRESS_WARNING_CLANG_POP _Pragma("clang diagnostic pop") -#define C4_SUPPRESS_WARNING_CLANG_WITH_PUSH(w) \ - C4_SUPPRESS_WARNING_CLANG_PUSH \ - C4_SUPPRESS_WARNING_CLANG(w) -#else // C4_CLANG -#define C4_SUPPRESS_WARNING_CLANG_PUSH -#define C4_SUPPRESS_WARNING_CLANG(w) -#define C4_SUPPRESS_WARNING_CLANG_POP -#define C4_SUPPRESS_WARNING_CLANG_WITH_PUSH(w) -#endif // C4_CLANG - - -#ifdef C4_GCC -#define C4_PRAGMA_TO_STR(x) _Pragma(#x) -#define C4_SUPPRESS_WARNING_GCC_PUSH _Pragma("GCC diagnostic push") -#define C4_SUPPRESS_WARNING_GCC(w) C4_PRAGMA_TO_STR(GCC diagnostic ignored w) -#define C4_SUPPRESS_WARNING_GCC_POP _Pragma("GCC diagnostic pop") -#define C4_SUPPRESS_WARNING_GCC_WITH_PUSH(w) \ - C4_SUPPRESS_WARNING_GCC_PUSH \ - C4_SUPPRESS_WARNING_GCC(w) -#else // C4_GCC -#define C4_SUPPRESS_WARNING_GCC_PUSH -#define C4_SUPPRESS_WARNING_GCC(w) -#define C4_SUPPRESS_WARNING_GCC_POP -#define C4_SUPPRESS_WARNING_GCC_WITH_PUSH(w) -#endif // C4_GCC - - -#define C4_SUPPRESS_WARNING_GCC_CLANG_PUSH \ - C4_SUPPRESS_WARNING_GCC_PUSH \ - C4_SUPPRESS_WARNING_CLANG_PUSH - -#define C4_SUPPRESS_WARNING_GCC_CLANG(w) \ - C4_SUPPRESS_WARNING_GCC(w) \ - C4_SUPPRESS_WARNING_CLANG(w) - -#define C4_SUPPRESS_WARNING_GCC_CLANG_WITH_PUSH(w) \ - C4_SUPPRESS_WARNING_GCC_WITH_PUSH(w) \ - C4_SUPPRESS_WARNING_CLANG_WITH_PUSH(w) - -#define C4_SUPPRESS_WARNING_GCC_CLANG_POP \ - C4_SUPPRESS_WARNING_GCC_POP \ - C4_SUPPRESS_WARNING_CLANG_POP - -} // namespace c4 - -#ifdef __clang__ -# pragma clang diagnostic pop -#endif - -#endif /* _C4_ERROR_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/export.hpp b/thirdparty/ryml/ext/c4core/src/c4/export.hpp deleted file mode 100644 index ffd02482f..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/export.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef C4_EXPORT_HPP_ -#define C4_EXPORT_HPP_ - -#ifdef _WIN32 - #ifdef C4CORE_SHARED - #ifdef C4CORE_EXPORTS - #define C4CORE_EXPORT __declspec(dllexport) - #else - #define C4CORE_EXPORT __declspec(dllimport) - #endif - #else - #define C4CORE_EXPORT - #endif -#else - #define C4CORE_EXPORT -#endif - -#endif /* C4CORE_EXPORT_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/.gitignore b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/.gitignore deleted file mode 100644 index e44cb5a54..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -*.o -fib -core -core.* -cscope.out -tags -.gdb_history -test/trap -test/break -test/break-c++ diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/COPYING b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/COPYING deleted file mode 100644 index 9e51088a5..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/COPYING +++ /dev/null @@ -1,23 +0,0 @@ -Copyright (c) 2011-2016, Scott Tsai - -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/GNUmakefile b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/GNUmakefile deleted file mode 100644 index 17ab01f9b..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/GNUmakefile +++ /dev/null @@ -1,28 +0,0 @@ -CFLAGS := -Os -Wall -g -CXXFLAGS := $(CFLAGS) - -PROGRAMS := $(basename $(wildcard *.c test/*.c test/*.cc *.S)) - -.PHONY: all clean -all: $(PROGRAMS) -clean: - rm -f $(PROGRAMS) cscope.out tags - -%: %.S - $(CC) $(CFLAGS) -nostdlib $< -o $@ - -# Not using builtin rules due to debugbreak.h dependency -%: %.c - $(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ - -%: %.cc - $(CXX) $(CXXFLAGS) $(LDFLAGS) $< -o $@ - -test/%: CFLAGS +=-I. -test/%: CXXFLAGS +=-I. -$(PROGRAMS): debugbreak.h - -GDB ?= gdb -.PHONY: gdb -gdb: - $(GDB) -q -x debugbreak-gdb.py diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/HOW-TO-USE-DEBUGBREAK-GDB-PY.md b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/HOW-TO-USE-DEBUGBREAK-GDB-PY.md deleted file mode 100644 index d5295c50c..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/HOW-TO-USE-DEBUGBREAK-GDB-PY.md +++ /dev/null @@ -1,33 +0,0 @@ -# How to use `debugbreak-gdb.py` - -Just add `-x debugbreak-gdb.py` to your usual GDB invocation or add `source debugbreak-gdb.py` to your `$HOME/.gdbinit`. - -Here's a sample session: - -``` -$ cd debugbreak -$ make -$ gdb -q -x debugbreak-gdb.py test/break -Reading symbols from test/break...done. - -(gdb) set disassemble-next-line 1 -(gdb) run -Starting program: /home/fedora/debugbreak/test/break - -Program received signal SIGTRAP, Trace/breakpoint trap. -main () at test/break.c:6 -6 debug_break(); - -Program received signal SIGTRAP, Trace/breakpoint trap. -main () at test/break.c:6 -6 debug_break(); -(gdb) debugbreak-step -7 printf("hello world\n"); -(gdb) debugbreak-continue -hello world -[Inferior 1 (process 12533) exited normally] -(gdb) - -``` - -On ARM and POWER, trying to use `step` or `stepi` in place of `debugbreak-step` in the sesion above wouldn't have worked as execution would be stock on the breakpoint instruction. diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/README.md b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/README.md deleted file mode 100644 index 965092c08..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# Debug Break - -[debugbreak.h](https://github.com/scottt/debugbreak/blob/master/debugbreak.h) allows you to put breakpoints in your C/C++ code with a call to **debug_break()**: -```C -#include -#include "debugbreak.h" - -int main() -{ - debug_break(); /* will break into debugger */ - printf("hello world\n"); - return 0; -} -``` -* Include one header file and insert calls to `debug_break()` in the code where you wish to break into the debugger. -* Supports GCC, Clang and MSVC. -* Works well on ARM, AArch64, i686, x86-64, POWER and has a fallback code path for other architectures. -* Works like the **DebugBreak()** fuction provided by [Windows](http://msdn.microsoft.com/en-us/library/ea9yy3ey.aspx) and [QNX](http://www.qnx.com/developers/docs/6.3.0SP3/neutrino/lib_ref/d/debugbreak.html). - -**License**: the very permissive [2-Clause BSD](https://github.com/scottt/debugbreak/blob/master/COPYING). - -Known Problem: if continuing execution after a debugbreak breakpoint hit doesn't work (e.g. on ARM or POWER), see [HOW-TO-USE-DEBUGBREAK-GDB-PY.md](HOW-TO-USE-DEBUGBREAK-GDB-PY.md) for a workaround. - -Implementation Notes -================================ - -The requirements for the **debug_break()** function are: -* Act as a compiler code motion barrier -* Don't cause the compiler optimizers to think the code following it can be removed -* Trigger a software breakpoint hit when executed (e.g. **SIGTRAP** on Linux) -* GDB commands like **continue**, **next**, **step**, **stepi** must work after a **debug_break()** hit - -Ideally, both GCC and Clang would provide a **__builtin_debugtrap()** that satisfies the above on all architectures and operating systems. Unfortunately, that is not the case (yet). -GCC's [__builtin_trap()](http://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html#index-g_t_005f_005fbuiltin_005ftrap-3278) causes the optimizers to think the code follwing can be removed ([test/trap.c](https://github.com/scottt/debugbreak/blob/master/test/trap.c)): -```C -#include - -int main() -{ - __builtin_trap(); - printf("hello world\n"); - return 0; -} -``` -compiles to: -``` -main -0x0000000000400390 <+0>: 0f 0b ud2 -``` -Notice how the call to `printf()` is not present in the assembly output. - -Further, on i386 / x86-64 **__builtin_trap()** generates an **ud2** instruction which triggers **SIGILL** instead of **SIGTRAP**. This makes it necessary to change GDB's default behavior on **SIGILL** to not terminate the process being debugged: -``` -(gdb) handle SIGILL stop nopass -``` -Even after this, continuing execution in GDB doesn't work well on some GCC, GDB combinations. See [GCC Bugzilla 84595](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84595). - -On ARM, **__builtin_trap()** generates a call to **abort()**, making it even less suitable. - -**debug_break()** generates an **int3** instruction on i386 / x86-64 ([test/break.c](https://github.com/scottt/debugbreak/blob/master/test/break.c)): -```C -#include -#include "debugbreak.h" - -int main() -{ - debug_break(); - printf("hello world\n"); - return 0; -} -``` -compiles to: -``` -main -0x00000000004003d0 <+0>: 50 push %rax -0x00000000004003d1 <+1>: cc int3 -0x00000000004003d2 <+2>: bf a0 05 40 00 mov $0x4005a0,%edi -0x00000000004003d7 <+7>: e8 d4 ff ff ff callq 0x4003b0 -0x00000000004003dc <+12>: 31 c0 xor %eax,%eax -0x00000000004003de <+14>: 5a pop %rdx -0x00000000004003df <+15>: c3 retq -``` -which correctly trigges **SIGTRAP** and single-stepping in GDB after a **debug_break()** hit works well. - -Clang / LLVM also has a **__builtin_trap()** that generates **ud2** but further provides **__builtin_debugtrap()** that generates **int3** on i386 / x86-64 ([original LLVM intrinsic](http://lists.llvm.org/pipermail/llvm-commits/Week-of-Mon-20120507/142621.html), [further fixes](https://reviews.llvm.org/rL166300#96cef7d3), [Clang builtin support](https://reviews.llvm.org/rL166298)). - -On ARM, **debug_break()** generates **.inst 0xe7f001f0** in ARM mode and **.inst 0xde01** in Thumb mode which correctly triggers **SIGTRAP** on Linux. Unfortunately, stepping in GDB after a **debug_break()** hit doesn't work and requires a workaround like: -``` -(gdb) set $l = 2 -(gdb) tbreak *($pc + $l) -(gdb) jump *($pc + $l) -(gdb) # Change $l from 2 to 4 for ARM mode -``` -to jump over the instruction. -A new GDB command, **debugbreak-step**, is defined in [debugbreak-gdb.py](https://github.com/scottt/debugbreak/blob/master/debugbreak-gdb.py) to automate the above. See [HOW-TO-USE-DEBUGBREAK-GDB-PY.md](HOW-TO-USE-DEBUGBREAK-GDB-PY.md) for sample usage. -``` -$ arm-none-linux-gnueabi-gdb -x debugbreak-gdb.py test/break-c++ -<...> -(gdb) run -Program received signal SIGTRAP, Trace/breakpoint trap. -main () at test/break-c++.cc:6 -6 debug_break(); - -(gdb) debugbreak-step - -7 std::cout << "hello, world\n"; -``` - -On AArch64, **debug_break()** generates **.inst 0xd4200000**. - -See table below for the behavior of **debug_break()** on other architecturs. - -Behavior on Different Architectures ----------------- - -| Architecture | debug_break() | -| ------------- | ------------- | -| x86/x86-64 | `int3` | -| ARM mode, 32-bit | `.inst 0xe7f001f0` | -| Thumb mode, 32-bit | `.inst 0xde01` | -| AArch64, ARMv8 | `.inst 0xd4200000` | -| POWER | `.4byte 0x7d821008` | -| RISC-V | `.4byte 0x00100073` | -| MSVC compiler | `__debugbreak` | -| Apple compiler on AArch64 | `__builtin_trap()` | -| Otherwise | `raise(SIGTRAP)` | - diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/debugbreak-gdb.py b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/debugbreak-gdb.py deleted file mode 100644 index 925f61a44..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/debugbreak-gdb.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (c) 2013, Scott Tsai -# -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - - -# Usage: gdb -x debugbreak-gdb.py -# (gdb) debugbreak-step -# (gdb) debugbreak-continue -# -# To debug: -# (gdb) set python print-stack full - -import gdb -import re - -def _gdb_show_version_parse(version_str): - ''' - >>> s0 = 'This GDB was configured as "x86_64-redhat-linux-gnu".' - >>> s1 = 'This GDB was configured as "--host=i686-build_pc-linux-gnu --target=arm-linux-gnueabihf".' - >>> s2 = 'This GDB was configured as "x86_64-unknown-linux-gnu".' - >>> _gdb_show_version_parse(s0) == dict(target='x86_64-redhat-linux-gnu') - True - >>> _gdb_show_version_parse(s1) == dict(host='i686-build_pc-linux-gnu', target='arm-linux-gnueabihf') - True - >>> _gdb_show_version_parse(s2) == dict(target='x86_64-unknown-linux-gnu') - True - ''' - - t = version_str - msg = 'This GDB was configured as "' - s = t.find(msg) - if s == -1: - raise ValueError - s += len(msg) - e = t.find('".', s) - if e == -1: - raise ValueError - - config = t[s:e] - d = {} - for i in config.split(): - i = i.strip() - if i.startswith('--'): - (k, v) = i[2:].split('=') - d[k] = v - else: - if not i: - continue - d['target'] = i - return d - -def _target_triplet(): - ''' - -> 'arm-linux-gnueabihf' or 'x86_64-redhat-linux-gnu' or ... - - >>> import re - >>> not not re.match(r'\w*-\w*-\w*', target_triplet()) - True - ''' - t = gdb.execute('show version', to_string=True) - return _gdb_show_version_parse(t)['target'] - -temp_breakpoint_num = None - -def on_stop_event(e): - global temp_breakpoint_num - if not isinstance(e, gdb.BreakpointEvent): - return - for bp in e.breakpoints: - if bp.number == temp_breakpoint_num: - bp.delete() - gdb.events.stop.disconnect(on_stop_event) - l = gdb.find_pc_line(int(gdb.parse_and_eval('$pc'))).line - gdb.execute('list %d, %d' % (l, l)) - break - -def _next_instn_jump_len(gdb_frame): - '-> None means don\'t jump' - try: - arch_name = gdb_frame.architecture().name() - except AttributeError: - arch_name = None - - if arch_name.startswith('powerpc:'): - # 'powerpc:common64' on ppc64 big endian - i = bytes(gdb.selected_inferior().read_memory(gdb.parse_and_eval('$pc'), 4)) - if (i == b'\x7d\x82\x10\x08') or (i == b'\x08\x10\x82\x7d'): - return 4 - else: # not stopped on a breakpoint instruction - return None - - triplet = _target_triplet() - if re.match(r'^arm-', triplet): - i = bytes(gdb.selected_inferior().read_memory(gdb.parse_and_eval('$pc'), 4)) - if i == b'\xf0\x01\xf0\xe7': - return 4 - elif i.startswith(b'\x01\xde'): - return 2 - elif i == b'\xf0\xf7\x00\xa0 ': - # 'arm_linux_thumb2_le_breakpoint' from arm-linux-tdep.c in GDB - return 4 - else: # not stopped on a breakpoint instruction - return None - return None - -def _debugbreak_step(): - global temp_breakpoint_num - try: - frame = gdb.selected_frame() - except gdb.error as e: - # 'No frame is currently selected.' - gdb.write(e.args[0] + '\n', gdb.STDERR) - return - instn_len = _next_instn_jump_len(frame) - - if instn_len is None: - gdb.execute('stepi') - else: - loc = '*($pc + %d)' % (instn_len,) - bp = gdb.Breakpoint(loc, gdb.BP_BREAKPOINT, internal=True) - bp.silent = True - temp_breakpoint_num = bp.number - gdb.events.stop.connect(on_stop_event) - gdb.execute('jump ' + loc) - -def _debugbreak_continue(): - try: - frame = gdb.selected_frame() - except gdb.error as e: - # 'No frame is currently selected.' - gdb.write(e.args[0] + '\n', gdb.STDERR) - return - instn_len = _next_instn_jump_len(frame) - - if instn_len is None: - gdb.execute('continue') - else: - loc = '*($pc + %d)' % (instn_len,) - gdb.execute('jump ' + loc) - -class _DebugBreakStep(gdb.Command): - '''Usage: debugbreak-step - Step one instruction after a debug_break() breakpoint hit''' - - def __init__(self): - gdb.Command.__init__(self, 'debugbreak-step', gdb.COMMAND_BREAKPOINTS, gdb.COMPLETE_NONE) - - def invoke(self, arg, from_tty): - _debugbreak_step() - -class _DebugBreakContinue(gdb.Command): - '''Usage: debugbreak-continue - Continue execution after a debug_break() breakpoint hit''' - - def __init__(self): - gdb.Command.__init__(self, 'debugbreak-continue', gdb.COMMAND_BREAKPOINTS, gdb.COMPLETE_NONE) - - def invoke(self, arg, from_tty): - _debugbreak_continue() - -_DebugBreakStep() -_DebugBreakContinue() diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/debugbreak.h b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/debugbreak.h deleted file mode 100644 index bfb828846..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/debugbreak.h +++ /dev/null @@ -1,174 +0,0 @@ -/* Copyright (c) 2011-2021, Scott Tsai - * - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ -#ifndef DEBUG_BREAK_H -#define DEBUG_BREAK_H - -#ifdef _MSC_VER - -#define debug_break __debugbreak - -#else - -#ifdef __cplusplus -extern "C" { -#endif - -#define DEBUG_BREAK_USE_TRAP_INSTRUCTION 1 -#define DEBUG_BREAK_USE_BULTIN_TRAP 2 -#define DEBUG_BREAK_USE_SIGTRAP 3 - -#if defined(__i386__) || defined(__x86_64__) - #define DEBUG_BREAK_IMPL DEBUG_BREAK_USE_TRAP_INSTRUCTION -__inline__ static void trap_instruction(void) -{ - __asm__ volatile("int $0x03"); -} -#elif defined(__thumb__) - #define DEBUG_BREAK_IMPL DEBUG_BREAK_USE_TRAP_INSTRUCTION -/* FIXME: handle __THUMB_INTERWORK__ */ -__attribute__((always_inline)) -__inline__ static void trap_instruction(void) -{ - /* See 'arm-linux-tdep.c' in GDB source. - * Both instruction sequences below work. */ -#if 1 - /* 'eabi_linux_thumb_le_breakpoint' */ - __asm__ volatile(".inst 0xde01"); -#else - /* 'eabi_linux_thumb2_le_breakpoint' */ - __asm__ volatile(".inst.w 0xf7f0a000"); -#endif - - /* Known problem: - * After a breakpoint hit, can't 'stepi', 'step', or 'continue' in GDB. - * 'step' would keep getting stuck on the same instruction. - * - * Workaround: use the new GDB commands 'debugbreak-step' and - * 'debugbreak-continue' that become available - * after you source the script from GDB: - * - * $ gdb -x debugbreak-gdb.py <... USUAL ARGUMENTS ...> - * - * 'debugbreak-step' would jump over the breakpoint instruction with - * roughly equivalent of: - * (gdb) set $instruction_len = 2 - * (gdb) tbreak *($pc + $instruction_len) - * (gdb) jump *($pc + $instruction_len) - */ -} -#elif defined(__arm__) && !defined(__thumb__) - #define DEBUG_BREAK_IMPL DEBUG_BREAK_USE_TRAP_INSTRUCTION -__attribute__((always_inline)) -__inline__ static void trap_instruction(void) -{ - /* See 'arm-linux-tdep.c' in GDB source, - * 'eabi_linux_arm_le_breakpoint' */ - __asm__ volatile(".inst 0xe7f001f0"); - /* Known problem: - * Same problem and workaround as Thumb mode */ -} -#elif defined(__aarch64__) && defined(__APPLE__) - #define DEBUG_BREAK_IMPL DEBUG_BREAK_USE_BULTIN_DEBUGTRAP -#elif defined(__aarch64__) - #define DEBUG_BREAK_IMPL DEBUG_BREAK_USE_TRAP_INSTRUCTION -__attribute__((always_inline)) -__inline__ static void trap_instruction(void) -{ - /* See 'aarch64-tdep.c' in GDB source, - * 'aarch64_default_breakpoint' */ - __asm__ volatile(".inst 0xd4200000"); -} -#elif defined(__powerpc__) - /* PPC 32 or 64-bit, big or little endian */ - #define DEBUG_BREAK_IMPL DEBUG_BREAK_USE_TRAP_INSTRUCTION -__attribute__((always_inline)) -__inline__ static void trap_instruction(void) -{ - /* See 'rs6000-tdep.c' in GDB source, - * 'rs6000_breakpoint' */ - __asm__ volatile(".4byte 0x7d821008"); - - /* Known problem: - * After a breakpoint hit, can't 'stepi', 'step', or 'continue' in GDB. - * 'step' stuck on the same instruction ("twge r2,r2"). - * - * The workaround is the same as ARM Thumb mode: use debugbreak-gdb.py - * or manually jump over the instruction. */ -} -#elif defined(__riscv) - /* RISC-V 32 or 64-bit, whether the "C" extension - * for compressed, 16-bit instructions are supported or not */ - #define DEBUG_BREAK_IMPL DEBUG_BREAK_USE_TRAP_INSTRUCTION -__attribute__((always_inline)) -__inline__ static void trap_instruction(void) -{ - /* See 'riscv-tdep.c' in GDB source, - * 'riscv_sw_breakpoint_from_kind' */ - __asm__ volatile(".4byte 0x00100073"); -} -#else - #define DEBUG_BREAK_IMPL DEBUG_BREAK_USE_SIGTRAP -#endif - - -#ifndef DEBUG_BREAK_IMPL -#error "debugbreak.h is not supported on this target" -#elif DEBUG_BREAK_IMPL == DEBUG_BREAK_USE_TRAP_INSTRUCTION -__attribute__((always_inline)) -__inline__ static void debug_break(void) -{ - trap_instruction(); -} -#elif DEBUG_BREAK_IMPL == DEBUG_BREAK_USE_BULTIN_DEBUGTRAP -__attribute__((always_inline)) -__inline__ static void debug_break(void) -{ - __builtin_debugtrap(); -} -#elif DEBUG_BREAK_IMPL == DEBUG_BREAK_USE_BULTIN_TRAP -__attribute__((always_inline)) -__inline__ static void debug_break(void) -{ - __builtin_trap(); -} -#elif DEBUG_BREAK_IMPL == DEBUG_BREAK_USE_SIGTRAP -#include -__attribute__((always_inline)) -__inline__ static void debug_break(void) -{ - raise(SIGTRAP); -} -#else -#error "invalid DEBUG_BREAK_IMPL value" -#endif - -#ifdef __cplusplus -} -#endif - -#endif /* ifdef _MSC_VER */ - -#endif /* ifndef DEBUG_BREAK_H */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break-c++.cc b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break-c++.cc deleted file mode 100644 index 0fcce848e..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break-c++.cc +++ /dev/null @@ -1,9 +0,0 @@ -#include -#include "debugbreak.h" - -int main() -{ - debug_break(); - std::cout << "hello, world\n"; - return 0; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break.c b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break.c deleted file mode 100644 index fde701e56..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break.c +++ /dev/null @@ -1,9 +0,0 @@ -#include -#include "debugbreak.h" - -int main() -{ - debug_break(); - printf("hello world\n"); - return 0; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break.gdb b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break.gdb deleted file mode 100644 index e8edab33a..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/break.gdb +++ /dev/null @@ -1,2 +0,0 @@ -file break -run diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/fib.c b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/fib.c deleted file mode 100644 index 0bd26bc22..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/fib.c +++ /dev/null @@ -1,21 +0,0 @@ -#include - -#include "debugbreak.h" - -int fib(int n) -{ - int r; - if (n == 0 || n == 1) - return 1; - r = fib(n-1) + fib(n-2); - if (r == 89) { - debug_break(); - } - return r; -} - -int main() -{ - printf("%d\n", fib(15)); - return 0; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/fib.gdb b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/fib.gdb deleted file mode 100644 index 2cd54a4cb..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/fib.gdb +++ /dev/null @@ -1,2 +0,0 @@ -file fib -run diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/test-debugbreak.gdb b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/test-debugbreak.gdb deleted file mode 100644 index 96e48484e..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/test-debugbreak.gdb +++ /dev/null @@ -1,6 +0,0 @@ -source ../debugbreak-gdb.py - -file break-c++ -# set remote exec-file break-c++ -run -#debugbreak-step diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/trap.c b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/trap.c deleted file mode 100644 index 9def30b68..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/trap.c +++ /dev/null @@ -1,8 +0,0 @@ -#include - -int main() -{ - __builtin_trap(); - printf("hello world\n"); - return 0; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/trap.gdb b/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/trap.gdb deleted file mode 100644 index 6cd59bd98..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/debugbreak/test/trap.gdb +++ /dev/null @@ -1,3 +0,0 @@ -file trap -handle SIGILL stop nopass -run diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float.hpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float.hpp deleted file mode 100644 index 9e75b5e14..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef _C4_EXT_FAST_FLOAT_HPP_ -#define _C4_EXT_FAST_FLOAT_HPP_ - -#ifdef _MSC_VER -# pragma warning(push) -# pragma warning(disable: 4996) // snprintf/scanf: this function or variable may be unsafe -#elif defined(__clang__) || defined(__APPLE_CC__) || defined(_LIBCPP_VERSION) -# pragma clang diagnostic push -# if (defined(__clang_major__) && _clang_major__ >= 9) || defined(__APPLE_CC__) -# pragma clang diagnostic ignored "-Wfortify-source" -# endif -# pragma clang diagnostic ignored "-Wshift-count-overflow" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wuseless-cast" -#endif - -#include "c4/ext/fast_float_all.h" - -#ifdef _MSC_VER -# pragma warning(pop) -#elif defined(__clang__) || defined(__APPLE_CC__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -#endif // _C4_EXT_FAST_FLOAT_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.cirrus.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.cirrus.yml deleted file mode 100644 index ad6c3458b..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.cirrus.yml +++ /dev/null @@ -1,22 +0,0 @@ - -task: - timeout_in: 120m - freebsd_instance: - matrix: - - image_family: freebsd-13-0-snap - - env: - ASSUME_ALWAYS_YES: YES - setup_script: - - pkg update -f - - pkg install bash - - pkg install cmake - - pkg install git - build_script: - - mkdir build - - cd build - - cmake -DFASTFLOAT_TEST=ON .. - - make - test_script: - - cd build - - ctest --output-on-failure -R basictest diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/alpine.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/alpine.yml deleted file mode 100644 index 825937673..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/alpine.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Alpine Linux -'on': - - push - - pull_request -jobs: - ubuntu-build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: start docker - run: | - docker run -w /src -dit --name alpine -v $PWD:/src alpine:latest - echo 'docker exec alpine "$@";' > ./alpine.sh - chmod +x ./alpine.sh - - name: install packages - run: | - ./alpine.sh apk update - ./alpine.sh apk add build-base cmake g++ linux-headers git bash - - name: cmake - run: | - ./alpine.sh cmake -DFASTFLOAT_TEST=ON -B build_for_alpine - - name: build - run: | - ./alpine.sh cmake --build build_for_alpine - - name: test - run: | - ./alpine.sh bash -c "cd build_for_alpine && ctest -R basictest" \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/amalgamate-ubuntu20.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/amalgamate-ubuntu20.yml deleted file mode 100644 index 3d4771431..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/amalgamate-ubuntu20.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Amalgamate Ubuntu 20.04 CI (GCC 9, 8) - -on: [push, pull_request] - -jobs: - ubuntu-build: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - include: - # Legacy/x86 compilers cause CI failures. - #- {cxx: -DCMAKE_CXX_COMPILER=g++-8, arch: } - - {cxx: , arch: } # default=gcc9 - #- {cxx: , arch: -DCMAKE_CXX_FLAGS="-m32"} # default=gcc9 - steps: - - uses: actions/checkout@v2 - - name: Compile with amalgamation - run: | - mkdir build && - mkdir build/fast_float && - python3 ./script/amalgamate.py > build/fast_float/fast_float.h && - cp tests/string_test.cpp build/ && - cd build && - g++ string_test.cpp diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/msys2-clang.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/msys2-clang.yml deleted file mode 100644 index ba16f4367..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/msys2-clang.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: MSYS2-CLANG-CI - -on: [push, pull_request] - -jobs: - windows-mingw: - name: ${{ matrix.msystem }} - runs-on: windows-latest - defaults: - run: - shell: msys2 {0} - strategy: - fail-fast: false - matrix: - include: - - msystem: "MINGW64" - install: mingw-w64-x86_64-libxml2 mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-clang - type: Release - - msystem: "MINGW32" - install: mingw-w64-i686-libxml2 mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-clang - type: Release - env: - CMAKE_GENERATOR: Ninja - - steps: - - uses: actions/checkout@v2 - - uses: msys2/setup-msys2@v2 - with: - update: true - msystem: ${{ matrix.msystem }} - install: ${{ matrix.install }} - - name: Prepare build dir - run: mkdir build - - name: Configure - run: cd build && cmake -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DFASTFLOAT_TEST=ON .. - - name: Build - run: cmake --build build - - name: Run basic tests - run: cd build && ctest --output-on-failure -R basictest diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/msys2.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/msys2.yml deleted file mode 100644 index 665c53916..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/msys2.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: MSYS2-CI - -on: [push, pull_request] - -jobs: - windows-mingw: - name: ${{ matrix.msystem }} - runs-on: windows-latest - defaults: - run: - shell: msys2 {0} - strategy: - fail-fast: false - matrix: - include: - - msystem: "MINGW64" - install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc - type: Release - - msystem: "MINGW32" - install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc - type: Release - - msystem: "MINGW64" - install: mingw-w64-x86_64-cmake mingw-w64-x86_64-ninja mingw-w64-x86_64-gcc - type: Debug - - msystem: "MINGW32" - install: mingw-w64-i686-cmake mingw-w64-i686-ninja mingw-w64-i686-gcc - type: Debug - env: - CMAKE_GENERATOR: Ninja - - steps: - - uses: actions/checkout@v2 - - uses: msys2/setup-msys2@v2 - with: - update: true - msystem: ${{ matrix.msystem }} - install: ${{ matrix.install }} - - name: Prepare build dir - run: mkdir build - - name: Configure - run: cd build && cmake -DCMAKE_BUILD_TYPE=${{ matrix.type }} -DFASTFLOAT_TEST=ON .. - - name: Build - run: cmake --build build - - name: Run basic tests - run: cd build && ctest --output-on-failure -R basictest diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu18.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu18.yml deleted file mode 100644 index dbdaa7adc..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu18.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Ubuntu 18.04 CI (GCC 7, 6, 5) - -on: [push, pull_request] - -jobs: - ubuntu-build: - runs-on: ubuntu-18.04 - strategy: - fail-fast: false - matrix: - include: - # Legacy/x86 compilers cause CI failures. - #- {cxx: -DCMAKE_CXX_COMPILER=g++-5, arch: } - #- {cxx: -DCMAKE_CXX_COMPILER=g++-6, arch: } - - {cxx: , arch: } # default=gcc7 - #- {cxx: , arch: -DCMAKE_CXX_FLAGS="-m32"} # default=gcc7 - steps: - - uses: actions/checkout@v2 - - name: Setup cmake - uses: jwlawson/actions-setup-cmake@v1.4 - with: - cmake-version: '3.11.x' - #- name: Install older compilers - # run: | - # sudo -E dpkg --add-architecture i386 - # sudo -E apt-get update - # sudo -E apt-get install -y --force-yes g++-5 g++-6 g++-5-multilib g++-6-multilib g++-multilib linux-libc-dev:i386 libc6:i386 libc6-dev:i386 libc6-dbg:i386 - - name: Prepare build dir - run: mkdir build - - name: Configure - run: cd build && cmake ${{matrix.cxx}} ${{matrix.arch}} -DFASTFLOAT_TEST=ON .. - - name: Build - run: cmake --build build - - name: Run basic tests - run: cd build && ctest --output-on-failure -R basictest diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu20-cxx20.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu20-cxx20.yml deleted file mode 100644 index a8b9acbc2..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu20-cxx20.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Ubuntu 20.04 CI (C++20) - -on: [push, pull_request] - -jobs: - ubuntu-build: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - steps: - - uses: actions/checkout@v2 - - name: Use cmake - run: | - mkdir build && - cd build && - cmake -DCMAKE_CXX_STANDARD=20 -DFASTFLOAT_TEST=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && - cmake --build . && - ctest --output-on-failure && - cmake --install . diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu20.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu20.yml deleted file mode 100644 index ebd8b23bc..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/ubuntu20.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Ubuntu 20.04 CI (GCC 9) - -on: [push, pull_request] - -jobs: - ubuntu-build: - runs-on: ubuntu-20.04 - strategy: - fail-fast: false - matrix: - include: - # Legacy/x86 compilers cause CI failures. - #- {cxx: -DCMAKE_CXX_COMPILER=g++-8, arch: } - - {cxx: , arch: } # default=gcc9 - #- {cxx: , arch: -DCMAKE_CXX_FLAGS="-m32"} # default=gcc9 - steps: - - uses: actions/checkout@v2 - - name: Use cmake - run: | - mkdir build && - cd build && - cmake ${{matrix.cxx}} ${{matrix.arch}} -DFASTFLOAT_TEST=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && - cmake --build . && - ctest --output-on-failure && - cmake --install . && - cd ../tests/installation_tests/find && - mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build . && - cd ../../issue72_installation && - mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build . diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs15-ci.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs15-ci.yml deleted file mode 100644 index a6207a7ad..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs15-ci.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: VS15-CI - -on: [push, pull_request] - -jobs: - ci: - if: >- - ! contains(toJSON(github.event.commits.*.message), '[skip ci]') && - ! contains(toJSON(github.event.commits.*.message), '[skip github]') - name: windows-vs15 - runs-on: windows-2016 - strategy: - fail-fast: false - matrix: - include: - - {gen: Visual Studio 15 2017, arch: Win32} - - {gen: Visual Studio 15 2017, arch: x64} - steps: - - uses: actions/checkout@v2 - - name: Configure - run: | - mkdir build - cd build && cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -DFASTFLOAT_TEST=ON .. - - name: Build - run: cmake --build build --verbose --config Release --parallel - - name: 'Run CTest' - run: | - cd build - ctest -C Release --output-on-failure -R basictest \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-arm-ci.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-arm-ci.yml deleted file mode 100644 index 7c69e38f1..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-arm-ci.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: VS16-ARM-CI - -on: [push, pull_request] - -jobs: - ci: - name: windows-vs16 - runs-on: windows-latest - strategy: - fail-fast: false - matrix: - include: - - {arch: ARM} - - {arch: ARM64} - steps: - - name: checkout - uses: actions/checkout@v2 - - name: Use cmake - run: | - cmake -A ${{ matrix.arch }} -DCMAKE_CROSSCOMPILING=1 -DFASTFLOAT_TEST=ON -B build && - cmake --build build --verbose \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-ci.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-ci.yml deleted file mode 100644 index fe40dddc9..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-ci.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: VS16-CI - -on: [push, pull_request] - -jobs: - ci: - name: windows-vs16 - runs-on: windows-latest - strategy: - fail-fast: false - matrix: - include: - - {gen: Visual Studio 16 2019, arch: Win32} - - {gen: Visual Studio 16 2019, arch: x64} - steps: - - name: checkout - uses: actions/checkout@v2 - - name: Use cmake - run: | - mkdir build && - cd build && - cmake ${{matrix.cxx}} ${{matrix.arch}} -DFASTFLOAT_TEST=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && - cmake --build . --verbose && - ctest --output-on-failure && - cmake --install . && - cd ../tests/installation_tests/find && - mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build . --verbose - cd ../../issue72_installation && - mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX:PATH=../../../build/destination .. && cmake --build . --verbose \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-clang-ci.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-clang-ci.yml deleted file mode 100644 index a32c6b822..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-clang-ci.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: VS16-CLANG-CI - -on: [push, pull_request] - -jobs: - ci: - name: windows-vs16 - runs-on: windows-latest - strategy: - fail-fast: false - matrix: - include: - - {gen: Visual Studio 16 2019, arch: Win32} - - {gen: Visual Studio 16 2019, arch: x64} - steps: - - name: checkout - uses: actions/checkout@v2 - - name: Configure - run: | - mkdir build - cd build && cmake -G "${{matrix.gen}}" -A ${{matrix.arch}} -T ClangCL -DFASTFLOAT_TEST=ON .. - - name: Build - run: cmake --build build --config Release --parallel - - name: Run basic tests - run: | - cd build - ctest -C Release --output-on-failure -R basictest diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-cxx20.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-cxx20.yml deleted file mode 100644 index 3a770935f..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.github/workflows/vs16-cxx20.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: VS16-CI C++20 - -on: [push, pull_request] - -jobs: - ci: - name: windows-vs16 - runs-on: windows-latest - strategy: - fail-fast: false - matrix: - include: - - {gen: Visual Studio 16 2019, arch: Win32} - - {gen: Visual Studio 16 2019, arch: x64} - steps: - - name: checkout - uses: actions/checkout@v2 - - name: Use cmake - run: | - mkdir build && - cd build && - cmake ${{matrix.cxx}} ${{matrix.arch}} -DCMAKE_CXX_STANDARD=20 -DFASTFLOAT_TEST=ON -DCMAKE_INSTALL_PREFIX:PATH=destination .. && - cmake --build . --verbose && - ctest --output-on-failure diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.gitignore b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.gitignore deleted file mode 100644 index 15665575e..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -build/* -Testing/* -.cache/ -compile_commands.json diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.travis.yml b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.travis.yml deleted file mode 100644 index ae854aed8..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/.travis.yml +++ /dev/null @@ -1,242 +0,0 @@ -language: cpp - -dist: bionic - -cache: - directories: - - $HOME/.dep_cache - -env: - global: - - fastfloat_DEPENDENCY_CACHE_DIR=$HOME/.dep_cache - -services: - - docker - -# the ppc64le and s390x images use cmake 3.10, but fast_float requires 3.11. -# so we compile cmake from source in those images. -# - tried the kitware ppa but that is using 3.10 as well -# - tried also using snap to get a more recent version but that failed with -# udev errors. - -matrix: - include: - - arch: ppc64le - os: linux - env: - - CMAKE_SRC="https://github.com/Kitware/CMake/releases/download/v3.11.4/cmake-3.11.4.tar.gz" - - - arch: s390x - os: linux - env: - - CMAKE_SRC="https://github.com/Kitware/CMake/releases/download/v3.11.4/cmake-3.11.4.tar.gz" - - - arch: amd64 - os: linux - - - arch: amd64 - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-8 - env: - - COMPILER="CC=gcc-8 && CXX=g++-8" - compiler: gcc-8 - - - arch: amd64 - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-9 - env: - - COMPILER="CC=gcc-9 && CXX=g++-9" - compiler: gcc-9 - - - arch: amd64 - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-10 - env: - - COMPILER="CC=gcc-10 && CXX=g++-10" - compiler: gcc-10 - - - arch: amd64 - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-10 - env: - - COMPILER="CC=gcc-10 && CXX=g++-10" - - SANITIZE="on" - compiler: gcc-10-sanitize - - - arch: amd64 - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-10 - env: - - COMPILER="CC=gcc-10 && CXX=g++-10" - - STATIC="on" - acompiler: gcc-10-static - - - arch: amd64 - os: linux - addons: - apt: - sources: - - llvm-toolchain-bionic-6.0 - packages: - - clang-6.0 - env: - - COMPILER="CC=clang-6.0 && CXX=clang++-6.0" - compiler: clang-6 - - - arch: amd64 - os: linux - addons: - apt: - sources: - - llvm-toolchain-bionic-7 - packages: - - clang-7 - env: - - COMPILER="CC=clang-7 && CXX=clang++-7" - compiler: clang-7 - - - arch: amd64 - os: linux - addons: - apt: - sources: - - llvm-toolchain-bionic-8 - packages: - - clang-8 - env: - - COMPILER="CC=clang-8 && CXX=clang++-8" - compiler: clang-8 - - - arch: amd64 - os: linux - addons: - apt: - sources: - - llvm-toolchain-bionic-9 - packages: - - clang-9 - env: - - COMPILER="CC=clang-9 && CXX=clang++-9" - compiler: clang-9 - - - arch: amd64 - os: linux - addons: - apt: - packages: - - clang-10 - sources: - - ubuntu-toolchain-r-test - - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' - key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - env: - - COMPILER="CC=clang-10 && CXX=clang++-10" - compiler: clang-10 - - - arch: amd64 - os: linux - addons: - apt: - packages: - - clang-10 - sources: - - ubuntu-toolchain-r-test - - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' - key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - env: - - COMPILER="CC=clang-10 && CXX=clang++-10" - - STATIC="on" - compiler: clang-10-static - - - arch: amd64 - os: linux - addons: - apt: - packages: - - clang-10 - sources: - - ubuntu-toolchain-r-test - - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' - key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' - env: - - COMPILER="CC=clang-10 && CXX=clang++-10" - - SANITIZE="on" - compiler: clang-10-sanitize - - - arch: amd64 - os: linux - env: - - TOOLCHAIN="mips64" - - - arch: amd64 - os: linux - env: - - TOOLCHAIN="riscv64" - -before_install: - - eval "${COMPILER}" - - | - if [ "$TOOLCHAIN" != "" ] ; then - docker pull ahuszagh/cross:"$TOOLCHAIN" - fi - -install: - - | - if [ "$CMAKE_SRC" != "" ] ; then - set -x - set -e - sudo -E apt remove --purge cmake - sudo -E apt-get update - sudo -E apt-get install -y build-essential libssl-dev - mkdir cmake_src - pushd cmake_src - wget "$CMAKE_SRC" - tar xfz $(basename "$CMAKE_SRC") - pushd $(basename "$CMAKE_SRC" | sed "s:.tar.gz::") - ./bootstrap - make -j2 - sudo make install - popd - popd - set +x - fi - - echo ${PATH} - - which cmake - - cmake --version - - which ${CC} - - ${CC} --version - - which ${CXX} - - ${CXX} --version - -script: - - | - if [ "$TOOLCHAIN" != "" ] ; then - docker run -v "$(pwd)":/ff ahuszagh/cross:"$TOOLCHAIN" /bin/bash -c "cd ff && ci/script.sh $TOOLCHAIN" - else - ci/script.sh - fi diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/AUTHORS b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/AUTHORS deleted file mode 100644 index 60c942582..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/AUTHORS +++ /dev/null @@ -1,2 +0,0 @@ -Daniel Lemire -João Paulo Magalhaes diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/CONTRIBUTORS b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/CONTRIBUTORS deleted file mode 100644 index 58a037cc5..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/CONTRIBUTORS +++ /dev/null @@ -1,6 +0,0 @@ -Eugene Golushkov -Maksim Kita -Marcin Wojdyr -Neal Richardson -Tim Paine -Fabio Pellacini diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/LICENSE-APACHE b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/LICENSE-APACHE deleted file mode 100644 index 79e59131d..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/LICENSE-APACHE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2020 The fast_float authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/LICENSE-MIT b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/LICENSE-MIT deleted file mode 100644 index 31aa79387..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/LICENSE-MIT +++ /dev/null @@ -1,23 +0,0 @@ -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/README.md b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/README.md deleted file mode 100644 index 059b9dade..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/README.md +++ /dev/null @@ -1,216 +0,0 @@ -## fast_float number parsing library: 4x faster than strtod - -![Ubuntu 20.04 CI (GCC 9)](https://github.com/lemire/fast_float/workflows/Ubuntu%2020.04%20CI%20(GCC%209)/badge.svg) -![Ubuntu 18.04 CI (GCC 7)](https://github.com/lemire/fast_float/workflows/Ubuntu%2018.04%20CI%20(GCC%207)/badge.svg) -![Alpine Linux](https://github.com/lemire/fast_float/workflows/Alpine%20Linux/badge.svg) -![MSYS2-CI](https://github.com/lemire/fast_float/workflows/MSYS2-CI/badge.svg) -![VS16-CLANG-CI](https://github.com/lemire/fast_float/workflows/VS16-CLANG-CI/badge.svg) -[![VS16-CI](https://github.com/fastfloat/fast_float/actions/workflows/vs16-ci.yml/badge.svg)](https://github.com/fastfloat/fast_float/actions/workflows/vs16-ci.yml) - -The fast_float library provides fast header-only implementations for the C++ from_chars -functions for `float` and `double` types. These functions convert ASCII strings representing -decimal values (e.g., `1.3e10`) into binary types. We provide exact rounding (including -round to even). In our experience, these `fast_float` functions many times faster than comparable number-parsing functions from existing C++ standard libraries. - -Specifically, `fast_float` provides the following two functions with a C++17-like syntax (the library itself only requires C++11): - -```C++ -from_chars_result from_chars(const char* first, const char* last, float& value, ...); -from_chars_result from_chars(const char* first, const char* last, double& value, ...); -``` - -The return type (`from_chars_result`) is defined as the struct: -```C++ -struct from_chars_result { - const char* ptr; - std::errc ec; -}; -``` - -It parses the character sequence [first,last) for a number. It parses floating-point numbers expecting -a locale-independent format equivalent to the C++17 from_chars function. -The resulting floating-point value is the closest floating-point values (using either float or double), -using the "round to even" convention for values that would otherwise fall right in-between two values. -That is, we provide exact parsing according to the IEEE standard. - - -Given a successful parse, the pointer (`ptr`) in the returned value is set to point right after the -parsed number, and the `value` referenced is set to the parsed value. In case of error, the returned -`ec` contains a representative error, otherwise the default (`std::errc()`) value is stored. - -The implementation does not throw and does not allocate memory (e.g., with `new` or `malloc`). - -It will parse infinity and nan values. - -Example: - -``` C++ -#include "fast_float/fast_float.h" -#include - -int main() { - const std::string input = "3.1416 xyz "; - double result; - auto answer = fast_float::from_chars(input.data(), input.data()+input.size(), result); - if(answer.ec != std::errc()) { std::cerr << "parsing failure\n"; return EXIT_FAILURE; } - std::cout << "parsed the number " << result << std::endl; - return EXIT_SUCCESS; -} -``` - - -Like the C++17 standard, the `fast_float::from_chars` functions take an optional last argument of -the type `fast_float::chars_format`. It is a bitset value: we check whether -`fmt & fast_float::chars_format::fixed` and `fmt & fast_float::chars_format::scientific` are set -to determine whether we allow the fixed point and scientific notation respectively. -The default is `fast_float::chars_format::general` which allows both `fixed` and `scientific`. - -The library seeks to follow the C++17 (see [20.19.3](http://eel.is/c++draft/charconv.from.chars).(7.1)) specification. -* The `from_chars` function does not skip leading white-space characters. -* [A leading `+` sign](https://en.cppreference.com/w/cpp/utility/from_chars) is forbidden. -* It is generally impossible to represent a decimal value exactly as binary floating-point number (`float` and `double` types). We seek the nearest value. We round to an even mantissa when we are in-between two binary floating-point numbers. - -Furthermore, we have the following restrictions: -* We only support `float` and `double` types at this time. -* We only support the decimal format: we do not support hexadecimal strings. -* For values that are either very large or very small (e.g., `1e9999`), we represent it using the infinity or negative infinity value. - -We support Visual Studio, macOS, Linux, freeBSD. We support big and little endian. We support 32-bit and 64-bit systems. - - - -## Using commas as decimal separator - - -The C++ standard stipulate that `from_chars` has to be locale-independent. In -particular, the decimal separator has to be the period (`.`). However, -some users still want to use the `fast_float` library with in a locale-dependent -manner. Using a separate function called `from_chars_advanced`, we allow the users -to pass a `parse_options` instance which contains a custom decimal separator (e.g., -the comma). You may use it as follows. - -```C++ -#include "fast_float/fast_float.h" -#include - -int main() { - const std::string input = "3,1416 xyz "; - double result; - fast_float::parse_options options{fast_float::chars_format::general, ','}; - auto answer = fast_float::from_chars_advanced(input.data(), input.data()+input.size(), result, options); - if((answer.ec != std::errc()) || ((result != 3.1416))) { std::cerr << "parsing failure\n"; return EXIT_FAILURE; } - std::cout << "parsed the number " << result << std::endl; - return EXIT_SUCCESS; -} -``` - - -## Reference - -- Daniel Lemire, [Number Parsing at a Gigabyte per Second](https://arxiv.org/abs/2101.11408), Software: Pratice and Experience 51 (8), 2021. - -## Other programming languages - -- [There is an R binding](https://github.com/eddelbuettel/rcppfastfloat) called `rcppfastfloat`. -- [There is a Rust port of the fast_float library](https://github.com/aldanor/fast-float-rust/) called `fast-float-rust`. -- [There is a Java port of the fast_float library](https://github.com/wrandelshofer/FastDoubleParser) called `FastDoubleParser`. -- [There is a C# port of the fast_float library](https://github.com/CarlVerret/csFastFloat) called `csFastFloat`. - - -## Relation With Other Work - -The fast_float library provides a performance similar to that of the [fast_double_parser](https://github.com/lemire/fast_double_parser) library but using an updated algorithm reworked from the ground up, and while offering an API more in line with the expectations of C++ programmers. The fast_double_parser library is part of the [Microsoft LightGBM machine-learning framework](https://github.com/microsoft/LightGBM). - -## Users - -The fast_float library is used by [Apache Arrow](https://github.com/apache/arrow/pull/8494) where it multiplied the number parsing speed by two or three times. It is also used by [Yandex ClickHouse](https://github.com/ClickHouse/ClickHouse) and by [Google Jsonnet](https://github.com/google/jsonnet). - - -## How fast is it? - -It can parse random floating-point numbers at a speed of 1 GB/s on some systems. We find that it is often twice as fast as the best available competitor, and many times faster than many standard-library implementations. - - - -``` -$ ./build/benchmarks/benchmark -# parsing random integers in the range [0,1) -volume = 2.09808 MB -netlib : 271.18 MB/s (+/- 1.2 %) 12.93 Mfloat/s -doubleconversion : 225.35 MB/s (+/- 1.2 %) 10.74 Mfloat/s -strtod : 190.94 MB/s (+/- 1.6 %) 9.10 Mfloat/s -abseil : 430.45 MB/s (+/- 2.2 %) 20.52 Mfloat/s -fastfloat : 1042.38 MB/s (+/- 9.9 %) 49.68 Mfloat/s -``` - -See https://github.com/lemire/simple_fastfloat_benchmark for our benchmarking code. - - -## Video - -[![Go Systems 2020](http://img.youtube.com/vi/AVXgvlMeIm4/0.jpg)](http://www.youtube.com/watch?v=AVXgvlMeIm4)
      - -## Using as a CMake dependency - -This library is header-only by design. The CMake file provides the `fast_float` target -which is merely a pointer to the `include` directory. - -If you drop the `fast_float` repository in your CMake project, you should be able to use -it in this manner: - -```cmake -add_subdirectory(fast_float) -target_link_libraries(myprogram PUBLIC fast_float) -``` - -Or you may want to retrieve the dependency automatically if you have a sufficiently recent version of CMake (3.11 or better at least): - -```cmake -FetchContent_Declare( - fast_float - GIT_REPOSITORY https://github.com/lemire/fast_float.git - GIT_TAG tags/v1.1.2 - GIT_SHALLOW TRUE) - -FetchContent_MakeAvailable(fast_float) -target_link_libraries(myprogram PUBLIC fast_float) - -``` - -You should change the `GIT_TAG` line so that you recover the version you wish to use. - -## Using as single header - -The script `script/amalgamate.py` may be used to generate a single header -version of the library if so desired. -Just run the script from the root directory of this repository. -You can customize the license type and output file if desired as described in -the command line help. - -You may directly download automatically generated single-header files: - -https://github.com/fastfloat/fast_float/releases/download/v1.1.2/fast_float.h - -## Credit - -Though this work is inspired by many different people, this work benefited especially from exchanges with -Michael Eisel, who motivated the original research with his key insights, and with Nigel Tao who provided -invaluable feedback. Rémy Oudompheng first implemented a fast path we use in the case of long digits. - -The library includes code adapted from Google Wuffs (written by Nigel Tao) which was originally published -under the Apache 2.0 license. - -## License - - -Licensed under either of Apache License, Version -2.0 or MIT license at your option. - - -
      - - -Unless you explicitly state otherwise, any contribution intentionally submitted -for inclusion in this repository by you, as defined in the Apache-2.0 license, -shall be dual licensed as above, without any additional terms or conditions. - diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/ci/script.sh b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/ci/script.sh deleted file mode 100755 index c9894c7a2..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/ci/script.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -TOOLCHAIN="$1" - -mkdir build -cd build - -if [ "$TOOLCHAIN" != "" ] ; then - cmake -DFASTFLOAT_TEST=ON .. -DCMAKE_TOOLCHAIN_FILE=/toolchains/"$TOOLCHAIN".cmake -else - cmake -DFASTFLOAT_TEST=ON .. -fi -make -j 2 -if [ "$TOOLCHAIN" != "" ] ; then - qemu-"$TOOLCHAIN" tests/basictest -else - ctest --output-on-failure -R basictest -fi diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/cmake/config.cmake.in b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/cmake/config.cmake.in deleted file mode 100644 index 035dc0fa0..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/cmake/config.cmake.in +++ /dev/null @@ -1,4 +0,0 @@ -@PACKAGE_INIT@ - -include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@-targets.cmake") -check_required_components("@PROJECT_NAME@") diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/ascii_number.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/ascii_number.h deleted file mode 100644 index 3e6bb3e9e..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/ascii_number.h +++ /dev/null @@ -1,231 +0,0 @@ -#ifndef FASTFLOAT_ASCII_NUMBER_H -#define FASTFLOAT_ASCII_NUMBER_H - -#include -#include -#include -#include - -#include "float_common.h" - -namespace fast_float { - -// Next function can be micro-optimized, but compilers are entirely -// able to optimize it well. -fastfloat_really_inline bool is_integer(char c) noexcept { return c >= '0' && c <= '9'; } - -fastfloat_really_inline uint64_t byteswap(uint64_t val) { - return (val & 0xFF00000000000000) >> 56 - | (val & 0x00FF000000000000) >> 40 - | (val & 0x0000FF0000000000) >> 24 - | (val & 0x000000FF00000000) >> 8 - | (val & 0x00000000FF000000) << 8 - | (val & 0x0000000000FF0000) << 24 - | (val & 0x000000000000FF00) << 40 - | (val & 0x00000000000000FF) << 56; -} - -fastfloat_really_inline uint64_t read_u64(const char *chars) { - uint64_t val; - ::memcpy(&val, chars, sizeof(uint64_t)); -#if FASTFLOAT_IS_BIG_ENDIAN == 1 - // Need to read as-if the number was in little-endian order. - val = byteswap(val); -#endif - return val; -} - -fastfloat_really_inline void write_u64(uint8_t *chars, uint64_t val) { -#if FASTFLOAT_IS_BIG_ENDIAN == 1 - // Need to read as-if the number was in little-endian order. - val = byteswap(val); -#endif - ::memcpy(chars, &val, sizeof(uint64_t)); -} - -// credit @aqrit -fastfloat_really_inline uint32_t parse_eight_digits_unrolled(uint64_t val) { - const uint64_t mask = 0x000000FF000000FF; - const uint64_t mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32) - const uint64_t mul2 = 0x0000271000000001; // 1 + (10000ULL << 32) - val -= 0x3030303030303030; - val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8; - val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; - return uint32_t(val); -} - -fastfloat_really_inline uint32_t parse_eight_digits_unrolled(const char *chars) noexcept { - return parse_eight_digits_unrolled(read_u64(chars)); -} - -// credit @aqrit -fastfloat_really_inline bool is_made_of_eight_digits_fast(uint64_t val) noexcept { - return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) & - 0x8080808080808080)); -} - -fastfloat_really_inline bool is_made_of_eight_digits_fast(const char *chars) noexcept { - return is_made_of_eight_digits_fast(read_u64(chars)); -} - -typedef span byte_span; - -struct parsed_number_string { - int64_t exponent{0}; - uint64_t mantissa{0}; - const char *lastmatch{nullptr}; - bool negative{false}; - bool valid{false}; - bool too_many_digits{false}; - // contains the range of the significant digits - byte_span integer{}; // non-nullable - byte_span fraction{}; // nullable -}; - -// Assuming that you use no more than 19 digits, this will -// parse an ASCII string. -fastfloat_really_inline -parsed_number_string parse_number_string(const char *p, const char *pend, parse_options options) noexcept { - const chars_format fmt = options.format; - const char decimal_point = options.decimal_point; - - parsed_number_string answer; - answer.valid = false; - answer.too_many_digits = false; - answer.negative = (*p == '-'); - if (*p == '-') { // C++17 20.19.3.(7.1) explicitly forbids '+' sign here - ++p; - if (p == pend) { - return answer; - } - if (!is_integer(*p) && (*p != decimal_point)) { // a sign must be followed by an integer or the dot - return answer; - } - } - const char *const start_digits = p; - - uint64_t i = 0; // an unsigned int avoids signed overflows (which are bad) - - while ((std::distance(p, pend) >= 8) && is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok - p += 8; - } - while ((p != pend) && is_integer(*p)) { - // a multiplication by 10 is cheaper than an arbitrary integer - // multiplication - i = 10 * i + - uint64_t(*p - '0'); // might overflow, we will handle the overflow later - ++p; - } - const char *const end_of_integer_part = p; - int64_t digit_count = int64_t(end_of_integer_part - start_digits); - answer.integer = byte_span(start_digits, size_t(digit_count)); - int64_t exponent = 0; - if ((p != pend) && (*p == decimal_point)) { - ++p; - const char* before = p; - // can occur at most twice without overflowing, but let it occur more, since - // for integers with many digits, digit parsing is the primary bottleneck. - while ((std::distance(p, pend) >= 8) && is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok - p += 8; - } - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - ++p; - i = i * 10 + digit; // in rare cases, this will overflow, but that's ok - } - exponent = before - p; - answer.fraction = byte_span(before, size_t(p - before)); - digit_count -= exponent; - } - // we must have encountered at least one integer! - if (digit_count == 0) { - return answer; - } - int64_t exp_number = 0; // explicit exponential part - if ((fmt & chars_format::scientific) && (p != pend) && (('e' == *p) || ('E' == *p))) { - const char * location_of_e = p; - ++p; - bool neg_exp = false; - if ((p != pend) && ('-' == *p)) { - neg_exp = true; - ++p; - } else if ((p != pend) && ('+' == *p)) { // '+' on exponent is allowed by C++17 20.19.3.(7.1) - ++p; - } - if ((p == pend) || !is_integer(*p)) { - if(!(fmt & chars_format::fixed)) { - // We are in error. - return answer; - } - // Otherwise, we will be ignoring the 'e'. - p = location_of_e; - } else { - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - if (exp_number < 0x10000000) { - exp_number = 10 * exp_number + digit; - } - ++p; - } - if(neg_exp) { exp_number = - exp_number; } - exponent += exp_number; - } - } else { - // If it scientific and not fixed, we have to bail out. - if((fmt & chars_format::scientific) && !(fmt & chars_format::fixed)) { return answer; } - } - answer.lastmatch = p; - answer.valid = true; - - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon. - // - // We can deal with up to 19 digits. - if (digit_count > 19) { // this is uncommon - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - // We need to be mindful of the case where we only have zeroes... - // E.g., 0.000000000...000. - const char *start = start_digits; - while ((start != pend) && (*start == '0' || *start == decimal_point)) { - if(*start == '0') { digit_count --; } - start++; - } - if (digit_count > 19) { - answer.too_many_digits = true; - // Let us start again, this time, avoiding overflows. - // We don't need to check if is_integer, since we use the - // pre-tokenized spans from above. - i = 0; - p = answer.integer.ptr; - const char* int_end = p + answer.integer.len(); - const uint64_t minimal_nineteen_digit_integer{1000000000000000000}; - while((i < minimal_nineteen_digit_integer) && (p != int_end)) { - i = i * 10 + uint64_t(*p - '0'); - ++p; - } - if (i >= minimal_nineteen_digit_integer) { // We have a big integers - exponent = end_of_integer_part - p + exp_number; - } else { // We have a value with a fractional component. - p = answer.fraction.ptr; - const char* frac_end = p + answer.fraction.len(); - while((i < minimal_nineteen_digit_integer) && (p != frac_end)) { - i = i * 10 + uint64_t(*p - '0'); - ++p; - } - exponent = answer.fraction.ptr - p + exp_number; - } - // We have now corrected both exponent and i, to a truncated value - } - } - answer.exponent = exponent; - answer.mantissa = i; - return answer; -} - -} // namespace fast_float - -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/bigint.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/bigint.h deleted file mode 100644 index b56cb9b03..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/bigint.h +++ /dev/null @@ -1,590 +0,0 @@ -#ifndef FASTFLOAT_BIGINT_H -#define FASTFLOAT_BIGINT_H - -#include -#include -#include -#include - -#include "float_common.h" - -namespace fast_float { - -// the limb width: we want efficient multiplication of double the bits in -// limb, or for 64-bit limbs, at least 64-bit multiplication where we can -// extract the high and low parts efficiently. this is every 64-bit -// architecture except for sparc, which emulates 128-bit multiplication. -// we might have platforms where `CHAR_BIT` is not 8, so let's avoid -// doing `8 * sizeof(limb)`. -#if defined(FASTFLOAT_64BIT) && !defined(__sparc) -#define FASTFLOAT_64BIT_LIMB -typedef uint64_t limb; -constexpr size_t limb_bits = 64; -#else -#define FASTFLOAT_32BIT_LIMB -typedef uint32_t limb; -constexpr size_t limb_bits = 32; -#endif - -typedef span limb_span; - -// number of bits in a bigint. this needs to be at least the number -// of bits required to store the largest bigint, which is -// `log2(10**(digits + max_exp))`, or `log2(10**(767 + 342))`, or -// ~3600 bits, so we round to 4000. -constexpr size_t bigint_bits = 4000; -constexpr size_t bigint_limbs = bigint_bits / limb_bits; - -// vector-like type that is allocated on the stack. the entire -// buffer is pre-allocated, and only the length changes. -template -struct stackvec { - limb data[size]; - // we never need more than 150 limbs - uint16_t length{0}; - - stackvec() = default; - stackvec(const stackvec &) = delete; - stackvec &operator=(const stackvec &) = delete; - stackvec(stackvec &&) = delete; - stackvec &operator=(stackvec &&other) = delete; - - // create stack vector from existing limb span. - stackvec(limb_span s) { - FASTFLOAT_ASSERT(try_extend(s)); - } - - limb& operator[](size_t index) noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - return data[index]; - } - const limb& operator[](size_t index) const noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - return data[index]; - } - // index from the end of the container - const limb& rindex(size_t index) const noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - size_t rindex = length - index - 1; - return data[rindex]; - } - - // set the length, without bounds checking. - void set_len(size_t len) noexcept { - length = uint16_t(len); - } - constexpr size_t len() const noexcept { - return length; - } - constexpr bool is_empty() const noexcept { - return length == 0; - } - constexpr size_t capacity() const noexcept { - return size; - } - // append item to vector, without bounds checking - void push_unchecked(limb value) noexcept { - data[length] = value; - length++; - } - // append item to vector, returning if item was added - bool try_push(limb value) noexcept { - if (len() < capacity()) { - push_unchecked(value); - return true; - } else { - return false; - } - } - // add items to the vector, from a span, without bounds checking - void extend_unchecked(limb_span s) noexcept { - limb* ptr = data + length; - ::memcpy((void*)ptr, (const void*)s.ptr, sizeof(limb) * s.len()); - set_len(len() + s.len()); - } - // try to add items to the vector, returning if items were added - bool try_extend(limb_span s) noexcept { - if (len() + s.len() <= capacity()) { - extend_unchecked(s); - return true; - } else { - return false; - } - } - // resize the vector, without bounds checking - // if the new size is longer than the vector, assign value to each - // appended item. - void resize_unchecked(size_t new_len, limb value) noexcept { - if (new_len > len()) { - size_t count = new_len - len(); - limb* first = data + len(); - limb* last = first + count; - ::std::fill(first, last, value); - set_len(new_len); - } else { - set_len(new_len); - } - } - // try to resize the vector, returning if the vector was resized. - bool try_resize(size_t new_len, limb value) noexcept { - if (new_len > capacity()) { - return false; - } else { - resize_unchecked(new_len, value); - return true; - } - } - // check if any limbs are non-zero after the given index. - // this needs to be done in reverse order, since the index - // is relative to the most significant limbs. - bool nonzero(size_t index) const noexcept { - while (index < len()) { - if (rindex(index) != 0) { - return true; - } - index++; - } - return false; - } - // normalize the big integer, so most-significant zero limbs are removed. - void normalize() noexcept { - while (len() > 0 && rindex(0) == 0) { - length--; - } - } -}; - -fastfloat_really_inline -uint64_t empty_hi64(bool& truncated) noexcept { - truncated = false; - return 0; -} - -fastfloat_really_inline -uint64_t uint64_hi64(uint64_t r0, bool& truncated) noexcept { - truncated = false; - int shl = leading_zeroes(r0); - return r0 << shl; -} - -fastfloat_really_inline -uint64_t uint64_hi64(uint64_t r0, uint64_t r1, bool& truncated) noexcept { - int shl = leading_zeroes(r0); - if (shl == 0) { - truncated = r1 != 0; - return r0; - } else { - int shr = 64 - shl; - truncated = (r1 << shl) != 0; - return (r0 << shl) | (r1 >> shr); - } -} - -fastfloat_really_inline -uint64_t uint32_hi64(uint32_t r0, bool& truncated) noexcept { - return uint64_hi64(r0, truncated); -} - -fastfloat_really_inline -uint64_t uint32_hi64(uint32_t r0, uint32_t r1, bool& truncated) noexcept { - uint64_t x0 = r0; - uint64_t x1 = r1; - return uint64_hi64((x0 << 32) | x1, truncated); -} - -fastfloat_really_inline -uint64_t uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool& truncated) noexcept { - uint64_t x0 = r0; - uint64_t x1 = r1; - uint64_t x2 = r2; - return uint64_hi64(x0, (x1 << 32) | x2, truncated); -} - -// add two small integers, checking for overflow. -// we want an efficient operation. for msvc, where -// we don't have built-in intrinsics, this is still -// pretty fast. -fastfloat_really_inline -limb scalar_add(limb x, limb y, bool& overflow) noexcept { - limb z; - -// gcc and clang -#if defined(__has_builtin) - #if __has_builtin(__builtin_add_overflow) - overflow = __builtin_add_overflow(x, y, &z); - return z; - #endif -#endif - - // generic, this still optimizes correctly on MSVC. - z = x + y; - overflow = z < x; - return z; -} - -// multiply two small integers, getting both the high and low bits. -fastfloat_really_inline -limb scalar_mul(limb x, limb y, limb& carry) noexcept { -#ifdef FASTFLOAT_64BIT_LIMB - #if defined(__SIZEOF_INT128__) - // GCC and clang both define it as an extension. - __uint128_t z = __uint128_t(x) * __uint128_t(y) + __uint128_t(carry); - carry = limb(z >> limb_bits); - return limb(z); - #else - // fallback, no native 128-bit integer multiplication with carry. - // on msvc, this optimizes identically, somehow. - value128 z = full_multiplication(x, y); - bool overflow; - z.low = scalar_add(z.low, carry, overflow); - z.high += uint64_t(overflow); // cannot overflow - carry = z.high; - return z.low; - #endif -#else - uint64_t z = uint64_t(x) * uint64_t(y) + uint64_t(carry); - carry = limb(z >> limb_bits); - return limb(z); -#endif -} - -// add scalar value to bigint starting from offset. -// used in grade school multiplication -template -inline bool small_add_from(stackvec& vec, limb y, size_t start) noexcept { - size_t index = start; - limb carry = y; - bool overflow; - while (carry != 0 && index < vec.len()) { - vec[index] = scalar_add(vec[index], carry, overflow); - carry = limb(overflow); - index += 1; - } - if (carry != 0) { - FASTFLOAT_TRY(vec.try_push(carry)); - } - return true; -} - -// add scalar value to bigint. -template -fastfloat_really_inline bool small_add(stackvec& vec, limb y) noexcept { - return small_add_from(vec, y, 0); -} - -// multiply bigint by scalar value. -template -inline bool small_mul(stackvec& vec, limb y) noexcept { - limb carry = 0; - for (size_t index = 0; index < vec.len(); index++) { - vec[index] = scalar_mul(vec[index], y, carry); - } - if (carry != 0) { - FASTFLOAT_TRY(vec.try_push(carry)); - } - return true; -} - -// add bigint to bigint starting from index. -// used in grade school multiplication -template -bool large_add_from(stackvec& x, limb_span y, size_t start) noexcept { - // the effective x buffer is from `xstart..x.len()`, so exit early - // if we can't get that current range. - if (x.len() < start || y.len() > x.len() - start) { - FASTFLOAT_TRY(x.try_resize(y.len() + start, 0)); - } - - bool carry = false; - for (size_t index = 0; index < y.len(); index++) { - limb xi = x[index + start]; - limb yi = y[index]; - bool c1 = false; - bool c2 = false; - xi = scalar_add(xi, yi, c1); - if (carry) { - xi = scalar_add(xi, 1, c2); - } - x[index + start] = xi; - carry = c1 | c2; - } - - // handle overflow - if (carry) { - FASTFLOAT_TRY(small_add_from(x, 1, y.len() + start)); - } - return true; -} - -// add bigint to bigint. -template -fastfloat_really_inline bool large_add_from(stackvec& x, limb_span y) noexcept { - return large_add_from(x, y, 0); -} - -// grade-school multiplication algorithm -template -bool long_mul(stackvec& x, limb_span y) noexcept { - limb_span xs = limb_span(x.data, x.len()); - stackvec z(xs); - limb_span zs = limb_span(z.data, z.len()); - - if (y.len() != 0) { - limb y0 = y[0]; - FASTFLOAT_TRY(small_mul(x, y0)); - for (size_t index = 1; index < y.len(); index++) { - limb yi = y[index]; - stackvec zi; - if (yi != 0) { - // re-use the same buffer throughout - zi.set_len(0); - FASTFLOAT_TRY(zi.try_extend(zs)); - FASTFLOAT_TRY(small_mul(zi, yi)); - limb_span zis = limb_span(zi.data, zi.len()); - FASTFLOAT_TRY(large_add_from(x, zis, index)); - } - } - } - - x.normalize(); - return true; -} - -// grade-school multiplication algorithm -template -bool large_mul(stackvec& x, limb_span y) noexcept { - if (y.len() == 1) { - FASTFLOAT_TRY(small_mul(x, y[0])); - } else { - FASTFLOAT_TRY(long_mul(x, y)); - } - return true; -} - -// big integer type. implements a small subset of big integer -// arithmetic, using simple algorithms since asymptotically -// faster algorithms are slower for a small number of limbs. -// all operations assume the big-integer is normalized. -struct bigint { - // storage of the limbs, in little-endian order. - stackvec vec; - - bigint(): vec() {} - bigint(const bigint &) = delete; - bigint &operator=(const bigint &) = delete; - bigint(bigint &&) = delete; - bigint &operator=(bigint &&other) = delete; - - bigint(uint64_t value): vec() { -#ifdef FASTFLOAT_64BIT_LIMB - vec.push_unchecked(value); -#else - vec.push_unchecked(uint32_t(value)); - vec.push_unchecked(uint32_t(value >> 32)); -#endif - vec.normalize(); - } - - // get the high 64 bits from the vector, and if bits were truncated. - // this is to get the significant digits for the float. - uint64_t hi64(bool& truncated) const noexcept { -#ifdef FASTFLOAT_64BIT_LIMB - if (vec.len() == 0) { - return empty_hi64(truncated); - } else if (vec.len() == 1) { - return uint64_hi64(vec.rindex(0), truncated); - } else { - uint64_t result = uint64_hi64(vec.rindex(0), vec.rindex(1), truncated); - truncated |= vec.nonzero(2); - return result; - } -#else - if (vec.len() == 0) { - return empty_hi64(truncated); - } else if (vec.len() == 1) { - return uint32_hi64(vec.rindex(0), truncated); - } else if (vec.len() == 2) { - return uint32_hi64(vec.rindex(0), vec.rindex(1), truncated); - } else { - uint64_t result = uint32_hi64(vec.rindex(0), vec.rindex(1), vec.rindex(2), truncated); - truncated |= vec.nonzero(3); - return result; - } -#endif - } - - // compare two big integers, returning the large value. - // assumes both are normalized. if the return value is - // negative, other is larger, if the return value is - // positive, this is larger, otherwise they are equal. - // the limbs are stored in little-endian order, so we - // must compare the limbs in ever order. - int compare(const bigint& other) const noexcept { - if (vec.len() > other.vec.len()) { - return 1; - } else if (vec.len() < other.vec.len()) { - return -1; - } else { - for (size_t index = vec.len(); index > 0; index--) { - limb xi = vec[index - 1]; - limb yi = other.vec[index - 1]; - if (xi > yi) { - return 1; - } else if (xi < yi) { - return -1; - } - } - return 0; - } - } - - // shift left each limb n bits, carrying over to the new limb - // returns true if we were able to shift all the digits. - bool shl_bits(size_t n) noexcept { - // Internally, for each item, we shift left by n, and add the previous - // right shifted limb-bits. - // For example, we transform (for u8) shifted left 2, to: - // b10100100 b01000010 - // b10 b10010001 b00001000 - FASTFLOAT_DEBUG_ASSERT(n != 0); - FASTFLOAT_DEBUG_ASSERT(n < sizeof(limb) * 8); - - size_t shl = n; - size_t shr = limb_bits - shl; - limb prev = 0; - for (size_t index = 0; index < vec.len(); index++) { - limb xi = vec[index]; - vec[index] = (xi << shl) | (prev >> shr); - prev = xi; - } - - limb carry = prev >> shr; - if (carry != 0) { - return vec.try_push(carry); - } - return true; - } - - // move the limbs left by `n` limbs. - bool shl_limbs(size_t n) noexcept { - FASTFLOAT_DEBUG_ASSERT(n != 0); - if (n + vec.len() > vec.capacity()) { - return false; - } else if (!vec.is_empty()) { - // move limbs - limb* dst = vec.data + n; - const limb* src = vec.data; - ::memmove(dst, src, sizeof(limb) * vec.len()); - // fill in empty limbs - limb* first = vec.data; - limb* last = first + n; - ::std::fill(first, last, 0); - vec.set_len(n + vec.len()); - return true; - } else { - return true; - } - } - - // move the limbs left by `n` bits. - bool shl(size_t n) noexcept { - size_t rem = n % limb_bits; - size_t div = n / limb_bits; - if (rem != 0) { - FASTFLOAT_TRY(shl_bits(rem)); - } - if (div != 0) { - FASTFLOAT_TRY(shl_limbs(div)); - } - return true; - } - - // get the number of leading zeros in the bigint. - int ctlz() const noexcept { - if (vec.is_empty()) { - return 0; - } else { -#ifdef FASTFLOAT_64BIT_LIMB - return leading_zeroes(vec.rindex(0)); -#else - // no use defining a specialized leading_zeroes for a 32-bit type. - uint64_t r0 = vec.rindex(0); - return leading_zeroes(r0 << 32); -#endif - } - } - - // get the number of bits in the bigint. - int bit_length() const noexcept { - int lz = ctlz(); - return int(limb_bits * vec.len()) - lz; - } - - bool mul(limb y) noexcept { - return small_mul(vec, y); - } - - bool add(limb y) noexcept { - return small_add(vec, y); - } - - // multiply as if by 2 raised to a power. - bool pow2(uint32_t exp) noexcept { - return shl(exp); - } - - // multiply as if by 5 raised to a power. - bool pow5(uint32_t exp) noexcept { - // multiply by a power of 5 - static constexpr uint32_t large_step = 135; - static constexpr uint64_t small_power_of_5[] = { - 1UL, 5UL, 25UL, 125UL, 625UL, 3125UL, 15625UL, 78125UL, 390625UL, - 1953125UL, 9765625UL, 48828125UL, 244140625UL, 1220703125UL, - 6103515625UL, 30517578125UL, 152587890625UL, 762939453125UL, - 3814697265625UL, 19073486328125UL, 95367431640625UL, 476837158203125UL, - 2384185791015625UL, 11920928955078125UL, 59604644775390625UL, - 298023223876953125UL, 1490116119384765625UL, 7450580596923828125UL, - }; -#ifdef FASTFLOAT_64BIT_LIMB - constexpr static limb large_power_of_5[] = { - 1414648277510068013UL, 9180637584431281687UL, 4539964771860779200UL, - 10482974169319127550UL, 198276706040285095UL}; -#else - constexpr static limb large_power_of_5[] = { - 4279965485U, 329373468U, 4020270615U, 2137533757U, 4287402176U, - 1057042919U, 1071430142U, 2440757623U, 381945767U, 46164893U}; -#endif - size_t large_length = sizeof(large_power_of_5) / sizeof(limb); - limb_span large = limb_span(large_power_of_5, large_length); - while (exp >= large_step) { - FASTFLOAT_TRY(large_mul(vec, large)); - exp -= large_step; - } -#ifdef FASTFLOAT_64BIT_LIMB - uint32_t small_step = 27; - limb max_native = 7450580596923828125UL; -#else - uint32_t small_step = 13; - limb max_native = 1220703125U; -#endif - while (exp >= small_step) { - FASTFLOAT_TRY(small_mul(vec, max_native)); - exp -= small_step; - } - if (exp != 0) { - FASTFLOAT_TRY(small_mul(vec, limb(small_power_of_5[exp]))); - } - - return true; - } - - // multiply as if by 10 raised to a power. - bool pow10(uint32_t exp) noexcept { - FASTFLOAT_TRY(pow5(exp)); - return pow2(exp); - } -}; - -} // namespace fast_float - -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/decimal_to_binary.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/decimal_to_binary.h deleted file mode 100644 index 6da6c66a3..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/decimal_to_binary.h +++ /dev/null @@ -1,194 +0,0 @@ -#ifndef FASTFLOAT_DECIMAL_TO_BINARY_H -#define FASTFLOAT_DECIMAL_TO_BINARY_H - -#include "float_common.h" -#include "fast_table.h" -#include -#include -#include -#include -#include -#include - -namespace fast_float { - -// This will compute or rather approximate w * 5**q and return a pair of 64-bit words approximating -// the result, with the "high" part corresponding to the most significant bits and the -// low part corresponding to the least significant bits. -// -template -fastfloat_really_inline -value128 compute_product_approximation(int64_t q, uint64_t w) { - const int index = 2 * int(q - powers::smallest_power_of_five); - // For small values of q, e.g., q in [0,27], the answer is always exact because - // The line value128 firstproduct = full_multiplication(w, power_of_five_128[index]); - // gives the exact answer. - value128 firstproduct = full_multiplication(w, powers::power_of_five_128[index]); - static_assert((bit_precision >= 0) && (bit_precision <= 64), " precision should be in (0,64]"); - constexpr uint64_t precision_mask = (bit_precision < 64) ? - (uint64_t(0xFFFFFFFFFFFFFFFF) >> bit_precision) - : uint64_t(0xFFFFFFFFFFFFFFFF); - if((firstproduct.high & precision_mask) == precision_mask) { // could further guard with (lower + w < lower) - // regarding the second product, we only need secondproduct.high, but our expectation is that the compiler will optimize this extra work away if needed. - value128 secondproduct = full_multiplication(w, powers::power_of_five_128[index + 1]); - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { - firstproduct.high++; - } - } - return firstproduct; -} - -namespace detail { -/** - * For q in (0,350), we have that - * f = (((152170 + 65536) * q ) >> 16); - * is equal to - * floor(p) + q - * where - * p = log(5**q)/log(2) = q * log(5)/log(2) - * - * For negative values of q in (-400,0), we have that - * f = (((152170 + 65536) * q ) >> 16); - * is equal to - * -ceil(p) + q - * where - * p = log(5**-q)/log(2) = -q * log(5)/log(2) - */ - constexpr fastfloat_really_inline int32_t power(int32_t q) noexcept { - return (((152170 + 65536) * q) >> 16) + 63; - } -} // namespace detail - -// create an adjusted mantissa, biased by the invalid power2 -// for significant digits already multiplied by 10 ** q. -template -fastfloat_really_inline -adjusted_mantissa compute_error_scaled(int64_t q, uint64_t w, int lz) noexcept { - int hilz = int(w >> 63) ^ 1; - adjusted_mantissa answer; - answer.mantissa = w << hilz; - int bias = binary::mantissa_explicit_bits() - binary::minimum_exponent(); - answer.power2 = int32_t(detail::power(int32_t(q)) + bias - hilz - lz - 62 + invalid_am_bias); - return answer; -} - -// w * 10 ** q, without rounding the representation up. -// the power2 in the exponent will be adjusted by invalid_am_bias. -template -fastfloat_really_inline -adjusted_mantissa compute_error(int64_t q, uint64_t w) noexcept { - int lz = leading_zeroes(w); - w <<= lz; - value128 product = compute_product_approximation(q, w); - return compute_error_scaled(q, product.high, lz); -} - -// w * 10 ** q -// The returned value should be a valid ieee64 number that simply need to be packed. -// However, in some very rare cases, the computation will fail. In such cases, we -// return an adjusted_mantissa with a negative power of 2: the caller should recompute -// in such cases. -template -fastfloat_really_inline -adjusted_mantissa compute_float(int64_t q, uint64_t w) noexcept { - adjusted_mantissa answer; - if ((w == 0) || (q < binary::smallest_power_of_ten())) { - answer.power2 = 0; - answer.mantissa = 0; - // result should be zero - return answer; - } - if (q > binary::largest_power_of_ten()) { - // we want to get infinity: - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } - // At this point in time q is in [powers::smallest_power_of_five, powers::largest_power_of_five]. - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(w); - w <<= lz; - - // The required precision is binary::mantissa_explicit_bits() + 3 because - // 1. We need the implicit bit - // 2. We need an extra bit for rounding purposes - // 3. We might lose a bit due to the "upperbit" routine (result too small, requiring a shift) - - value128 product = compute_product_approximation(q, w); - if(product.low == 0xFFFFFFFFFFFFFFFF) { // could guard it further - // In some very rare cases, this could happen, in which case we might need a more accurate - // computation that what we can provide cheaply. This is very, very unlikely. - // - const bool inside_safe_exponent = (q >= -27) && (q <= 55); // always good because 5**q <2**128 when q>=0, - // and otherwise, for q<0, we have 5**-q<2**64 and the 128-bit reciprocal allows for exact computation. - if(!inside_safe_exponent) { - return compute_error_scaled(q, product.high, lz); - } - } - // The "compute_product_approximation" function can be slightly slower than a branchless approach: - // value128 product = compute_product(q, w); - // but in practice, we can win big with the compute_product_approximation if its additional branch - // is easily predicted. Which is best is data specific. - int upperbit = int(product.high >> 63); - - answer.mantissa = product.high >> (upperbit + 64 - binary::mantissa_explicit_bits() - 3); - - answer.power2 = int32_t(detail::power(int32_t(q)) + upperbit - lz - binary::minimum_exponent()); - if (answer.power2 <= 0) { // we have a subnormal? - // Here have that answer.power2 <= 0 so -answer.power2 >= 0 - if(-answer.power2 + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - answer.power2 = 0; - answer.mantissa = 0; - // result should be zero - return answer; - } - // next line is safe because -answer.power2 + 1 < 64 - answer.mantissa >>= -answer.power2 + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - answer.mantissa += (answer.mantissa & 1); // round up - answer.mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - answer.power2 = (answer.mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) ? 0 : 1; - return answer; - } - - // usually, we round *up*, but if we fall right in between and and we have an - // even basis, we need to round down - // We are only concerned with the cases where 5**q fits in single 64-bit word. - if ((product.low <= 1) && (q >= binary::min_exponent_round_to_even()) && (q <= binary::max_exponent_round_to_even()) && - ((answer.mantissa & 3) == 1) ) { // we may fall between two floats! - // To be in-between two floats we need that in doing - // answer.mantissa = product.high >> (upperbit + 64 - binary::mantissa_explicit_bits() - 3); - // ... we dropped out only zeroes. But if this happened, then we can go back!!! - if((answer.mantissa << (upperbit + 64 - binary::mantissa_explicit_bits() - 3)) == product.high) { - answer.mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - answer.mantissa += (answer.mantissa & 1); // round up - answer.mantissa >>= 1; - if (answer.mantissa >= (uint64_t(2) << binary::mantissa_explicit_bits())) { - answer.mantissa = (uint64_t(1) << binary::mantissa_explicit_bits()); - answer.power2++; // undo previous addition - } - - answer.mantissa &= ~(uint64_t(1) << binary::mantissa_explicit_bits()); - if (answer.power2 >= binary::infinite_power()) { // infinity - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - } - return answer; -} - -} // namespace fast_float - -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/digit_comparison.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/digit_comparison.h deleted file mode 100644 index 7ffe87430..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/digit_comparison.h +++ /dev/null @@ -1,423 +0,0 @@ -#ifndef FASTFLOAT_DIGIT_COMPARISON_H -#define FASTFLOAT_DIGIT_COMPARISON_H - -#include -#include -#include -#include - -#include "float_common.h" -#include "bigint.h" -#include "ascii_number.h" - -namespace fast_float { - -// 1e0 to 1e19 -constexpr static uint64_t powers_of_ten_uint64[] = { - 1UL, 10UL, 100UL, 1000UL, 10000UL, 100000UL, 1000000UL, 10000000UL, 100000000UL, - 1000000000UL, 10000000000UL, 100000000000UL, 1000000000000UL, 10000000000000UL, - 100000000000000UL, 1000000000000000UL, 10000000000000000UL, 100000000000000000UL, - 1000000000000000000UL, 10000000000000000000UL}; - -// calculate the exponent, in scientific notation, of the number. -// this algorithm is not even close to optimized, but it has no practical -// effect on performance: in order to have a faster algorithm, we'd need -// to slow down performance for faster algorithms, and this is still fast. -fastfloat_really_inline int32_t scientific_exponent(parsed_number_string& num) noexcept { - uint64_t mantissa = num.mantissa; - int32_t exponent = int32_t(num.exponent); - while (mantissa >= 10000) { - mantissa /= 10000; - exponent += 4; - } - while (mantissa >= 100) { - mantissa /= 100; - exponent += 2; - } - while (mantissa >= 10) { - mantissa /= 10; - exponent += 1; - } - return exponent; -} - -// this converts a native floating-point number to an extended-precision float. -template -fastfloat_really_inline adjusted_mantissa to_extended(T value) noexcept { - adjusted_mantissa am; - int32_t bias = binary_format::mantissa_explicit_bits() - binary_format::minimum_exponent(); - if (std::is_same::value) { - constexpr uint32_t exponent_mask = 0x7F800000; - constexpr uint32_t mantissa_mask = 0x007FFFFF; - constexpr uint64_t hidden_bit_mask = 0x00800000; - uint32_t bits; - ::memcpy(&bits, &value, sizeof(T)); - if ((bits & exponent_mask) == 0) { - // denormal - am.power2 = 1 - bias; - am.mantissa = bits & mantissa_mask; - } else { - // normal - am.power2 = int32_t((bits & exponent_mask) >> binary_format::mantissa_explicit_bits()); - am.power2 -= bias; - am.mantissa = (bits & mantissa_mask) | hidden_bit_mask; - } - } else { - constexpr uint64_t exponent_mask = 0x7FF0000000000000; - constexpr uint64_t mantissa_mask = 0x000FFFFFFFFFFFFF; - constexpr uint64_t hidden_bit_mask = 0x0010000000000000; - uint64_t bits; - ::memcpy(&bits, &value, sizeof(T)); - if ((bits & exponent_mask) == 0) { - // denormal - am.power2 = 1 - bias; - am.mantissa = bits & mantissa_mask; - } else { - // normal - am.power2 = int32_t((bits & exponent_mask) >> binary_format::mantissa_explicit_bits()); - am.power2 -= bias; - am.mantissa = (bits & mantissa_mask) | hidden_bit_mask; - } - } - - return am; -} - -// get the extended precision value of the halfway point between b and b+u. -// we are given a native float that represents b, so we need to adjust it -// halfway between b and b+u. -template -fastfloat_really_inline adjusted_mantissa to_extended_halfway(T value) noexcept { - adjusted_mantissa am = to_extended(value); - am.mantissa <<= 1; - am.mantissa += 1; - am.power2 -= 1; - return am; -} - -// round an extended-precision float to the nearest machine float. -template -fastfloat_really_inline void round(adjusted_mantissa& am, callback cb) noexcept { - int32_t mantissa_shift = 64 - binary_format::mantissa_explicit_bits() - 1; - if (-am.power2 >= mantissa_shift) { - // have a denormal float - int32_t shift = -am.power2 + 1; - cb(am, std::min(shift, 64)); - // check for round-up: if rounding-nearest carried us to the hidden bit. - am.power2 = (am.mantissa < (uint64_t(1) << binary_format::mantissa_explicit_bits())) ? 0 : 1; - return; - } - - // have a normal float, use the default shift. - cb(am, mantissa_shift); - - // check for carry - if (am.mantissa >= (uint64_t(2) << binary_format::mantissa_explicit_bits())) { - am.mantissa = (uint64_t(1) << binary_format::mantissa_explicit_bits()); - am.power2++; - } - - // check for infinite: we could have carried to an infinite power - am.mantissa &= ~(uint64_t(1) << binary_format::mantissa_explicit_bits()); - if (am.power2 >= binary_format::infinite_power()) { - am.power2 = binary_format::infinite_power(); - am.mantissa = 0; - } -} - -template -fastfloat_really_inline -void round_nearest_tie_even(adjusted_mantissa& am, int32_t shift, callback cb) noexcept { - uint64_t mask; - uint64_t halfway; - if (shift == 64) { - mask = UINT64_MAX; - } else { - mask = (uint64_t(1) << shift) - 1; - } - if (shift == 0) { - halfway = 0; - } else { - halfway = uint64_t(1) << (shift - 1); - } - uint64_t truncated_bits = am.mantissa & mask; - uint64_t is_above = truncated_bits > halfway; - uint64_t is_halfway = truncated_bits == halfway; - - // shift digits into position - if (shift == 64) { - am.mantissa = 0; - } else { - am.mantissa >>= shift; - } - am.power2 += shift; - - bool is_odd = (am.mantissa & 1) == 1; - am.mantissa += uint64_t(cb(is_odd, is_halfway, is_above)); -} - -fastfloat_really_inline void round_down(adjusted_mantissa& am, int32_t shift) noexcept { - if (shift == 64) { - am.mantissa = 0; - } else { - am.mantissa >>= shift; - } - am.power2 += shift; -} - -fastfloat_really_inline void skip_zeros(const char*& first, const char* last) noexcept { - uint64_t val; - while (std::distance(first, last) >= 8) { - ::memcpy(&val, first, sizeof(uint64_t)); - if (val != 0x3030303030303030) { - break; - } - first += 8; - } - while (first != last) { - if (*first != '0') { - break; - } - first++; - } -} - -// determine if any non-zero digits were truncated. -// all characters must be valid digits. -fastfloat_really_inline bool is_truncated(const char* first, const char* last) noexcept { - // do 8-bit optimizations, can just compare to 8 literal 0s. - uint64_t val; - while (std::distance(first, last) >= 8) { - ::memcpy(&val, first, sizeof(uint64_t)); - if (val != 0x3030303030303030) { - return true; - } - first += 8; - } - while (first != last) { - if (*first != '0') { - return true; - } - first++; - } - return false; -} - -fastfloat_really_inline bool is_truncated(byte_span s) noexcept { - return is_truncated(s.ptr, s.ptr + s.len()); -} - -fastfloat_really_inline -void parse_eight_digits(const char*& p, limb& value, size_t& counter, size_t& count) noexcept { - value = value * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - counter += 8; - count += 8; -} - -fastfloat_really_inline -void parse_one_digit(const char*& p, limb& value, size_t& counter, size_t& count) noexcept { - value = value * 10 + limb(*p - '0'); - p++; - counter++; - count++; -} - -fastfloat_really_inline -void add_native(bigint& big, limb power, limb value) noexcept { - big.mul(power); - big.add(value); -} - -fastfloat_really_inline void round_up_bigint(bigint& big, size_t& count) noexcept { - // need to round-up the digits, but need to avoid rounding - // ....9999 to ...10000, which could cause a false halfway point. - add_native(big, 10, 1); - count++; -} - -// parse the significant digits into a big integer -inline void parse_mantissa(bigint& result, parsed_number_string& num, size_t max_digits, size_t& digits) noexcept { - // try to minimize the number of big integer and scalar multiplication. - // therefore, try to parse 8 digits at a time, and multiply by the largest - // scalar value (9 or 19 digits) for each step. - size_t counter = 0; - digits = 0; - limb value = 0; -#ifdef FASTFLOAT_64BIT_LIMB - size_t step = 19; -#else - size_t step = 9; -#endif - - // process all integer digits. - const char* p = num.integer.ptr; - const char* pend = p + num.integer.len(); - skip_zeros(p, pend); - // process all digits, in increments of step per loop - while (p != pend) { - while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && (max_digits - digits >= 8)) { - parse_eight_digits(p, value, counter, digits); - } - while (counter < step && p != pend && digits < max_digits) { - parse_one_digit(p, value, counter, digits); - } - if (digits == max_digits) { - // add the temporary value, then check if we've truncated any digits - add_native(result, limb(powers_of_ten_uint64[counter]), value); - bool truncated = is_truncated(p, pend); - if (num.fraction.ptr != nullptr) { - truncated |= is_truncated(num.fraction); - } - if (truncated) { - round_up_bigint(result, digits); - } - return; - } else { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - counter = 0; - value = 0; - } - } - - // add our fraction digits, if they're available. - if (num.fraction.ptr != nullptr) { - p = num.fraction.ptr; - pend = p + num.fraction.len(); - if (digits == 0) { - skip_zeros(p, pend); - } - // process all digits, in increments of step per loop - while (p != pend) { - while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && (max_digits - digits >= 8)) { - parse_eight_digits(p, value, counter, digits); - } - while (counter < step && p != pend && digits < max_digits) { - parse_one_digit(p, value, counter, digits); - } - if (digits == max_digits) { - // add the temporary value, then check if we've truncated any digits - add_native(result, limb(powers_of_ten_uint64[counter]), value); - bool truncated = is_truncated(p, pend); - if (truncated) { - round_up_bigint(result, digits); - } - return; - } else { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - counter = 0; - value = 0; - } - } - } - - if (counter != 0) { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - } -} - -template -inline adjusted_mantissa positive_digit_comp(bigint& bigmant, int32_t exponent) noexcept { - FASTFLOAT_ASSERT(bigmant.pow10(uint32_t(exponent))); - adjusted_mantissa answer; - bool truncated; - answer.mantissa = bigmant.hi64(truncated); - int bias = binary_format::mantissa_explicit_bits() - binary_format::minimum_exponent(); - answer.power2 = bigmant.bit_length() - 64 + bias; - - round(answer, [truncated](adjusted_mantissa& a, int32_t shift) { - round_nearest_tie_even(a, shift, [truncated](bool is_odd, bool is_halfway, bool is_above) -> bool { - return is_above || (is_halfway && truncated) || (is_odd && is_halfway); - }); - }); - - return answer; -} - -// the scaling here is quite simple: we have, for the real digits `m * 10^e`, -// and for the theoretical digits `n * 2^f`. Since `e` is always negative, -// to scale them identically, we do `n * 2^f * 5^-f`, so we now have `m * 2^e`. -// we then need to scale by `2^(f- e)`, and then the two significant digits -// are of the same magnitude. -template -inline adjusted_mantissa negative_digit_comp(bigint& bigmant, adjusted_mantissa am, int32_t exponent) noexcept { - bigint& real_digits = bigmant; - int32_t real_exp = exponent; - - // get the value of `b`, rounded down, and get a bigint representation of b+h - adjusted_mantissa am_b = am; - // gcc7 buf: use a lambda to remove the noexcept qualifier bug with -Wnoexcept-type. - round(am_b, [](adjusted_mantissa&a, int32_t shift) { round_down(a, shift); }); - T b; - to_float(false, am_b, b); - adjusted_mantissa theor = to_extended_halfway(b); - bigint theor_digits(theor.mantissa); - int32_t theor_exp = theor.power2; - - // scale real digits and theor digits to be same power. - int32_t pow2_exp = theor_exp - real_exp; - uint32_t pow5_exp = uint32_t(-real_exp); - if (pow5_exp != 0) { - FASTFLOAT_ASSERT(theor_digits.pow5(pow5_exp)); - } - if (pow2_exp > 0) { - FASTFLOAT_ASSERT(theor_digits.pow2(uint32_t(pow2_exp))); - } else if (pow2_exp < 0) { - FASTFLOAT_ASSERT(real_digits.pow2(uint32_t(-pow2_exp))); - } - - // compare digits, and use it to director rounding - int ord = real_digits.compare(theor_digits); - adjusted_mantissa answer = am; - round(answer, [ord](adjusted_mantissa& a, int32_t shift) { - round_nearest_tie_even(a, shift, [ord](bool is_odd, bool _, bool __) -> bool { - (void)_; // not needed, since we've done our comparison - (void)__; // not needed, since we've done our comparison - if (ord > 0) { - return true; - } else if (ord < 0) { - return false; - } else { - return is_odd; - } - }); - }); - - return answer; -} - -// parse the significant digits as a big integer to unambiguously round the -// the significant digits. here, we are trying to determine how to round -// an extended float representation close to `b+h`, halfway between `b` -// (the float rounded-down) and `b+u`, the next positive float. this -// algorithm is always correct, and uses one of two approaches. when -// the exponent is positive relative to the significant digits (such as -// 1234), we create a big-integer representation, get the high 64-bits, -// determine if any lower bits are truncated, and use that to direct -// rounding. in case of a negative exponent relative to the significant -// digits (such as 1.2345), we create a theoretical representation of -// `b` as a big-integer type, scaled to the same binary exponent as -// the actual digits. we then compare the big integer representations -// of both, and use that to direct rounding. -template -inline adjusted_mantissa digit_comp(parsed_number_string& num, adjusted_mantissa am) noexcept { - // remove the invalid exponent bias - am.power2 -= invalid_am_bias; - - int32_t sci_exp = scientific_exponent(num); - size_t max_digits = binary_format::max_digits(); - size_t digits = 0; - bigint bigmant; - parse_mantissa(bigmant, num, max_digits, digits); - // can't underflow, since digits is at most max_digits. - int32_t exponent = sci_exp + 1 - int32_t(digits); - if (exponent >= 0) { - return positive_digit_comp(bigmant, exponent); - } else { - return negative_digit_comp(bigmant, am, exponent); - } -} - -} // namespace fast_float - -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/fast_float.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/fast_float.h deleted file mode 100644 index 3c483803a..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/fast_float.h +++ /dev/null @@ -1,63 +0,0 @@ -#ifndef FASTFLOAT_FAST_FLOAT_H -#define FASTFLOAT_FAST_FLOAT_H - -#include - -namespace fast_float { -enum chars_format { - scientific = 1<<0, - fixed = 1<<2, - hex = 1<<3, - general = fixed | scientific -}; - - -struct from_chars_result { - const char *ptr; - std::errc ec; -}; - -struct parse_options { - constexpr explicit parse_options(chars_format fmt = chars_format::general, - char dot = '.') - : format(fmt), decimal_point(dot) {} - - /** Which number formats are accepted */ - chars_format format; - /** The character used as decimal point */ - char decimal_point; -}; - -/** - * This function parses the character sequence [first,last) for a number. It parses floating-point numbers expecting - * a locale-indepent format equivalent to what is used by std::strtod in the default ("C") locale. - * The resulting floating-point value is the closest floating-point values (using either float or double), - * using the "round to even" convention for values that would otherwise fall right in-between two values. - * That is, we provide exact parsing according to the IEEE standard. - * - * Given a successful parse, the pointer (`ptr`) in the returned value is set to point right after the - * parsed number, and the `value` referenced is set to the parsed value. In case of error, the returned - * `ec` contains a representative error, otherwise the default (`std::errc()`) value is stored. - * - * The implementation does not throw and does not allocate memory (e.g., with `new` or `malloc`). - * - * Like the C++17 standard, the `fast_float::from_chars` functions take an optional last argument of - * the type `fast_float::chars_format`. It is a bitset value: we check whether - * `fmt & fast_float::chars_format::fixed` and `fmt & fast_float::chars_format::scientific` are set - * to determine whether we allowe the fixed point and scientific notation respectively. - * The default is `fast_float::chars_format::general` which allows both `fixed` and `scientific`. - */ -template -from_chars_result from_chars(const char *first, const char *last, - T &value, chars_format fmt = chars_format::general) noexcept; - -/** - * Like from_chars, but accepts an `options` argument to govern number parsing. - */ -template -from_chars_result from_chars_advanced(const char *first, const char *last, - T &value, parse_options options) noexcept; - -} -#include "parse_number.h" -#endif // FASTFLOAT_FAST_FLOAT_H diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/fast_table.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/fast_table.h deleted file mode 100644 index 5766274ca..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/fast_table.h +++ /dev/null @@ -1,699 +0,0 @@ -#ifndef FASTFLOAT_FAST_TABLE_H -#define FASTFLOAT_FAST_TABLE_H - -#include - -namespace fast_float { - -/** - * When mapping numbers from decimal to binary, - * we go from w * 10^q to m * 2^p but we have - * 10^q = 5^q * 2^q, so effectively - * we are trying to match - * w * 2^q * 5^q to m * 2^p. Thus the powers of two - * are not a concern since they can be represented - * exactly using the binary notation, only the powers of five - * affect the binary significand. - */ - -/** - * The smallest non-zero float (binary64) is 2^−1074. - * We take as input numbers of the form w x 10^q where w < 2^64. - * We have that w * 10^-343 < 2^(64-344) 5^-343 < 2^-1076. - * However, we have that - * (2^64-1) * 10^-342 = (2^64-1) * 2^-342 * 5^-342 > 2^−1074. - * Thus it is possible for a number of the form w * 10^-342 where - * w is a 64-bit value to be a non-zero floating-point number. - ********* - * Any number of form w * 10^309 where w>= 1 is going to be - * infinite in binary64 so we never need to worry about powers - * of 5 greater than 308. - */ -template -struct powers_template { - -constexpr static int smallest_power_of_five = binary_format::smallest_power_of_ten(); -constexpr static int largest_power_of_five = binary_format::largest_power_of_ten(); -constexpr static int number_of_entries = 2 * (largest_power_of_five - smallest_power_of_five + 1); -// Powers of five from 5^-342 all the way to 5^308 rounded toward one. -static const uint64_t power_of_five_128[number_of_entries]; -}; - -template -const uint64_t powers_template::power_of_five_128[number_of_entries] = { - 0xeef453d6923bd65a,0x113faa2906a13b3f, - 0x9558b4661b6565f8,0x4ac7ca59a424c507, - 0xbaaee17fa23ebf76,0x5d79bcf00d2df649, - 0xe95a99df8ace6f53,0xf4d82c2c107973dc, - 0x91d8a02bb6c10594,0x79071b9b8a4be869, - 0xb64ec836a47146f9,0x9748e2826cdee284, - 0xe3e27a444d8d98b7,0xfd1b1b2308169b25, - 0x8e6d8c6ab0787f72,0xfe30f0f5e50e20f7, - 0xb208ef855c969f4f,0xbdbd2d335e51a935, - 0xde8b2b66b3bc4723,0xad2c788035e61382, - 0x8b16fb203055ac76,0x4c3bcb5021afcc31, - 0xaddcb9e83c6b1793,0xdf4abe242a1bbf3d, - 0xd953e8624b85dd78,0xd71d6dad34a2af0d, - 0x87d4713d6f33aa6b,0x8672648c40e5ad68, - 0xa9c98d8ccb009506,0x680efdaf511f18c2, - 0xd43bf0effdc0ba48,0x212bd1b2566def2, - 0x84a57695fe98746d,0x14bb630f7604b57, - 0xa5ced43b7e3e9188,0x419ea3bd35385e2d, - 0xcf42894a5dce35ea,0x52064cac828675b9, - 0x818995ce7aa0e1b2,0x7343efebd1940993, - 0xa1ebfb4219491a1f,0x1014ebe6c5f90bf8, - 0xca66fa129f9b60a6,0xd41a26e077774ef6, - 0xfd00b897478238d0,0x8920b098955522b4, - 0x9e20735e8cb16382,0x55b46e5f5d5535b0, - 0xc5a890362fddbc62,0xeb2189f734aa831d, - 0xf712b443bbd52b7b,0xa5e9ec7501d523e4, - 0x9a6bb0aa55653b2d,0x47b233c92125366e, - 0xc1069cd4eabe89f8,0x999ec0bb696e840a, - 0xf148440a256e2c76,0xc00670ea43ca250d, - 0x96cd2a865764dbca,0x380406926a5e5728, - 0xbc807527ed3e12bc,0xc605083704f5ecf2, - 0xeba09271e88d976b,0xf7864a44c633682e, - 0x93445b8731587ea3,0x7ab3ee6afbe0211d, - 0xb8157268fdae9e4c,0x5960ea05bad82964, - 0xe61acf033d1a45df,0x6fb92487298e33bd, - 0x8fd0c16206306bab,0xa5d3b6d479f8e056, - 0xb3c4f1ba87bc8696,0x8f48a4899877186c, - 0xe0b62e2929aba83c,0x331acdabfe94de87, - 0x8c71dcd9ba0b4925,0x9ff0c08b7f1d0b14, - 0xaf8e5410288e1b6f,0x7ecf0ae5ee44dd9, - 0xdb71e91432b1a24a,0xc9e82cd9f69d6150, - 0x892731ac9faf056e,0xbe311c083a225cd2, - 0xab70fe17c79ac6ca,0x6dbd630a48aaf406, - 0xd64d3d9db981787d,0x92cbbccdad5b108, - 0x85f0468293f0eb4e,0x25bbf56008c58ea5, - 0xa76c582338ed2621,0xaf2af2b80af6f24e, - 0xd1476e2c07286faa,0x1af5af660db4aee1, - 0x82cca4db847945ca,0x50d98d9fc890ed4d, - 0xa37fce126597973c,0xe50ff107bab528a0, - 0xcc5fc196fefd7d0c,0x1e53ed49a96272c8, - 0xff77b1fcbebcdc4f,0x25e8e89c13bb0f7a, - 0x9faacf3df73609b1,0x77b191618c54e9ac, - 0xc795830d75038c1d,0xd59df5b9ef6a2417, - 0xf97ae3d0d2446f25,0x4b0573286b44ad1d, - 0x9becce62836ac577,0x4ee367f9430aec32, - 0xc2e801fb244576d5,0x229c41f793cda73f, - 0xf3a20279ed56d48a,0x6b43527578c1110f, - 0x9845418c345644d6,0x830a13896b78aaa9, - 0xbe5691ef416bd60c,0x23cc986bc656d553, - 0xedec366b11c6cb8f,0x2cbfbe86b7ec8aa8, - 0x94b3a202eb1c3f39,0x7bf7d71432f3d6a9, - 0xb9e08a83a5e34f07,0xdaf5ccd93fb0cc53, - 0xe858ad248f5c22c9,0xd1b3400f8f9cff68, - 0x91376c36d99995be,0x23100809b9c21fa1, - 0xb58547448ffffb2d,0xabd40a0c2832a78a, - 0xe2e69915b3fff9f9,0x16c90c8f323f516c, - 0x8dd01fad907ffc3b,0xae3da7d97f6792e3, - 0xb1442798f49ffb4a,0x99cd11cfdf41779c, - 0xdd95317f31c7fa1d,0x40405643d711d583, - 0x8a7d3eef7f1cfc52,0x482835ea666b2572, - 0xad1c8eab5ee43b66,0xda3243650005eecf, - 0xd863b256369d4a40,0x90bed43e40076a82, - 0x873e4f75e2224e68,0x5a7744a6e804a291, - 0xa90de3535aaae202,0x711515d0a205cb36, - 0xd3515c2831559a83,0xd5a5b44ca873e03, - 0x8412d9991ed58091,0xe858790afe9486c2, - 0xa5178fff668ae0b6,0x626e974dbe39a872, - 0xce5d73ff402d98e3,0xfb0a3d212dc8128f, - 0x80fa687f881c7f8e,0x7ce66634bc9d0b99, - 0xa139029f6a239f72,0x1c1fffc1ebc44e80, - 0xc987434744ac874e,0xa327ffb266b56220, - 0xfbe9141915d7a922,0x4bf1ff9f0062baa8, - 0x9d71ac8fada6c9b5,0x6f773fc3603db4a9, - 0xc4ce17b399107c22,0xcb550fb4384d21d3, - 0xf6019da07f549b2b,0x7e2a53a146606a48, - 0x99c102844f94e0fb,0x2eda7444cbfc426d, - 0xc0314325637a1939,0xfa911155fefb5308, - 0xf03d93eebc589f88,0x793555ab7eba27ca, - 0x96267c7535b763b5,0x4bc1558b2f3458de, - 0xbbb01b9283253ca2,0x9eb1aaedfb016f16, - 0xea9c227723ee8bcb,0x465e15a979c1cadc, - 0x92a1958a7675175f,0xbfacd89ec191ec9, - 0xb749faed14125d36,0xcef980ec671f667b, - 0xe51c79a85916f484,0x82b7e12780e7401a, - 0x8f31cc0937ae58d2,0xd1b2ecb8b0908810, - 0xb2fe3f0b8599ef07,0x861fa7e6dcb4aa15, - 0xdfbdcece67006ac9,0x67a791e093e1d49a, - 0x8bd6a141006042bd,0xe0c8bb2c5c6d24e0, - 0xaecc49914078536d,0x58fae9f773886e18, - 0xda7f5bf590966848,0xaf39a475506a899e, - 0x888f99797a5e012d,0x6d8406c952429603, - 0xaab37fd7d8f58178,0xc8e5087ba6d33b83, - 0xd5605fcdcf32e1d6,0xfb1e4a9a90880a64, - 0x855c3be0a17fcd26,0x5cf2eea09a55067f, - 0xa6b34ad8c9dfc06f,0xf42faa48c0ea481e, - 0xd0601d8efc57b08b,0xf13b94daf124da26, - 0x823c12795db6ce57,0x76c53d08d6b70858, - 0xa2cb1717b52481ed,0x54768c4b0c64ca6e, - 0xcb7ddcdda26da268,0xa9942f5dcf7dfd09, - 0xfe5d54150b090b02,0xd3f93b35435d7c4c, - 0x9efa548d26e5a6e1,0xc47bc5014a1a6daf, - 0xc6b8e9b0709f109a,0x359ab6419ca1091b, - 0xf867241c8cc6d4c0,0xc30163d203c94b62, - 0x9b407691d7fc44f8,0x79e0de63425dcf1d, - 0xc21094364dfb5636,0x985915fc12f542e4, - 0xf294b943e17a2bc4,0x3e6f5b7b17b2939d, - 0x979cf3ca6cec5b5a,0xa705992ceecf9c42, - 0xbd8430bd08277231,0x50c6ff782a838353, - 0xece53cec4a314ebd,0xa4f8bf5635246428, - 0x940f4613ae5ed136,0x871b7795e136be99, - 0xb913179899f68584,0x28e2557b59846e3f, - 0xe757dd7ec07426e5,0x331aeada2fe589cf, - 0x9096ea6f3848984f,0x3ff0d2c85def7621, - 0xb4bca50b065abe63,0xfed077a756b53a9, - 0xe1ebce4dc7f16dfb,0xd3e8495912c62894, - 0x8d3360f09cf6e4bd,0x64712dd7abbbd95c, - 0xb080392cc4349dec,0xbd8d794d96aacfb3, - 0xdca04777f541c567,0xecf0d7a0fc5583a0, - 0x89e42caaf9491b60,0xf41686c49db57244, - 0xac5d37d5b79b6239,0x311c2875c522ced5, - 0xd77485cb25823ac7,0x7d633293366b828b, - 0x86a8d39ef77164bc,0xae5dff9c02033197, - 0xa8530886b54dbdeb,0xd9f57f830283fdfc, - 0xd267caa862a12d66,0xd072df63c324fd7b, - 0x8380dea93da4bc60,0x4247cb9e59f71e6d, - 0xa46116538d0deb78,0x52d9be85f074e608, - 0xcd795be870516656,0x67902e276c921f8b, - 0x806bd9714632dff6,0xba1cd8a3db53b6, - 0xa086cfcd97bf97f3,0x80e8a40eccd228a4, - 0xc8a883c0fdaf7df0,0x6122cd128006b2cd, - 0xfad2a4b13d1b5d6c,0x796b805720085f81, - 0x9cc3a6eec6311a63,0xcbe3303674053bb0, - 0xc3f490aa77bd60fc,0xbedbfc4411068a9c, - 0xf4f1b4d515acb93b,0xee92fb5515482d44, - 0x991711052d8bf3c5,0x751bdd152d4d1c4a, - 0xbf5cd54678eef0b6,0xd262d45a78a0635d, - 0xef340a98172aace4,0x86fb897116c87c34, - 0x9580869f0e7aac0e,0xd45d35e6ae3d4da0, - 0xbae0a846d2195712,0x8974836059cca109, - 0xe998d258869facd7,0x2bd1a438703fc94b, - 0x91ff83775423cc06,0x7b6306a34627ddcf, - 0xb67f6455292cbf08,0x1a3bc84c17b1d542, - 0xe41f3d6a7377eeca,0x20caba5f1d9e4a93, - 0x8e938662882af53e,0x547eb47b7282ee9c, - 0xb23867fb2a35b28d,0xe99e619a4f23aa43, - 0xdec681f9f4c31f31,0x6405fa00e2ec94d4, - 0x8b3c113c38f9f37e,0xde83bc408dd3dd04, - 0xae0b158b4738705e,0x9624ab50b148d445, - 0xd98ddaee19068c76,0x3badd624dd9b0957, - 0x87f8a8d4cfa417c9,0xe54ca5d70a80e5d6, - 0xa9f6d30a038d1dbc,0x5e9fcf4ccd211f4c, - 0xd47487cc8470652b,0x7647c3200069671f, - 0x84c8d4dfd2c63f3b,0x29ecd9f40041e073, - 0xa5fb0a17c777cf09,0xf468107100525890, - 0xcf79cc9db955c2cc,0x7182148d4066eeb4, - 0x81ac1fe293d599bf,0xc6f14cd848405530, - 0xa21727db38cb002f,0xb8ada00e5a506a7c, - 0xca9cf1d206fdc03b,0xa6d90811f0e4851c, - 0xfd442e4688bd304a,0x908f4a166d1da663, - 0x9e4a9cec15763e2e,0x9a598e4e043287fe, - 0xc5dd44271ad3cdba,0x40eff1e1853f29fd, - 0xf7549530e188c128,0xd12bee59e68ef47c, - 0x9a94dd3e8cf578b9,0x82bb74f8301958ce, - 0xc13a148e3032d6e7,0xe36a52363c1faf01, - 0xf18899b1bc3f8ca1,0xdc44e6c3cb279ac1, - 0x96f5600f15a7b7e5,0x29ab103a5ef8c0b9, - 0xbcb2b812db11a5de,0x7415d448f6b6f0e7, - 0xebdf661791d60f56,0x111b495b3464ad21, - 0x936b9fcebb25c995,0xcab10dd900beec34, - 0xb84687c269ef3bfb,0x3d5d514f40eea742, - 0xe65829b3046b0afa,0xcb4a5a3112a5112, - 0x8ff71a0fe2c2e6dc,0x47f0e785eaba72ab, - 0xb3f4e093db73a093,0x59ed216765690f56, - 0xe0f218b8d25088b8,0x306869c13ec3532c, - 0x8c974f7383725573,0x1e414218c73a13fb, - 0xafbd2350644eeacf,0xe5d1929ef90898fa, - 0xdbac6c247d62a583,0xdf45f746b74abf39, - 0x894bc396ce5da772,0x6b8bba8c328eb783, - 0xab9eb47c81f5114f,0x66ea92f3f326564, - 0xd686619ba27255a2,0xc80a537b0efefebd, - 0x8613fd0145877585,0xbd06742ce95f5f36, - 0xa798fc4196e952e7,0x2c48113823b73704, - 0xd17f3b51fca3a7a0,0xf75a15862ca504c5, - 0x82ef85133de648c4,0x9a984d73dbe722fb, - 0xa3ab66580d5fdaf5,0xc13e60d0d2e0ebba, - 0xcc963fee10b7d1b3,0x318df905079926a8, - 0xffbbcfe994e5c61f,0xfdf17746497f7052, - 0x9fd561f1fd0f9bd3,0xfeb6ea8bedefa633, - 0xc7caba6e7c5382c8,0xfe64a52ee96b8fc0, - 0xf9bd690a1b68637b,0x3dfdce7aa3c673b0, - 0x9c1661a651213e2d,0x6bea10ca65c084e, - 0xc31bfa0fe5698db8,0x486e494fcff30a62, - 0xf3e2f893dec3f126,0x5a89dba3c3efccfa, - 0x986ddb5c6b3a76b7,0xf89629465a75e01c, - 0xbe89523386091465,0xf6bbb397f1135823, - 0xee2ba6c0678b597f,0x746aa07ded582e2c, - 0x94db483840b717ef,0xa8c2a44eb4571cdc, - 0xba121a4650e4ddeb,0x92f34d62616ce413, - 0xe896a0d7e51e1566,0x77b020baf9c81d17, - 0x915e2486ef32cd60,0xace1474dc1d122e, - 0xb5b5ada8aaff80b8,0xd819992132456ba, - 0xe3231912d5bf60e6,0x10e1fff697ed6c69, - 0x8df5efabc5979c8f,0xca8d3ffa1ef463c1, - 0xb1736b96b6fd83b3,0xbd308ff8a6b17cb2, - 0xddd0467c64bce4a0,0xac7cb3f6d05ddbde, - 0x8aa22c0dbef60ee4,0x6bcdf07a423aa96b, - 0xad4ab7112eb3929d,0x86c16c98d2c953c6, - 0xd89d64d57a607744,0xe871c7bf077ba8b7, - 0x87625f056c7c4a8b,0x11471cd764ad4972, - 0xa93af6c6c79b5d2d,0xd598e40d3dd89bcf, - 0xd389b47879823479,0x4aff1d108d4ec2c3, - 0x843610cb4bf160cb,0xcedf722a585139ba, - 0xa54394fe1eedb8fe,0xc2974eb4ee658828, - 0xce947a3da6a9273e,0x733d226229feea32, - 0x811ccc668829b887,0x806357d5a3f525f, - 0xa163ff802a3426a8,0xca07c2dcb0cf26f7, - 0xc9bcff6034c13052,0xfc89b393dd02f0b5, - 0xfc2c3f3841f17c67,0xbbac2078d443ace2, - 0x9d9ba7832936edc0,0xd54b944b84aa4c0d, - 0xc5029163f384a931,0xa9e795e65d4df11, - 0xf64335bcf065d37d,0x4d4617b5ff4a16d5, - 0x99ea0196163fa42e,0x504bced1bf8e4e45, - 0xc06481fb9bcf8d39,0xe45ec2862f71e1d6, - 0xf07da27a82c37088,0x5d767327bb4e5a4c, - 0x964e858c91ba2655,0x3a6a07f8d510f86f, - 0xbbe226efb628afea,0x890489f70a55368b, - 0xeadab0aba3b2dbe5,0x2b45ac74ccea842e, - 0x92c8ae6b464fc96f,0x3b0b8bc90012929d, - 0xb77ada0617e3bbcb,0x9ce6ebb40173744, - 0xe55990879ddcaabd,0xcc420a6a101d0515, - 0x8f57fa54c2a9eab6,0x9fa946824a12232d, - 0xb32df8e9f3546564,0x47939822dc96abf9, - 0xdff9772470297ebd,0x59787e2b93bc56f7, - 0x8bfbea76c619ef36,0x57eb4edb3c55b65a, - 0xaefae51477a06b03,0xede622920b6b23f1, - 0xdab99e59958885c4,0xe95fab368e45eced, - 0x88b402f7fd75539b,0x11dbcb0218ebb414, - 0xaae103b5fcd2a881,0xd652bdc29f26a119, - 0xd59944a37c0752a2,0x4be76d3346f0495f, - 0x857fcae62d8493a5,0x6f70a4400c562ddb, - 0xa6dfbd9fb8e5b88e,0xcb4ccd500f6bb952, - 0xd097ad07a71f26b2,0x7e2000a41346a7a7, - 0x825ecc24c873782f,0x8ed400668c0c28c8, - 0xa2f67f2dfa90563b,0x728900802f0f32fa, - 0xcbb41ef979346bca,0x4f2b40a03ad2ffb9, - 0xfea126b7d78186bc,0xe2f610c84987bfa8, - 0x9f24b832e6b0f436,0xdd9ca7d2df4d7c9, - 0xc6ede63fa05d3143,0x91503d1c79720dbb, - 0xf8a95fcf88747d94,0x75a44c6397ce912a, - 0x9b69dbe1b548ce7c,0xc986afbe3ee11aba, - 0xc24452da229b021b,0xfbe85badce996168, - 0xf2d56790ab41c2a2,0xfae27299423fb9c3, - 0x97c560ba6b0919a5,0xdccd879fc967d41a, - 0xbdb6b8e905cb600f,0x5400e987bbc1c920, - 0xed246723473e3813,0x290123e9aab23b68, - 0x9436c0760c86e30b,0xf9a0b6720aaf6521, - 0xb94470938fa89bce,0xf808e40e8d5b3e69, - 0xe7958cb87392c2c2,0xb60b1d1230b20e04, - 0x90bd77f3483bb9b9,0xb1c6f22b5e6f48c2, - 0xb4ecd5f01a4aa828,0x1e38aeb6360b1af3, - 0xe2280b6c20dd5232,0x25c6da63c38de1b0, - 0x8d590723948a535f,0x579c487e5a38ad0e, - 0xb0af48ec79ace837,0x2d835a9df0c6d851, - 0xdcdb1b2798182244,0xf8e431456cf88e65, - 0x8a08f0f8bf0f156b,0x1b8e9ecb641b58ff, - 0xac8b2d36eed2dac5,0xe272467e3d222f3f, - 0xd7adf884aa879177,0x5b0ed81dcc6abb0f, - 0x86ccbb52ea94baea,0x98e947129fc2b4e9, - 0xa87fea27a539e9a5,0x3f2398d747b36224, - 0xd29fe4b18e88640e,0x8eec7f0d19a03aad, - 0x83a3eeeef9153e89,0x1953cf68300424ac, - 0xa48ceaaab75a8e2b,0x5fa8c3423c052dd7, - 0xcdb02555653131b6,0x3792f412cb06794d, - 0x808e17555f3ebf11,0xe2bbd88bbee40bd0, - 0xa0b19d2ab70e6ed6,0x5b6aceaeae9d0ec4, - 0xc8de047564d20a8b,0xf245825a5a445275, - 0xfb158592be068d2e,0xeed6e2f0f0d56712, - 0x9ced737bb6c4183d,0x55464dd69685606b, - 0xc428d05aa4751e4c,0xaa97e14c3c26b886, - 0xf53304714d9265df,0xd53dd99f4b3066a8, - 0x993fe2c6d07b7fab,0xe546a8038efe4029, - 0xbf8fdb78849a5f96,0xde98520472bdd033, - 0xef73d256a5c0f77c,0x963e66858f6d4440, - 0x95a8637627989aad,0xdde7001379a44aa8, - 0xbb127c53b17ec159,0x5560c018580d5d52, - 0xe9d71b689dde71af,0xaab8f01e6e10b4a6, - 0x9226712162ab070d,0xcab3961304ca70e8, - 0xb6b00d69bb55c8d1,0x3d607b97c5fd0d22, - 0xe45c10c42a2b3b05,0x8cb89a7db77c506a, - 0x8eb98a7a9a5b04e3,0x77f3608e92adb242, - 0xb267ed1940f1c61c,0x55f038b237591ed3, - 0xdf01e85f912e37a3,0x6b6c46dec52f6688, - 0x8b61313bbabce2c6,0x2323ac4b3b3da015, - 0xae397d8aa96c1b77,0xabec975e0a0d081a, - 0xd9c7dced53c72255,0x96e7bd358c904a21, - 0x881cea14545c7575,0x7e50d64177da2e54, - 0xaa242499697392d2,0xdde50bd1d5d0b9e9, - 0xd4ad2dbfc3d07787,0x955e4ec64b44e864, - 0x84ec3c97da624ab4,0xbd5af13bef0b113e, - 0xa6274bbdd0fadd61,0xecb1ad8aeacdd58e, - 0xcfb11ead453994ba,0x67de18eda5814af2, - 0x81ceb32c4b43fcf4,0x80eacf948770ced7, - 0xa2425ff75e14fc31,0xa1258379a94d028d, - 0xcad2f7f5359a3b3e,0x96ee45813a04330, - 0xfd87b5f28300ca0d,0x8bca9d6e188853fc, - 0x9e74d1b791e07e48,0x775ea264cf55347e, - 0xc612062576589dda,0x95364afe032a819e, - 0xf79687aed3eec551,0x3a83ddbd83f52205, - 0x9abe14cd44753b52,0xc4926a9672793543, - 0xc16d9a0095928a27,0x75b7053c0f178294, - 0xf1c90080baf72cb1,0x5324c68b12dd6339, - 0x971da05074da7bee,0xd3f6fc16ebca5e04, - 0xbce5086492111aea,0x88f4bb1ca6bcf585, - 0xec1e4a7db69561a5,0x2b31e9e3d06c32e6, - 0x9392ee8e921d5d07,0x3aff322e62439fd0, - 0xb877aa3236a4b449,0x9befeb9fad487c3, - 0xe69594bec44de15b,0x4c2ebe687989a9b4, - 0x901d7cf73ab0acd9,0xf9d37014bf60a11, - 0xb424dc35095cd80f,0x538484c19ef38c95, - 0xe12e13424bb40e13,0x2865a5f206b06fba, - 0x8cbccc096f5088cb,0xf93f87b7442e45d4, - 0xafebff0bcb24aafe,0xf78f69a51539d749, - 0xdbe6fecebdedd5be,0xb573440e5a884d1c, - 0x89705f4136b4a597,0x31680a88f8953031, - 0xabcc77118461cefc,0xfdc20d2b36ba7c3e, - 0xd6bf94d5e57a42bc,0x3d32907604691b4d, - 0x8637bd05af6c69b5,0xa63f9a49c2c1b110, - 0xa7c5ac471b478423,0xfcf80dc33721d54, - 0xd1b71758e219652b,0xd3c36113404ea4a9, - 0x83126e978d4fdf3b,0x645a1cac083126ea, - 0xa3d70a3d70a3d70a,0x3d70a3d70a3d70a4, - 0xcccccccccccccccc,0xcccccccccccccccd, - 0x8000000000000000,0x0, - 0xa000000000000000,0x0, - 0xc800000000000000,0x0, - 0xfa00000000000000,0x0, - 0x9c40000000000000,0x0, - 0xc350000000000000,0x0, - 0xf424000000000000,0x0, - 0x9896800000000000,0x0, - 0xbebc200000000000,0x0, - 0xee6b280000000000,0x0, - 0x9502f90000000000,0x0, - 0xba43b74000000000,0x0, - 0xe8d4a51000000000,0x0, - 0x9184e72a00000000,0x0, - 0xb5e620f480000000,0x0, - 0xe35fa931a0000000,0x0, - 0x8e1bc9bf04000000,0x0, - 0xb1a2bc2ec5000000,0x0, - 0xde0b6b3a76400000,0x0, - 0x8ac7230489e80000,0x0, - 0xad78ebc5ac620000,0x0, - 0xd8d726b7177a8000,0x0, - 0x878678326eac9000,0x0, - 0xa968163f0a57b400,0x0, - 0xd3c21bcecceda100,0x0, - 0x84595161401484a0,0x0, - 0xa56fa5b99019a5c8,0x0, - 0xcecb8f27f4200f3a,0x0, - 0x813f3978f8940984,0x4000000000000000, - 0xa18f07d736b90be5,0x5000000000000000, - 0xc9f2c9cd04674ede,0xa400000000000000, - 0xfc6f7c4045812296,0x4d00000000000000, - 0x9dc5ada82b70b59d,0xf020000000000000, - 0xc5371912364ce305,0x6c28000000000000, - 0xf684df56c3e01bc6,0xc732000000000000, - 0x9a130b963a6c115c,0x3c7f400000000000, - 0xc097ce7bc90715b3,0x4b9f100000000000, - 0xf0bdc21abb48db20,0x1e86d40000000000, - 0x96769950b50d88f4,0x1314448000000000, - 0xbc143fa4e250eb31,0x17d955a000000000, - 0xeb194f8e1ae525fd,0x5dcfab0800000000, - 0x92efd1b8d0cf37be,0x5aa1cae500000000, - 0xb7abc627050305ad,0xf14a3d9e40000000, - 0xe596b7b0c643c719,0x6d9ccd05d0000000, - 0x8f7e32ce7bea5c6f,0xe4820023a2000000, - 0xb35dbf821ae4f38b,0xdda2802c8a800000, - 0xe0352f62a19e306e,0xd50b2037ad200000, - 0x8c213d9da502de45,0x4526f422cc340000, - 0xaf298d050e4395d6,0x9670b12b7f410000, - 0xdaf3f04651d47b4c,0x3c0cdd765f114000, - 0x88d8762bf324cd0f,0xa5880a69fb6ac800, - 0xab0e93b6efee0053,0x8eea0d047a457a00, - 0xd5d238a4abe98068,0x72a4904598d6d880, - 0x85a36366eb71f041,0x47a6da2b7f864750, - 0xa70c3c40a64e6c51,0x999090b65f67d924, - 0xd0cf4b50cfe20765,0xfff4b4e3f741cf6d, - 0x82818f1281ed449f,0xbff8f10e7a8921a4, - 0xa321f2d7226895c7,0xaff72d52192b6a0d, - 0xcbea6f8ceb02bb39,0x9bf4f8a69f764490, - 0xfee50b7025c36a08,0x2f236d04753d5b4, - 0x9f4f2726179a2245,0x1d762422c946590, - 0xc722f0ef9d80aad6,0x424d3ad2b7b97ef5, - 0xf8ebad2b84e0d58b,0xd2e0898765a7deb2, - 0x9b934c3b330c8577,0x63cc55f49f88eb2f, - 0xc2781f49ffcfa6d5,0x3cbf6b71c76b25fb, - 0xf316271c7fc3908a,0x8bef464e3945ef7a, - 0x97edd871cfda3a56,0x97758bf0e3cbb5ac, - 0xbde94e8e43d0c8ec,0x3d52eeed1cbea317, - 0xed63a231d4c4fb27,0x4ca7aaa863ee4bdd, - 0x945e455f24fb1cf8,0x8fe8caa93e74ef6a, - 0xb975d6b6ee39e436,0xb3e2fd538e122b44, - 0xe7d34c64a9c85d44,0x60dbbca87196b616, - 0x90e40fbeea1d3a4a,0xbc8955e946fe31cd, - 0xb51d13aea4a488dd,0x6babab6398bdbe41, - 0xe264589a4dcdab14,0xc696963c7eed2dd1, - 0x8d7eb76070a08aec,0xfc1e1de5cf543ca2, - 0xb0de65388cc8ada8,0x3b25a55f43294bcb, - 0xdd15fe86affad912,0x49ef0eb713f39ebe, - 0x8a2dbf142dfcc7ab,0x6e3569326c784337, - 0xacb92ed9397bf996,0x49c2c37f07965404, - 0xd7e77a8f87daf7fb,0xdc33745ec97be906, - 0x86f0ac99b4e8dafd,0x69a028bb3ded71a3, - 0xa8acd7c0222311bc,0xc40832ea0d68ce0c, - 0xd2d80db02aabd62b,0xf50a3fa490c30190, - 0x83c7088e1aab65db,0x792667c6da79e0fa, - 0xa4b8cab1a1563f52,0x577001b891185938, - 0xcde6fd5e09abcf26,0xed4c0226b55e6f86, - 0x80b05e5ac60b6178,0x544f8158315b05b4, - 0xa0dc75f1778e39d6,0x696361ae3db1c721, - 0xc913936dd571c84c,0x3bc3a19cd1e38e9, - 0xfb5878494ace3a5f,0x4ab48a04065c723, - 0x9d174b2dcec0e47b,0x62eb0d64283f9c76, - 0xc45d1df942711d9a,0x3ba5d0bd324f8394, - 0xf5746577930d6500,0xca8f44ec7ee36479, - 0x9968bf6abbe85f20,0x7e998b13cf4e1ecb, - 0xbfc2ef456ae276e8,0x9e3fedd8c321a67e, - 0xefb3ab16c59b14a2,0xc5cfe94ef3ea101e, - 0x95d04aee3b80ece5,0xbba1f1d158724a12, - 0xbb445da9ca61281f,0x2a8a6e45ae8edc97, - 0xea1575143cf97226,0xf52d09d71a3293bd, - 0x924d692ca61be758,0x593c2626705f9c56, - 0xb6e0c377cfa2e12e,0x6f8b2fb00c77836c, - 0xe498f455c38b997a,0xb6dfb9c0f956447, - 0x8edf98b59a373fec,0x4724bd4189bd5eac, - 0xb2977ee300c50fe7,0x58edec91ec2cb657, - 0xdf3d5e9bc0f653e1,0x2f2967b66737e3ed, - 0x8b865b215899f46c,0xbd79e0d20082ee74, - 0xae67f1e9aec07187,0xecd8590680a3aa11, - 0xda01ee641a708de9,0xe80e6f4820cc9495, - 0x884134fe908658b2,0x3109058d147fdcdd, - 0xaa51823e34a7eede,0xbd4b46f0599fd415, - 0xd4e5e2cdc1d1ea96,0x6c9e18ac7007c91a, - 0x850fadc09923329e,0x3e2cf6bc604ddb0, - 0xa6539930bf6bff45,0x84db8346b786151c, - 0xcfe87f7cef46ff16,0xe612641865679a63, - 0x81f14fae158c5f6e,0x4fcb7e8f3f60c07e, - 0xa26da3999aef7749,0xe3be5e330f38f09d, - 0xcb090c8001ab551c,0x5cadf5bfd3072cc5, - 0xfdcb4fa002162a63,0x73d9732fc7c8f7f6, - 0x9e9f11c4014dda7e,0x2867e7fddcdd9afa, - 0xc646d63501a1511d,0xb281e1fd541501b8, - 0xf7d88bc24209a565,0x1f225a7ca91a4226, - 0x9ae757596946075f,0x3375788de9b06958, - 0xc1a12d2fc3978937,0x52d6b1641c83ae, - 0xf209787bb47d6b84,0xc0678c5dbd23a49a, - 0x9745eb4d50ce6332,0xf840b7ba963646e0, - 0xbd176620a501fbff,0xb650e5a93bc3d898, - 0xec5d3fa8ce427aff,0xa3e51f138ab4cebe, - 0x93ba47c980e98cdf,0xc66f336c36b10137, - 0xb8a8d9bbe123f017,0xb80b0047445d4184, - 0xe6d3102ad96cec1d,0xa60dc059157491e5, - 0x9043ea1ac7e41392,0x87c89837ad68db2f, - 0xb454e4a179dd1877,0x29babe4598c311fb, - 0xe16a1dc9d8545e94,0xf4296dd6fef3d67a, - 0x8ce2529e2734bb1d,0x1899e4a65f58660c, - 0xb01ae745b101e9e4,0x5ec05dcff72e7f8f, - 0xdc21a1171d42645d,0x76707543f4fa1f73, - 0x899504ae72497eba,0x6a06494a791c53a8, - 0xabfa45da0edbde69,0x487db9d17636892, - 0xd6f8d7509292d603,0x45a9d2845d3c42b6, - 0x865b86925b9bc5c2,0xb8a2392ba45a9b2, - 0xa7f26836f282b732,0x8e6cac7768d7141e, - 0xd1ef0244af2364ff,0x3207d795430cd926, - 0x8335616aed761f1f,0x7f44e6bd49e807b8, - 0xa402b9c5a8d3a6e7,0x5f16206c9c6209a6, - 0xcd036837130890a1,0x36dba887c37a8c0f, - 0x802221226be55a64,0xc2494954da2c9789, - 0xa02aa96b06deb0fd,0xf2db9baa10b7bd6c, - 0xc83553c5c8965d3d,0x6f92829494e5acc7, - 0xfa42a8b73abbf48c,0xcb772339ba1f17f9, - 0x9c69a97284b578d7,0xff2a760414536efb, - 0xc38413cf25e2d70d,0xfef5138519684aba, - 0xf46518c2ef5b8cd1,0x7eb258665fc25d69, - 0x98bf2f79d5993802,0xef2f773ffbd97a61, - 0xbeeefb584aff8603,0xaafb550ffacfd8fa, - 0xeeaaba2e5dbf6784,0x95ba2a53f983cf38, - 0x952ab45cfa97a0b2,0xdd945a747bf26183, - 0xba756174393d88df,0x94f971119aeef9e4, - 0xe912b9d1478ceb17,0x7a37cd5601aab85d, - 0x91abb422ccb812ee,0xac62e055c10ab33a, - 0xb616a12b7fe617aa,0x577b986b314d6009, - 0xe39c49765fdf9d94,0xed5a7e85fda0b80b, - 0x8e41ade9fbebc27d,0x14588f13be847307, - 0xb1d219647ae6b31c,0x596eb2d8ae258fc8, - 0xde469fbd99a05fe3,0x6fca5f8ed9aef3bb, - 0x8aec23d680043bee,0x25de7bb9480d5854, - 0xada72ccc20054ae9,0xaf561aa79a10ae6a, - 0xd910f7ff28069da4,0x1b2ba1518094da04, - 0x87aa9aff79042286,0x90fb44d2f05d0842, - 0xa99541bf57452b28,0x353a1607ac744a53, - 0xd3fa922f2d1675f2,0x42889b8997915ce8, - 0x847c9b5d7c2e09b7,0x69956135febada11, - 0xa59bc234db398c25,0x43fab9837e699095, - 0xcf02b2c21207ef2e,0x94f967e45e03f4bb, - 0x8161afb94b44f57d,0x1d1be0eebac278f5, - 0xa1ba1ba79e1632dc,0x6462d92a69731732, - 0xca28a291859bbf93,0x7d7b8f7503cfdcfe, - 0xfcb2cb35e702af78,0x5cda735244c3d43e, - 0x9defbf01b061adab,0x3a0888136afa64a7, - 0xc56baec21c7a1916,0x88aaa1845b8fdd0, - 0xf6c69a72a3989f5b,0x8aad549e57273d45, - 0x9a3c2087a63f6399,0x36ac54e2f678864b, - 0xc0cb28a98fcf3c7f,0x84576a1bb416a7dd, - 0xf0fdf2d3f3c30b9f,0x656d44a2a11c51d5, - 0x969eb7c47859e743,0x9f644ae5a4b1b325, - 0xbc4665b596706114,0x873d5d9f0dde1fee, - 0xeb57ff22fc0c7959,0xa90cb506d155a7ea, - 0x9316ff75dd87cbd8,0x9a7f12442d588f2, - 0xb7dcbf5354e9bece,0xc11ed6d538aeb2f, - 0xe5d3ef282a242e81,0x8f1668c8a86da5fa, - 0x8fa475791a569d10,0xf96e017d694487bc, - 0xb38d92d760ec4455,0x37c981dcc395a9ac, - 0xe070f78d3927556a,0x85bbe253f47b1417, - 0x8c469ab843b89562,0x93956d7478ccec8e, - 0xaf58416654a6babb,0x387ac8d1970027b2, - 0xdb2e51bfe9d0696a,0x6997b05fcc0319e, - 0x88fcf317f22241e2,0x441fece3bdf81f03, - 0xab3c2fddeeaad25a,0xd527e81cad7626c3, - 0xd60b3bd56a5586f1,0x8a71e223d8d3b074, - 0x85c7056562757456,0xf6872d5667844e49, - 0xa738c6bebb12d16c,0xb428f8ac016561db, - 0xd106f86e69d785c7,0xe13336d701beba52, - 0x82a45b450226b39c,0xecc0024661173473, - 0xa34d721642b06084,0x27f002d7f95d0190, - 0xcc20ce9bd35c78a5,0x31ec038df7b441f4, - 0xff290242c83396ce,0x7e67047175a15271, - 0x9f79a169bd203e41,0xf0062c6e984d386, - 0xc75809c42c684dd1,0x52c07b78a3e60868, - 0xf92e0c3537826145,0xa7709a56ccdf8a82, - 0x9bbcc7a142b17ccb,0x88a66076400bb691, - 0xc2abf989935ddbfe,0x6acff893d00ea435, - 0xf356f7ebf83552fe,0x583f6b8c4124d43, - 0x98165af37b2153de,0xc3727a337a8b704a, - 0xbe1bf1b059e9a8d6,0x744f18c0592e4c5c, - 0xeda2ee1c7064130c,0x1162def06f79df73, - 0x9485d4d1c63e8be7,0x8addcb5645ac2ba8, - 0xb9a74a0637ce2ee1,0x6d953e2bd7173692, - 0xe8111c87c5c1ba99,0xc8fa8db6ccdd0437, - 0x910ab1d4db9914a0,0x1d9c9892400a22a2, - 0xb54d5e4a127f59c8,0x2503beb6d00cab4b, - 0xe2a0b5dc971f303a,0x2e44ae64840fd61d, - 0x8da471a9de737e24,0x5ceaecfed289e5d2, - 0xb10d8e1456105dad,0x7425a83e872c5f47, - 0xdd50f1996b947518,0xd12f124e28f77719, - 0x8a5296ffe33cc92f,0x82bd6b70d99aaa6f, - 0xace73cbfdc0bfb7b,0x636cc64d1001550b, - 0xd8210befd30efa5a,0x3c47f7e05401aa4e, - 0x8714a775e3e95c78,0x65acfaec34810a71, - 0xa8d9d1535ce3b396,0x7f1839a741a14d0d, - 0xd31045a8341ca07c,0x1ede48111209a050, - 0x83ea2b892091e44d,0x934aed0aab460432, - 0xa4e4b66b68b65d60,0xf81da84d5617853f, - 0xce1de40642e3f4b9,0x36251260ab9d668e, - 0x80d2ae83e9ce78f3,0xc1d72b7c6b426019, - 0xa1075a24e4421730,0xb24cf65b8612f81f, - 0xc94930ae1d529cfc,0xdee033f26797b627, - 0xfb9b7cd9a4a7443c,0x169840ef017da3b1, - 0x9d412e0806e88aa5,0x8e1f289560ee864e, - 0xc491798a08a2ad4e,0xf1a6f2bab92a27e2, - 0xf5b5d7ec8acb58a2,0xae10af696774b1db, - 0x9991a6f3d6bf1765,0xacca6da1e0a8ef29, - 0xbff610b0cc6edd3f,0x17fd090a58d32af3, - 0xeff394dcff8a948e,0xddfc4b4cef07f5b0, - 0x95f83d0a1fb69cd9,0x4abdaf101564f98e, - 0xbb764c4ca7a4440f,0x9d6d1ad41abe37f1, - 0xea53df5fd18d5513,0x84c86189216dc5ed, - 0x92746b9be2f8552c,0x32fd3cf5b4e49bb4, - 0xb7118682dbb66a77,0x3fbc8c33221dc2a1, - 0xe4d5e82392a40515,0xfabaf3feaa5334a, - 0x8f05b1163ba6832d,0x29cb4d87f2a7400e, - 0xb2c71d5bca9023f8,0x743e20e9ef511012, - 0xdf78e4b2bd342cf6,0x914da9246b255416, - 0x8bab8eefb6409c1a,0x1ad089b6c2f7548e, - 0xae9672aba3d0c320,0xa184ac2473b529b1, - 0xda3c0f568cc4f3e8,0xc9e5d72d90a2741e, - 0x8865899617fb1871,0x7e2fa67c7a658892, - 0xaa7eebfb9df9de8d,0xddbb901b98feeab7, - 0xd51ea6fa85785631,0x552a74227f3ea565, - 0x8533285c936b35de,0xd53a88958f87275f, - 0xa67ff273b8460356,0x8a892abaf368f137, - 0xd01fef10a657842c,0x2d2b7569b0432d85, - 0x8213f56a67f6b29b,0x9c3b29620e29fc73, - 0xa298f2c501f45f42,0x8349f3ba91b47b8f, - 0xcb3f2f7642717713,0x241c70a936219a73, - 0xfe0efb53d30dd4d7,0xed238cd383aa0110, - 0x9ec95d1463e8a506,0xf4363804324a40aa, - 0xc67bb4597ce2ce48,0xb143c6053edcd0d5, - 0xf81aa16fdc1b81da,0xdd94b7868e94050a, - 0x9b10a4e5e9913128,0xca7cf2b4191c8326, - 0xc1d4ce1f63f57d72,0xfd1c2f611f63a3f0, - 0xf24a01a73cf2dccf,0xbc633b39673c8cec, - 0x976e41088617ca01,0xd5be0503e085d813, - 0xbd49d14aa79dbc82,0x4b2d8644d8a74e18, - 0xec9c459d51852ba2,0xddf8e7d60ed1219e, - 0x93e1ab8252f33b45,0xcabb90e5c942b503, - 0xb8da1662e7b00a17,0x3d6a751f3b936243, - 0xe7109bfba19c0c9d,0xcc512670a783ad4, - 0x906a617d450187e2,0x27fb2b80668b24c5, - 0xb484f9dc9641e9da,0xb1f9f660802dedf6, - 0xe1a63853bbd26451,0x5e7873f8a0396973, - 0x8d07e33455637eb2,0xdb0b487b6423e1e8, - 0xb049dc016abc5e5f,0x91ce1a9a3d2cda62, - 0xdc5c5301c56b75f7,0x7641a140cc7810fb, - 0x89b9b3e11b6329ba,0xa9e904c87fcb0a9d, - 0xac2820d9623bf429,0x546345fa9fbdcd44, - 0xd732290fbacaf133,0xa97c177947ad4095, - 0x867f59a9d4bed6c0,0x49ed8eabcccc485d, - 0xa81f301449ee8c70,0x5c68f256bfff5a74, - 0xd226fc195c6a2f8c,0x73832eec6fff3111, - 0x83585d8fd9c25db7,0xc831fd53c5ff7eab, - 0xa42e74f3d032f525,0xba3e7ca8b77f5e55, - 0xcd3a1230c43fb26f,0x28ce1bd2e55f35eb, - 0x80444b5e7aa7cf85,0x7980d163cf5b81b3, - 0xa0555e361951c366,0xd7e105bcc332621f, - 0xc86ab5c39fa63440,0x8dd9472bf3fefaa7, - 0xfa856334878fc150,0xb14f98f6f0feb951, - 0x9c935e00d4b9d8d2,0x6ed1bf9a569f33d3, - 0xc3b8358109e84f07,0xa862f80ec4700c8, - 0xf4a642e14c6262c8,0xcd27bb612758c0fa, - 0x98e7e9cccfbd7dbd,0x8038d51cb897789c, - 0xbf21e44003acdd2c,0xe0470a63e6bd56c3, - 0xeeea5d5004981478,0x1858ccfce06cac74, - 0x95527a5202df0ccb,0xf37801e0c43ebc8, - 0xbaa718e68396cffd,0xd30560258f54e6ba, - 0xe950df20247c83fd,0x47c6b82ef32a2069, - 0x91d28b7416cdd27e,0x4cdc331d57fa5441, - 0xb6472e511c81471d,0xe0133fe4adf8e952, - 0xe3d8f9e563a198e5,0x58180fddd97723a6, - 0x8e679c2f5e44ff8f,0x570f09eaa7ea7648,}; -using powers = powers_template<>; - -} - -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/float_common.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/float_common.h deleted file mode 100644 index c8e7e4fc9..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/float_common.h +++ /dev/null @@ -1,362 +0,0 @@ -#ifndef FASTFLOAT_FLOAT_COMMON_H -#define FASTFLOAT_FLOAT_COMMON_H - -#include -#include -#include -#include - -#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) \ - || defined(__amd64) || defined(__aarch64__) || defined(_M_ARM64) \ - || defined(__MINGW64__) \ - || defined(__s390x__) \ - || (defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || defined(__PPC64LE__)) \ - || defined(__EMSCRIPTEN__)) -#define FASTFLOAT_64BIT -#elif (defined(__i386) || defined(__i386__) || defined(_M_IX86) \ - || defined(__arm__) || defined(_M_ARM) \ - || defined(__MINGW32__)) -#define FASTFLOAT_32BIT -#else - // Need to check incrementally, since SIZE_MAX is a size_t, avoid overflow. - // We can never tell the register width, but the SIZE_MAX is a good approximation. - // UINTPTR_MAX and INTPTR_MAX are optional, so avoid them for max portability. - #if SIZE_MAX == 0xffff - #error Unknown platform (16-bit, unsupported) - #elif SIZE_MAX == 0xffffffff - #define FASTFLOAT_32BIT - #elif SIZE_MAX == 0xffffffffffffffff - #define FASTFLOAT_64BIT - #else - #error Unknown platform (not 32-bit, not 64-bit?) - #endif -#endif - -#if ((defined(_WIN32) || defined(_WIN64)) && !defined(__clang__)) -#include -#endif - -#if defined(_MSC_VER) && !defined(__clang__) -#define FASTFLOAT_VISUAL_STUDIO 1 -#endif - -#ifdef _WIN32 -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#else -#if defined(__APPLE__) || defined(__FreeBSD__) -#include -#elif defined(sun) || defined(__sun) -#include -#else -#include -#endif -# -#ifndef __BYTE_ORDER__ -// safe choice -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#endif -# -#ifndef __ORDER_LITTLE_ENDIAN__ -// safe choice -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#endif -# -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#else -#define FASTFLOAT_IS_BIG_ENDIAN 1 -#endif -#endif - -#ifdef FASTFLOAT_VISUAL_STUDIO -#define fastfloat_really_inline __forceinline -#else -#define fastfloat_really_inline inline __attribute__((always_inline)) -#endif - -#ifndef FASTFLOAT_ASSERT -#define FASTFLOAT_ASSERT(x) { if (!(x)) abort(); } -#endif - -#ifndef FASTFLOAT_DEBUG_ASSERT -#include -#define FASTFLOAT_DEBUG_ASSERT(x) assert(x) -#endif - -// rust style `try!()` macro, or `?` operator -#define FASTFLOAT_TRY(x) { if (!(x)) return false; } - -namespace fast_float { - -// Compares two ASCII strings in a case insensitive manner. -inline bool fastfloat_strncasecmp(const char *input1, const char *input2, - size_t length) { - char running_diff{0}; - for (size_t i = 0; i < length; i++) { - running_diff |= (input1[i] ^ input2[i]); - } - return (running_diff == 0) || (running_diff == 32); -} - -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif - -// a pointer and a length to a contiguous block of memory -template -struct span { - const T* ptr; - size_t length; - span(const T* _ptr, size_t _length) : ptr(_ptr), length(_length) {} - span() : ptr(nullptr), length(0) {} - - constexpr size_t len() const noexcept { - return length; - } - - const T& operator[](size_t index) const noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - return ptr[index]; - } -}; - -struct value128 { - uint64_t low; - uint64_t high; - value128(uint64_t _low, uint64_t _high) : low(_low), high(_high) {} - value128() : low(0), high(0) {} -}; - -/* result might be undefined when input_num is zero */ -fastfloat_really_inline int leading_zeroes(uint64_t input_num) { - assert(input_num > 0); -#ifdef FASTFLOAT_VISUAL_STUDIO - #if defined(_M_X64) || defined(_M_ARM64) - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - _BitScanReverse64(&leading_zero, input_num); - return (int)(63 - leading_zero); - #else - int last_bit = 0; - if(input_num & uint64_t(0xffffffff00000000)) input_num >>= 32, last_bit |= 32; - if(input_num & uint64_t( 0xffff0000)) input_num >>= 16, last_bit |= 16; - if(input_num & uint64_t( 0xff00)) input_num >>= 8, last_bit |= 8; - if(input_num & uint64_t( 0xf0)) input_num >>= 4, last_bit |= 4; - if(input_num & uint64_t( 0xc)) input_num >>= 2, last_bit |= 2; - if(input_num & uint64_t( 0x2)) input_num >>= 1, last_bit |= 1; - return 63 - last_bit; - #endif -#else - return __builtin_clzll(input_num); -#endif -} - -#ifdef FASTFLOAT_32BIT - -// slow emulation routine for 32-bit -fastfloat_really_inline uint64_t emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} - -// slow emulation routine for 32-bit -#if !defined(__MINGW64__) -fastfloat_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, - uint64_t *hi) { - uint64_t ad = emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif // !__MINGW64__ - -#endif // FASTFLOAT_32BIT - - -// compute 64-bit a*b -fastfloat_really_inline value128 full_multiplication(uint64_t a, - uint64_t b) { - value128 answer; -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emulate - answer.high = __umulh(a, b); - answer.low = a * b; -#elif defined(FASTFLOAT_32BIT) || (defined(_WIN64) && !defined(__clang__)) - answer.low = _umul128(a, b, &answer.high); // _umul128 not available on ARM64 -#elif defined(FASTFLOAT_64BIT) - __uint128_t r = ((__uint128_t)a) * b; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#else - #error Not implemented -#endif - return answer; -} - -struct adjusted_mantissa { - uint64_t mantissa{0}; - int32_t power2{0}; // a negative value indicates an invalid result - adjusted_mantissa() = default; - bool operator==(const adjusted_mantissa &o) const { - return mantissa == o.mantissa && power2 == o.power2; - } - bool operator!=(const adjusted_mantissa &o) const { - return mantissa != o.mantissa || power2 != o.power2; - } -}; - -// Bias so we can get the real exponent with an invalid adjusted_mantissa. -constexpr static int32_t invalid_am_bias = -0x8000; - -constexpr static double powers_of_ten_double[] = { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, - 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22}; -constexpr static float powers_of_ten_float[] = {1e0, 1e1, 1e2, 1e3, 1e4, 1e5, - 1e6, 1e7, 1e8, 1e9, 1e10}; - -template struct binary_format { - static inline constexpr int mantissa_explicit_bits(); - static inline constexpr int minimum_exponent(); - static inline constexpr int infinite_power(); - static inline constexpr int sign_index(); - static inline constexpr int min_exponent_fast_path(); - static inline constexpr int max_exponent_fast_path(); - static inline constexpr int max_exponent_round_to_even(); - static inline constexpr int min_exponent_round_to_even(); - static inline constexpr uint64_t max_mantissa_fast_path(); - static inline constexpr int largest_power_of_ten(); - static inline constexpr int smallest_power_of_ten(); - static inline constexpr T exact_power_of_ten(int64_t power); - static inline constexpr size_t max_digits(); -}; - -template <> inline constexpr int binary_format::mantissa_explicit_bits() { - return 52; -} -template <> inline constexpr int binary_format::mantissa_explicit_bits() { - return 23; -} - -template <> inline constexpr int binary_format::max_exponent_round_to_even() { - return 23; -} - -template <> inline constexpr int binary_format::max_exponent_round_to_even() { - return 10; -} - -template <> inline constexpr int binary_format::min_exponent_round_to_even() { - return -4; -} - -template <> inline constexpr int binary_format::min_exponent_round_to_even() { - return -17; -} - -template <> inline constexpr int binary_format::minimum_exponent() { - return -1023; -} -template <> inline constexpr int binary_format::minimum_exponent() { - return -127; -} - -template <> inline constexpr int binary_format::infinite_power() { - return 0x7FF; -} -template <> inline constexpr int binary_format::infinite_power() { - return 0xFF; -} - -template <> inline constexpr int binary_format::sign_index() { return 63; } -template <> inline constexpr int binary_format::sign_index() { return 31; } - -template <> inline constexpr int binary_format::min_exponent_fast_path() { -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return 0; -#else - return -22; -#endif -} -template <> inline constexpr int binary_format::min_exponent_fast_path() { -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return 0; -#else - return -10; -#endif -} - -template <> inline constexpr int binary_format::max_exponent_fast_path() { - return 22; -} -template <> inline constexpr int binary_format::max_exponent_fast_path() { - return 10; -} - -template <> inline constexpr uint64_t binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} -template <> inline constexpr uint64_t binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} - -template <> -inline constexpr double binary_format::exact_power_of_ten(int64_t power) { - return powers_of_ten_double[power]; -} -template <> -inline constexpr float binary_format::exact_power_of_ten(int64_t power) { - - return powers_of_ten_float[power]; -} - - -template <> -inline constexpr int binary_format::largest_power_of_ten() { - return 308; -} -template <> -inline constexpr int binary_format::largest_power_of_ten() { - return 38; -} - -template <> -inline constexpr int binary_format::smallest_power_of_ten() { - return -342; -} -template <> -inline constexpr int binary_format::smallest_power_of_ten() { - return -65; -} - -template <> inline constexpr size_t binary_format::max_digits() { - return 769; -} -template <> inline constexpr size_t binary_format::max_digits() { - return 114; -} - -template -fastfloat_really_inline void to_float(bool negative, adjusted_mantissa am, T &value) { - uint64_t word = am.mantissa; - word |= uint64_t(am.power2) << binary_format::mantissa_explicit_bits(); - word = negative - ? word | (uint64_t(1) << binary_format::sign_index()) : word; -#if FASTFLOAT_IS_BIG_ENDIAN == 1 - if (std::is_same::value) { - ::memcpy(&value, (char *)&word + 4, sizeof(T)); // extract value at offset 4-7 if float on big-endian - } else { - ::memcpy(&value, &word, sizeof(T)); - } -#else - // For little-endian systems: - ::memcpy(&value, &word, sizeof(T)); -#endif -} - -} // namespace fast_float - -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/parse_number.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/parse_number.h deleted file mode 100644 index 62ae3b039..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/parse_number.h +++ /dev/null @@ -1,113 +0,0 @@ -#ifndef FASTFLOAT_PARSE_NUMBER_H -#define FASTFLOAT_PARSE_NUMBER_H - -#include "ascii_number.h" -#include "decimal_to_binary.h" -#include "digit_comparison.h" - -#include -#include -#include -#include - -namespace fast_float { - - -namespace detail { -/** - * Special case +inf, -inf, nan, infinity, -infinity. - * The case comparisons could be made much faster given that we know that the - * strings a null-free and fixed. - **/ -template -from_chars_result parse_infnan(const char *first, const char *last, T &value) noexcept { - from_chars_result answer; - answer.ptr = first; - answer.ec = std::errc(); // be optimistic - bool minusSign = false; - if (*first == '-') { // assume first < last, so dereference without checks; C++17 20.19.3.(7.1) explicitly forbids '+' here - minusSign = true; - ++first; - } - if (last - first >= 3) { - if (fastfloat_strncasecmp(first, "nan", 3)) { - answer.ptr = (first += 3); - value = minusSign ? -std::numeric_limits::quiet_NaN() : std::numeric_limits::quiet_NaN(); - // Check for possible nan(n-char-seq-opt), C++17 20.19.3.7, C11 7.20.1.3.3. At least MSVC produces nan(ind) and nan(snan). - if(first != last && *first == '(') { - for(const char* ptr = first + 1; ptr != last; ++ptr) { - if (*ptr == ')') { - answer.ptr = ptr + 1; // valid nan(n-char-seq-opt) - break; - } - else if(!(('a' <= *ptr && *ptr <= 'z') || ('A' <= *ptr && *ptr <= 'Z') || ('0' <= *ptr && *ptr <= '9') || *ptr == '_')) - break; // forbidden char, not nan(n-char-seq-opt) - } - } - return answer; - } - if (fastfloat_strncasecmp(first, "inf", 3)) { - if ((last - first >= 8) && fastfloat_strncasecmp(first + 3, "inity", 5)) { - answer.ptr = first + 8; - } else { - answer.ptr = first + 3; - } - value = minusSign ? -std::numeric_limits::infinity() : std::numeric_limits::infinity(); - return answer; - } - } - answer.ec = std::errc::invalid_argument; - return answer; -} - -} // namespace detail - -template -from_chars_result from_chars(const char *first, const char *last, - T &value, chars_format fmt /*= chars_format::general*/) noexcept { - return from_chars_advanced(first, last, value, parse_options{fmt}); -} - -template -from_chars_result from_chars_advanced(const char *first, const char *last, - T &value, parse_options options) noexcept { - - static_assert (std::is_same::value || std::is_same::value, "only float and double are supported"); - - - from_chars_result answer; - if (first == last) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } - parsed_number_string pns = parse_number_string(first, last, options); - if (!pns.valid) { - return detail::parse_infnan(first, last, value); - } - answer.ec = std::errc(); // be optimistic - answer.ptr = pns.lastmatch; - // Next is Clinger's fast path. - if (binary_format::min_exponent_fast_path() <= pns.exponent && pns.exponent <= binary_format::max_exponent_fast_path() && pns.mantissa <=binary_format::max_mantissa_fast_path() && !pns.too_many_digits) { - value = T(pns.mantissa); - if (pns.exponent < 0) { value = value / binary_format::exact_power_of_ten(-pns.exponent); } - else { value = value * binary_format::exact_power_of_ten(pns.exponent); } - if (pns.negative) { value = -value; } - return answer; - } - adjusted_mantissa am = compute_float>(pns.exponent, pns.mantissa); - if(pns.too_many_digits && am.power2 >= 0) { - if(am != compute_float>(pns.exponent, pns.mantissa + 1)) { - am = compute_error>(pns.exponent, pns.mantissa); - } - } - // If we called compute_float>(pns.exponent, pns.mantissa) and we have an invalid power (am.power2 < 0), - // then we need to go the long way around again. This is very uncommon. - if(am.power2 < 0) { am = digit_comp(pns, am); } - to_float(pns.negative, am, value); - return answer; -} - -} // namespace fast_float - -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/simple_decimal_conversion.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/simple_decimal_conversion.h deleted file mode 100644 index e87801480..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/include/fast_float/simple_decimal_conversion.h +++ /dev/null @@ -1,360 +0,0 @@ -#ifndef FASTFLOAT_GENERIC_DECIMAL_TO_BINARY_H -#define FASTFLOAT_GENERIC_DECIMAL_TO_BINARY_H - -/** - * This code is meant to handle the case where we have more than 19 digits. - * - * It is based on work by Nigel Tao (at https://github.com/google/wuffs/) - * who credits Ken Thompson for the design (via a reference to the Go source - * code). - * - * Rob Pike suggested that this algorithm be called "Simple Decimal Conversion". - * - * It is probably not very fast but it is a fallback that should almost never - * be used in real life. Though it is not fast, it is "easily" understood and debugged. - **/ -#include "ascii_number.h" -#include "decimal_to_binary.h" -#include - -namespace fast_float { - -namespace detail { - -// remove all final zeroes -inline void trim(decimal &h) { - while ((h.num_digits > 0) && (h.digits[h.num_digits - 1] == 0)) { - h.num_digits--; - } -} - - - -inline uint32_t number_of_digits_decimal_left_shift(const decimal &h, uint32_t shift) { - shift &= 63; - constexpr uint16_t number_of_digits_decimal_left_shift_table[65] = { - 0x0000, 0x0800, 0x0801, 0x0803, 0x1006, 0x1009, 0x100D, 0x1812, 0x1817, - 0x181D, 0x2024, 0x202B, 0x2033, 0x203C, 0x2846, 0x2850, 0x285B, 0x3067, - 0x3073, 0x3080, 0x388E, 0x389C, 0x38AB, 0x38BB, 0x40CC, 0x40DD, 0x40EF, - 0x4902, 0x4915, 0x4929, 0x513E, 0x5153, 0x5169, 0x5180, 0x5998, 0x59B0, - 0x59C9, 0x61E3, 0x61FD, 0x6218, 0x6A34, 0x6A50, 0x6A6D, 0x6A8B, 0x72AA, - 0x72C9, 0x72E9, 0x7B0A, 0x7B2B, 0x7B4D, 0x8370, 0x8393, 0x83B7, 0x83DC, - 0x8C02, 0x8C28, 0x8C4F, 0x9477, 0x949F, 0x94C8, 0x9CF2, 0x051C, 0x051C, - 0x051C, 0x051C, - }; - uint32_t x_a = number_of_digits_decimal_left_shift_table[shift]; - uint32_t x_b = number_of_digits_decimal_left_shift_table[shift + 1]; - uint32_t num_new_digits = x_a >> 11; - uint32_t pow5_a = 0x7FF & x_a; - uint32_t pow5_b = 0x7FF & x_b; - constexpr uint8_t - number_of_digits_decimal_left_shift_table_powers_of_5[0x051C] = { - 5, 2, 5, 1, 2, 5, 6, 2, 5, 3, 1, 2, 5, 1, 5, 6, 2, 5, 7, 8, 1, 2, 5, 3, - 9, 0, 6, 2, 5, 1, 9, 5, 3, 1, 2, 5, 9, 7, 6, 5, 6, 2, 5, 4, 8, 8, 2, 8, - 1, 2, 5, 2, 4, 4, 1, 4, 0, 6, 2, 5, 1, 2, 2, 0, 7, 0, 3, 1, 2, 5, 6, 1, - 0, 3, 5, 1, 5, 6, 2, 5, 3, 0, 5, 1, 7, 5, 7, 8, 1, 2, 5, 1, 5, 2, 5, 8, - 7, 8, 9, 0, 6, 2, 5, 7, 6, 2, 9, 3, 9, 4, 5, 3, 1, 2, 5, 3, 8, 1, 4, 6, - 9, 7, 2, 6, 5, 6, 2, 5, 1, 9, 0, 7, 3, 4, 8, 6, 3, 2, 8, 1, 2, 5, 9, 5, - 3, 6, 7, 4, 3, 1, 6, 4, 0, 6, 2, 5, 4, 7, 6, 8, 3, 7, 1, 5, 8, 2, 0, 3, - 1, 2, 5, 2, 3, 8, 4, 1, 8, 5, 7, 9, 1, 0, 1, 5, 6, 2, 5, 1, 1, 9, 2, 0, - 9, 2, 8, 9, 5, 5, 0, 7, 8, 1, 2, 5, 5, 9, 6, 0, 4, 6, 4, 4, 7, 7, 5, 3, - 9, 0, 6, 2, 5, 2, 9, 8, 0, 2, 3, 2, 2, 3, 8, 7, 6, 9, 5, 3, 1, 2, 5, 1, - 4, 9, 0, 1, 1, 6, 1, 1, 9, 3, 8, 4, 7, 6, 5, 6, 2, 5, 7, 4, 5, 0, 5, 8, - 0, 5, 9, 6, 9, 2, 3, 8, 2, 8, 1, 2, 5, 3, 7, 2, 5, 2, 9, 0, 2, 9, 8, 4, - 6, 1, 9, 1, 4, 0, 6, 2, 5, 1, 8, 6, 2, 6, 4, 5, 1, 4, 9, 2, 3, 0, 9, 5, - 7, 0, 3, 1, 2, 5, 9, 3, 1, 3, 2, 2, 5, 7, 4, 6, 1, 5, 4, 7, 8, 5, 1, 5, - 6, 2, 5, 4, 6, 5, 6, 6, 1, 2, 8, 7, 3, 0, 7, 7, 3, 9, 2, 5, 7, 8, 1, 2, - 5, 2, 3, 2, 8, 3, 0, 6, 4, 3, 6, 5, 3, 8, 6, 9, 6, 2, 8, 9, 0, 6, 2, 5, - 1, 1, 6, 4, 1, 5, 3, 2, 1, 8, 2, 6, 9, 3, 4, 8, 1, 4, 4, 5, 3, 1, 2, 5, - 5, 8, 2, 0, 7, 6, 6, 0, 9, 1, 3, 4, 6, 7, 4, 0, 7, 2, 2, 6, 5, 6, 2, 5, - 2, 9, 1, 0, 3, 8, 3, 0, 4, 5, 6, 7, 3, 3, 7, 0, 3, 6, 1, 3, 2, 8, 1, 2, - 5, 1, 4, 5, 5, 1, 9, 1, 5, 2, 2, 8, 3, 6, 6, 8, 5, 1, 8, 0, 6, 6, 4, 0, - 6, 2, 5, 7, 2, 7, 5, 9, 5, 7, 6, 1, 4, 1, 8, 3, 4, 2, 5, 9, 0, 3, 3, 2, - 0, 3, 1, 2, 5, 3, 6, 3, 7, 9, 7, 8, 8, 0, 7, 0, 9, 1, 7, 1, 2, 9, 5, 1, - 6, 6, 0, 1, 5, 6, 2, 5, 1, 8, 1, 8, 9, 8, 9, 4, 0, 3, 5, 4, 5, 8, 5, 6, - 4, 7, 5, 8, 3, 0, 0, 7, 8, 1, 2, 5, 9, 0, 9, 4, 9, 4, 7, 0, 1, 7, 7, 2, - 9, 2, 8, 2, 3, 7, 9, 1, 5, 0, 3, 9, 0, 6, 2, 5, 4, 5, 4, 7, 4, 7, 3, 5, - 0, 8, 8, 6, 4, 6, 4, 1, 1, 8, 9, 5, 7, 5, 1, 9, 5, 3, 1, 2, 5, 2, 2, 7, - 3, 7, 3, 6, 7, 5, 4, 4, 3, 2, 3, 2, 0, 5, 9, 4, 7, 8, 7, 5, 9, 7, 6, 5, - 6, 2, 5, 1, 1, 3, 6, 8, 6, 8, 3, 7, 7, 2, 1, 6, 1, 6, 0, 2, 9, 7, 3, 9, - 3, 7, 9, 8, 8, 2, 8, 1, 2, 5, 5, 6, 8, 4, 3, 4, 1, 8, 8, 6, 0, 8, 0, 8, - 0, 1, 4, 8, 6, 9, 6, 8, 9, 9, 4, 1, 4, 0, 6, 2, 5, 2, 8, 4, 2, 1, 7, 0, - 9, 4, 3, 0, 4, 0, 4, 0, 0, 7, 4, 3, 4, 8, 4, 4, 9, 7, 0, 7, 0, 3, 1, 2, - 5, 1, 4, 2, 1, 0, 8, 5, 4, 7, 1, 5, 2, 0, 2, 0, 0, 3, 7, 1, 7, 4, 2, 2, - 4, 8, 5, 3, 5, 1, 5, 6, 2, 5, 7, 1, 0, 5, 4, 2, 7, 3, 5, 7, 6, 0, 1, 0, - 0, 1, 8, 5, 8, 7, 1, 1, 2, 4, 2, 6, 7, 5, 7, 8, 1, 2, 5, 3, 5, 5, 2, 7, - 1, 3, 6, 7, 8, 8, 0, 0, 5, 0, 0, 9, 2, 9, 3, 5, 5, 6, 2, 1, 3, 3, 7, 8, - 9, 0, 6, 2, 5, 1, 7, 7, 6, 3, 5, 6, 8, 3, 9, 4, 0, 0, 2, 5, 0, 4, 6, 4, - 6, 7, 7, 8, 1, 0, 6, 6, 8, 9, 4, 5, 3, 1, 2, 5, 8, 8, 8, 1, 7, 8, 4, 1, - 9, 7, 0, 0, 1, 2, 5, 2, 3, 2, 3, 3, 8, 9, 0, 5, 3, 3, 4, 4, 7, 2, 6, 5, - 6, 2, 5, 4, 4, 4, 0, 8, 9, 2, 0, 9, 8, 5, 0, 0, 6, 2, 6, 1, 6, 1, 6, 9, - 4, 5, 2, 6, 6, 7, 2, 3, 6, 3, 2, 8, 1, 2, 5, 2, 2, 2, 0, 4, 4, 6, 0, 4, - 9, 2, 5, 0, 3, 1, 3, 0, 8, 0, 8, 4, 7, 2, 6, 3, 3, 3, 6, 1, 8, 1, 6, 4, - 0, 6, 2, 5, 1, 1, 1, 0, 2, 2, 3, 0, 2, 4, 6, 2, 5, 1, 5, 6, 5, 4, 0, 4, - 2, 3, 6, 3, 1, 6, 6, 8, 0, 9, 0, 8, 2, 0, 3, 1, 2, 5, 5, 5, 5, 1, 1, 1, - 5, 1, 2, 3, 1, 2, 5, 7, 8, 2, 7, 0, 2, 1, 1, 8, 1, 5, 8, 3, 4, 0, 4, 5, - 4, 1, 0, 1, 5, 6, 2, 5, 2, 7, 7, 5, 5, 5, 7, 5, 6, 1, 5, 6, 2, 8, 9, 1, - 3, 5, 1, 0, 5, 9, 0, 7, 9, 1, 7, 0, 2, 2, 7, 0, 5, 0, 7, 8, 1, 2, 5, 1, - 3, 8, 7, 7, 7, 8, 7, 8, 0, 7, 8, 1, 4, 4, 5, 6, 7, 5, 5, 2, 9, 5, 3, 9, - 5, 8, 5, 1, 1, 3, 5, 2, 5, 3, 9, 0, 6, 2, 5, 6, 9, 3, 8, 8, 9, 3, 9, 0, - 3, 9, 0, 7, 2, 2, 8, 3, 7, 7, 6, 4, 7, 6, 9, 7, 9, 2, 5, 5, 6, 7, 6, 2, - 6, 9, 5, 3, 1, 2, 5, 3, 4, 6, 9, 4, 4, 6, 9, 5, 1, 9, 5, 3, 6, 1, 4, 1, - 8, 8, 8, 2, 3, 8, 4, 8, 9, 6, 2, 7, 8, 3, 8, 1, 3, 4, 7, 6, 5, 6, 2, 5, - 1, 7, 3, 4, 7, 2, 3, 4, 7, 5, 9, 7, 6, 8, 0, 7, 0, 9, 4, 4, 1, 1, 9, 2, - 4, 4, 8, 1, 3, 9, 1, 9, 0, 6, 7, 3, 8, 2, 8, 1, 2, 5, 8, 6, 7, 3, 6, 1, - 7, 3, 7, 9, 8, 8, 4, 0, 3, 5, 4, 7, 2, 0, 5, 9, 6, 2, 2, 4, 0, 6, 9, 5, - 9, 5, 3, 3, 6, 9, 1, 4, 0, 6, 2, 5, - }; - const uint8_t *pow5 = - &number_of_digits_decimal_left_shift_table_powers_of_5[pow5_a]; - uint32_t i = 0; - uint32_t n = pow5_b - pow5_a; - for (; i < n; i++) { - if (i >= h.num_digits) { - return num_new_digits - 1; - } else if (h.digits[i] == pow5[i]) { - continue; - } else if (h.digits[i] < pow5[i]) { - return num_new_digits - 1; - } else { - return num_new_digits; - } - } - return num_new_digits; -} - -inline uint64_t round(decimal &h) { - if ((h.num_digits == 0) || (h.decimal_point < 0)) { - return 0; - } else if (h.decimal_point > 18) { - return UINT64_MAX; - } - // at this point, we know that h.decimal_point >= 0 - uint32_t dp = uint32_t(h.decimal_point); - uint64_t n = 0; - for (uint32_t i = 0; i < dp; i++) { - n = (10 * n) + ((i < h.num_digits) ? h.digits[i] : 0); - } - bool round_up = false; - if (dp < h.num_digits) { - round_up = h.digits[dp] >= 5; // normally, we round up - // but we may need to round to even! - if ((h.digits[dp] == 5) && (dp + 1 == h.num_digits)) { - round_up = h.truncated || ((dp > 0) && (1 & h.digits[dp - 1])); - } - } - if (round_up) { - n++; - } - return n; -} - -// computes h * 2^-shift -inline void decimal_left_shift(decimal &h, uint32_t shift) { - if (h.num_digits == 0) { - return; - } - uint32_t num_new_digits = number_of_digits_decimal_left_shift(h, shift); - int32_t read_index = int32_t(h.num_digits - 1); - uint32_t write_index = h.num_digits - 1 + num_new_digits; - uint64_t n = 0; - - while (read_index >= 0) { - n += uint64_t(h.digits[read_index]) << shift; - uint64_t quotient = n / 10; - uint64_t remainder = n - (10 * quotient); - if (write_index < max_digits) { - h.digits[write_index] = uint8_t(remainder); - } else if (remainder > 0) { - h.truncated = true; - } - n = quotient; - write_index--; - read_index--; - } - while (n > 0) { - uint64_t quotient = n / 10; - uint64_t remainder = n - (10 * quotient); - if (write_index < max_digits) { - h.digits[write_index] = uint8_t(remainder); - } else if (remainder > 0) { - h.truncated = true; - } - n = quotient; - write_index--; - } - h.num_digits += num_new_digits; - if (h.num_digits > max_digits) { - h.num_digits = max_digits; - } - h.decimal_point += int32_t(num_new_digits); - trim(h); -} - -// computes h * 2^shift -inline void decimal_right_shift(decimal &h, uint32_t shift) { - uint32_t read_index = 0; - uint32_t write_index = 0; - - uint64_t n = 0; - - while ((n >> shift) == 0) { - if (read_index < h.num_digits) { - n = (10 * n) + h.digits[read_index++]; - } else if (n == 0) { - return; - } else { - while ((n >> shift) == 0) { - n = 10 * n; - read_index++; - } - break; - } - } - h.decimal_point -= int32_t(read_index - 1); - if (h.decimal_point < -decimal_point_range) { // it is zero - h.num_digits = 0; - h.decimal_point = 0; - h.negative = false; - h.truncated = false; - return; - } - uint64_t mask = (uint64_t(1) << shift) - 1; - while (read_index < h.num_digits) { - uint8_t new_digit = uint8_t(n >> shift); - n = (10 * (n & mask)) + h.digits[read_index++]; - h.digits[write_index++] = new_digit; - } - while (n > 0) { - uint8_t new_digit = uint8_t(n >> shift); - n = 10 * (n & mask); - if (write_index < max_digits) { - h.digits[write_index++] = new_digit; - } else if (new_digit > 0) { - h.truncated = true; - } - } - h.num_digits = write_index; - trim(h); -} - -} // namespace detail - -template -adjusted_mantissa compute_float(decimal &d) { - adjusted_mantissa answer; - if (d.num_digits == 0) { - // should be zero - answer.power2 = 0; - answer.mantissa = 0; - return answer; - } - // At this point, going further, we can assume that d.num_digits > 0. - // - // We want to guard against excessive decimal point values because - // they can result in long running times. Indeed, we do - // shifts by at most 60 bits. We have that log(10**400)/log(2**60) ~= 22 - // which is fine, but log(10**299995)/log(2**60) ~= 16609 which is not - // fine (runs for a long time). - // - if(d.decimal_point < -324) { - // We have something smaller than 1e-324 which is always zero - // in binary64 and binary32. - // It should be zero. - answer.power2 = 0; - answer.mantissa = 0; - return answer; - } else if(d.decimal_point >= 310) { - // We have something at least as large as 0.1e310 which is - // always infinite. - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } - constexpr uint32_t max_shift = 60; - constexpr uint32_t num_powers = 19; - constexpr uint8_t decimal_powers[19] = { - 0, 3, 6, 9, 13, 16, 19, 23, 26, 29, // - 33, 36, 39, 43, 46, 49, 53, 56, 59, // - }; - int32_t exp2 = 0; - while (d.decimal_point > 0) { - uint32_t n = uint32_t(d.decimal_point); - uint32_t shift = (n < num_powers) ? decimal_powers[n] : max_shift; - detail::decimal_right_shift(d, shift); - if (d.decimal_point < -decimal_point_range) { - // should be zero - answer.power2 = 0; - answer.mantissa = 0; - return answer; - } - exp2 += int32_t(shift); - } - // We shift left toward [1/2 ... 1]. - while (d.decimal_point <= 0) { - uint32_t shift; - if (d.decimal_point == 0) { - if (d.digits[0] >= 5) { - break; - } - shift = (d.digits[0] < 2) ? 2 : 1; - } else { - uint32_t n = uint32_t(-d.decimal_point); - shift = (n < num_powers) ? decimal_powers[n] : max_shift; - } - detail::decimal_left_shift(d, shift); - if (d.decimal_point > decimal_point_range) { - // we want to get infinity: - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } - exp2 -= int32_t(shift); - } - // We are now in the range [1/2 ... 1] but the binary format uses [1 ... 2]. - exp2--; - constexpr int32_t minimum_exponent = binary::minimum_exponent(); - while ((minimum_exponent + 1) > exp2) { - uint32_t n = uint32_t((minimum_exponent + 1) - exp2); - if (n > max_shift) { - n = max_shift; - } - detail::decimal_right_shift(d, n); - exp2 += int32_t(n); - } - if ((exp2 - minimum_exponent) >= binary::infinite_power()) { - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } - - const int mantissa_size_in_bits = binary::mantissa_explicit_bits() + 1; - detail::decimal_left_shift(d, mantissa_size_in_bits); - - uint64_t mantissa = detail::round(d); - // It is possible that we have an overflow, in which case we need - // to shift back. - if(mantissa >= (uint64_t(1) << mantissa_size_in_bits)) { - detail::decimal_right_shift(d, 1); - exp2 += 1; - mantissa = detail::round(d); - if ((exp2 - minimum_exponent) >= binary::infinite_power()) { - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } - } - answer.power2 = exp2 - binary::minimum_exponent(); - if(mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) { answer.power2--; } - answer.mantissa = mantissa & ((uint64_t(1) << binary::mantissa_explicit_bits()) - 1); - return answer; -} - -template -adjusted_mantissa parse_long_mantissa(const char *first, const char* last, parse_options options) { - decimal d = parse_decimal(first, last, options); - return compute_float(d); -} - -} // namespace fast_float -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/amalgamate.py b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/amalgamate.py deleted file mode 100644 index b360d9988..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/amalgamate.py +++ /dev/null @@ -1,56 +0,0 @@ -# text parts -processed_files = { } - -# authors -for filename in ['AUTHORS', 'CONTRIBUTORS']: - with open(filename) as f: - text = '' - for line in f: - if filename == 'AUTHORS': - text += '// fast_float by ' + line - if filename == 'CONTRIBUTORS': - text += '// with contributions from ' + line - processed_files[filename] = text - -# licenses -for filename in ['LICENSE-MIT', 'LICENSE-APACHE']: - with open(filename) as f: - text = '' - for line in f: - text += '// ' + line - processed_files[filename] = text - -# code -for filename in [ 'fast_float.h', 'float_common.h', 'ascii_number.h', - 'fast_table.h', 'decimal_to_binary.h', 'bigint.h', - 'ascii_number.h', 'digit_comparison.h', 'parse_number.h']: - with open('include/fast_float/' + filename) as f: - text = '' - for line in f: - if line.startswith('#include "'): continue - text += line - processed_files[filename] = text - -# command line -import argparse - -parser = argparse.ArgumentParser(description='Amalgamate fast_float.') -parser.add_argument('--license', default='MIT', help='choose license') -parser.add_argument('--output', default='', help='output file (stdout if none') - -args = parser.parse_args() - -text = '\n\n'.join([ - processed_files['AUTHORS'], processed_files['CONTRIBUTORS'], - processed_files['LICENSE-' + args.license], - processed_files['fast_float.h'], processed_files['float_common.h'], - processed_files['ascii_number.h'], processed_files['fast_table.h'], - processed_files['decimal_to_binary.h'], processed_files['bigint.h'], - processed_files['ascii_number.h'], processed_files['digit_comparison.h'], - processed_files['parse_number.h']]) - -if args.output: - with open(args.output, 'wt') as f: - f.write(text) -else: - print(text) diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/analysis.py b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/analysis.py deleted file mode 100644 index 8dcbcd533..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/analysis.py +++ /dev/null @@ -1,36 +0,0 @@ -from math import floor - -def log2(x): - """returns ceil(log2(x)))""" - y = 0 - while((1<= 2**127 - K = 2**127 - if(not(c * K * d<=( K + 1) * t)): - print(q) - top = floor(t/(c * d - t)) - sys.exit(-1) - -for q in range(18, 344+1): - d = 5**q - b = 64 + 2*log2(d) - t = 2**b - c = t//d + 1 - assert c > 2**(64 +log2(d)) - K = 2**64 - if(not(c * K * d<=( K + 1) * t)): - print(q) - top = floor(t/(c * d - t)) - sys.exit(-1) - -print("all good") \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/table_generation.py b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/table_generation.py deleted file mode 100644 index 24fec7cdd..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/script/table_generation.py +++ /dev/null @@ -1,31 +0,0 @@ -def format(number): - upper = number // (1<<64) - lower = number % (1<<64) - print(""+hex(upper)+","+hex(lower)+",") - -for q in range(-342,0): - power5 = 5 ** -q - z = 0 - while( (1<= -27): - b = z + 127 - c = 2 ** b // power5 + 1 - format(c) - else: - b = 2 * z + 2 * 64 - c = 2 ** b // power5 + 1 - # truncate - while(c >= (1<<128)): - c //= 2 - format(c) - -for q in range(0,308+1): - power5 = 5 ** q - # move the most significant bit in position - while(power5 < (1<<127)): - power5 *= 2 - # *truncate* - while(power5 >= (1<<128)): - power5 //= 2 - format(power5) diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/basictest.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/basictest.cpp deleted file mode 100644 index 1dd924e5b..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/basictest.cpp +++ /dev/null @@ -1,704 +0,0 @@ -#define DOCTEST_CONFIG_SUPER_FAST_ASSERTS -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN -#include - -#include "fast_float/fast_float.h" -#include -#include -#include -#include -#include -#include -#include - -#ifndef SUPPLEMENTAL_TEST_DATA_DIR -#define SUPPLEMENTAL_TEST_DATA_DIR "data/" -#endif - -#ifndef __cplusplus -#error fastfloat requires a C++ compiler -#endif - -#ifndef FASTFLOAT_CPLUSPLUS -#if defined(_MSVC_LANG) && !defined(__clang__) -#define FASTFLOAT_CPLUSPLUS (_MSC_VER == 1900 ? 201103L : _MSVC_LANG) -#else -#define FASTFLOAT_CPLUSPLUS __cplusplus -#endif -#endif - - -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) -#define FASTFLOAT_ODDPLATFORM 1 -#endif -#if defined __has_include -#if __has_include () -#else -// filesystem is not available -#define FASTFLOAT_ODDPLATFORM 1 -#endif -#else -// __has_include is not available -#define FASTFLOAT_ODDPLATFORM 1 -#endif - -// C++ 17 because it is otherwise annoying to browse all files in a directory. -// We also only run these tests on little endian systems. -#if (FASTFLOAT_CPLUSPLUS >= 201703L) && (FASTFLOAT_IS_BIG_ENDIAN == 0) && !defined(FASTFLOAT_ODDPLATFORM) - -#include -#include -#include - -// return true on succcess -bool check_file(std::string file_name) { - std::cout << "Checking " << file_name << std::endl; - size_t number{0}; - std::fstream newfile(file_name, std::ios::in); - if (newfile.is_open()) { - std::string str; - while (std::getline(newfile, str)) { - if (str.size() > 0) { - // Read 32-bit hex - uint32_t float32; - auto r32 = std::from_chars(str.data() + 5, str.data() + str.size(), - float32, 16); - if(r32.ec != std::errc()) { std::cerr << "32-bit parsing failure\n"; return false; } - // Read 64-bit hex - uint64_t float64; - auto r64 = std::from_chars(str.data() + 14, str.data() + str.size(), - float64, 16); - if(r64.ec != std::errc()) { std::cerr << "64-bit parsing failure\n"; return false; } - // The string to parse: - const char *number_string = str.data() + 31; - const char *end_of_string = str.data() + str.size(); - // Parse as 32-bit float - float parsed_32; - auto fast_float_r32 = fast_float::from_chars(number_string, end_of_string, parsed_32); - if(fast_float_r32.ec != std::errc()) { std::cerr << "parsing failure\n"; return false; } - // Parse as 64-bit float - double parsed_64; - auto fast_float_r64 = fast_float::from_chars(number_string, end_of_string, parsed_64); - if(fast_float_r64.ec != std::errc()) { std::cerr << "parsing failure\n"; return false; } - // Convert the floats to unsigned ints. - uint32_t float32_parsed; - uint64_t float64_parsed; - ::memcpy(&float32_parsed, &parsed_32, sizeof(parsed_32)); - ::memcpy(&float64_parsed, &parsed_64, sizeof(parsed_64)); - // Compare with expected results - if (float32_parsed != float32) { - std::cout << "bad 32 " << str << std::endl; - return false; - } - if (float64_parsed != float64) { - std::cout << "bad 64 " << str << std::endl; - return false; - } - number++; - } - } - std::cout << "checked " << std::defaultfloat << number << " values" << std::endl; - newfile.close(); // close the file object - } else { - std::cout << "Could not read " << file_name << std::endl; - return false; - } - return true; -} - - -TEST_CASE("supplemental") { - std::string path = SUPPLEMENTAL_TEST_DATA_DIR; - for (const auto & entry : std::filesystem::directory_iterator(path)) { - CHECK(check_file(entry.path().string())); - } -} -#endif - -TEST_CASE("leading_zeroes") { - constexpr const uint64_t bit = 1; - CHECK(fast_float::leading_zeroes(bit << 0) == 63); - CHECK(fast_float::leading_zeroes(bit << 1) == 62); - CHECK(fast_float::leading_zeroes(bit << 2) == 61); - CHECK(fast_float::leading_zeroes(bit << 61) == 2); - CHECK(fast_float::leading_zeroes(bit << 62) == 1); - CHECK(fast_float::leading_zeroes(bit << 63) == 0); -} - -#define iHexAndDec(v) std::hex << "0x" << (v) << " (" << std::dec << (v) << ")" -#define fHexAndDec(v) std::hexfloat << (v) << " (" << std::defaultfloat << (v) << ")" - -void test_full_multiplication(uint64_t lhs, uint64_t rhs, uint64_t expected_lo, uint64_t expected_hi) { - fast_float::value128 v; - v = fast_float::full_multiplication(lhs, rhs); - INFO("lhs=" << iHexAndDec(lhs) << " " << "rhs=" << iHexAndDec(rhs) - << "\n actualLo=" << iHexAndDec(v.low) << " " << "actualHi=" << iHexAndDec(v.high) - << "\n expectedLo=" << iHexAndDec(expected_lo) << " " << "expectedHi=" << iHexAndDec(expected_hi)); - CHECK_EQ(v.low, expected_lo); - CHECK_EQ(v.high, expected_hi); - v = fast_float::full_multiplication(rhs, lhs); - CHECK_EQ(v.low, expected_lo); - CHECK_EQ(v.high, expected_hi); -} - -TEST_CASE("full_multiplication") { - constexpr const uint64_t bit = 1; - // lhs rhs lo hi - test_full_multiplication(bit << 0 , bit << 0, 1u , 0u); - test_full_multiplication(bit << 0 , bit << 63, bit << 63, 0u); - test_full_multiplication(bit << 1 , bit << 63, 0u , 1u); - test_full_multiplication(bit << 63, bit << 0, bit << 63, 0u); - test_full_multiplication(bit << 63, bit << 1, 0u , 1u); - test_full_multiplication(bit << 63, bit << 2, 0u , 2u); - test_full_multiplication(bit << 63, bit << 63, 0u , bit << 62); -} - - -TEST_CASE("issue8") { - const char* s = - "3." - "141592653589793238462643383279502884197169399375105820974944592307816406" - "286208998628034825342117067982148086513282306647093844609550582231725359" - "408128481117450284102701938521105559644622948954930381964428810975665933" - "446128475648233786783165271201909145648566923460348610454326648213393607" - "260249141273724587006606315588174881520920962829254091715364367892590360" - "011330530548820466521384146951941511609433057270365759591953092186117381" - "932611793105118548074462379962749567351885752724891227938183011949129833" - "673362440656643086021394946395224737190702179860943702770539217176293176" - "752384674818467669405132000568127145263560827785771342757789609173637178" - "721468440901224953430146549585371050792279689258923542019956112129021960" - "864034418159813629774771309960518707211349999998372978"; - for (int i = 0; i < 16; i++) { - // Parse all but the last i chars. We should still get 3.141ish. - double d = 0.0; - auto answer = fast_float::from_chars(s, s + strlen(s) - i, d); - CHECK_MESSAGE(answer.ec == std::errc(), "i=" << i); - CHECK_MESSAGE(d == 0x1.921fb54442d18p+1, "i=" << i << "\n" - << std::string(s, strlen(s) - size_t(i)) << "\n" - << std::hexfloat << d << "\n" - << std::defaultfloat << "\n"); - } -} - -TEST_CASE("check_behavior") { - const std::string input = "abc"; - double result; - auto answer = fast_float::from_chars(input.data(), input.data()+input.size(), result); - CHECK_MESSAGE(answer.ec != std::errc(), "expected parse failure"); - CHECK_MESSAGE(answer.ptr == input.data(), "If there is no pattern match, we should have ptr equals first"); -} - - -TEST_CASE("decimal_point_parsing") { - double result; - fast_float::parse_options options{}; - { - const std::string input = "1,25"; - auto answer = fast_float::from_chars_advanced(input.data(), input.data()+input.size(), result, options); - CHECK_MESSAGE(answer.ec == std::errc(), "expected parse success"); - CHECK_MESSAGE(answer.ptr == input.data() + 1, - "Parsing should have stopped at comma"); - CHECK_EQ(result, 1.0); - - options.decimal_point = ','; - answer = fast_float::from_chars_advanced(input.data(), input.data()+input.size(), result, options); - CHECK_MESSAGE(answer.ec == std::errc(), "expected parse success"); - CHECK_MESSAGE(answer.ptr == input.data() + input.size(), - "Parsing should have stopped at end"); - CHECK_EQ(result, 1.25); - } - { - const std::string input = "1.25"; - auto answer = fast_float::from_chars_advanced(input.data(), input.data()+input.size(), result, options); - CHECK_MESSAGE(answer.ec == std::errc(), "expected parse success"); - CHECK_MESSAGE(answer.ptr == input.data() + 1, - "Parsing should have stopped at dot"); - CHECK_EQ(result, 1.0); - - options.decimal_point = '.'; - answer = fast_float::from_chars_advanced(input.data(), input.data()+input.size(), result, options); - CHECK_MESSAGE(answer.ec == std::errc(), "expected parse success"); - CHECK_MESSAGE(answer.ptr == input.data() + input.size(), - "Parsing should have stopped at end"); - CHECK_EQ(result, 1.25); - } -} - -TEST_CASE("issue19") { - const std::string input = "3.14e"; - double result; - auto answer = fast_float::from_chars(input.data(), input.data()+input.size(), result); - CHECK_MESSAGE(answer.ec == std::errc(), "We want to parse up to 3.14\n"); - CHECK_MESSAGE(answer.ptr == input.data() + 4, - "Parsed the number " << result - << " and stopped at the wrong character: after " << (answer.ptr - input.data()) << " characters"); -} - - -TEST_CASE("scientific_only") { - // first, we try with something that should fail... - std::string input = "3.14"; - double result; - auto answer = fast_float::from_chars(input.data(), input.data()+input.size(), result, fast_float::chars_format::scientific); - CHECK_MESSAGE(answer.ec != std::errc(), "It is not scientific! Parsed: " << result); - - input = "3.14e10"; - answer = fast_float::from_chars(input.data(), input.data()+input.size(), result, fast_float::chars_format::scientific); - CHECK_MESSAGE(answer.ec == std::errc(), "It is scientific! Parsed: " << result); - CHECK_MESSAGE(answer.ptr == input.data() + input.size(), - "Parsed the number " << result - << " and stopped at the wrong character: after " << (answer.ptr - input.data()) << " characters"); -} - - -TEST_CASE("test_fixed_only") { - const std::string input = "3.14e10"; - double result; - auto answer = fast_float::from_chars(input.data(), input.data()+input.size(), result, fast_float::chars_format::fixed); - CHECK_MESSAGE(answer.ec == std::errc(), "We want to parse up to 3.14; parsed: " << result); - CHECK_MESSAGE(answer.ptr == input.data() + 4, - "Parsed the number " << result - << " and stopped at the wrong character: after " << (answer.ptr - input.data()) << " characters"); -} - - -static const double testing_power_of_ten[] = { - 1e-307, 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, 1e-299, - 1e-298, 1e-297, 1e-296, 1e-295, 1e-294, 1e-293, 1e-292, 1e-291, 1e-290, - 1e-289, 1e-288, 1e-287, 1e-286, 1e-285, 1e-284, 1e-283, 1e-282, 1e-281, - 1e-280, 1e-279, 1e-278, 1e-277, 1e-276, 1e-275, 1e-274, 1e-273, 1e-272, - 1e-271, 1e-270, 1e-269, 1e-268, 1e-267, 1e-266, 1e-265, 1e-264, 1e-263, - 1e-262, 1e-261, 1e-260, 1e-259, 1e-258, 1e-257, 1e-256, 1e-255, 1e-254, - 1e-253, 1e-252, 1e-251, 1e-250, 1e-249, 1e-248, 1e-247, 1e-246, 1e-245, - 1e-244, 1e-243, 1e-242, 1e-241, 1e-240, 1e-239, 1e-238, 1e-237, 1e-236, - 1e-235, 1e-234, 1e-233, 1e-232, 1e-231, 1e-230, 1e-229, 1e-228, 1e-227, - 1e-226, 1e-225, 1e-224, 1e-223, 1e-222, 1e-221, 1e-220, 1e-219, 1e-218, - 1e-217, 1e-216, 1e-215, 1e-214, 1e-213, 1e-212, 1e-211, 1e-210, 1e-209, - 1e-208, 1e-207, 1e-206, 1e-205, 1e-204, 1e-203, 1e-202, 1e-201, 1e-200, - 1e-199, 1e-198, 1e-197, 1e-196, 1e-195, 1e-194, 1e-193, 1e-192, 1e-191, - 1e-190, 1e-189, 1e-188, 1e-187, 1e-186, 1e-185, 1e-184, 1e-183, 1e-182, - 1e-181, 1e-180, 1e-179, 1e-178, 1e-177, 1e-176, 1e-175, 1e-174, 1e-173, - 1e-172, 1e-171, 1e-170, 1e-169, 1e-168, 1e-167, 1e-166, 1e-165, 1e-164, - 1e-163, 1e-162, 1e-161, 1e-160, 1e-159, 1e-158, 1e-157, 1e-156, 1e-155, - 1e-154, 1e-153, 1e-152, 1e-151, 1e-150, 1e-149, 1e-148, 1e-147, 1e-146, - 1e-145, 1e-144, 1e-143, 1e-142, 1e-141, 1e-140, 1e-139, 1e-138, 1e-137, - 1e-136, 1e-135, 1e-134, 1e-133, 1e-132, 1e-131, 1e-130, 1e-129, 1e-128, - 1e-127, 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, 1e-119, - 1e-118, 1e-117, 1e-116, 1e-115, 1e-114, 1e-113, 1e-112, 1e-111, 1e-110, - 1e-109, 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, 1e-101, - 1e-100, 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, 1e-92, - 1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 1e-85, 1e-84, 1e-83, - 1e-82, 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, 1e-74, - 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, 1e-65, - 1e-64, 1e-63, 1e-62, 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, 1e-56, - 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, 1e-47, - 1e-46, 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, 1e-38, - 1e-37, 1e-36, 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, 1e-29, - 1e-28, 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, 1e-20, - 1e-19, 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, 1e-11, - 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, - 1e-1, 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, - 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, - 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, 1e23, 1e24, 1e25, - 1e26, 1e27, 1e28, 1e29, 1e30, 1e31, 1e32, 1e33, 1e34, - 1e35, 1e36, 1e37, 1e38, 1e39, 1e40, 1e41, 1e42, 1e43, - 1e44, 1e45, 1e46, 1e47, 1e48, 1e49, 1e50, 1e51, 1e52, - 1e53, 1e54, 1e55, 1e56, 1e57, 1e58, 1e59, 1e60, 1e61, - 1e62, 1e63, 1e64, 1e65, 1e66, 1e67, 1e68, 1e69, 1e70, - 1e71, 1e72, 1e73, 1e74, 1e75, 1e76, 1e77, 1e78, 1e79, - 1e80, 1e81, 1e82, 1e83, 1e84, 1e85, 1e86, 1e87, 1e88, - 1e89, 1e90, 1e91, 1e92, 1e93, 1e94, 1e95, 1e96, 1e97, - 1e98, 1e99, 1e100, 1e101, 1e102, 1e103, 1e104, 1e105, 1e106, - 1e107, 1e108, 1e109, 1e110, 1e111, 1e112, 1e113, 1e114, 1e115, - 1e116, 1e117, 1e118, 1e119, 1e120, 1e121, 1e122, 1e123, 1e124, - 1e125, 1e126, 1e127, 1e128, 1e129, 1e130, 1e131, 1e132, 1e133, - 1e134, 1e135, 1e136, 1e137, 1e138, 1e139, 1e140, 1e141, 1e142, - 1e143, 1e144, 1e145, 1e146, 1e147, 1e148, 1e149, 1e150, 1e151, - 1e152, 1e153, 1e154, 1e155, 1e156, 1e157, 1e158, 1e159, 1e160, - 1e161, 1e162, 1e163, 1e164, 1e165, 1e166, 1e167, 1e168, 1e169, - 1e170, 1e171, 1e172, 1e173, 1e174, 1e175, 1e176, 1e177, 1e178, - 1e179, 1e180, 1e181, 1e182, 1e183, 1e184, 1e185, 1e186, 1e187, - 1e188, 1e189, 1e190, 1e191, 1e192, 1e193, 1e194, 1e195, 1e196, - 1e197, 1e198, 1e199, 1e200, 1e201, 1e202, 1e203, 1e204, 1e205, - 1e206, 1e207, 1e208, 1e209, 1e210, 1e211, 1e212, 1e213, 1e214, - 1e215, 1e216, 1e217, 1e218, 1e219, 1e220, 1e221, 1e222, 1e223, - 1e224, 1e225, 1e226, 1e227, 1e228, 1e229, 1e230, 1e231, 1e232, - 1e233, 1e234, 1e235, 1e236, 1e237, 1e238, 1e239, 1e240, 1e241, - 1e242, 1e243, 1e244, 1e245, 1e246, 1e247, 1e248, 1e249, 1e250, - 1e251, 1e252, 1e253, 1e254, 1e255, 1e256, 1e257, 1e258, 1e259, - 1e260, 1e261, 1e262, 1e263, 1e264, 1e265, 1e266, 1e267, 1e268, - 1e269, 1e270, 1e271, 1e272, 1e273, 1e274, 1e275, 1e276, 1e277, - 1e278, 1e279, 1e280, 1e281, 1e282, 1e283, 1e284, 1e285, 1e286, - 1e287, 1e288, 1e289, 1e290, 1e291, 1e292, 1e293, 1e294, 1e295, - 1e296, 1e297, 1e298, 1e299, 1e300, 1e301, 1e302, 1e303, 1e304, - 1e305, 1e306, 1e307, 1e308}; - - -TEST_CASE("powers_of_ten") { - char buf[1024]; - WARN_MESSAGE(1e-308 == std::pow(10, -308), "On your system, the pow function is busted. Sorry about that."); - bool is_pow_correct{1e-308 == std::pow(10,-308)}; - // large negative values should be zero. - int start_point = is_pow_correct ? -1000 : -307; - for (int i = start_point; i <= 308; ++i) { - INFO("i=" << i); - size_t n = size_t(snprintf(buf, sizeof(buf), "1e%d", i)); - REQUIRE(n < sizeof(buf)); // if false, fails the test and exits - double actual; - auto result = fast_float::from_chars(buf, buf + 1000, actual); - CHECK_MESSAGE(result.ec == std::errc(), " I could not parse " << buf); - double expected = ((i >= -307) ? testing_power_of_ten[i + 307] : std::pow(10, i)); - CHECK_MESSAGE(actual == expected, "String '" << buf << "'parsed to " << actual); - } -} - - -template std::string to_string(T d) { - std::string s(64, '\0'); - auto written = std::snprintf(&s[0], s.size(), "%.*e", - std::numeric_limits::max_digits10 - 1, d); - s.resize(size_t(written)); - return s; -} - -template std::string to_long_string(T d) { - std::string s(4096, '\0'); - auto written = std::snprintf(&s[0], s.size(), "%.*e", - std::numeric_limits::max_digits10 * 10, d); - s.resize(size_t(written)); - return s; -} - -uint32_t get_mantissa(float f) { - uint32_t m; - memcpy(&m, &f, sizeof(f)); - return (m & ((uint32_t(1)<<23)-1)); -} - -uint64_t get_mantissa(double f) { - uint64_t m; - memcpy(&m, &f, sizeof(f)); - return (m & ((uint64_t(1)<<57)-1)); -} - - -std::string append_zeros(std::string str, size_t number_of_zeros) { - std::string answer(str); - for(size_t i = 0; i < number_of_zeros; i++) { answer += "0"; } - return answer; -} - -template -void check_basic_test_result(const std::string& str, fast_float::from_chars_result result, - T actual, T expected) { - INFO("str=" << str << "\n" - << " expected=" << fHexAndDec(expected) << "\n" - << " ..actual=" << fHexAndDec(actual) << "\n" - << " expected mantissa=" << iHexAndDec(get_mantissa(expected)) << "\n" - << " ..actual mantissa=" << iHexAndDec(get_mantissa(actual))); - CHECK_EQ(result.ec, std::errc()); - CHECK_EQ(result.ptr, str.data() + str.size()); - CHECK_EQ(copysign(1, actual), copysign(1, expected)); - CHECK_EQ(std::isnan(actual), std::isnan(expected)); - CHECK_EQ(actual, expected); -} - -template -void basic_test(std::string str, T expected) { - T actual; - auto result = fast_float::from_chars(str.data(), str.data() + str.size(), actual); - check_basic_test_result(str, result, actual, expected); -} - -template -void basic_test(std::string str, T expected, fast_float::parse_options options) { - T actual; - auto result = fast_float::from_chars_advanced(str.data(), str.data() + str.size(), actual, options); - check_basic_test_result(str, result, actual, expected); -} - -void basic_test(float val) { - { - std::string long_vals = to_long_string(val); - INFO("long vals: " << long_vals); - basic_test(long_vals, val); - } - { - std::string vals = to_string(val); - INFO("vals: " << vals); - basic_test(vals, val); - } -} - -#define verify(lhs, rhs) { INFO(lhs); basic_test(lhs, rhs); } -#define verify32(val) { INFO(#val); basic_test(val); } - -#define verify_options(lhs, rhs) { INFO(lhs); basic_test(lhs, rhs, options); } - -TEST_CASE("64bit.inf") { - verify("INF", std::numeric_limits::infinity()); - verify("-INF", -std::numeric_limits::infinity()); - verify("INFINITY", std::numeric_limits::infinity()); - verify("-INFINITY", -std::numeric_limits::infinity()); - verify("infinity", std::numeric_limits::infinity()); - verify("-infinity", -std::numeric_limits::infinity()); - verify("inf", std::numeric_limits::infinity()); - verify("-inf", -std::numeric_limits::infinity()); - verify("1234456789012345678901234567890e9999999999999999999999999999", std::numeric_limits::infinity()); - verify("-2139879401095466344511101915470454744.9813888656856943E+272", -std::numeric_limits::infinity()); - verify("1.8e308", std::numeric_limits::infinity()); - verify("1.832312213213213232132132143451234453123412321321312e308", std::numeric_limits::infinity()); - verify("2e30000000000000000", std::numeric_limits::infinity()); - verify("2e3000", std::numeric_limits::infinity()); - verify("1.9e308", std::numeric_limits::infinity()); -} - -TEST_CASE("64bit.general") { - verify("-2.2222222222223e-322",-0x1.68p-1069); - verify("9007199254740993.0", 0x1p+53); - verify("860228122.6654514319E+90", 0x1.92bb20990715fp+328); - verify(append_zeros("9007199254740993.0",1000), 0x1p+53); - verify("10000000000000000000", 0x1.158e460913dp+63); - verify("10000000000000000000000000000001000000000000", 0x1.cb2d6f618c879p+142); - verify("10000000000000000000000000000000000000000001", 0x1.cb2d6f618c879p+142); - verify("1.1920928955078125e-07", 1.1920928955078125e-07); - verify("9355950000000000000.00000000000000000000000000000000001844674407370955161600000184467440737095516161844674407370955161407370955161618446744073709551616000184467440737095516166000001844674407370955161618446744073709551614073709551616184467440737095516160001844674407370955161601844674407370955674451616184467440737095516140737095516161844674407370955161600018446744073709551616018446744073709551611616000184467440737095001844674407370955161600184467440737095516160018446744073709551168164467440737095516160001844073709551616018446744073709551616184467440737095516160001844674407536910751601611616000184467440737095001844674407370955161600184467440737095516160018446744073709551616184467440737095516160001844955161618446744073709551616000184467440753691075160018446744073709",0x1.03ae05e8fca1cp+63); - verify("-0",-0.0); - verify("2.22507385850720212418870147920222032907240528279439037814303133837435107319244194686754406432563881851382188218502438069999947733013005649884107791928741341929297200970481951993067993290969042784064731682041565926728632933630474670123316852983422152744517260835859654566319282835244787787799894310779783833699159288594555213714181128458251145584319223079897504395086859412457230891738946169368372321191373658977977723286698840356390251044443035457396733706583981055420456693824658413747607155981176573877626747665912387199931904006317334709003012790188175203447190250028061277777916798391090578584006464715943810511489154282775041174682194133952466682503431306181587829379004205392375072083366693241580002758391118854188641513168478436313080237596295773983001708984375e-308", 0x1.0000000000002p-1022); - verify("1.0000000000000006661338147750939242541790008544921875",1.0000000000000007); - verify("1090544144181609348835077142190",0x1.b8779f2474dfbp+99); - verify("2.2250738585072013e-308", 2.2250738585072013e-308); - verify("-92666518056446206563E3", -92666518056446206563E3); - verify("-92666518056446206563E3", -92666518056446206563E3); - verify("-42823146028335318693e-128",-42823146028335318693e-128); - verify("90054602635948575728E72",90054602635948575728E72); - verify("1.00000000000000188558920870223463870174566020691753515394643550663070558368373221972569761144603605635692374830246134201063722058e-309", 1.00000000000000188558920870223463870174566020691753515394643550663070558368373221972569761144603605635692374830246134201063722058e-309); - verify("0e9999999999999999999999999999", 0.0); - verify("-2402844368454405395.2", -2402844368454405395.2); - verify("2402844368454405395.2", 2402844368454405395.2); - verify("7.0420557077594588669468784357561207962098443483187940792729600000e+59", 7.0420557077594588669468784357561207962098443483187940792729600000e+59); - verify("7.0420557077594588669468784357561207962098443483187940792729600000e+59", 7.0420557077594588669468784357561207962098443483187940792729600000e+59); - verify("-1.7339253062092163730578609458683877051596800000000000000000000000e+42", -1.7339253062092163730578609458683877051596800000000000000000000000e+42); - verify("-2.0972622234386619214559824785284023792871122537545728000000000000e+52", -2.0972622234386619214559824785284023792871122537545728000000000000e+52); - verify("-1.0001803374372191849407179462120053338028379051879898808320000000e+57", -1.0001803374372191849407179462120053338028379051879898808320000000e+57); - verify("-1.8607245283054342363818436991534856973992070520151142825984000000e+58", -1.8607245283054342363818436991534856973992070520151142825984000000e+58); - verify("-1.9189205311132686907264385602245237137907390376574976000000000000e+52", -1.9189205311132686907264385602245237137907390376574976000000000000e+52); - verify("-2.8184483231688951563253238886553506793085187889855201280000000000e+54", -2.8184483231688951563253238886553506793085187889855201280000000000e+54); - verify("-1.7664960224650106892054063261344555646357024359107788800000000000e+53", -1.7664960224650106892054063261344555646357024359107788800000000000e+53); - verify("-2.1470977154320536489471030463761883783915110400000000000000000000e+45", -2.1470977154320536489471030463761883783915110400000000000000000000e+45); - verify("-4.4900312744003159009338275160799498340862630046359789166919680000e+61", -4.4900312744003159009338275160799498340862630046359789166919680000e+61); - verify("1", 1.0); - verify("1.797693134862315700000000000000001e308", 1.7976931348623157e308); - verify("3e-324", 0x0.0000000000001p-1022); - verify("1.00000006e+09", 0x1.dcd651ep+29); - verify("4.9406564584124653e-324", 0x0.0000000000001p-1022); - verify("4.9406564584124654e-324", 0x0.0000000000001p-1022); - verify("2.2250738585072009e-308", 0x0.fffffffffffffp-1022); - verify("2.2250738585072014e-308", 0x1p-1022); - verify("1.7976931348623157e308", 0x1.fffffffffffffp+1023); - verify("1.7976931348623158e308", 0x1.fffffffffffffp+1023); - verify("4503599627370496.5", 4503599627370496.5); - verify("4503599627475352.5", 4503599627475352.5); - verify("4503599627475353.5", 4503599627475353.5); - verify("2251799813685248.25", 2251799813685248.25); - verify("1125899906842624.125", 1125899906842624.125); - verify("1125899906842901.875", 1125899906842901.875); - verify("2251799813685803.75", 2251799813685803.75); - verify("4503599627370497.5", 4503599627370497.5); - verify("45035996.273704995", 45035996.273704995); - verify("45035996.273704985", 45035996.273704985); - verify("0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000044501477170144022721148195934182639518696390927032912960468522194496444440421538910330590478162701758282983178260792422137401728773891892910553144148156412434867599762821265346585071045737627442980259622449029037796981144446145705102663115100318287949527959668236039986479250965780342141637013812613333119898765515451440315261253813266652951306000184917766328660755595837392240989947807556594098101021612198814605258742579179000071675999344145086087205681577915435923018910334964869420614052182892431445797605163650903606514140377217442262561590244668525767372446430075513332450079650686719491377688478005309963967709758965844137894433796621993967316936280457084866613206797017728916080020698679408551343728867675409720757232455434770912461317493580281734466552734375", 0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000044501477170144022721148195934182639518696390927032912960468522194496444440421538910330590478162701758282983178260792422137401728773891892910553144148156412434867599762821265346585071045737627442980259622449029037796981144446145705102663115100318287949527959668236039986479250965780342141637013812613333119898765515451440315261253813266652951306000184917766328660755595837392240989947807556594098101021612198814605258742579179000071675999344145086087205681577915435923018910334964869420614052182892431445797605163650903606514140377217442262561590244668525767372446430075513332450079650686719491377688478005309963967709758965844137894433796621993967316936280457084866613206797017728916080020698679408551343728867675409720757232455434770912461317493580281734466552734375); - verify("0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000022250738585072008890245868760858598876504231122409594654935248025624400092282356951787758888037591552642309780950434312085877387158357291821993020294379224223559819827501242041788969571311791082261043971979604000454897391938079198936081525613113376149842043271751033627391549782731594143828136275113838604094249464942286316695429105080201815926642134996606517803095075913058719846423906068637102005108723282784678843631944515866135041223479014792369585208321597621066375401613736583044193603714778355306682834535634005074073040135602968046375918583163124224521599262546494300836851861719422417646455137135420132217031370496583210154654068035397417906022589503023501937519773030945763173210852507299305089761582519159720757232455434770912461317493580281734466552734375", 0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000022250738585072008890245868760858598876504231122409594654935248025624400092282356951787758888037591552642309780950434312085877387158357291821993020294379224223559819827501242041788969571311791082261043971979604000454897391938079198936081525613113376149842043271751033627391549782731594143828136275113838604094249464942286316695429105080201815926642134996606517803095075913058719846423906068637102005108723282784678843631944515866135041223479014792369585208321597621066375401613736583044193603714778355306682834535634005074073040135602968046375918583163124224521599262546494300836851861719422417646455137135420132217031370496583210154654068035397417906022589503023501937519773030945763173210852507299305089761582519159720757232455434770912461317493580281734466552734375); - verify("1438456663141390273526118207642235581183227845246331231162636653790368152091394196930365828634687637948157940776599182791387527135353034738357134110310609455693900824193549772792016543182680519740580354365467985440183598701312257624545562331397018329928613196125590274187720073914818062530830316533158098624984118889298281371812288789537310599037529113415438738954894752124724983067241108764488346454376699018673078404751121414804937224240805993123816932326223683090770561597570457793932985826162604255884529134126396282202126526253389383421806727954588525596114379801269094096329805054803089299736996870951258573010877404407451953846698609198213926882692078557033228265259305481198526059813164469187586693257335779522020407645498684263339921905227556616698129967412891282231685504660671277927198290009824680186319750978665734576683784255802269708917361719466043175201158849097881370477111850171579869056016061666173029059588433776015644439705050377554277696143928278093453792803846252715966016733222646442382892123940052441346822429721593884378212558701004356924243030059517489346646577724622498919752597382095222500311124181823512251071356181769376577651390028297796156208815375089159128394945710515861334486267101797497111125909272505194792870889617179758703442608016143343262159998149700606597792535574457560429226974273443630323818747730771316763398572110874959981923732463076884528677392654150010269822239401993427482376513231389212353583573566376915572650916866553612366187378959554983566712767093372906030188976220169058025354973622211666504549316958271880975697143546564469806791358707318873075708383345004090151974068325838177531266954177406661392229801349994695941509935655355652985723782153570084089560139142231.738475042362596875449154552392299548947138162081694168675340677843807613129780449323363759027012972466987370921816813162658754726545121090545507240267000456594786540949605260722461937870630634874991729398208026467698131898691830012167897399682179601734569071423681e-733", std::numeric_limits::infinity()); - verify("-2240084132271013504.131248280843119943687942846658579428", -0x1.f1660a65b00bfp+60); -} - -TEST_CASE("64bit.decimal_point") { - fast_float::parse_options options{}; - options.decimal_point = ','; - - // infinities - verify_options("1,8e308", std::numeric_limits::infinity()); - verify_options("1,832312213213213232132132143451234453123412321321312e308", std::numeric_limits::infinity()); - verify_options("2e30000000000000000", std::numeric_limits::infinity()); - verify_options("2e3000", std::numeric_limits::infinity()); - verify_options("1,9e308", std::numeric_limits::infinity()); - - // finites - verify_options("-2,2222222222223e-322",-0x1.68p-1069); - verify_options("9007199254740993,0", 0x1p+53); - verify_options("860228122,6654514319E+90", 0x1.92bb20990715fp+328); - verify_options(append_zeros("9007199254740993,0",1000), 0x1p+53); - verify_options("1,1920928955078125e-07", 1.1920928955078125e-07); - verify_options("9355950000000000000,00000000000000000000000000000000001844674407370955161600000184467440737095516161844674407370955161407370955161618446744073709551616000184467440737095516166000001844674407370955161618446744073709551614073709551616184467440737095516160001844674407370955161601844674407370955674451616184467440737095516140737095516161844674407370955161600018446744073709551616018446744073709551611616000184467440737095001844674407370955161600184467440737095516160018446744073709551168164467440737095516160001844073709551616018446744073709551616184467440737095516160001844674407536910751601611616000184467440737095001844674407370955161600184467440737095516160018446744073709551616184467440737095516160001844955161618446744073709551616000184467440753691075160018446744073709",0x1.03ae05e8fca1cp+63); - verify_options("2,22507385850720212418870147920222032907240528279439037814303133837435107319244194686754406432563881851382188218502438069999947733013005649884107791928741341929297200970481951993067993290969042784064731682041565926728632933630474670123316852983422152744517260835859654566319282835244787787799894310779783833699159288594555213714181128458251145584319223079897504395086859412457230891738946169368372321191373658977977723286698840356390251044443035457396733706583981055420456693824658413747607155981176573877626747665912387199931904006317334709003012790188175203447190250028061277777916798391090578584006464715943810511489154282775041174682194133952466682503431306181587829379004205392375072083366693241580002758391118854188641513168478436313080237596295773983001708984375e-308", 0x1.0000000000002p-1022); - verify_options("1,0000000000000006661338147750939242541790008544921875",1.0000000000000007); - verify_options("2,2250738585072013e-308", 2.2250738585072013e-308); - verify_options("1,00000000000000188558920870223463870174566020691753515394643550663070558368373221972569761144603605635692374830246134201063722058e-309", 1.00000000000000188558920870223463870174566020691753515394643550663070558368373221972569761144603605635692374830246134201063722058e-309); - verify_options("-2402844368454405395,2", -2402844368454405395.2); - verify_options("2402844368454405395,2", 2402844368454405395.2); - verify_options("7,0420557077594588669468784357561207962098443483187940792729600000e+59", 7.0420557077594588669468784357561207962098443483187940792729600000e+59); - verify_options("7,0420557077594588669468784357561207962098443483187940792729600000e+59", 7.0420557077594588669468784357561207962098443483187940792729600000e+59); - verify_options("-1,7339253062092163730578609458683877051596800000000000000000000000e+42", -1.7339253062092163730578609458683877051596800000000000000000000000e+42); - verify_options("-2,0972622234386619214559824785284023792871122537545728000000000000e+52", -2.0972622234386619214559824785284023792871122537545728000000000000e+52); - verify_options("-1,0001803374372191849407179462120053338028379051879898808320000000e+57", -1.0001803374372191849407179462120053338028379051879898808320000000e+57); - verify_options("-1,8607245283054342363818436991534856973992070520151142825984000000e+58", -1.8607245283054342363818436991534856973992070520151142825984000000e+58); - verify_options("-1,9189205311132686907264385602245237137907390376574976000000000000e+52", -1.9189205311132686907264385602245237137907390376574976000000000000e+52); - verify_options("-2,8184483231688951563253238886553506793085187889855201280000000000e+54", -2.8184483231688951563253238886553506793085187889855201280000000000e+54); - verify_options("-1,7664960224650106892054063261344555646357024359107788800000000000e+53", -1.7664960224650106892054063261344555646357024359107788800000000000e+53); - verify_options("-2,1470977154320536489471030463761883783915110400000000000000000000e+45", -2.1470977154320536489471030463761883783915110400000000000000000000e+45); - verify_options("-4,4900312744003159009338275160799498340862630046359789166919680000e+61", -4.4900312744003159009338275160799498340862630046359789166919680000e+61); - verify_options("1", 1.0); - verify_options("1,797693134862315700000000000000001e308", 1.7976931348623157e308); - verify_options("3e-324", 0x0.0000000000001p-1022); - verify_options("1,00000006e+09", 0x1.dcd651ep+29); - verify_options("4,9406564584124653e-324", 0x0.0000000000001p-1022); - verify_options("4,9406564584124654e-324", 0x0.0000000000001p-1022); - verify_options("2,2250738585072009e-308", 0x0.fffffffffffffp-1022); - verify_options("2,2250738585072014e-308", 0x1p-1022); - verify_options("1,7976931348623157e308", 0x1.fffffffffffffp+1023); - verify_options("1,7976931348623158e308", 0x1.fffffffffffffp+1023); - verify_options("4503599627370496,5", 4503599627370496.5); - verify_options("4503599627475352,5", 4503599627475352.5); - verify_options("4503599627475353,5", 4503599627475353.5); - verify_options("2251799813685248,25", 2251799813685248.25); - verify_options("1125899906842624,125", 1125899906842624.125); - verify_options("1125899906842901,875", 1125899906842901.875); - verify_options("2251799813685803,75", 2251799813685803.75); - verify_options("4503599627370497,5", 4503599627370497.5); - verify_options("45035996,273704995", 45035996.273704995); - verify_options("45035996,273704985", 45035996.273704985); - verify_options("0,000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000044501477170144022721148195934182639518696390927032912960468522194496444440421538910330590478162701758282983178260792422137401728773891892910553144148156412434867599762821265346585071045737627442980259622449029037796981144446145705102663115100318287949527959668236039986479250965780342141637013812613333119898765515451440315261253813266652951306000184917766328660755595837392240989947807556594098101021612198814605258742579179000071675999344145086087205681577915435923018910334964869420614052182892431445797605163650903606514140377217442262561590244668525767372446430075513332450079650686719491377688478005309963967709758965844137894433796621993967316936280457084866613206797017728916080020698679408551343728867675409720757232455434770912461317493580281734466552734375", 0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000044501477170144022721148195934182639518696390927032912960468522194496444440421538910330590478162701758282983178260792422137401728773891892910553144148156412434867599762821265346585071045737627442980259622449029037796981144446145705102663115100318287949527959668236039986479250965780342141637013812613333119898765515451440315261253813266652951306000184917766328660755595837392240989947807556594098101021612198814605258742579179000071675999344145086087205681577915435923018910334964869420614052182892431445797605163650903606514140377217442262561590244668525767372446430075513332450079650686719491377688478005309963967709758965844137894433796621993967316936280457084866613206797017728916080020698679408551343728867675409720757232455434770912461317493580281734466552734375); - verify_options("0,000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000022250738585072008890245868760858598876504231122409594654935248025624400092282356951787758888037591552642309780950434312085877387158357291821993020294379224223559819827501242041788969571311791082261043971979604000454897391938079198936081525613113376149842043271751033627391549782731594143828136275113838604094249464942286316695429105080201815926642134996606517803095075913058719846423906068637102005108723282784678843631944515866135041223479014792369585208321597621066375401613736583044193603714778355306682834535634005074073040135602968046375918583163124224521599262546494300836851861719422417646455137135420132217031370496583210154654068035397417906022589503023501937519773030945763173210852507299305089761582519159720757232455434770912461317493580281734466552734375", 0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000022250738585072008890245868760858598876504231122409594654935248025624400092282356951787758888037591552642309780950434312085877387158357291821993020294379224223559819827501242041788969571311791082261043971979604000454897391938079198936081525613113376149842043271751033627391549782731594143828136275113838604094249464942286316695429105080201815926642134996606517803095075913058719846423906068637102005108723282784678843631944515866135041223479014792369585208321597621066375401613736583044193603714778355306682834535634005074073040135602968046375918583163124224521599262546494300836851861719422417646455137135420132217031370496583210154654068035397417906022589503023501937519773030945763173210852507299305089761582519159720757232455434770912461317493580281734466552734375); -} - - -TEST_CASE("32bit.inf") { - verify("INF", std::numeric_limits::infinity()); - verify("-INF", -std::numeric_limits::infinity()); - verify("INFINITY", std::numeric_limits::infinity()); - verify("-INFINITY", -std::numeric_limits::infinity()); - verify("infinity", std::numeric_limits::infinity()); - verify("-infinity", -std::numeric_limits::infinity()); - verify("inf", std::numeric_limits::infinity()); - verify("-inf", -std::numeric_limits::infinity()); - verify("1234456789012345678901234567890e9999999999999999999999999999", std::numeric_limits::infinity()); - verify("2e3000", std::numeric_limits::infinity()); - verify("3.5028234666e38", std::numeric_limits::infinity()); -} - -TEST_CASE("32bit.general") { - verify("1.1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125", 0x1.2ced3p+0f); - verify("1.1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125e-38", 0x1.fffff8p-127f); - verify(append_zeros("1.1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",655), 0x1.2ced3p+0f); - verify(append_zeros("1.1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",656), 0x1.2ced3p+0f); - verify(append_zeros("1.1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",1000), 0x1.2ced3p+0f); - std::string test_string; - test_string = append_zeros("1.1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",655) + std::string("e-38"); - verify(test_string, 0x1.fffff8p-127f); - test_string = append_zeros("1.1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",656) + std::string("e-38"); - verify(test_string, 0x1.fffff8p-127f); - test_string = append_zeros("1.1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",1000) + std::string("e-38"); - verify(test_string, 0x1.fffff8p-127f); - verify32(1.00000006e+09f); - verify32(1.4012984643e-45f); - verify32(1.1754942107e-38f); - verify32(1.1754943508e-45f); - verify("-0", -0.0f); - verify("1090544144181609348835077142190", 0x1.b877ap+99f); - verify("1.1754943508e-38", 1.1754943508e-38f); - verify("30219.0830078125", 30219.0830078125f); - verify("16252921.5", 16252921.5f); - verify("5322519.25", 5322519.25f); - verify("3900245.875", 3900245.875f); - verify("1510988.3125", 1510988.3125f); - verify("782262.28125", 782262.28125f); - verify("328381.484375", 328381.484375f); - verify("156782.0703125", 156782.0703125f); - verify("85003.24609375", 85003.24609375f); - verify("17419.6494140625", 17419.6494140625f); - verify("15498.36376953125", 15498.36376953125f); - verify("6318.580322265625", 6318.580322265625f); - verify("2525.2840576171875", 2525.2840576171875f); - verify("1370.9265747070312", 1370.9265747070312f); - verify("936.3702087402344", 936.3702087402344f); - verify("411.88682556152344", 411.88682556152344f); - verify("206.50310516357422", 206.50310516357422f); - verify("124.16878890991211", 124.16878890991211f); - verify("50.811574935913086", 50.811574935913086f); - verify("17.486443519592285", 17.486443519592285f); - verify("13.91745138168335", 13.91745138168335f); - verify("7.5464513301849365", 0x1.e2f90ep+2f); - verify("2.687217116355896", 2.687217116355896f); - verify("1.1877630352973938", 0x1.30113ep+0f); - verify("0.7622503340244293", 0.7622503340244293f); - verify("0.30531780421733856", 0x1.38a53ap-2f); - verify("0.21791061013936996", 0x1.be47eap-3f); - verify("0.09289376810193062", 0x1.7c7e2ep-4f); - verify("0.03706067614257336", 0.03706067614257336f); - verify("0.028068351559340954", 0.028068351559340954f); - verify("0.012114629615098238", 0x1.8cf8e2p-7f); - verify("0.004221370676532388", 0x1.14a6dap-8f); - verify("0.002153817447833717", 0.002153817447833717f); - verify("0.0015924838953651488", 0x1.a175cap-10f); - verify("0.0008602388261351734", 0.0008602388261351734f); - verify("0.00036393293703440577", 0x1.7d9c82p-12f); - verify("0.00013746770127909258", 0.00013746770127909258f); - verify("16407.9462890625", 16407.9462890625f); - verify("1.1754947011469036e-38", 0x1.000006p-126f); - verify("7.0064923216240854e-46", 0x1p-149f); - verify("8388614.5", 8388614.5f); - verify("0e9999999999999999999999999999", 0.f); - verify("4.7019774032891500318749461488889827112746622270883500860350068251e-38",4.7019774032891500318749461488889827112746622270883500860350068251e-38f); - verify("3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679", 3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679f); - verify("2.3509887016445750159374730744444913556373311135441750430175034126e-38", 2.3509887016445750159374730744444913556373311135441750430175034126e-38f); - verify("1", 1.f); - verify("7.0060e-46", 0.f); - verify("3.4028234664e38", 0x1.fffffep+127f); - verify("3.4028234665e38", 0x1.fffffep+127f); - verify("3.4028234666e38", 0x1.fffffep+127f); - verify("0.000000000000000000000000000000000000011754943508222875079687365372222456778186655567720875215087517062784172594547271728515625", 0.000000000000000000000000000000000000011754943508222875079687365372222456778186655567720875215087517062784172594547271728515625); - verify("0.00000000000000000000000000000000000000000000140129846432481707092372958328991613128026194187651577175706828388979108268586060148663818836212158203125", 0.00000000000000000000000000000000000000000000140129846432481707092372958328991613128026194187651577175706828388979108268586060148663818836212158203125f); - verify("0.00000000000000000000000000000000000002350988561514728583455765982071533026645717985517980855365926236850006129930346077117064851336181163787841796875", 0.00000000000000000000000000000000000002350988561514728583455765982071533026645717985517980855365926236850006129930346077117064851336181163787841796875f); - verify("0.00000000000000000000000000000000000001175494210692441075487029444849287348827052428745893333857174530571588870475618904265502351336181163787841796875", 0.00000000000000000000000000000000000001175494210692441075487029444849287348827052428745893333857174530571588870475618904265502351336181163787841796875f); -} - -TEST_CASE("32bit.decimal_point") { - fast_float::parse_options options{}; - options.decimal_point = ','; - - // infinity - verify_options("3,5028234666e38", std::numeric_limits::infinity()); - - // finites - verify_options("1,1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125", 0x1.2ced3p+0f); - verify_options("1,1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125e-38", 0x1.fffff8p-127f); - verify_options(append_zeros("1,1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",655), 0x1.2ced3p+0f); - verify_options(append_zeros("1,1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",656), 0x1.2ced3p+0f); - verify_options(append_zeros("1,1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",1000), 0x1.2ced3p+0f); - std::string test_string; - test_string = append_zeros("1,1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",655) + std::string("e-38"); - verify_options(test_string, 0x1.fffff8p-127f); - test_string = append_zeros("1,1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",656) + std::string("e-38"); - verify_options(test_string, 0x1.fffff8p-127f); - test_string = append_zeros("1,1754941406275178592461758986628081843312458647327962400313859427181746759860647699724722770042717456817626953125",1000) + std::string("e-38"); - verify_options(test_string, 0x1.fffff8p-127f); - verify_options("1,1754943508e-38", 1.1754943508e-38f); - verify_options("30219,0830078125", 30219.0830078125f); - verify_options("1,1754947011469036e-38", 0x1.000006p-126f); - verify_options("7,0064923216240854e-46", 0x1p-149f); - verify_options("8388614,5", 8388614.5f); - verify_options("0e9999999999999999999999999999", 0.f); - verify_options("4,7019774032891500318749461488889827112746622270883500860350068251e-38",4.7019774032891500318749461488889827112746622270883500860350068251e-38f); - verify_options("3,1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679", 3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679f); - verify_options("2,3509887016445750159374730744444913556373311135441750430175034126e-38", 2.3509887016445750159374730744444913556373311135441750430175034126e-38f); - verify_options("1", 1.f); - verify_options("7,0060e-46", 0.f); - verify_options("3,4028234664e38", 0x1.fffffep+127f); - verify_options("3,4028234665e38", 0x1.fffffep+127f); - verify_options("3,4028234666e38", 0x1.fffffep+127f); - verify_options("0,000000000000000000000000000000000000011754943508222875079687365372222456778186655567720875215087517062784172594547271728515625", 0.000000000000000000000000000000000000011754943508222875079687365372222456778186655567720875215087517062784172594547271728515625); - verify_options("0,00000000000000000000000000000000000000000000140129846432481707092372958328991613128026194187651577175706828388979108268586060148663818836212158203125", 0.00000000000000000000000000000000000000000000140129846432481707092372958328991613128026194187651577175706828388979108268586060148663818836212158203125f); - verify_options("0,00000000000000000000000000000000000002350988561514728583455765982071533026645717985517980855365926236850006129930346077117064851336181163787841796875", 0.00000000000000000000000000000000000002350988561514728583455765982071533026645717985517980855365926236850006129930346077117064851336181163787841796875f); - verify_options("0,00000000000000000000000000000000000001175494210692441075487029444849287348827052428745893333857174530571588870475618904265502351336181163787841796875", 0.00000000000000000000000000000000000001175494210692441075487029444849287348827052428745893333857174530571588870475618904265502351336181163787841796875f); -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/foo.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/foo.cpp deleted file mode 100644 index cbe241275..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/foo.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#include "test.h" -void foo() { } \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/main.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/main.cpp deleted file mode 100644 index 3d46c0abc..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/main.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#include "test.h" -int main() { return 0; } \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/test.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/test.h deleted file mode 100644 index 550a31b5a..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/build_tests/issue72/test.h +++ /dev/null @@ -1,2 +0,0 @@ -#pragma once -#include "fast_float/fast_float.h" \ No newline at end of file diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/example_comma_test.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/example_comma_test.cpp deleted file mode 100644 index 12f488d1a..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/example_comma_test.cpp +++ /dev/null @@ -1,15 +0,0 @@ - -#include "fast_float/fast_float.h" -#include -#include -#include - -int main() { - const std::string input = "3,1416 xyz "; - double result; - fast_float::parse_options options{fast_float::chars_format::general, ','}; - auto answer = fast_float::from_chars_advanced(input.data(), input.data()+input.size(), result, options); - if((answer.ec != std::errc()) || ((result != 3.1416))) { std::cerr << "parsing failure\n"; return EXIT_FAILURE; } - std::cout << "parsed the number " << result << std::endl; - return EXIT_SUCCESS; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/example_test.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/example_test.cpp deleted file mode 100644 index db1bae829..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/example_test.cpp +++ /dev/null @@ -1,14 +0,0 @@ - -#include "fast_float/fast_float.h" -#include -#include -#include - -int main() { - const std::string input = "3.1416 xyz "; - double result; - auto answer = fast_float::from_chars(input.data(), input.data()+input.size(), result); - if((answer.ec != std::errc()) || ((result != 3.1416))) { std::cerr << "parsing failure\n"; return EXIT_FAILURE; } - std::cout << "parsed the number " << result << std::endl; - return EXIT_SUCCESS; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32.cpp deleted file mode 100644 index 9e1e42f26..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32.cpp +++ /dev/null @@ -1,63 +0,0 @@ - -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include -#include - -template char *to_string(T d, char *buffer) { - auto written = std::snprintf(buffer, 64, "%.*e", - std::numeric_limits::max_digits10 - 1, d); - return buffer + written; -} - -void allvalues() { - char buffer[64]; - for (uint64_t w = 0; w <= 0xFFFFFFFF; w++) { - float v; - if ((w % 1048576) == 0) { - std::cout << "."; - std::cout.flush(); - } - uint32_t word = uint32_t(w); - memcpy(&v, &word, sizeof(v)); - - { - const char *string_end = to_string(v, buffer); - float result_value; - auto result = fast_float::from_chars(buffer, string_end, result_value); - if (result.ec != std::errc()) { - std::cerr << "parsing error ? " << buffer << std::endl; - abort(); - } - if (std::isnan(v)) { - if (!std::isnan(result_value)) { - std::cerr << "not nan" << buffer << std::endl; - abort(); - } - } else if(copysign(1,result_value) != copysign(1,v)) { - std::cerr << "I got " << std::hexfloat << result_value << " but I was expecting " << v - << std::endl; - abort(); - } else if (result_value != v) { - std::cerr << "no match ? " << buffer << std::endl; - std::cout << "started with " << std::hexfloat << v << std::endl; - std::cout << "got back " << std::hexfloat << result_value << std::endl; - std::cout << std::dec; - abort(); - } - } - } - std::cout << std::endl; -} - -int main() { - allvalues(); - std::cout << std::endl; - std::cout << "all ok" << std::endl; - return EXIT_SUCCESS; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32_64.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32_64.cpp deleted file mode 100644 index e02757bd3..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32_64.cpp +++ /dev/null @@ -1,78 +0,0 @@ - -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -template char *to_string(T d, char *buffer) { - auto written = std::snprintf(buffer, 64, "%.*e", - std::numeric_limits::max_digits10 - 1, d); - return buffer + written; -} - - -bool basic_test_64bit(std::string vals, double val) { - double result_value; - auto result = fast_float::from_chars(vals.data(), vals.data() + vals.size(), - result_value); - if (result.ec != std::errc()) { - std::cerr << " I could not parse " << vals << std::endl; - return false; - } - if (std::isnan(val)) { - if (!std::isnan(result_value)) { - std::cerr << vals << std::endl; - std::cerr << "not nan" << result_value << std::endl; - return false; - } - } else if(copysign(1,result_value) != copysign(1,val)) { - std::cerr << "I got " << std::hexfloat << result_value << " but I was expecting " << val - << std::endl; - return false; - } else if (result_value != val) { - std::cerr << vals << std::endl; - std::cerr << "I got " << std::hexfloat << result_value << " but I was expecting " << val - << std::endl; - std::cerr << std::dec; - std::cerr << "string: " << vals << std::endl; - return false; - } - return true; -} - - -void all_32bit_values() { - char buffer[64]; - for (uint64_t w = 0; w <= 0xFFFFFFFF; w++) { - float v32; - if ((w % 1048576) == 0) { - std::cout << "."; - std::cout.flush(); - } - uint32_t word = uint32_t(w); - memcpy(&v32, &word, sizeof(v32)); - double v = v32; - - { - const char *string_end = to_string(v, buffer); - std::string s(buffer, size_t(string_end-buffer)); - if(!basic_test_64bit(s,v)) { - return; - } - } - } - std::cout << std::endl; -} - -int main() { - all_32bit_values(); - std::cout << std::endl; - std::cout << "all ok" << std::endl; - return EXIT_SUCCESS; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32_midpoint.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32_midpoint.cpp deleted file mode 100644 index f64f4e481..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/exhaustive32_midpoint.cpp +++ /dev/null @@ -1,148 +0,0 @@ -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include -#include - -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) -// Anything at all that is related to cygwin, msys and so forth will -// always use this fallback because we cannot rely on it behaving as normal -// gcc. -#include -#include -// workaround for CYGWIN -double cygwin_strtod_l(const char* start, char** end) { - double d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -float cygwin_strtof_l(const char* start, char** end) { - float d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -#endif - -template char *to_string(T d, char *buffer) { - auto written = std::snprintf(buffer, 64, "%.*e", - std::numeric_limits::max_digits10 - 1, d); - return buffer + written; -} - -void strtof_from_string(const char * st, float& d) { - char *pr = (char *)st; -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - d = cygwin_strtof_l(st, &pr); -#elif defined(_WIN32) - static _locale_t c_locale = _create_locale(LC_ALL, "C"); - d = _strtof_l(st, &pr, c_locale); -#else - static locale_t c_locale = newlocale(LC_ALL_MASK, "C", NULL); - d = strtof_l(st, &pr, c_locale); -#endif - if (pr == st) { - throw std::runtime_error("bug in strtod_from_string"); - } -} - -bool allvalues() { - char buffer[64]; - for (uint64_t w = 0; w <= 0xFFFFFFFF; w++) { - float v; - if ((w % 1048576) == 0) { - std::cout << "."; - std::cout.flush(); - } - uint32_t word = uint32_t(w); - memcpy(&v, &word, sizeof(v)); - if(std::isfinite(v)) { - float nextf = std::nextafterf(v, INFINITY); - if(copysign(1,v) != copysign(1,nextf)) { continue; } - if(!std::isfinite(nextf)) { continue; } - double v1{v}; - assert(float(v1) == v); - double v2{nextf}; - assert(float(v2) == nextf); - double midv{v1 + (v2 - v1) / 2}; - float expected_midv = float(midv); - - const char *string_end = to_string(midv, buffer); - float str_answer; - strtof_from_string(buffer, str_answer); - - float result_value; - auto result = fast_float::from_chars(buffer, string_end, result_value); - if (result.ec != std::errc()) { - std::cerr << "parsing error ? " << buffer << std::endl; - return false; - } - if (std::isnan(v)) { - if (!std::isnan(result_value)) { - std::cerr << "not nan" << buffer << std::endl; - std::cerr << "v " << std::hexfloat << v << std::endl; - std::cerr << "v2 " << std::hexfloat << v2 << std::endl; - std::cerr << "midv " << std::hexfloat << midv << std::endl; - std::cerr << "expected_midv " << std::hexfloat << expected_midv << std::endl; - return false; - } - } else if(copysign(1,result_value) != copysign(1,v)) { - std::cerr << buffer << std::endl; - std::cerr << "v " << std::hexfloat << v << std::endl; - std::cerr << "v2 " << std::hexfloat << v2 << std::endl; - std::cerr << "midv " << std::hexfloat << midv << std::endl; - std::cerr << "expected_midv " << std::hexfloat << expected_midv << std::endl; - std::cerr << "I got " << std::hexfloat << result_value << " but I was expecting " << v - << std::endl; - return false; - } else if (result_value != str_answer) { - std::cerr << "no match ? " << buffer << std::endl; - std::cerr << "v " << std::hexfloat << v << std::endl; - std::cerr << "v2 " << std::hexfloat << v2 << std::endl; - std::cerr << "midv " << std::hexfloat << midv << std::endl; - std::cerr << "expected_midv " << std::hexfloat << expected_midv << std::endl; - std::cout << "started with " << std::hexfloat << midv << std::endl; - std::cout << "round down to " << std::hexfloat << str_answer << std::endl; - std::cout << "got back " << std::hexfloat << result_value << std::endl; - std::cout << std::dec; - return false; - } - } - } - std::cout << std::endl; - return true; -} - -inline void Assert(bool Assertion) { -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - if (!Assertion) { std::cerr << "Omitting hard falure on msys/cygwin/sun systems."; } -#else - if (!Assertion) { throw std::runtime_error("bug"); } -#endif -} -int main() { -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - std::cout << "Warning: msys/cygwin or solaris detected. This particular test is likely to generate false failures due to our reliance on the underlying runtime library as a gold standard." << std::endl; -#endif - Assert(allvalues()); - std::cout << std::endl; - std::cout << "all ok" << std::endl; - return EXIT_SUCCESS; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_exhaustive32.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_exhaustive32.cpp deleted file mode 100644 index 0a6b53d81..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_exhaustive32.cpp +++ /dev/null @@ -1,63 +0,0 @@ - -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include - -template char *to_string(T d, char *buffer) { - auto written = std::snprintf(buffer, 128, "%.*e", - 64, d); - return buffer + written; -} - -void allvalues() { - char buffer[128]; - for (uint64_t w = 0; w <= 0xFFFFFFFF; w++) { - float v; - if ((w % 1048576) == 0) { - std::cout << "."; - std::cout.flush(); - } - uint32_t word = uint32_t(w); - memcpy(&v, &word, sizeof(v)); - - { - const char *string_end = to_string(v, buffer); - float result_value; - auto result = fast_float::from_chars(buffer, string_end, result_value); - if (result.ec != std::errc()) { - std::cerr << "parsing error ? " << buffer << std::endl; - abort(); - } - if (std::isnan(v)) { - if (!std::isnan(result_value)) { - std::cerr << "not nan" << buffer << std::endl; - abort(); - } - } else if(copysign(1,result_value) != copysign(1,v)) { - std::cerr << buffer << std::endl; - std::cerr << "I got " << std::hexfloat << result_value << " but I was expecting " << v - << std::endl; - abort(); - } else if (result_value != v) { - std::cerr << "no match ? " << buffer << " got " << result_value << " expected " << v << std::endl; - std::cout << "started with " << std::hexfloat << v << std::endl; - std::cout << "got back " << std::hexfloat << result_value << std::endl; - std::cout << std::dec; - abort(); - } - } - } - std::cout << std::endl; -} - -int main() { - allvalues(); - std::cout << std::endl; - std::cout << "all ok" << std::endl; - return EXIT_SUCCESS; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_exhaustive32_64.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_exhaustive32_64.cpp deleted file mode 100644 index cea8497f9..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_exhaustive32_64.cpp +++ /dev/null @@ -1,66 +0,0 @@ -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include - -template char *to_string(T d, char *buffer) { - auto written = std::snprintf(buffer, 128, "%.*e", - 64, d); - return buffer + written; -} - -void all_32bit_values() { - char buffer[128]; - for (uint64_t w = 0; w <= 0xFFFFFFFF; w++) { - float v32; - if ((w % 1048576) == 0) { - std::cout << "."; - std::cout.flush(); - } - uint32_t word = uint32_t(w); - memcpy(&v32, &word, sizeof(v32)); - double v = v32; - - { - const char *string_end = to_string(v, buffer); - double result_value; - auto result = fast_float::from_chars(buffer, string_end, result_value); - if (result.ec != std::errc()) { - std::cerr << "parsing error ? " << buffer << std::endl; - abort(); - } - if (std::isnan(v)) { - if (!std::isnan(result_value)) { - std::cerr << "not nan" << buffer << std::endl; - abort(); - } - } else if(copysign(1,result_value) != copysign(1,v)) { - std::cerr << "I got " << std::hexfloat << result_value << " but I was expecting " << v - << std::endl; - abort(); - } else if (std::isnan(v)) { - if (!std::isnan(result_value)) { - std::cerr << "not nan" << buffer << std::endl; - abort(); - } - } else if (result_value != v) { - std::cerr << "no match ? " << buffer << std::endl; - std::cout << "started with " << std::hexfloat << v << std::endl; - std::cout << "got back " << std::hexfloat << result_value << std::endl; - std::cout << std::dec; - abort(); - } - } - } - std::cout << std::endl; -} - -int main() { - all_32bit_values(); - std::cout << std::endl; - std::cout << "all ok" << std::endl; - return EXIT_SUCCESS; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_random64.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_random64.cpp deleted file mode 100644 index a6680e82c..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_random64.cpp +++ /dev/null @@ -1,106 +0,0 @@ -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include - -template char *to_string(T d, char *buffer) { - auto written = std::snprintf(buffer, 128, "%.*e", - 64, d); - return buffer + written; -} - -static fast_float::value128 g_lehmer64_state; - -/** - * D. H. Lehmer, Mathematical methods in large-scale computing units. - * Proceedings of a Second Symposium on Large Scale Digital Calculating - * Machinery; - * Annals of the Computation Laboratory, Harvard Univ. 26 (1951), pp. 141-146. - * - * P L'Ecuyer, Tables of linear congruential generators of different sizes and - * good lattice structure. Mathematics of Computation of the American - * Mathematical - * Society 68.225 (1999): 249-260. - */ - -static inline void lehmer64_seed(uint64_t seed) { - g_lehmer64_state.high = 0; - g_lehmer64_state.low = seed; -} - -static inline uint64_t lehmer64() { - fast_float::value128 v = fast_float::full_multiplication(g_lehmer64_state.low,UINT64_C(0xda942042e4dd58b5)); - v.high += g_lehmer64_state.high * UINT64_C(0xda942042e4dd58b5); - g_lehmer64_state = v; - return v.high; -} - -size_t errors; - -void random_values(size_t N) { - char buffer[128]; - lehmer64_seed(N); - for (size_t t = 0; t < N; t++) { - if ((t % 1048576) == 0) { - std::cout << "."; - std::cout.flush(); - } - uint64_t word = lehmer64(); - double v; - memcpy(&v, &word, sizeof(v)); - { - const char *string_end = to_string(v, buffer); - double result_value; - auto result = fast_float::from_chars(buffer, string_end, result_value); - if (result.ec != std::errc()) { - std::cerr << "parsing error ? " << buffer << std::endl; - errors++; - if (errors > 10) { - abort(); - } - } - if (std::isnan(v)) { - if (!std::isnan(result_value)) { - std::cerr << "not nan" << buffer << std::endl; - errors++; - if (errors > 10) { - abort(); - } - } - } else if(copysign(1,result_value) != copysign(1,v)) { - std::cerr << buffer << std::endl; - std::cerr << "I got " << std::hexfloat << result_value << " but I was expecting " << v - << std::endl; - abort(); - } else if (result_value != v) { - std::cerr << "no match ? '" << buffer << "'" << std::endl; - std::cout << "started with " << std::hexfloat << v << std::endl; - std::cout << "got back " << std::hexfloat << result_value << std::endl; - std::cout << std::dec; - errors++; - if (errors > 10) { - abort(); - } - } - } - } - std::cout << std::endl; -} - -int main() { - errors = 0; - size_t N = size_t(1) << (sizeof(size_t) * 4); // shift: 32 for 64bit, 16 for 32bit - random_values(N); - if (errors == 0) { - std::cout << std::endl; - std::cout << "all ok" << std::endl; - return EXIT_SUCCESS; - } - std::cerr << std::endl; - std::cerr << "errors were encountered" << std::endl; - return EXIT_FAILURE; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_test.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_test.cpp deleted file mode 100644 index 36b92104e..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/long_test.cpp +++ /dev/null @@ -1,58 +0,0 @@ -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include - -inline void Assert(bool Assertion) { - if (!Assertion) { throw std::runtime_error("bug"); } -} - -template -bool test() { - std::string input = "0.156250000000000000000000000000000000000000 3.14159265358979323846264338327950288419716939937510 2.71828182845904523536028747135266249775724709369995"; - std::vector answers = {T(0.15625), T(3.141592653589793), T(2.718281828459045)}; - const char * begin = input.data(); - const char * end = input.data() + input.size(); - for(size_t i = 0; i < answers.size(); i++) { - T result_value; - while((begin < end) && (std::isspace(*begin))) { begin++; } - auto result = fast_float::from_chars(begin, end, - result_value); - if (result.ec != std::errc()) { - printf("parsing %.*s\n", int(end - begin), begin); - std::cerr << " I could not parse " << std::endl; - return false; - } - if(result_value != answers[i]) { - printf("parsing %.*s\n", int(end - begin), begin); - std::cerr << " Mismatch " << std::endl; - std::cerr << " Expected " << answers[i] << std::endl; - std::cerr << " Got " << result_value << std::endl; - - return false; - - } - begin = result.ptr; - } - if(begin != end) { - std::cerr << " bad ending " << std::endl; - return false; - } - return true; -} - -int main() { - - std::cout << "32 bits checks" << std::endl; - Assert(test()); - - std::cout << "64 bits checks" << std::endl; - Assert(test()); - - std::cout << "All ok" << std::endl; - return EXIT_SUCCESS; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/powersoffive_hardround.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/powersoffive_hardround.cpp deleted file mode 100644 index 09b95bd59..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/powersoffive_hardround.cpp +++ /dev/null @@ -1,134 +0,0 @@ -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include -#include -#include - - -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) -// Anything at all that is related to cygwin, msys and so forth will -// always use this fallback because we cannot rely on it behaving as normal -// gcc. -#include -// workaround for CYGWIN -double cygwin_strtod_l(const char* start, char** end) { - double d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -float cygwin_strtof_l(const char* start, char** end) { - float d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -#endif - - -std::pair strtod_from_string(const char *st) { - double d; - char *pr; -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - d = cygwin_strtod_l(st, &pr); -#elif defined(_WIN32) - static _locale_t c_locale = _create_locale(LC_ALL, "C"); - d = _strtod_l(st, &pr, c_locale); -#else - static locale_t c_locale = newlocale(LC_ALL_MASK, "C", NULL); - d = strtod_l(st, &pr, c_locale); -#endif - if (st == pr) { - std::cerr << "strtod_l could not parse '" << st << std::endl; - return std::make_pair(0, false); - } - return std::make_pair(d, true); -} - -std::pair strtof_from_string(char *st) { - float d; - char *pr; -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - d = cygwin_strtof_l(st, &pr); -#elif defined(_WIN32) - static _locale_t c_locale = _create_locale(LC_ALL, "C"); - d = _strtof_l(st, &pr, c_locale); -#else - static locale_t c_locale = newlocale(LC_ALL_MASK, "C", NULL); - d = strtof_l(st, &pr, c_locale); -#endif - if (st == pr) { - std::cerr << "strtof_l could not parse '" << st << std::endl; - return std::make_pair(0.0f, false); - } - return std::make_pair(d, true); -} - -bool tester() { - std::random_device rd; - std::mt19937 gen(rd()); - for (int q = 18; q <= 27; q++) { - std::cout << "q = " << -q << std::endl; - uint64_t power5 = 1; - for (int k = 0; k < q; k++) { - power5 *= 5; - } - uint64_t low_threshold = 0x20000000000000 / power5 + 1; - uint64_t threshold = 0xFFFFFFFFFFFFFFFF / power5; - std::uniform_int_distribution dis(low_threshold, threshold); - for (size_t i = 0; i < 10000; i++) { - uint64_t mantissa = dis(gen) * power5; - std::stringstream ss; - ss << mantissa; - ss << "e"; - ss << -q; - std::string to_be_parsed = ss.str(); - std::pair expected_double = - strtod_from_string(to_be_parsed.c_str()); - double result_value; - auto result = - fast_float::from_chars(to_be_parsed.data(), to_be_parsed.data() + to_be_parsed.size(), result_value); - if (result.ec != std::errc()) { - std::cout << to_be_parsed << std::endl; - std::cerr << " I could not parse " << std::endl; - return false; - } - if (result_value != expected_double.first) { - std::cout << to_be_parsed << std::endl; - std::cerr << std::hexfloat << result_value << std::endl; - std::cerr << std::hexfloat << expected_double.first << std::endl; - std::cerr << " Mismatch " << std::endl; - return false; - } - } - } - return true; -} - -int main() { - if (tester()) { - std::cout << std::endl; - std::cout << "all ok" << std::endl; - return EXIT_SUCCESS; - } - std::cerr << std::endl; - std::cerr << "errors were encountered" << std::endl; - return EXIT_FAILURE; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/random64.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/random64.cpp deleted file mode 100644 index 0775993a4..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/random64.cpp +++ /dev/null @@ -1,109 +0,0 @@ -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include -#include - -template char *to_string(T d, char *buffer) { - auto written = std::snprintf(buffer, 64, "%.*e", - std::numeric_limits::max_digits10 - 1, d); - return buffer + written; -} - - -static fast_float::value128 g_lehmer64_state; - -/** - * D. H. Lehmer, Mathematical methods in large-scale computing units. - * Proceedings of a Second Symposium on Large Scale Digital Calculating - * Machinery; - * Annals of the Computation Laboratory, Harvard Univ. 26 (1951), pp. 141-146. - * - * P L'Ecuyer, Tables of linear congruential generators of different sizes and - * good lattice structure. Mathematics of Computation of the American - * Mathematical - * Society 68.225 (1999): 249-260. - */ - -static inline void lehmer64_seed(uint64_t seed) { - g_lehmer64_state.high = 0; - g_lehmer64_state.low = seed; -} - -static inline uint64_t lehmer64() { - fast_float::value128 v = fast_float::full_multiplication(g_lehmer64_state.low,UINT64_C(0xda942042e4dd58b5)); - v.high += g_lehmer64_state.high * UINT64_C(0xda942042e4dd58b5); - g_lehmer64_state = v; - return v.high; -} - -size_t errors; - -void random_values(size_t N) { - char buffer[64]; - lehmer64_seed(N); - for (size_t t = 0; t < N; t++) { - if ((t % 1048576) == 0) { - std::cout << "."; - std::cout.flush(); - } - uint64_t word = lehmer64(); - double v; - memcpy(&v, &word, sizeof(v)); - // if (!std::isnormal(v)) - { - const char *string_end = to_string(v, buffer); - double result_value; - auto result = fast_float::from_chars(buffer, string_end, result_value); - if (result.ec != std::errc()) { - std::cerr << "parsing error ? " << buffer << std::endl; - errors++; - if (errors > 10) { - abort(); - } - } - if (std::isnan(v)) { - if (!std::isnan(result_value)) { - std::cerr << "not nan" << buffer << std::endl; - errors++; - if (errors > 10) { - abort(); - } - } - } else if(copysign(1,result_value) != copysign(1,v)) { - std::cerr << buffer << std::endl; - std::cerr << "I got " << std::hexfloat << result_value << " but I was expecting " << v - << std::endl; - abort(); - } else if (result_value != v) { - std::cerr << "no match ? " << buffer << std::endl; - std::cout << "started with " << std::hexfloat << v << std::endl; - std::cout << "got back " << std::hexfloat << result_value << std::endl; - std::cout << std::dec; - errors++; - if (errors > 10) { - abort(); - } - } - } - } - std::cout << std::endl; -} - -int main() { - errors = 0; - size_t N = size_t(1) << (sizeof(size_t) * 4); // shift: 32 for 64bit, 16 for 32bit - random_values(N); - if (errors == 0) { - std::cout << std::endl; - std::cout << "all ok" << std::endl; - return EXIT_SUCCESS; - } - std::cerr << std::endl; - std::cerr << "errors were encountered" << std::endl; - return EXIT_FAILURE; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/random_string.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/random_string.cpp deleted file mode 100644 index 8cabf5f4d..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/random_string.cpp +++ /dev/null @@ -1,240 +0,0 @@ -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include - -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) -// Anything at all that is related to cygwin, msys and so forth will -// always use this fallback because we cannot rely on it behaving as normal -// gcc. -#include -#include -// workaround for CYGWIN -double cygwin_strtod_l(const char* start, char** end) { - double d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -float cygwin_strtof_l(const char* start, char** end) { - float d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -#endif - -class RandomEngine { -public: - RandomEngine() = delete; - RandomEngine(uint64_t new_seed) : wyhash64_x_(new_seed) {}; - uint64_t next() { - // Adapted from https://github.com/wangyi-fudan/wyhash/blob/master/wyhash.h - // Inspired from - // https://github.com/lemire/testingRNG/blob/master/source/wyhash.h - wyhash64_x_ += UINT64_C(0x60bee2bee120fc15); - fast_float::value128 tmp = fast_float::full_multiplication(wyhash64_x_, UINT64_C(0xa3b195354a39b70d)); - uint64_t m1 = (tmp.high) ^ tmp.low; - tmp = fast_float::full_multiplication(m1, UINT64_C(0x1b03738712fad5c9)); - uint64_t m2 = (tmp.high) ^ tmp.low; - return m2; - } - bool next_bool() { return (next() & 1) == 1; } - int next_int() { return static_cast(next()); } - char next_char() { return static_cast(next()); } - double next_double() { return static_cast(next()); } - - int next_ranged_int(int min, int max) { // min and max are included - // Adapted from - // https://lemire.me/blog/2019/06/06/nearly-divisionless-random-integer-generation-on-various-systems/ - /* if (min == max) { - return min; - }*/ - uint64_t s = uint64_t(max - min + 1); - uint64_t x = next(); - fast_float::value128 m = fast_float::full_multiplication(x, s); - uint64_t l = m.low; - if (l < s) { - uint64_t t = -s % s; - while (l < t) { - x = next(); - m = fast_float::full_multiplication(x, s); - l = m.low; - } - } - return int(m.high) + min; - } - int next_digit() { return next_ranged_int(0, 9); } - -private: - uint64_t wyhash64_x_; -}; - -size_t build_random_string(RandomEngine &rand, char *buffer) { - size_t pos{0}; - if (rand.next_bool()) { - buffer[pos++] = '-'; - } - int number_of_digits = rand.next_ranged_int(1, 100); - if(number_of_digits == 100) { - // With low probability, we want to allow very long strings just to stress the system. - number_of_digits = rand.next_ranged_int(1, 2000); - } - int location_of_decimal_separator = rand.next_ranged_int(1, number_of_digits); - for (size_t i = 0; i < size_t(number_of_digits); i++) { - if (i == size_t(location_of_decimal_separator)) { - buffer[pos++] = '.'; - } - buffer[pos++] = char(rand.next_digit() + '0'); - } - if (rand.next_bool()) { - if (rand.next_bool()) { - buffer[pos++] = 'e'; - } else { - buffer[pos++] = 'E'; - } - if (rand.next_bool()) { - buffer[pos++] = '-'; - } else { - if (rand.next_bool()) { - buffer[pos++] = '+'; - } - } - number_of_digits = rand.next_ranged_int(1, 3); - for (size_t i = 0; i < size_t(number_of_digits); i++) { - buffer[pos++] = char(rand.next_digit() + '0'); - } - } - buffer[pos] = '\0'; // null termination - return pos; -} - -std::pair strtod_from_string(char *st) { - double d; - char *pr; -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - d = cygwin_strtod_l(st, &pr); -#elif defined(_WIN32) - static _locale_t c_locale = _create_locale(LC_ALL, "C"); - d = _strtod_l(st, &pr, c_locale); -#else - static locale_t c_locale = newlocale(LC_ALL_MASK, "C", NULL); - d = strtod_l(st, &pr, c_locale); -#endif - if (st == pr) { - std::cerr << "strtod_l could not parse '" << st << std::endl; - return std::make_pair(0, false); - } - return std::make_pair(d, true); -} - -std::pair strtof_from_string(char *st) { - float d; - char *pr; -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - d = cygwin_strtof_l(st, &pr); -#elif defined(_WIN32) - static _locale_t c_locale = _create_locale(LC_ALL, "C"); - d = _strtof_l(st, &pr, c_locale); -#else - static locale_t c_locale = newlocale(LC_ALL_MASK, "C", NULL); - d = strtof_l(st, &pr, c_locale); -#endif - if (st == pr) { - std::cerr << "strtof_l could not parse '" << st << std::endl; - return std::make_pair(0.0f, false); - } - return std::make_pair(d, true); -} - -/** - * We generate random strings and we try to parse them with both strtod/strtof, - * and we verify that we get the same answer with with fast_float::from_chars. - */ -bool tester(uint64_t seed, size_t volume) { - char buffer[4096]; // large buffer (can't overflow) - RandomEngine rand(seed); - for (size_t i = 0; i < volume; i++) { - if((i%100000) == 0) { std::cout << "."; std::cout.flush(); } - size_t length = build_random_string(rand, buffer); - std::pair expected_double = strtod_from_string(buffer); - if (expected_double.second) { - double result_value; - auto result = - fast_float::from_chars(buffer, buffer + length, result_value); - if (result.ec != std::errc()) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << " I could not parse " << std::endl; - return false; - } - if (result.ptr != buffer + length) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << " Did not get to the end " << std::endl; - return false; - } - if (result_value != expected_double.first) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << std::hexfloat << result_value << std::endl; - std::cerr << std::hexfloat << expected_double.first << std::endl; - std::cerr << " Mismatch " << std::endl; - return false; - } - } - std::pair expected_float = strtof_from_string(buffer); - if (expected_float.second) { - float result_value; - auto result = - fast_float::from_chars(buffer, buffer + length, result_value); - if (result.ec != std::errc()) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << " I could not parse " << std::endl; - return false; - } - if (result.ptr != buffer + length) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << " Did not get to the end " << std::endl; - return false; - } - if (result_value != expected_float.first) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << std::hexfloat << result_value << std::endl; - std::cerr << std::hexfloat << expected_float.first << std::endl; - std::cerr << " Mismatch " << std::endl; - return false; - } - } - } - return true; -} - -int main() { - -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - std::cout << "Warning: msys/cygwin or solaris detected." << std::endl; - return EXIT_SUCCESS; -#else - if (tester(1234344, 100000000)) { - std::cout << "All tests ok." << std::endl; - return EXIT_SUCCESS; - } - std::cout << "Failure." << std::endl; - return EXIT_FAILURE; - -#endif -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/short_random_string.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/short_random_string.cpp deleted file mode 100644 index 3051b749d..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/short_random_string.cpp +++ /dev/null @@ -1,234 +0,0 @@ -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include - -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) -// Anything at all that is related to cygwin, msys and so forth will -// always use this fallback because we cannot rely on it behaving as normal -// gcc. -#include -#include -// workaround for CYGWIN -double cygwin_strtod_l(const char* start, char** end) { - double d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -float cygwin_strtof_l(const char* start, char** end) { - float d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -#endif - -class RandomEngine { -public: - RandomEngine() = delete; - RandomEngine(uint64_t new_seed) : wyhash64_x_(new_seed) { }; - uint64_t next() { - // Adapted from https://github.com/wangyi-fudan/wyhash/blob/master/wyhash.h - // Inspired from - // https://github.com/lemire/testingRNG/blob/master/source/wyhash.h - wyhash64_x_ += UINT64_C(0x60bee2bee120fc15); - fast_float::value128 tmp = fast_float::full_multiplication(wyhash64_x_, UINT64_C(0xa3b195354a39b70d)); - uint64_t m1 = (tmp.high) ^ tmp.low; - tmp = fast_float::full_multiplication(m1, UINT64_C(0x1b03738712fad5c9)); - uint64_t m2 = (tmp.high) ^ tmp.low; - return m2; - } - bool next_bool() { return (next() & 1) == 1; } - int next_int() { return static_cast(next()); } - char next_char() { return static_cast(next()); } - double next_double() { return static_cast(next()); } - - int next_ranged_int(int min, int max) { // min and max are included - // Adapted from - // https://lemire.me/blog/2019/06/06/nearly-divisionless-random-integer-generation-on-various-systems/ - /* if (min == max) { - return min; - }*/ - uint64_t s = uint64_t(max - min + 1); - uint64_t x = next(); - fast_float::value128 m = fast_float::full_multiplication(x, s); - uint64_t l = m.low; - if (l < s) { - uint64_t t = -s % s; - while (l < t) { - x = next(); - m = fast_float::full_multiplication(x, s); - l = m.low; - } - } - return int(m.high) + min; - } - int next_digit() { return next_ranged_int(0, 9); } - -private: - uint64_t wyhash64_x_; -}; - -size_t build_random_string(RandomEngine &rand, char *buffer) { - size_t pos{0}; - if (rand.next_bool()) { - buffer[pos++] = '-'; - } - int number_of_digits = rand.next_ranged_int(1, 19); - int location_of_decimal_separator = rand.next_ranged_int(1, number_of_digits); - for (size_t i = 0; i < size_t(number_of_digits); i++) { - if (i == size_t(location_of_decimal_separator)) { - buffer[pos++] = '.'; - } - buffer[pos++] = char(rand.next_digit() + '0'); - } - if (rand.next_bool()) { - if (rand.next_bool()) { - buffer[pos++] = 'e'; - } else { - buffer[pos++] = 'E'; - } - if (rand.next_bool()) { - buffer[pos++] = '-'; - } else { - if (rand.next_bool()) { - buffer[pos++] = '+'; - } - } - number_of_digits = rand.next_ranged_int(1, 3); - for (size_t i = 0; i < size_t(number_of_digits); i++) { - buffer[pos++] = char(rand.next_digit() + '0'); - } - } - buffer[pos] = '\0'; // null termination - return pos; -} - -std::pair strtod_from_string(char *st) { - double d; - char *pr; -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - d = cygwin_strtod_l(st, &pr); -#elif defined(_WIN32) - static _locale_t c_locale = _create_locale(LC_ALL, "C"); - d = _strtod_l(st, &pr, c_locale); -#else - static locale_t c_locale = newlocale(LC_ALL_MASK, "C", NULL); - d = strtod_l(st, &pr, c_locale); -#endif - if (st == pr) { - std::cerr << "strtod_l could not parse '" << st << std::endl; - return std::make_pair(0, false); - } - return std::make_pair(d, true); -} - -std::pair strtof_from_string(char *st) { - float d; - char *pr; -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - d = cygwin_strtof_l(st, &pr); -#elif defined(_WIN32) - static _locale_t c_locale = _create_locale(LC_ALL, "C"); - d = _strtof_l(st, &pr, c_locale); -#else - static locale_t c_locale = newlocale(LC_ALL_MASK, "C", NULL); - d = strtof_l(st, &pr, c_locale); -#endif - if (st == pr) { - std::cerr << "strtof_l could not parse '" << st << std::endl; - return std::make_pair(0.0f, false); - } - return std::make_pair(d, true); -} - -/** - * We generate random strings and we try to parse them with both strtod/strtof, - * and we verify that we get the same answer with with fast_float::from_chars. - */ -bool tester(uint64_t seed, size_t volume) { - char buffer[4096]; // large buffer (can't overflow) - RandomEngine rand(seed); - for (size_t i = 0; i < volume; i++) { - if((i%1000000) == 0) { std::cout << "."; std::cout.flush(); } - size_t length = build_random_string(rand, buffer); - std::pair expected_double = strtod_from_string(buffer); - if (expected_double.second) { - double result_value; - auto result = - fast_float::from_chars(buffer, buffer + length, result_value); - if (result.ec != std::errc()) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << " I could not parse " << std::endl; - return false; - } - if (result.ptr != buffer + length) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << " Did not get to the end " << std::endl; - return false; - } - if (result_value != expected_double.first) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << std::hexfloat << result_value << std::endl; - std::cerr << std::hexfloat << expected_double.first << std::endl; - std::cerr << " Mismatch " << std::endl; - return false; - } - } - std::pair expected_float = strtof_from_string(buffer); - if (expected_float.second) { - float result_value; - auto result = - fast_float::from_chars(buffer, buffer + length, result_value); - if (result.ec != std::errc()) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << " I could not parse " << std::endl; - return false; - } - if (result.ptr != buffer + length) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << " Did not get to the end " << std::endl; - return false; - } - if (result_value != expected_float.first) { - printf("parsing %.*s\n", int(length), buffer); - std::cerr << std::hexfloat << result_value << std::endl; - std::cerr << std::hexfloat << expected_float.first << std::endl; - std::cerr << " Mismatch " << std::endl; - return false; - } - } - } - return true; -} - -int main() { -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - std::cout << "Warning: msys/cygwin detected. This particular test is likely to generate false failures due to our reliance on the underlying runtime library." << std::endl; - return EXIT_SUCCESS; -#else - if (tester(1234344, 100000000)) { - std::cout << "All tests ok." << std::endl; - return EXIT_SUCCESS; - } - return EXIT_FAILURE; - -#endif -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/string_test.cpp b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/string_test.cpp deleted file mode 100644 index 05871dbe3..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float/tests/string_test.cpp +++ /dev/null @@ -1,279 +0,0 @@ -#include "fast_float/fast_float.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) -// Anything at all that is related to cygwin, msys and so forth will -// always use this fallback because we cannot rely on it behaving as normal -// gcc. -#include -#include -// workaround for CYGWIN -double cygwin_strtod_l(const char* start, char** end) { - double d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -float cygwin_strtof_l(const char* start, char** end) { - float d; - std::stringstream ss; - ss.imbue(std::locale::classic()); - ss << start; - ss >> d; - if(ss.fail()) { *end = nullptr; } - if(ss.eof()) { ss.clear(); } - auto nread = ss.tellg(); - *end = const_cast(start) + nread; - return d; -} -#endif - -inline void Assert(bool Assertion) { -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - if (!Assertion) { std::cerr << "Omitting hard falure on msys/cygwin/sun systems."; } -#else - if (!Assertion) { throw std::runtime_error("bug"); } -#endif -} - -template std::string to_string(T d) { - std::string s(64, '\0'); - auto written = std::snprintf(&s[0], s.size(), "%.*e", - std::numeric_limits::max_digits10 - 1, d); - s.resize(size_t(written)); - return s; -} - -template -bool test() { - std::string input = "0.1 1e1000 100000 3.14159265359 -1e-500 001 1e01 1e0000001 -inf"; - std::vector answers = {T(0.1), std::numeric_limits::infinity(), 100000, T(3.14159265359), -0.0, 1, 10, 10, -std::numeric_limits::infinity()}; - const char * begin = input.data(); - const char * end = input.data() + input.size(); - for(size_t i = 0; i < answers.size(); i++) { - T result_value; - while((begin < end) && (std::isspace(*begin))) { begin++; } - auto result = fast_float::from_chars(begin, end, - result_value); - if (result.ec != std::errc()) { - printf("parsing %.*s\n", int(end - begin), begin); - std::cerr << " I could not parse " << std::endl; - return false; - } - if(result_value != answers[i]) { - printf("parsing %.*s\n", int(end - begin), begin); - std::cerr << " Mismatch " << std::endl; - return false; - - } - begin = result.ptr; - } - if(begin != end) { - std::cerr << " bad ending " << std::endl; - return false; - } - return true; -} - -template -void strtod_from_string(const std::string &st, T& d); - -template <> -void strtod_from_string(const std::string &st, double& d) { - char *pr = (char *)st.c_str(); -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - d = cygwin_strtod_l(pr, &pr); -#elif defined(_WIN32) - static _locale_t c_locale = _create_locale(LC_ALL, "C"); - d = _strtod_l(st.c_str(), &pr, c_locale); -#else - static locale_t c_locale = newlocale(LC_ALL_MASK, "C", NULL); - d = strtod_l(st.c_str(), &pr, c_locale); -#endif - if (pr == st.c_str()) { - throw std::runtime_error("bug in strtod_from_string"); - } -} - -template <> -void strtod_from_string(const std::string &st, float& d) { - char *pr = (char *)st.c_str(); -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - d = cygwin_strtof_l(st.c_str(), &pr); -#elif defined(_WIN32) - static _locale_t c_locale = _create_locale(LC_ALL, "C"); - d = _strtof_l(st.c_str(), &pr, c_locale); -#else - static locale_t c_locale = newlocale(LC_ALL_MASK, "C", NULL); - d = strtof_l(st.c_str(), &pr, c_locale); -#endif - if (pr == st.c_str()) { - throw std::runtime_error("bug in strtod_from_string"); - } -} - -template -bool partow_test() { - // credit: https://github.com/ArashPartow/strtk/blob/master/strtk_tokenizer_cmp.cpp#L568 - // MIT license - const std::string strint_list[] = { "9007199254740993", "9007199254740994", "9007199254740995" , - "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", - "917049", "4931205", "6768064", "6884243", "5647132", "7371203", "-8629878", "4941840", "4543268", "1075600", - "290", "823", "111", "715", "-866", "367", "666", "-706", "850", "-161", - "9922547", "6960207", "1883152", "2300759", "-279294", "4187292", "3699841", "8386395", "-1441129", "-887892", - "-635422", "9742573", "2326186", "-5903851", "5648486", "3057647", "2980079", "2957468", "7929158", "1925615", - "879", "130", "292", "705", "817", "446", "576", "750", "523", "-527", - "4365041", "5624958", "8990205", "2652177", "3993588", "-298316", "2901599", "3887387", "-5202979", "1196268", - "5968501", "7619928", "3565643", "1885272", "-749485", "2961381", "2982579", "2387454", "4250081", "5958205", - "00000", "00001", "00002", "00003", "00004", "00005", "00006", "00007", "00008", "00009", - "4907034", "2592882", "3269234", "549815", "6256292", "9721039", "-595225", "5587491", "4596297", "-3885009", - "673", "-899", "174", "354", "870", "147", "898", "-510", "369", "859", - "6518423", "5149762", "8834164", "-8085586", "3233120", "8166948", "4172345", "6735549", "-934295", "9481935", - "-430406", "6932717", "4087292", "4047263", "3236400", "-3863050", "4312079", "6956261", "5689446", "3871332", - "535", "691", "326", "-409", "704", "-568", "301", "951", "121", "384", - "4969414", "9378599", "7971781", "5380630", "5001363", "1715827", "6044615", "9118925", "9956168", "-8865496", - "5962464", "7408980", "6646513", "-634564", "4188330", "9805948", "5625691", "7641113", "-4212929", "7802447", - "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", - "2174248", "7449361", "9896659", "-25961", "1706598", "2412368", "-4617035", "6314554", "2225957", "7521434", - "-9530566", "3914164", "2394759", "7157744", "9919392", "6406949", "-744004", "9899789", "8380325", "-1416284", - "3402833", "2150043", "5191009", "8979538", "9565778", "3750211", "7304823", "2829359", "6544236", "-615740", - "363", "-627", "129", "656", "135", "113", "381", "646", "198", "38", - "8060564", "-176752", "1184717", "-666343", "-1273292", "-485827", "6241066", "6579411", "8093119", "7481306", - "-4924485", "7467889", "9813178", "7927100", "3614859", "7293354", "9232973", "4323115", "1133911", "9511638", - "4443188", "2289448", "5639726", "9073898", "8540394", "5389992", "1397726", "-589230", "1017086", "1852330", - "-840", "267", "201", "533", "-675", "494", "315", "706", "-920", "784", - "9097353", "6002251", "-308780", "-3830169", "4340467", "2235284", "3314444", "1085967", "4152107", "5431117", - "-0000", "-0001", "-0002", "-0003", "-0004", "-0005", "-0006", "-0007", "-0008", "-0009", - "-444999", "2136400", "6925907", "6990614", "3588271", "8422028", "-4034772", "5804039", "-6740545", "9381873", - "-924923", "1652367", "2302616", "6776663", "2567821", "-248935", "2587688", "7076742", "-6461467", "1562896", - "-768116", "2338768", "9887307", "9992184", "2045182", "2797589", "9784597", "9696554", "5113329", "1067216", - "-76247763", "58169007", "29408062", "85342511", "42092201", "-95817703", "-1912517", "-26275135", "54656606", "-58188878", - "473", "74", "374", "-64", "266", "715", "937", "-249", "249", "780", - "3907360", "-23063423", "59062754", "83711047", "-95221044", "34894840", "-38562139", "-82018330", "14226223", "-10799717", - "8529722", "88961903", "25608618", "-39988247", "33228241", "38598533", "21161480", "-33723784", "8873948", "96505557", - "-47385048", "-79413272", "-85904404", "87791158", "49194195", "13051222", "57773302", "31904423", "3142966", "27846156", - "7420011", "-72376922", "-68873971", "23765361", "4040725", "-22359806", "85777219", "10099223", "-90364256", "-40158172", - "-7948696", "-64344821", "34404238", "84037448", "-85084788", "-42078409", "-56550310", "96898389", "-595829", "-73166703", - "-0", "-1", "-2", "-3", "-4", "-5", "-6", "-7", "-8", "-9", - "2147483647", "31", "2147483610", "33", "2147483573", "37", "2147483536", - "-82838342", "64441808", "43641062", "-64419642", "-44421934", "75232413", "-75773725", "-89139509", "12812089", "-97633526", - "36090916", "-57706234", "17804655", "4189936", "-4100124", "38803710", "-39735126", "-62397437", "75801648", "51302332", - "73433906", "13015224", "-12624818", "91360377", "11576319", "-54467535", "8892431", "36319780", "38832042", "50172572", - "-317", "109", "-888", "302", "-463", "716", "916", "665", "826", "513", - "42423473", "41078812", "40445652", "-76722281", "95092224", "12075234", "-4045888", "-74396490", "-57304222", "-21726885", - "92038121", "-31899682", "21589254", "-30260046", "56000244", "69686659", "93327838", "96882881", "-91419389", "77529147", - "43288506", "1192435", "-74095920", "76756590", "-31184683", "-35716724", "9451980", "-63168350", "62864002", "26283194", - "37188395", "29151634", "99343471", "-69450330", "-55680090", "-64957599", "47577948", "47107924", "2490477", "48633003", - "-82740809", "-24122215", "67301713", "-63649610", "75499016", "82746620", "17052193", "4602244", "-32721165", "20837836", - "674", "467", "706", "889", "172", "282", "-795", "188", "87", "153", - "64501793", "53146328", "5152287", "-9674493", "68105580", "57245637", "39740229", "-74071854", "86777268", "86484437", - "-86962508", "12644427", "-62944073", "59539680", "43340539", "30661534", "20143968", "-68183731", "-48250926", "42669063", - "000", "001", "002", "003", "004", "005", "006", "007", "008", "009", - "2147483499", "71", "2147483462", "73", "2147483425", "77", "2147483388", - "87736852", "-4444906", "-48094147", "54774735", "54571890", "-22473078", "95053418", "393654", "-33229960", "32276798", - "-48361110", "44295939", "-79813406", "11630865", "38544571", "70972830", "-9821748", "-60965384", "-13096675", "-24569041", - "708", "-467", "-794", "610", "929", "766", "152", "482", "397", "-191", - "97233152", "51028396", "-13796948", "95437272", "71352512", "-83233730", "-68517318", "61832742", "-42667174", "-18002395", - "-92239407", "12701336", "-63830875", "41514172", "-5726049", "18668677", "69555144", "-13737009", "-22626233", "-55078143", - "00", "11", "22", "33", "44", "-00", "-11", "-22", "-33", "-44", - "000", "111", "222", "333", "444", "-000", "-111", "-222", "-333", "-444", - "0000", "1111", "2222", "3333", "4444", "-0000", "-1111", "-2222", "-3333", "-4444", - "00000", "11111", "22222", "33333", "44444", "-00000", "-11111", "-22222", "-33333", "-44444", - "000000", "111111", "222222", "333333", "444444", "-000000", "-111111", "-222222", "-333333", "-444444", - "0000000", "1111111", "2222222", "3333333", "4444444", "-0000000", "-1111111", "-2222222", "-3333333", "-4444444", - "00000000", "11111111", "22222222", "33333333", "44444444", "-00000000", "-11111111", "-22222222", "-33333333", "-44444444", - "000000000", "111111111", "222222222", "333333333", "444444444","-000000000","-111111111","-222222222","-333333333","-444444444", - "2147483351", "51", "2147483314", "53", "-2147483648", "57", "-2147483611", - "55", "66", "77", "88", "99", "-55", "-66", "-77", "-88", "-99", - "555", "666", "777", "888", "999", "-555", "-666", "-777", "-888", "-999", - "5555", "6666", "7777", "8888", "9999", "-5555", "-6666", "-7777", "-8888", "-9999", - "55555", "66666", "77777", "88888", "99999", "-55555", "-66666", "-77777", "-88888", "-99999", - "555555", "666666", "777777", "888888", "999999", "-555555", "-666666", "-777777", "-888888", "-999999", - "5555555", "6666666", "7777777", "8888888", "9999999", "-5555555", "-6666666", "-7777777", "-8888888", "-9999999", - "55555555", "66666666", "77777777", "88888888", "99999999", "-55555555", "-66666666", "-77777777", "-88888888", "-99999999", - "555555555", "666666666", "777777777", "888888888", "999999999","-555555555","-666666666","-777777777","-888888888","-999999999", - "-2147483574", "91", "-2147483537", "93", "-2147483500", "97", "-2147483463", - "0000000011", "0000000022", "0000000033", "0000000044", "-000000011", "-000000022", "-000000033", "-000000044", "-000000088", - "0000000111", "0000000222", "0000000333", "0000000444", "-000000111", "-000000222", "-000000333", "-000000444", "-000000888", - "0000001111", "0000002222", "0000003333", "0000004444", "-000001111", "-000002222", "-000003333", "-000004444", "-000008888", - "0000011111", "0000022222", "0000033333", "0000044444", "-000011111", "-000022222", "-000033333", "-000044444", "-000088888", - "0000111111", "0000222222", "0000333333", "0000444444", "-000111111", "-000222222", "-000333333", "-000444444", "-000888888", - "0001111111", "0002222222", "0003333333", "0004444444", "-001111111", "-002222222", "-003333333", "-004444444", "-008888888", - "0011111111", "0022222222", "0033333333", "0044444444", "-011111111", "-022222222", "-033333333", "-044444444", "-088888888", - "0111111111", "0222222222", "0333333333", "0444444444", "-111111111", "-222222222", "-333333333", "-444444444", "-888888888", - "0000000055", "0000000066", "0000000077", "0000000088", "0000000099", "-000000055", "-000000066", "-000000077", "-000000099", - "0000000555", "0000000666", "0000000777", "0000000888", "0000000999", "-000000555", "-000000666", "-000000777", "-000000999", - "0000005555", "0000006666", "0000007777", "0000008888", "0000009999", "-000005555", "-000006666", "-000007777", "-000009999", - "0000055555", "0000066666", "0000077777", "0000088888", "0000099999", "-000055555", "-000066666", "-000077777", "-000099999", - "0000555555", "0000666666", "0000777777", "0000888888", "0000999999", "-000555555", "-000666666", "-000777777", "-000999999", - "0005555555", "0006666666", "0007777777", "0008888888", "0009999999", "-005555555", "-006666666", "-007777777", "-009999999", - "0055555555", "0066666666", "0077777777", "0088888888", "0099999999", "-055555555", "-066666666", "-077777777", "-099999999", - "0555555555", "0666666666", "0777777777", "0888888888", "0999999999", "-555555555", "-666666666", "-777777777", "-999999999", - "-2147483426", "101", "-2147483389", "103", "-2147483352", "105", "-2147483315", - "0000001234567890", "0000001234567890", "-0000001234567890", - "000001234567890", "000001234567890", "-000001234567890", - "00001234567890", "00001234567890", "-00001234567890", - "0001234567890", "0001234567890", "-0001234567890", - "001234567890", "001234567890", "-001234567890", - "01234567890", "01234567890", "-01234567890", - "1234567890", "1234567890", "-1234567890", - }; - for(const std::string& st : strint_list) { - T expected_value; - strtod_from_string(st, expected_value); - T result_value; - auto result = fast_float::from_chars(st.data(), st.data() + st.size(), - result_value); - if (result.ec != std::errc()) { - printf("parsing %.*s\n", int(st.size()), st.data()); - std::cerr << " I could not parse " << std::endl; - return false; - } - if(result.ptr != st.data() + st.size()) { - printf("parsing %.*s\n", int(st.size()), st.data()); - std::cerr << " Did not get to the end " << std::endl; - return false; - } - if(result_value != expected_value) { - printf("parsing %.*s\n", int(st.size()), st.data()); - std::cerr << "expected value : " << to_string(expected_value) << std::endl; - std::cerr << "result value : " << to_string(result_value) << std::endl; - std::cerr << " Mismatch " << std::endl; - return false; - } - - } - return true; - -} - - -int main() { -#if defined(__CYGWIN__) || defined(__MINGW32__) || defined(__MINGW64__) || defined(sun) || defined(__sun) - std::cout << "Warning: msys/cygwin or solaris detected." << std::endl; -#endif - std::cout << "32 bits checks" << std::endl; - Assert(partow_test()); - Assert(test()); - - std::cout << "64 bits checks" << std::endl; - Assert(partow_test()); - Assert(test()); - - std::cout << "All ok" << std::endl; - return EXIT_SUCCESS; -} diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float_all.h b/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float_all.h deleted file mode 100644 index 098aa17ab..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/fast_float_all.h +++ /dev/null @@ -1,2947 +0,0 @@ -// fast_float by Daniel Lemire -// fast_float by João Paulo Magalhaes - - -// with contributions from Eugene Golushkov -// with contributions from Maksim Kita -// with contributions from Marcin Wojdyr -// with contributions from Neal Richardson -// with contributions from Tim Paine -// with contributions from Fabio Pellacini - - -// Permission is hereby granted, free of charge, to any -// person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the -// Software without restriction, including without -// limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of -// the Software, and to permit persons to whom the Software -// is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice -// shall be included in all copies or substantial portions -// of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - - -#ifndef FASTFLOAT_FAST_FLOAT_H -#define FASTFLOAT_FAST_FLOAT_H - -#include - -namespace fast_float { -enum chars_format { - scientific = 1<<0, - fixed = 1<<2, - hex = 1<<3, - general = fixed | scientific -}; - - -struct from_chars_result { - const char *ptr; - std::errc ec; -}; - -struct parse_options { - constexpr explicit parse_options(chars_format fmt = chars_format::general, - char dot = '.') - : format(fmt), decimal_point(dot) {} - - /** Which number formats are accepted */ - chars_format format; - /** The character used as decimal point */ - char decimal_point; -}; - -/** - * This function parses the character sequence [first,last) for a number. It parses floating-point numbers expecting - * a locale-indepent format equivalent to what is used by std::strtod in the default ("C") locale. - * The resulting floating-point value is the closest floating-point values (using either float or double), - * using the "round to even" convention for values that would otherwise fall right in-between two values. - * That is, we provide exact parsing according to the IEEE standard. - * - * Given a successful parse, the pointer (`ptr`) in the returned value is set to point right after the - * parsed number, and the `value` referenced is set to the parsed value. In case of error, the returned - * `ec` contains a representative error, otherwise the default (`std::errc()`) value is stored. - * - * The implementation does not throw and does not allocate memory (e.g., with `new` or `malloc`). - * - * Like the C++17 standard, the `fast_float::from_chars` functions take an optional last argument of - * the type `fast_float::chars_format`. It is a bitset value: we check whether - * `fmt & fast_float::chars_format::fixed` and `fmt & fast_float::chars_format::scientific` are set - * to determine whether we allowe the fixed point and scientific notation respectively. - * The default is `fast_float::chars_format::general` which allows both `fixed` and `scientific`. - */ -template -from_chars_result from_chars(const char *first, const char *last, - T &value, chars_format fmt = chars_format::general) noexcept; - -/** - * Like from_chars, but accepts an `options` argument to govern number parsing. - */ -template -from_chars_result from_chars_advanced(const char *first, const char *last, - T &value, parse_options options) noexcept; - -} -#endif // FASTFLOAT_FAST_FLOAT_H - - -#ifndef FASTFLOAT_FLOAT_COMMON_H -#define FASTFLOAT_FLOAT_COMMON_H - -#include -#include -#include -#include - -#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) \ - || defined(__amd64) || defined(__aarch64__) || defined(_M_ARM64) \ - || defined(__MINGW64__) \ - || defined(__s390x__) \ - || (defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || defined(__PPC64LE__)) \ - || defined(__EMSCRIPTEN__)) -#define FASTFLOAT_64BIT -#elif (defined(__i386) || defined(__i386__) || defined(_M_IX86) \ - || defined(__arm__) || defined(_M_ARM) \ - || defined(__MINGW32__)) -#define FASTFLOAT_32BIT -#else - // Need to check incrementally, since SIZE_MAX is a size_t, avoid overflow. - // We can never tell the register width, but the SIZE_MAX is a good approximation. - // UINTPTR_MAX and INTPTR_MAX are optional, so avoid them for max portability. - #if SIZE_MAX == 0xffff - #error Unknown platform (16-bit, unsupported) - #elif SIZE_MAX == 0xffffffff - #define FASTFLOAT_32BIT - #elif SIZE_MAX == 0xffffffffffffffff - #define FASTFLOAT_64BIT - #else - #error Unknown platform (not 32-bit, not 64-bit?) - #endif -#endif - -#if ((defined(_WIN32) || defined(_WIN64)) && !defined(__clang__)) -#include -#endif - -#if defined(_MSC_VER) && !defined(__clang__) -#define FASTFLOAT_VISUAL_STUDIO 1 -#endif - -#ifdef _WIN32 -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#else -#if defined(__APPLE__) || defined(__FreeBSD__) -#include -#elif defined(sun) || defined(__sun) -#include -#else -#include -#endif -# -#ifndef __BYTE_ORDER__ -// safe choice -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#endif -# -#ifndef __ORDER_LITTLE_ENDIAN__ -// safe choice -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#endif -# -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -#define FASTFLOAT_IS_BIG_ENDIAN 0 -#else -#define FASTFLOAT_IS_BIG_ENDIAN 1 -#endif -#endif - -#ifdef FASTFLOAT_VISUAL_STUDIO -#define fastfloat_really_inline __forceinline -#else -#define fastfloat_really_inline inline __attribute__((always_inline)) -#endif - -#ifndef FASTFLOAT_ASSERT -#define FASTFLOAT_ASSERT(x) { if (!(x)) abort(); } -#endif - -#ifndef FASTFLOAT_DEBUG_ASSERT -#include -#define FASTFLOAT_DEBUG_ASSERT(x) assert(x) -#endif - -// rust style `try!()` macro, or `?` operator -#define FASTFLOAT_TRY(x) { if (!(x)) return false; } - -namespace fast_float { - -// Compares two ASCII strings in a case insensitive manner. -inline bool fastfloat_strncasecmp(const char *input1, const char *input2, - size_t length) { - char running_diff{0}; - for (size_t i = 0; i < length; i++) { - running_diff |= (input1[i] ^ input2[i]); - } - return (running_diff == 0) || (running_diff == 32); -} - -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif - -// a pointer and a length to a contiguous block of memory -template -struct span { - const T* ptr; - size_t length; - span(const T* _ptr, size_t _length) : ptr(_ptr), length(_length) {} - span() : ptr(nullptr), length(0) {} - - constexpr size_t len() const noexcept { - return length; - } - - const T& operator[](size_t index) const noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - return ptr[index]; - } -}; - -struct value128 { - uint64_t low; - uint64_t high; - value128(uint64_t _low, uint64_t _high) : low(_low), high(_high) {} - value128() : low(0), high(0) {} -}; - -/* result might be undefined when input_num is zero */ -fastfloat_really_inline int leading_zeroes(uint64_t input_num) { - assert(input_num > 0); -#ifdef FASTFLOAT_VISUAL_STUDIO - #if defined(_M_X64) || defined(_M_ARM64) - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - _BitScanReverse64(&leading_zero, input_num); - return (int)(63 - leading_zero); - #else - int last_bit = 0; - if(input_num & uint64_t(0xffffffff00000000)) input_num >>= 32, last_bit |= 32; - if(input_num & uint64_t( 0xffff0000)) input_num >>= 16, last_bit |= 16; - if(input_num & uint64_t( 0xff00)) input_num >>= 8, last_bit |= 8; - if(input_num & uint64_t( 0xf0)) input_num >>= 4, last_bit |= 4; - if(input_num & uint64_t( 0xc)) input_num >>= 2, last_bit |= 2; - if(input_num & uint64_t( 0x2)) input_num >>= 1, last_bit |= 1; - return 63 - last_bit; - #endif -#else - return __builtin_clzll(input_num); -#endif -} - -#ifdef FASTFLOAT_32BIT - -// slow emulation routine for 32-bit -fastfloat_really_inline uint64_t emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; -} - -// slow emulation routine for 32-bit -#if !defined(__MINGW64__) -fastfloat_really_inline uint64_t _umul128(uint64_t ab, uint64_t cd, - uint64_t *hi) { - uint64_t ad = emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; -} -#endif // !__MINGW64__ - -#endif // FASTFLOAT_32BIT - - -// compute 64-bit a*b -fastfloat_really_inline value128 full_multiplication(uint64_t a, - uint64_t b) { - value128 answer; -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emulate - answer.high = __umulh(a, b); - answer.low = a * b; -#elif defined(FASTFLOAT_32BIT) || (defined(_WIN64) && !defined(__clang__)) - answer.low = _umul128(a, b, &answer.high); // _umul128 not available on ARM64 -#elif defined(FASTFLOAT_64BIT) - __uint128_t r = ((__uint128_t)a) * b; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#else - #error Not implemented -#endif - return answer; -} - -struct adjusted_mantissa { - uint64_t mantissa{0}; - int32_t power2{0}; // a negative value indicates an invalid result - adjusted_mantissa() = default; - bool operator==(const adjusted_mantissa &o) const { - return mantissa == o.mantissa && power2 == o.power2; - } - bool operator!=(const adjusted_mantissa &o) const { - return mantissa != o.mantissa || power2 != o.power2; - } -}; - -// Bias so we can get the real exponent with an invalid adjusted_mantissa. -constexpr static int32_t invalid_am_bias = -0x8000; - -constexpr static double powers_of_ten_double[] = { - 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, - 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22}; -constexpr static float powers_of_ten_float[] = {1e0, 1e1, 1e2, 1e3, 1e4, 1e5, - 1e6, 1e7, 1e8, 1e9, 1e10}; - -template struct binary_format { - static inline constexpr int mantissa_explicit_bits(); - static inline constexpr int minimum_exponent(); - static inline constexpr int infinite_power(); - static inline constexpr int sign_index(); - static inline constexpr int min_exponent_fast_path(); - static inline constexpr int max_exponent_fast_path(); - static inline constexpr int max_exponent_round_to_even(); - static inline constexpr int min_exponent_round_to_even(); - static inline constexpr uint64_t max_mantissa_fast_path(); - static inline constexpr int largest_power_of_ten(); - static inline constexpr int smallest_power_of_ten(); - static inline constexpr T exact_power_of_ten(int64_t power); - static inline constexpr size_t max_digits(); -}; - -template <> inline constexpr int binary_format::mantissa_explicit_bits() { - return 52; -} -template <> inline constexpr int binary_format::mantissa_explicit_bits() { - return 23; -} - -template <> inline constexpr int binary_format::max_exponent_round_to_even() { - return 23; -} - -template <> inline constexpr int binary_format::max_exponent_round_to_even() { - return 10; -} - -template <> inline constexpr int binary_format::min_exponent_round_to_even() { - return -4; -} - -template <> inline constexpr int binary_format::min_exponent_round_to_even() { - return -17; -} - -template <> inline constexpr int binary_format::minimum_exponent() { - return -1023; -} -template <> inline constexpr int binary_format::minimum_exponent() { - return -127; -} - -template <> inline constexpr int binary_format::infinite_power() { - return 0x7FF; -} -template <> inline constexpr int binary_format::infinite_power() { - return 0xFF; -} - -template <> inline constexpr int binary_format::sign_index() { return 63; } -template <> inline constexpr int binary_format::sign_index() { return 31; } - -template <> inline constexpr int binary_format::min_exponent_fast_path() { -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return 0; -#else - return -22; -#endif -} -template <> inline constexpr int binary_format::min_exponent_fast_path() { -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - return 0; -#else - return -10; -#endif -} - -template <> inline constexpr int binary_format::max_exponent_fast_path() { - return 22; -} -template <> inline constexpr int binary_format::max_exponent_fast_path() { - return 10; -} - -template <> inline constexpr uint64_t binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} -template <> inline constexpr uint64_t binary_format::max_mantissa_fast_path() { - return uint64_t(2) << mantissa_explicit_bits(); -} - -template <> -inline constexpr double binary_format::exact_power_of_ten(int64_t power) { - return powers_of_ten_double[power]; -} -template <> -inline constexpr float binary_format::exact_power_of_ten(int64_t power) { - - return powers_of_ten_float[power]; -} - - -template <> -inline constexpr int binary_format::largest_power_of_ten() { - return 308; -} -template <> -inline constexpr int binary_format::largest_power_of_ten() { - return 38; -} - -template <> -inline constexpr int binary_format::smallest_power_of_ten() { - return -342; -} -template <> -inline constexpr int binary_format::smallest_power_of_ten() { - return -65; -} - -template <> inline constexpr size_t binary_format::max_digits() { - return 769; -} -template <> inline constexpr size_t binary_format::max_digits() { - return 114; -} - -template -fastfloat_really_inline void to_float(bool negative, adjusted_mantissa am, T &value) { - uint64_t word = am.mantissa; - word |= uint64_t(am.power2) << binary_format::mantissa_explicit_bits(); - word = negative - ? word | (uint64_t(1) << binary_format::sign_index()) : word; -#if FASTFLOAT_IS_BIG_ENDIAN == 1 - if (std::is_same::value) { - ::memcpy(&value, (char *)&word + 4, sizeof(T)); // extract value at offset 4-7 if float on big-endian - } else { - ::memcpy(&value, &word, sizeof(T)); - } -#else - // For little-endian systems: - ::memcpy(&value, &word, sizeof(T)); -#endif -} - -} // namespace fast_float - -#endif - - -#ifndef FASTFLOAT_ASCII_NUMBER_H -#define FASTFLOAT_ASCII_NUMBER_H - -#include -#include -#include -#include - - -namespace fast_float { - -// Next function can be micro-optimized, but compilers are entirely -// able to optimize it well. -fastfloat_really_inline bool is_integer(char c) noexcept { return c >= '0' && c <= '9'; } - -fastfloat_really_inline uint64_t byteswap(uint64_t val) { - return (val & 0xFF00000000000000) >> 56 - | (val & 0x00FF000000000000) >> 40 - | (val & 0x0000FF0000000000) >> 24 - | (val & 0x000000FF00000000) >> 8 - | (val & 0x00000000FF000000) << 8 - | (val & 0x0000000000FF0000) << 24 - | (val & 0x000000000000FF00) << 40 - | (val & 0x00000000000000FF) << 56; -} - -fastfloat_really_inline uint64_t read_u64(const char *chars) { - uint64_t val; - ::memcpy(&val, chars, sizeof(uint64_t)); -#if FASTFLOAT_IS_BIG_ENDIAN == 1 - // Need to read as-if the number was in little-endian order. - val = byteswap(val); -#endif - return val; -} - -fastfloat_really_inline void write_u64(uint8_t *chars, uint64_t val) { -#if FASTFLOAT_IS_BIG_ENDIAN == 1 - // Need to read as-if the number was in little-endian order. - val = byteswap(val); -#endif - ::memcpy(chars, &val, sizeof(uint64_t)); -} - -// credit @aqrit -fastfloat_really_inline uint32_t parse_eight_digits_unrolled(uint64_t val) { - const uint64_t mask = 0x000000FF000000FF; - const uint64_t mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32) - const uint64_t mul2 = 0x0000271000000001; // 1 + (10000ULL << 32) - val -= 0x3030303030303030; - val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8; - val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; - return uint32_t(val); -} - -fastfloat_really_inline uint32_t parse_eight_digits_unrolled(const char *chars) noexcept { - return parse_eight_digits_unrolled(read_u64(chars)); -} - -// credit @aqrit -fastfloat_really_inline bool is_made_of_eight_digits_fast(uint64_t val) noexcept { - return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) & - 0x8080808080808080)); -} - -fastfloat_really_inline bool is_made_of_eight_digits_fast(const char *chars) noexcept { - return is_made_of_eight_digits_fast(read_u64(chars)); -} - -typedef span byte_span; - -struct parsed_number_string { - int64_t exponent{0}; - uint64_t mantissa{0}; - const char *lastmatch{nullptr}; - bool negative{false}; - bool valid{false}; - bool too_many_digits{false}; - // contains the range of the significant digits - byte_span integer{}; // non-nullable - byte_span fraction{}; // nullable -}; - -// Assuming that you use no more than 19 digits, this will -// parse an ASCII string. -fastfloat_really_inline -parsed_number_string parse_number_string(const char *p, const char *pend, parse_options options) noexcept { - const chars_format fmt = options.format; - const char decimal_point = options.decimal_point; - - parsed_number_string answer; - answer.valid = false; - answer.too_many_digits = false; - answer.negative = (*p == '-'); - if (*p == '-') { // C++17 20.19.3.(7.1) explicitly forbids '+' sign here - ++p; - if (p == pend) { - return answer; - } - if (!is_integer(*p) && (*p != decimal_point)) { // a sign must be followed by an integer or the dot - return answer; - } - } - const char *const start_digits = p; - - uint64_t i = 0; // an unsigned int avoids signed overflows (which are bad) - - while ((std::distance(p, pend) >= 8) && is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok - p += 8; - } - while ((p != pend) && is_integer(*p)) { - // a multiplication by 10 is cheaper than an arbitrary integer - // multiplication - i = 10 * i + - uint64_t(*p - '0'); // might overflow, we will handle the overflow later - ++p; - } - const char *const end_of_integer_part = p; - int64_t digit_count = int64_t(end_of_integer_part - start_digits); - answer.integer = byte_span(start_digits, size_t(digit_count)); - int64_t exponent = 0; - if ((p != pend) && (*p == decimal_point)) { - ++p; - const char* before = p; - // can occur at most twice without overflowing, but let it occur more, since - // for integers with many digits, digit parsing is the primary bottleneck. - while ((std::distance(p, pend) >= 8) && is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok - p += 8; - } - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - ++p; - i = i * 10 + digit; // in rare cases, this will overflow, but that's ok - } - exponent = before - p; - answer.fraction = byte_span(before, size_t(p - before)); - digit_count -= exponent; - } - // we must have encountered at least one integer! - if (digit_count == 0) { - return answer; - } - int64_t exp_number = 0; // explicit exponential part - if ((fmt & chars_format::scientific) && (p != pend) && (('e' == *p) || ('E' == *p))) { - const char * location_of_e = p; - ++p; - bool neg_exp = false; - if ((p != pend) && ('-' == *p)) { - neg_exp = true; - ++p; - } else if ((p != pend) && ('+' == *p)) { // '+' on exponent is allowed by C++17 20.19.3.(7.1) - ++p; - } - if ((p == pend) || !is_integer(*p)) { - if(!(fmt & chars_format::fixed)) { - // We are in error. - return answer; - } - // Otherwise, we will be ignoring the 'e'. - p = location_of_e; - } else { - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - if (exp_number < 0x10000000) { - exp_number = 10 * exp_number + digit; - } - ++p; - } - if(neg_exp) { exp_number = - exp_number; } - exponent += exp_number; - } - } else { - // If it scientific and not fixed, we have to bail out. - if((fmt & chars_format::scientific) && !(fmt & chars_format::fixed)) { return answer; } - } - answer.lastmatch = p; - answer.valid = true; - - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon. - // - // We can deal with up to 19 digits. - if (digit_count > 19) { // this is uncommon - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - // We need to be mindful of the case where we only have zeroes... - // E.g., 0.000000000...000. - const char *start = start_digits; - while ((start != pend) && (*start == '0' || *start == decimal_point)) { - if(*start == '0') { digit_count --; } - start++; - } - if (digit_count > 19) { - answer.too_many_digits = true; - // Let us start again, this time, avoiding overflows. - // We don't need to check if is_integer, since we use the - // pre-tokenized spans from above. - i = 0; - p = answer.integer.ptr; - const char* int_end = p + answer.integer.len(); - const uint64_t minimal_nineteen_digit_integer{1000000000000000000}; - while((i < minimal_nineteen_digit_integer) && (p != int_end)) { - i = i * 10 + uint64_t(*p - '0'); - ++p; - } - if (i >= minimal_nineteen_digit_integer) { // We have a big integers - exponent = end_of_integer_part - p + exp_number; - } else { // We have a value with a fractional component. - p = answer.fraction.ptr; - const char* frac_end = p + answer.fraction.len(); - while((i < minimal_nineteen_digit_integer) && (p != frac_end)) { - i = i * 10 + uint64_t(*p - '0'); - ++p; - } - exponent = answer.fraction.ptr - p + exp_number; - } - // We have now corrected both exponent and i, to a truncated value - } - } - answer.exponent = exponent; - answer.mantissa = i; - return answer; -} - -} // namespace fast_float - -#endif - - -#ifndef FASTFLOAT_FAST_TABLE_H -#define FASTFLOAT_FAST_TABLE_H - -#include - -namespace fast_float { - -/** - * When mapping numbers from decimal to binary, - * we go from w * 10^q to m * 2^p but we have - * 10^q = 5^q * 2^q, so effectively - * we are trying to match - * w * 2^q * 5^q to m * 2^p. Thus the powers of two - * are not a concern since they can be represented - * exactly using the binary notation, only the powers of five - * affect the binary significand. - */ - -/** - * The smallest non-zero float (binary64) is 2^−1074. - * We take as input numbers of the form w x 10^q where w < 2^64. - * We have that w * 10^-343 < 2^(64-344) 5^-343 < 2^-1076. - * However, we have that - * (2^64-1) * 10^-342 = (2^64-1) * 2^-342 * 5^-342 > 2^−1074. - * Thus it is possible for a number of the form w * 10^-342 where - * w is a 64-bit value to be a non-zero floating-point number. - ********* - * Any number of form w * 10^309 where w>= 1 is going to be - * infinite in binary64 so we never need to worry about powers - * of 5 greater than 308. - */ -template -struct powers_template { - -constexpr static int smallest_power_of_five = binary_format::smallest_power_of_ten(); -constexpr static int largest_power_of_five = binary_format::largest_power_of_ten(); -constexpr static int number_of_entries = 2 * (largest_power_of_five - smallest_power_of_five + 1); -// Powers of five from 5^-342 all the way to 5^308 rounded toward one. -static const uint64_t power_of_five_128[number_of_entries]; -}; - -template -const uint64_t powers_template::power_of_five_128[number_of_entries] = { - 0xeef453d6923bd65a,0x113faa2906a13b3f, - 0x9558b4661b6565f8,0x4ac7ca59a424c507, - 0xbaaee17fa23ebf76,0x5d79bcf00d2df649, - 0xe95a99df8ace6f53,0xf4d82c2c107973dc, - 0x91d8a02bb6c10594,0x79071b9b8a4be869, - 0xb64ec836a47146f9,0x9748e2826cdee284, - 0xe3e27a444d8d98b7,0xfd1b1b2308169b25, - 0x8e6d8c6ab0787f72,0xfe30f0f5e50e20f7, - 0xb208ef855c969f4f,0xbdbd2d335e51a935, - 0xde8b2b66b3bc4723,0xad2c788035e61382, - 0x8b16fb203055ac76,0x4c3bcb5021afcc31, - 0xaddcb9e83c6b1793,0xdf4abe242a1bbf3d, - 0xd953e8624b85dd78,0xd71d6dad34a2af0d, - 0x87d4713d6f33aa6b,0x8672648c40e5ad68, - 0xa9c98d8ccb009506,0x680efdaf511f18c2, - 0xd43bf0effdc0ba48,0x212bd1b2566def2, - 0x84a57695fe98746d,0x14bb630f7604b57, - 0xa5ced43b7e3e9188,0x419ea3bd35385e2d, - 0xcf42894a5dce35ea,0x52064cac828675b9, - 0x818995ce7aa0e1b2,0x7343efebd1940993, - 0xa1ebfb4219491a1f,0x1014ebe6c5f90bf8, - 0xca66fa129f9b60a6,0xd41a26e077774ef6, - 0xfd00b897478238d0,0x8920b098955522b4, - 0x9e20735e8cb16382,0x55b46e5f5d5535b0, - 0xc5a890362fddbc62,0xeb2189f734aa831d, - 0xf712b443bbd52b7b,0xa5e9ec7501d523e4, - 0x9a6bb0aa55653b2d,0x47b233c92125366e, - 0xc1069cd4eabe89f8,0x999ec0bb696e840a, - 0xf148440a256e2c76,0xc00670ea43ca250d, - 0x96cd2a865764dbca,0x380406926a5e5728, - 0xbc807527ed3e12bc,0xc605083704f5ecf2, - 0xeba09271e88d976b,0xf7864a44c633682e, - 0x93445b8731587ea3,0x7ab3ee6afbe0211d, - 0xb8157268fdae9e4c,0x5960ea05bad82964, - 0xe61acf033d1a45df,0x6fb92487298e33bd, - 0x8fd0c16206306bab,0xa5d3b6d479f8e056, - 0xb3c4f1ba87bc8696,0x8f48a4899877186c, - 0xe0b62e2929aba83c,0x331acdabfe94de87, - 0x8c71dcd9ba0b4925,0x9ff0c08b7f1d0b14, - 0xaf8e5410288e1b6f,0x7ecf0ae5ee44dd9, - 0xdb71e91432b1a24a,0xc9e82cd9f69d6150, - 0x892731ac9faf056e,0xbe311c083a225cd2, - 0xab70fe17c79ac6ca,0x6dbd630a48aaf406, - 0xd64d3d9db981787d,0x92cbbccdad5b108, - 0x85f0468293f0eb4e,0x25bbf56008c58ea5, - 0xa76c582338ed2621,0xaf2af2b80af6f24e, - 0xd1476e2c07286faa,0x1af5af660db4aee1, - 0x82cca4db847945ca,0x50d98d9fc890ed4d, - 0xa37fce126597973c,0xe50ff107bab528a0, - 0xcc5fc196fefd7d0c,0x1e53ed49a96272c8, - 0xff77b1fcbebcdc4f,0x25e8e89c13bb0f7a, - 0x9faacf3df73609b1,0x77b191618c54e9ac, - 0xc795830d75038c1d,0xd59df5b9ef6a2417, - 0xf97ae3d0d2446f25,0x4b0573286b44ad1d, - 0x9becce62836ac577,0x4ee367f9430aec32, - 0xc2e801fb244576d5,0x229c41f793cda73f, - 0xf3a20279ed56d48a,0x6b43527578c1110f, - 0x9845418c345644d6,0x830a13896b78aaa9, - 0xbe5691ef416bd60c,0x23cc986bc656d553, - 0xedec366b11c6cb8f,0x2cbfbe86b7ec8aa8, - 0x94b3a202eb1c3f39,0x7bf7d71432f3d6a9, - 0xb9e08a83a5e34f07,0xdaf5ccd93fb0cc53, - 0xe858ad248f5c22c9,0xd1b3400f8f9cff68, - 0x91376c36d99995be,0x23100809b9c21fa1, - 0xb58547448ffffb2d,0xabd40a0c2832a78a, - 0xe2e69915b3fff9f9,0x16c90c8f323f516c, - 0x8dd01fad907ffc3b,0xae3da7d97f6792e3, - 0xb1442798f49ffb4a,0x99cd11cfdf41779c, - 0xdd95317f31c7fa1d,0x40405643d711d583, - 0x8a7d3eef7f1cfc52,0x482835ea666b2572, - 0xad1c8eab5ee43b66,0xda3243650005eecf, - 0xd863b256369d4a40,0x90bed43e40076a82, - 0x873e4f75e2224e68,0x5a7744a6e804a291, - 0xa90de3535aaae202,0x711515d0a205cb36, - 0xd3515c2831559a83,0xd5a5b44ca873e03, - 0x8412d9991ed58091,0xe858790afe9486c2, - 0xa5178fff668ae0b6,0x626e974dbe39a872, - 0xce5d73ff402d98e3,0xfb0a3d212dc8128f, - 0x80fa687f881c7f8e,0x7ce66634bc9d0b99, - 0xa139029f6a239f72,0x1c1fffc1ebc44e80, - 0xc987434744ac874e,0xa327ffb266b56220, - 0xfbe9141915d7a922,0x4bf1ff9f0062baa8, - 0x9d71ac8fada6c9b5,0x6f773fc3603db4a9, - 0xc4ce17b399107c22,0xcb550fb4384d21d3, - 0xf6019da07f549b2b,0x7e2a53a146606a48, - 0x99c102844f94e0fb,0x2eda7444cbfc426d, - 0xc0314325637a1939,0xfa911155fefb5308, - 0xf03d93eebc589f88,0x793555ab7eba27ca, - 0x96267c7535b763b5,0x4bc1558b2f3458de, - 0xbbb01b9283253ca2,0x9eb1aaedfb016f16, - 0xea9c227723ee8bcb,0x465e15a979c1cadc, - 0x92a1958a7675175f,0xbfacd89ec191ec9, - 0xb749faed14125d36,0xcef980ec671f667b, - 0xe51c79a85916f484,0x82b7e12780e7401a, - 0x8f31cc0937ae58d2,0xd1b2ecb8b0908810, - 0xb2fe3f0b8599ef07,0x861fa7e6dcb4aa15, - 0xdfbdcece67006ac9,0x67a791e093e1d49a, - 0x8bd6a141006042bd,0xe0c8bb2c5c6d24e0, - 0xaecc49914078536d,0x58fae9f773886e18, - 0xda7f5bf590966848,0xaf39a475506a899e, - 0x888f99797a5e012d,0x6d8406c952429603, - 0xaab37fd7d8f58178,0xc8e5087ba6d33b83, - 0xd5605fcdcf32e1d6,0xfb1e4a9a90880a64, - 0x855c3be0a17fcd26,0x5cf2eea09a55067f, - 0xa6b34ad8c9dfc06f,0xf42faa48c0ea481e, - 0xd0601d8efc57b08b,0xf13b94daf124da26, - 0x823c12795db6ce57,0x76c53d08d6b70858, - 0xa2cb1717b52481ed,0x54768c4b0c64ca6e, - 0xcb7ddcdda26da268,0xa9942f5dcf7dfd09, - 0xfe5d54150b090b02,0xd3f93b35435d7c4c, - 0x9efa548d26e5a6e1,0xc47bc5014a1a6daf, - 0xc6b8e9b0709f109a,0x359ab6419ca1091b, - 0xf867241c8cc6d4c0,0xc30163d203c94b62, - 0x9b407691d7fc44f8,0x79e0de63425dcf1d, - 0xc21094364dfb5636,0x985915fc12f542e4, - 0xf294b943e17a2bc4,0x3e6f5b7b17b2939d, - 0x979cf3ca6cec5b5a,0xa705992ceecf9c42, - 0xbd8430bd08277231,0x50c6ff782a838353, - 0xece53cec4a314ebd,0xa4f8bf5635246428, - 0x940f4613ae5ed136,0x871b7795e136be99, - 0xb913179899f68584,0x28e2557b59846e3f, - 0xe757dd7ec07426e5,0x331aeada2fe589cf, - 0x9096ea6f3848984f,0x3ff0d2c85def7621, - 0xb4bca50b065abe63,0xfed077a756b53a9, - 0xe1ebce4dc7f16dfb,0xd3e8495912c62894, - 0x8d3360f09cf6e4bd,0x64712dd7abbbd95c, - 0xb080392cc4349dec,0xbd8d794d96aacfb3, - 0xdca04777f541c567,0xecf0d7a0fc5583a0, - 0x89e42caaf9491b60,0xf41686c49db57244, - 0xac5d37d5b79b6239,0x311c2875c522ced5, - 0xd77485cb25823ac7,0x7d633293366b828b, - 0x86a8d39ef77164bc,0xae5dff9c02033197, - 0xa8530886b54dbdeb,0xd9f57f830283fdfc, - 0xd267caa862a12d66,0xd072df63c324fd7b, - 0x8380dea93da4bc60,0x4247cb9e59f71e6d, - 0xa46116538d0deb78,0x52d9be85f074e608, - 0xcd795be870516656,0x67902e276c921f8b, - 0x806bd9714632dff6,0xba1cd8a3db53b6, - 0xa086cfcd97bf97f3,0x80e8a40eccd228a4, - 0xc8a883c0fdaf7df0,0x6122cd128006b2cd, - 0xfad2a4b13d1b5d6c,0x796b805720085f81, - 0x9cc3a6eec6311a63,0xcbe3303674053bb0, - 0xc3f490aa77bd60fc,0xbedbfc4411068a9c, - 0xf4f1b4d515acb93b,0xee92fb5515482d44, - 0x991711052d8bf3c5,0x751bdd152d4d1c4a, - 0xbf5cd54678eef0b6,0xd262d45a78a0635d, - 0xef340a98172aace4,0x86fb897116c87c34, - 0x9580869f0e7aac0e,0xd45d35e6ae3d4da0, - 0xbae0a846d2195712,0x8974836059cca109, - 0xe998d258869facd7,0x2bd1a438703fc94b, - 0x91ff83775423cc06,0x7b6306a34627ddcf, - 0xb67f6455292cbf08,0x1a3bc84c17b1d542, - 0xe41f3d6a7377eeca,0x20caba5f1d9e4a93, - 0x8e938662882af53e,0x547eb47b7282ee9c, - 0xb23867fb2a35b28d,0xe99e619a4f23aa43, - 0xdec681f9f4c31f31,0x6405fa00e2ec94d4, - 0x8b3c113c38f9f37e,0xde83bc408dd3dd04, - 0xae0b158b4738705e,0x9624ab50b148d445, - 0xd98ddaee19068c76,0x3badd624dd9b0957, - 0x87f8a8d4cfa417c9,0xe54ca5d70a80e5d6, - 0xa9f6d30a038d1dbc,0x5e9fcf4ccd211f4c, - 0xd47487cc8470652b,0x7647c3200069671f, - 0x84c8d4dfd2c63f3b,0x29ecd9f40041e073, - 0xa5fb0a17c777cf09,0xf468107100525890, - 0xcf79cc9db955c2cc,0x7182148d4066eeb4, - 0x81ac1fe293d599bf,0xc6f14cd848405530, - 0xa21727db38cb002f,0xb8ada00e5a506a7c, - 0xca9cf1d206fdc03b,0xa6d90811f0e4851c, - 0xfd442e4688bd304a,0x908f4a166d1da663, - 0x9e4a9cec15763e2e,0x9a598e4e043287fe, - 0xc5dd44271ad3cdba,0x40eff1e1853f29fd, - 0xf7549530e188c128,0xd12bee59e68ef47c, - 0x9a94dd3e8cf578b9,0x82bb74f8301958ce, - 0xc13a148e3032d6e7,0xe36a52363c1faf01, - 0xf18899b1bc3f8ca1,0xdc44e6c3cb279ac1, - 0x96f5600f15a7b7e5,0x29ab103a5ef8c0b9, - 0xbcb2b812db11a5de,0x7415d448f6b6f0e7, - 0xebdf661791d60f56,0x111b495b3464ad21, - 0x936b9fcebb25c995,0xcab10dd900beec34, - 0xb84687c269ef3bfb,0x3d5d514f40eea742, - 0xe65829b3046b0afa,0xcb4a5a3112a5112, - 0x8ff71a0fe2c2e6dc,0x47f0e785eaba72ab, - 0xb3f4e093db73a093,0x59ed216765690f56, - 0xe0f218b8d25088b8,0x306869c13ec3532c, - 0x8c974f7383725573,0x1e414218c73a13fb, - 0xafbd2350644eeacf,0xe5d1929ef90898fa, - 0xdbac6c247d62a583,0xdf45f746b74abf39, - 0x894bc396ce5da772,0x6b8bba8c328eb783, - 0xab9eb47c81f5114f,0x66ea92f3f326564, - 0xd686619ba27255a2,0xc80a537b0efefebd, - 0x8613fd0145877585,0xbd06742ce95f5f36, - 0xa798fc4196e952e7,0x2c48113823b73704, - 0xd17f3b51fca3a7a0,0xf75a15862ca504c5, - 0x82ef85133de648c4,0x9a984d73dbe722fb, - 0xa3ab66580d5fdaf5,0xc13e60d0d2e0ebba, - 0xcc963fee10b7d1b3,0x318df905079926a8, - 0xffbbcfe994e5c61f,0xfdf17746497f7052, - 0x9fd561f1fd0f9bd3,0xfeb6ea8bedefa633, - 0xc7caba6e7c5382c8,0xfe64a52ee96b8fc0, - 0xf9bd690a1b68637b,0x3dfdce7aa3c673b0, - 0x9c1661a651213e2d,0x6bea10ca65c084e, - 0xc31bfa0fe5698db8,0x486e494fcff30a62, - 0xf3e2f893dec3f126,0x5a89dba3c3efccfa, - 0x986ddb5c6b3a76b7,0xf89629465a75e01c, - 0xbe89523386091465,0xf6bbb397f1135823, - 0xee2ba6c0678b597f,0x746aa07ded582e2c, - 0x94db483840b717ef,0xa8c2a44eb4571cdc, - 0xba121a4650e4ddeb,0x92f34d62616ce413, - 0xe896a0d7e51e1566,0x77b020baf9c81d17, - 0x915e2486ef32cd60,0xace1474dc1d122e, - 0xb5b5ada8aaff80b8,0xd819992132456ba, - 0xe3231912d5bf60e6,0x10e1fff697ed6c69, - 0x8df5efabc5979c8f,0xca8d3ffa1ef463c1, - 0xb1736b96b6fd83b3,0xbd308ff8a6b17cb2, - 0xddd0467c64bce4a0,0xac7cb3f6d05ddbde, - 0x8aa22c0dbef60ee4,0x6bcdf07a423aa96b, - 0xad4ab7112eb3929d,0x86c16c98d2c953c6, - 0xd89d64d57a607744,0xe871c7bf077ba8b7, - 0x87625f056c7c4a8b,0x11471cd764ad4972, - 0xa93af6c6c79b5d2d,0xd598e40d3dd89bcf, - 0xd389b47879823479,0x4aff1d108d4ec2c3, - 0x843610cb4bf160cb,0xcedf722a585139ba, - 0xa54394fe1eedb8fe,0xc2974eb4ee658828, - 0xce947a3da6a9273e,0x733d226229feea32, - 0x811ccc668829b887,0x806357d5a3f525f, - 0xa163ff802a3426a8,0xca07c2dcb0cf26f7, - 0xc9bcff6034c13052,0xfc89b393dd02f0b5, - 0xfc2c3f3841f17c67,0xbbac2078d443ace2, - 0x9d9ba7832936edc0,0xd54b944b84aa4c0d, - 0xc5029163f384a931,0xa9e795e65d4df11, - 0xf64335bcf065d37d,0x4d4617b5ff4a16d5, - 0x99ea0196163fa42e,0x504bced1bf8e4e45, - 0xc06481fb9bcf8d39,0xe45ec2862f71e1d6, - 0xf07da27a82c37088,0x5d767327bb4e5a4c, - 0x964e858c91ba2655,0x3a6a07f8d510f86f, - 0xbbe226efb628afea,0x890489f70a55368b, - 0xeadab0aba3b2dbe5,0x2b45ac74ccea842e, - 0x92c8ae6b464fc96f,0x3b0b8bc90012929d, - 0xb77ada0617e3bbcb,0x9ce6ebb40173744, - 0xe55990879ddcaabd,0xcc420a6a101d0515, - 0x8f57fa54c2a9eab6,0x9fa946824a12232d, - 0xb32df8e9f3546564,0x47939822dc96abf9, - 0xdff9772470297ebd,0x59787e2b93bc56f7, - 0x8bfbea76c619ef36,0x57eb4edb3c55b65a, - 0xaefae51477a06b03,0xede622920b6b23f1, - 0xdab99e59958885c4,0xe95fab368e45eced, - 0x88b402f7fd75539b,0x11dbcb0218ebb414, - 0xaae103b5fcd2a881,0xd652bdc29f26a119, - 0xd59944a37c0752a2,0x4be76d3346f0495f, - 0x857fcae62d8493a5,0x6f70a4400c562ddb, - 0xa6dfbd9fb8e5b88e,0xcb4ccd500f6bb952, - 0xd097ad07a71f26b2,0x7e2000a41346a7a7, - 0x825ecc24c873782f,0x8ed400668c0c28c8, - 0xa2f67f2dfa90563b,0x728900802f0f32fa, - 0xcbb41ef979346bca,0x4f2b40a03ad2ffb9, - 0xfea126b7d78186bc,0xe2f610c84987bfa8, - 0x9f24b832e6b0f436,0xdd9ca7d2df4d7c9, - 0xc6ede63fa05d3143,0x91503d1c79720dbb, - 0xf8a95fcf88747d94,0x75a44c6397ce912a, - 0x9b69dbe1b548ce7c,0xc986afbe3ee11aba, - 0xc24452da229b021b,0xfbe85badce996168, - 0xf2d56790ab41c2a2,0xfae27299423fb9c3, - 0x97c560ba6b0919a5,0xdccd879fc967d41a, - 0xbdb6b8e905cb600f,0x5400e987bbc1c920, - 0xed246723473e3813,0x290123e9aab23b68, - 0x9436c0760c86e30b,0xf9a0b6720aaf6521, - 0xb94470938fa89bce,0xf808e40e8d5b3e69, - 0xe7958cb87392c2c2,0xb60b1d1230b20e04, - 0x90bd77f3483bb9b9,0xb1c6f22b5e6f48c2, - 0xb4ecd5f01a4aa828,0x1e38aeb6360b1af3, - 0xe2280b6c20dd5232,0x25c6da63c38de1b0, - 0x8d590723948a535f,0x579c487e5a38ad0e, - 0xb0af48ec79ace837,0x2d835a9df0c6d851, - 0xdcdb1b2798182244,0xf8e431456cf88e65, - 0x8a08f0f8bf0f156b,0x1b8e9ecb641b58ff, - 0xac8b2d36eed2dac5,0xe272467e3d222f3f, - 0xd7adf884aa879177,0x5b0ed81dcc6abb0f, - 0x86ccbb52ea94baea,0x98e947129fc2b4e9, - 0xa87fea27a539e9a5,0x3f2398d747b36224, - 0xd29fe4b18e88640e,0x8eec7f0d19a03aad, - 0x83a3eeeef9153e89,0x1953cf68300424ac, - 0xa48ceaaab75a8e2b,0x5fa8c3423c052dd7, - 0xcdb02555653131b6,0x3792f412cb06794d, - 0x808e17555f3ebf11,0xe2bbd88bbee40bd0, - 0xa0b19d2ab70e6ed6,0x5b6aceaeae9d0ec4, - 0xc8de047564d20a8b,0xf245825a5a445275, - 0xfb158592be068d2e,0xeed6e2f0f0d56712, - 0x9ced737bb6c4183d,0x55464dd69685606b, - 0xc428d05aa4751e4c,0xaa97e14c3c26b886, - 0xf53304714d9265df,0xd53dd99f4b3066a8, - 0x993fe2c6d07b7fab,0xe546a8038efe4029, - 0xbf8fdb78849a5f96,0xde98520472bdd033, - 0xef73d256a5c0f77c,0x963e66858f6d4440, - 0x95a8637627989aad,0xdde7001379a44aa8, - 0xbb127c53b17ec159,0x5560c018580d5d52, - 0xe9d71b689dde71af,0xaab8f01e6e10b4a6, - 0x9226712162ab070d,0xcab3961304ca70e8, - 0xb6b00d69bb55c8d1,0x3d607b97c5fd0d22, - 0xe45c10c42a2b3b05,0x8cb89a7db77c506a, - 0x8eb98a7a9a5b04e3,0x77f3608e92adb242, - 0xb267ed1940f1c61c,0x55f038b237591ed3, - 0xdf01e85f912e37a3,0x6b6c46dec52f6688, - 0x8b61313bbabce2c6,0x2323ac4b3b3da015, - 0xae397d8aa96c1b77,0xabec975e0a0d081a, - 0xd9c7dced53c72255,0x96e7bd358c904a21, - 0x881cea14545c7575,0x7e50d64177da2e54, - 0xaa242499697392d2,0xdde50bd1d5d0b9e9, - 0xd4ad2dbfc3d07787,0x955e4ec64b44e864, - 0x84ec3c97da624ab4,0xbd5af13bef0b113e, - 0xa6274bbdd0fadd61,0xecb1ad8aeacdd58e, - 0xcfb11ead453994ba,0x67de18eda5814af2, - 0x81ceb32c4b43fcf4,0x80eacf948770ced7, - 0xa2425ff75e14fc31,0xa1258379a94d028d, - 0xcad2f7f5359a3b3e,0x96ee45813a04330, - 0xfd87b5f28300ca0d,0x8bca9d6e188853fc, - 0x9e74d1b791e07e48,0x775ea264cf55347e, - 0xc612062576589dda,0x95364afe032a819e, - 0xf79687aed3eec551,0x3a83ddbd83f52205, - 0x9abe14cd44753b52,0xc4926a9672793543, - 0xc16d9a0095928a27,0x75b7053c0f178294, - 0xf1c90080baf72cb1,0x5324c68b12dd6339, - 0x971da05074da7bee,0xd3f6fc16ebca5e04, - 0xbce5086492111aea,0x88f4bb1ca6bcf585, - 0xec1e4a7db69561a5,0x2b31e9e3d06c32e6, - 0x9392ee8e921d5d07,0x3aff322e62439fd0, - 0xb877aa3236a4b449,0x9befeb9fad487c3, - 0xe69594bec44de15b,0x4c2ebe687989a9b4, - 0x901d7cf73ab0acd9,0xf9d37014bf60a11, - 0xb424dc35095cd80f,0x538484c19ef38c95, - 0xe12e13424bb40e13,0x2865a5f206b06fba, - 0x8cbccc096f5088cb,0xf93f87b7442e45d4, - 0xafebff0bcb24aafe,0xf78f69a51539d749, - 0xdbe6fecebdedd5be,0xb573440e5a884d1c, - 0x89705f4136b4a597,0x31680a88f8953031, - 0xabcc77118461cefc,0xfdc20d2b36ba7c3e, - 0xd6bf94d5e57a42bc,0x3d32907604691b4d, - 0x8637bd05af6c69b5,0xa63f9a49c2c1b110, - 0xa7c5ac471b478423,0xfcf80dc33721d54, - 0xd1b71758e219652b,0xd3c36113404ea4a9, - 0x83126e978d4fdf3b,0x645a1cac083126ea, - 0xa3d70a3d70a3d70a,0x3d70a3d70a3d70a4, - 0xcccccccccccccccc,0xcccccccccccccccd, - 0x8000000000000000,0x0, - 0xa000000000000000,0x0, - 0xc800000000000000,0x0, - 0xfa00000000000000,0x0, - 0x9c40000000000000,0x0, - 0xc350000000000000,0x0, - 0xf424000000000000,0x0, - 0x9896800000000000,0x0, - 0xbebc200000000000,0x0, - 0xee6b280000000000,0x0, - 0x9502f90000000000,0x0, - 0xba43b74000000000,0x0, - 0xe8d4a51000000000,0x0, - 0x9184e72a00000000,0x0, - 0xb5e620f480000000,0x0, - 0xe35fa931a0000000,0x0, - 0x8e1bc9bf04000000,0x0, - 0xb1a2bc2ec5000000,0x0, - 0xde0b6b3a76400000,0x0, - 0x8ac7230489e80000,0x0, - 0xad78ebc5ac620000,0x0, - 0xd8d726b7177a8000,0x0, - 0x878678326eac9000,0x0, - 0xa968163f0a57b400,0x0, - 0xd3c21bcecceda100,0x0, - 0x84595161401484a0,0x0, - 0xa56fa5b99019a5c8,0x0, - 0xcecb8f27f4200f3a,0x0, - 0x813f3978f8940984,0x4000000000000000, - 0xa18f07d736b90be5,0x5000000000000000, - 0xc9f2c9cd04674ede,0xa400000000000000, - 0xfc6f7c4045812296,0x4d00000000000000, - 0x9dc5ada82b70b59d,0xf020000000000000, - 0xc5371912364ce305,0x6c28000000000000, - 0xf684df56c3e01bc6,0xc732000000000000, - 0x9a130b963a6c115c,0x3c7f400000000000, - 0xc097ce7bc90715b3,0x4b9f100000000000, - 0xf0bdc21abb48db20,0x1e86d40000000000, - 0x96769950b50d88f4,0x1314448000000000, - 0xbc143fa4e250eb31,0x17d955a000000000, - 0xeb194f8e1ae525fd,0x5dcfab0800000000, - 0x92efd1b8d0cf37be,0x5aa1cae500000000, - 0xb7abc627050305ad,0xf14a3d9e40000000, - 0xe596b7b0c643c719,0x6d9ccd05d0000000, - 0x8f7e32ce7bea5c6f,0xe4820023a2000000, - 0xb35dbf821ae4f38b,0xdda2802c8a800000, - 0xe0352f62a19e306e,0xd50b2037ad200000, - 0x8c213d9da502de45,0x4526f422cc340000, - 0xaf298d050e4395d6,0x9670b12b7f410000, - 0xdaf3f04651d47b4c,0x3c0cdd765f114000, - 0x88d8762bf324cd0f,0xa5880a69fb6ac800, - 0xab0e93b6efee0053,0x8eea0d047a457a00, - 0xd5d238a4abe98068,0x72a4904598d6d880, - 0x85a36366eb71f041,0x47a6da2b7f864750, - 0xa70c3c40a64e6c51,0x999090b65f67d924, - 0xd0cf4b50cfe20765,0xfff4b4e3f741cf6d, - 0x82818f1281ed449f,0xbff8f10e7a8921a4, - 0xa321f2d7226895c7,0xaff72d52192b6a0d, - 0xcbea6f8ceb02bb39,0x9bf4f8a69f764490, - 0xfee50b7025c36a08,0x2f236d04753d5b4, - 0x9f4f2726179a2245,0x1d762422c946590, - 0xc722f0ef9d80aad6,0x424d3ad2b7b97ef5, - 0xf8ebad2b84e0d58b,0xd2e0898765a7deb2, - 0x9b934c3b330c8577,0x63cc55f49f88eb2f, - 0xc2781f49ffcfa6d5,0x3cbf6b71c76b25fb, - 0xf316271c7fc3908a,0x8bef464e3945ef7a, - 0x97edd871cfda3a56,0x97758bf0e3cbb5ac, - 0xbde94e8e43d0c8ec,0x3d52eeed1cbea317, - 0xed63a231d4c4fb27,0x4ca7aaa863ee4bdd, - 0x945e455f24fb1cf8,0x8fe8caa93e74ef6a, - 0xb975d6b6ee39e436,0xb3e2fd538e122b44, - 0xe7d34c64a9c85d44,0x60dbbca87196b616, - 0x90e40fbeea1d3a4a,0xbc8955e946fe31cd, - 0xb51d13aea4a488dd,0x6babab6398bdbe41, - 0xe264589a4dcdab14,0xc696963c7eed2dd1, - 0x8d7eb76070a08aec,0xfc1e1de5cf543ca2, - 0xb0de65388cc8ada8,0x3b25a55f43294bcb, - 0xdd15fe86affad912,0x49ef0eb713f39ebe, - 0x8a2dbf142dfcc7ab,0x6e3569326c784337, - 0xacb92ed9397bf996,0x49c2c37f07965404, - 0xd7e77a8f87daf7fb,0xdc33745ec97be906, - 0x86f0ac99b4e8dafd,0x69a028bb3ded71a3, - 0xa8acd7c0222311bc,0xc40832ea0d68ce0c, - 0xd2d80db02aabd62b,0xf50a3fa490c30190, - 0x83c7088e1aab65db,0x792667c6da79e0fa, - 0xa4b8cab1a1563f52,0x577001b891185938, - 0xcde6fd5e09abcf26,0xed4c0226b55e6f86, - 0x80b05e5ac60b6178,0x544f8158315b05b4, - 0xa0dc75f1778e39d6,0x696361ae3db1c721, - 0xc913936dd571c84c,0x3bc3a19cd1e38e9, - 0xfb5878494ace3a5f,0x4ab48a04065c723, - 0x9d174b2dcec0e47b,0x62eb0d64283f9c76, - 0xc45d1df942711d9a,0x3ba5d0bd324f8394, - 0xf5746577930d6500,0xca8f44ec7ee36479, - 0x9968bf6abbe85f20,0x7e998b13cf4e1ecb, - 0xbfc2ef456ae276e8,0x9e3fedd8c321a67e, - 0xefb3ab16c59b14a2,0xc5cfe94ef3ea101e, - 0x95d04aee3b80ece5,0xbba1f1d158724a12, - 0xbb445da9ca61281f,0x2a8a6e45ae8edc97, - 0xea1575143cf97226,0xf52d09d71a3293bd, - 0x924d692ca61be758,0x593c2626705f9c56, - 0xb6e0c377cfa2e12e,0x6f8b2fb00c77836c, - 0xe498f455c38b997a,0xb6dfb9c0f956447, - 0x8edf98b59a373fec,0x4724bd4189bd5eac, - 0xb2977ee300c50fe7,0x58edec91ec2cb657, - 0xdf3d5e9bc0f653e1,0x2f2967b66737e3ed, - 0x8b865b215899f46c,0xbd79e0d20082ee74, - 0xae67f1e9aec07187,0xecd8590680a3aa11, - 0xda01ee641a708de9,0xe80e6f4820cc9495, - 0x884134fe908658b2,0x3109058d147fdcdd, - 0xaa51823e34a7eede,0xbd4b46f0599fd415, - 0xd4e5e2cdc1d1ea96,0x6c9e18ac7007c91a, - 0x850fadc09923329e,0x3e2cf6bc604ddb0, - 0xa6539930bf6bff45,0x84db8346b786151c, - 0xcfe87f7cef46ff16,0xe612641865679a63, - 0x81f14fae158c5f6e,0x4fcb7e8f3f60c07e, - 0xa26da3999aef7749,0xe3be5e330f38f09d, - 0xcb090c8001ab551c,0x5cadf5bfd3072cc5, - 0xfdcb4fa002162a63,0x73d9732fc7c8f7f6, - 0x9e9f11c4014dda7e,0x2867e7fddcdd9afa, - 0xc646d63501a1511d,0xb281e1fd541501b8, - 0xf7d88bc24209a565,0x1f225a7ca91a4226, - 0x9ae757596946075f,0x3375788de9b06958, - 0xc1a12d2fc3978937,0x52d6b1641c83ae, - 0xf209787bb47d6b84,0xc0678c5dbd23a49a, - 0x9745eb4d50ce6332,0xf840b7ba963646e0, - 0xbd176620a501fbff,0xb650e5a93bc3d898, - 0xec5d3fa8ce427aff,0xa3e51f138ab4cebe, - 0x93ba47c980e98cdf,0xc66f336c36b10137, - 0xb8a8d9bbe123f017,0xb80b0047445d4184, - 0xe6d3102ad96cec1d,0xa60dc059157491e5, - 0x9043ea1ac7e41392,0x87c89837ad68db2f, - 0xb454e4a179dd1877,0x29babe4598c311fb, - 0xe16a1dc9d8545e94,0xf4296dd6fef3d67a, - 0x8ce2529e2734bb1d,0x1899e4a65f58660c, - 0xb01ae745b101e9e4,0x5ec05dcff72e7f8f, - 0xdc21a1171d42645d,0x76707543f4fa1f73, - 0x899504ae72497eba,0x6a06494a791c53a8, - 0xabfa45da0edbde69,0x487db9d17636892, - 0xd6f8d7509292d603,0x45a9d2845d3c42b6, - 0x865b86925b9bc5c2,0xb8a2392ba45a9b2, - 0xa7f26836f282b732,0x8e6cac7768d7141e, - 0xd1ef0244af2364ff,0x3207d795430cd926, - 0x8335616aed761f1f,0x7f44e6bd49e807b8, - 0xa402b9c5a8d3a6e7,0x5f16206c9c6209a6, - 0xcd036837130890a1,0x36dba887c37a8c0f, - 0x802221226be55a64,0xc2494954da2c9789, - 0xa02aa96b06deb0fd,0xf2db9baa10b7bd6c, - 0xc83553c5c8965d3d,0x6f92829494e5acc7, - 0xfa42a8b73abbf48c,0xcb772339ba1f17f9, - 0x9c69a97284b578d7,0xff2a760414536efb, - 0xc38413cf25e2d70d,0xfef5138519684aba, - 0xf46518c2ef5b8cd1,0x7eb258665fc25d69, - 0x98bf2f79d5993802,0xef2f773ffbd97a61, - 0xbeeefb584aff8603,0xaafb550ffacfd8fa, - 0xeeaaba2e5dbf6784,0x95ba2a53f983cf38, - 0x952ab45cfa97a0b2,0xdd945a747bf26183, - 0xba756174393d88df,0x94f971119aeef9e4, - 0xe912b9d1478ceb17,0x7a37cd5601aab85d, - 0x91abb422ccb812ee,0xac62e055c10ab33a, - 0xb616a12b7fe617aa,0x577b986b314d6009, - 0xe39c49765fdf9d94,0xed5a7e85fda0b80b, - 0x8e41ade9fbebc27d,0x14588f13be847307, - 0xb1d219647ae6b31c,0x596eb2d8ae258fc8, - 0xde469fbd99a05fe3,0x6fca5f8ed9aef3bb, - 0x8aec23d680043bee,0x25de7bb9480d5854, - 0xada72ccc20054ae9,0xaf561aa79a10ae6a, - 0xd910f7ff28069da4,0x1b2ba1518094da04, - 0x87aa9aff79042286,0x90fb44d2f05d0842, - 0xa99541bf57452b28,0x353a1607ac744a53, - 0xd3fa922f2d1675f2,0x42889b8997915ce8, - 0x847c9b5d7c2e09b7,0x69956135febada11, - 0xa59bc234db398c25,0x43fab9837e699095, - 0xcf02b2c21207ef2e,0x94f967e45e03f4bb, - 0x8161afb94b44f57d,0x1d1be0eebac278f5, - 0xa1ba1ba79e1632dc,0x6462d92a69731732, - 0xca28a291859bbf93,0x7d7b8f7503cfdcfe, - 0xfcb2cb35e702af78,0x5cda735244c3d43e, - 0x9defbf01b061adab,0x3a0888136afa64a7, - 0xc56baec21c7a1916,0x88aaa1845b8fdd0, - 0xf6c69a72a3989f5b,0x8aad549e57273d45, - 0x9a3c2087a63f6399,0x36ac54e2f678864b, - 0xc0cb28a98fcf3c7f,0x84576a1bb416a7dd, - 0xf0fdf2d3f3c30b9f,0x656d44a2a11c51d5, - 0x969eb7c47859e743,0x9f644ae5a4b1b325, - 0xbc4665b596706114,0x873d5d9f0dde1fee, - 0xeb57ff22fc0c7959,0xa90cb506d155a7ea, - 0x9316ff75dd87cbd8,0x9a7f12442d588f2, - 0xb7dcbf5354e9bece,0xc11ed6d538aeb2f, - 0xe5d3ef282a242e81,0x8f1668c8a86da5fa, - 0x8fa475791a569d10,0xf96e017d694487bc, - 0xb38d92d760ec4455,0x37c981dcc395a9ac, - 0xe070f78d3927556a,0x85bbe253f47b1417, - 0x8c469ab843b89562,0x93956d7478ccec8e, - 0xaf58416654a6babb,0x387ac8d1970027b2, - 0xdb2e51bfe9d0696a,0x6997b05fcc0319e, - 0x88fcf317f22241e2,0x441fece3bdf81f03, - 0xab3c2fddeeaad25a,0xd527e81cad7626c3, - 0xd60b3bd56a5586f1,0x8a71e223d8d3b074, - 0x85c7056562757456,0xf6872d5667844e49, - 0xa738c6bebb12d16c,0xb428f8ac016561db, - 0xd106f86e69d785c7,0xe13336d701beba52, - 0x82a45b450226b39c,0xecc0024661173473, - 0xa34d721642b06084,0x27f002d7f95d0190, - 0xcc20ce9bd35c78a5,0x31ec038df7b441f4, - 0xff290242c83396ce,0x7e67047175a15271, - 0x9f79a169bd203e41,0xf0062c6e984d386, - 0xc75809c42c684dd1,0x52c07b78a3e60868, - 0xf92e0c3537826145,0xa7709a56ccdf8a82, - 0x9bbcc7a142b17ccb,0x88a66076400bb691, - 0xc2abf989935ddbfe,0x6acff893d00ea435, - 0xf356f7ebf83552fe,0x583f6b8c4124d43, - 0x98165af37b2153de,0xc3727a337a8b704a, - 0xbe1bf1b059e9a8d6,0x744f18c0592e4c5c, - 0xeda2ee1c7064130c,0x1162def06f79df73, - 0x9485d4d1c63e8be7,0x8addcb5645ac2ba8, - 0xb9a74a0637ce2ee1,0x6d953e2bd7173692, - 0xe8111c87c5c1ba99,0xc8fa8db6ccdd0437, - 0x910ab1d4db9914a0,0x1d9c9892400a22a2, - 0xb54d5e4a127f59c8,0x2503beb6d00cab4b, - 0xe2a0b5dc971f303a,0x2e44ae64840fd61d, - 0x8da471a9de737e24,0x5ceaecfed289e5d2, - 0xb10d8e1456105dad,0x7425a83e872c5f47, - 0xdd50f1996b947518,0xd12f124e28f77719, - 0x8a5296ffe33cc92f,0x82bd6b70d99aaa6f, - 0xace73cbfdc0bfb7b,0x636cc64d1001550b, - 0xd8210befd30efa5a,0x3c47f7e05401aa4e, - 0x8714a775e3e95c78,0x65acfaec34810a71, - 0xa8d9d1535ce3b396,0x7f1839a741a14d0d, - 0xd31045a8341ca07c,0x1ede48111209a050, - 0x83ea2b892091e44d,0x934aed0aab460432, - 0xa4e4b66b68b65d60,0xf81da84d5617853f, - 0xce1de40642e3f4b9,0x36251260ab9d668e, - 0x80d2ae83e9ce78f3,0xc1d72b7c6b426019, - 0xa1075a24e4421730,0xb24cf65b8612f81f, - 0xc94930ae1d529cfc,0xdee033f26797b627, - 0xfb9b7cd9a4a7443c,0x169840ef017da3b1, - 0x9d412e0806e88aa5,0x8e1f289560ee864e, - 0xc491798a08a2ad4e,0xf1a6f2bab92a27e2, - 0xf5b5d7ec8acb58a2,0xae10af696774b1db, - 0x9991a6f3d6bf1765,0xacca6da1e0a8ef29, - 0xbff610b0cc6edd3f,0x17fd090a58d32af3, - 0xeff394dcff8a948e,0xddfc4b4cef07f5b0, - 0x95f83d0a1fb69cd9,0x4abdaf101564f98e, - 0xbb764c4ca7a4440f,0x9d6d1ad41abe37f1, - 0xea53df5fd18d5513,0x84c86189216dc5ed, - 0x92746b9be2f8552c,0x32fd3cf5b4e49bb4, - 0xb7118682dbb66a77,0x3fbc8c33221dc2a1, - 0xe4d5e82392a40515,0xfabaf3feaa5334a, - 0x8f05b1163ba6832d,0x29cb4d87f2a7400e, - 0xb2c71d5bca9023f8,0x743e20e9ef511012, - 0xdf78e4b2bd342cf6,0x914da9246b255416, - 0x8bab8eefb6409c1a,0x1ad089b6c2f7548e, - 0xae9672aba3d0c320,0xa184ac2473b529b1, - 0xda3c0f568cc4f3e8,0xc9e5d72d90a2741e, - 0x8865899617fb1871,0x7e2fa67c7a658892, - 0xaa7eebfb9df9de8d,0xddbb901b98feeab7, - 0xd51ea6fa85785631,0x552a74227f3ea565, - 0x8533285c936b35de,0xd53a88958f87275f, - 0xa67ff273b8460356,0x8a892abaf368f137, - 0xd01fef10a657842c,0x2d2b7569b0432d85, - 0x8213f56a67f6b29b,0x9c3b29620e29fc73, - 0xa298f2c501f45f42,0x8349f3ba91b47b8f, - 0xcb3f2f7642717713,0x241c70a936219a73, - 0xfe0efb53d30dd4d7,0xed238cd383aa0110, - 0x9ec95d1463e8a506,0xf4363804324a40aa, - 0xc67bb4597ce2ce48,0xb143c6053edcd0d5, - 0xf81aa16fdc1b81da,0xdd94b7868e94050a, - 0x9b10a4e5e9913128,0xca7cf2b4191c8326, - 0xc1d4ce1f63f57d72,0xfd1c2f611f63a3f0, - 0xf24a01a73cf2dccf,0xbc633b39673c8cec, - 0x976e41088617ca01,0xd5be0503e085d813, - 0xbd49d14aa79dbc82,0x4b2d8644d8a74e18, - 0xec9c459d51852ba2,0xddf8e7d60ed1219e, - 0x93e1ab8252f33b45,0xcabb90e5c942b503, - 0xb8da1662e7b00a17,0x3d6a751f3b936243, - 0xe7109bfba19c0c9d,0xcc512670a783ad4, - 0x906a617d450187e2,0x27fb2b80668b24c5, - 0xb484f9dc9641e9da,0xb1f9f660802dedf6, - 0xe1a63853bbd26451,0x5e7873f8a0396973, - 0x8d07e33455637eb2,0xdb0b487b6423e1e8, - 0xb049dc016abc5e5f,0x91ce1a9a3d2cda62, - 0xdc5c5301c56b75f7,0x7641a140cc7810fb, - 0x89b9b3e11b6329ba,0xa9e904c87fcb0a9d, - 0xac2820d9623bf429,0x546345fa9fbdcd44, - 0xd732290fbacaf133,0xa97c177947ad4095, - 0x867f59a9d4bed6c0,0x49ed8eabcccc485d, - 0xa81f301449ee8c70,0x5c68f256bfff5a74, - 0xd226fc195c6a2f8c,0x73832eec6fff3111, - 0x83585d8fd9c25db7,0xc831fd53c5ff7eab, - 0xa42e74f3d032f525,0xba3e7ca8b77f5e55, - 0xcd3a1230c43fb26f,0x28ce1bd2e55f35eb, - 0x80444b5e7aa7cf85,0x7980d163cf5b81b3, - 0xa0555e361951c366,0xd7e105bcc332621f, - 0xc86ab5c39fa63440,0x8dd9472bf3fefaa7, - 0xfa856334878fc150,0xb14f98f6f0feb951, - 0x9c935e00d4b9d8d2,0x6ed1bf9a569f33d3, - 0xc3b8358109e84f07,0xa862f80ec4700c8, - 0xf4a642e14c6262c8,0xcd27bb612758c0fa, - 0x98e7e9cccfbd7dbd,0x8038d51cb897789c, - 0xbf21e44003acdd2c,0xe0470a63e6bd56c3, - 0xeeea5d5004981478,0x1858ccfce06cac74, - 0x95527a5202df0ccb,0xf37801e0c43ebc8, - 0xbaa718e68396cffd,0xd30560258f54e6ba, - 0xe950df20247c83fd,0x47c6b82ef32a2069, - 0x91d28b7416cdd27e,0x4cdc331d57fa5441, - 0xb6472e511c81471d,0xe0133fe4adf8e952, - 0xe3d8f9e563a198e5,0x58180fddd97723a6, - 0x8e679c2f5e44ff8f,0x570f09eaa7ea7648,}; -using powers = powers_template<>; - -} - -#endif - - -#ifndef FASTFLOAT_DECIMAL_TO_BINARY_H -#define FASTFLOAT_DECIMAL_TO_BINARY_H - -#include -#include -#include -#include -#include -#include - -namespace fast_float { - -// This will compute or rather approximate w * 5**q and return a pair of 64-bit words approximating -// the result, with the "high" part corresponding to the most significant bits and the -// low part corresponding to the least significant bits. -// -template -fastfloat_really_inline -value128 compute_product_approximation(int64_t q, uint64_t w) { - const int index = 2 * int(q - powers::smallest_power_of_five); - // For small values of q, e.g., q in [0,27], the answer is always exact because - // The line value128 firstproduct = full_multiplication(w, power_of_five_128[index]); - // gives the exact answer. - value128 firstproduct = full_multiplication(w, powers::power_of_five_128[index]); - static_assert((bit_precision >= 0) && (bit_precision <= 64), " precision should be in (0,64]"); - constexpr uint64_t precision_mask = (bit_precision < 64) ? - (uint64_t(0xFFFFFFFFFFFFFFFF) >> bit_precision) - : uint64_t(0xFFFFFFFFFFFFFFFF); - if((firstproduct.high & precision_mask) == precision_mask) { // could further guard with (lower + w < lower) - // regarding the second product, we only need secondproduct.high, but our expectation is that the compiler will optimize this extra work away if needed. - value128 secondproduct = full_multiplication(w, powers::power_of_five_128[index + 1]); - firstproduct.low += secondproduct.high; - if(secondproduct.high > firstproduct.low) { - firstproduct.high++; - } - } - return firstproduct; -} - -namespace detail { -/** - * For q in (0,350), we have that - * f = (((152170 + 65536) * q ) >> 16); - * is equal to - * floor(p) + q - * where - * p = log(5**q)/log(2) = q * log(5)/log(2) - * - * For negative values of q in (-400,0), we have that - * f = (((152170 + 65536) * q ) >> 16); - * is equal to - * -ceil(p) + q - * where - * p = log(5**-q)/log(2) = -q * log(5)/log(2) - */ - constexpr fastfloat_really_inline int32_t power(int32_t q) noexcept { - return (((152170 + 65536) * q) >> 16) + 63; - } -} // namespace detail - -// create an adjusted mantissa, biased by the invalid power2 -// for significant digits already multiplied by 10 ** q. -template -fastfloat_really_inline -adjusted_mantissa compute_error_scaled(int64_t q, uint64_t w, int lz) noexcept { - int hilz = int(w >> 63) ^ 1; - adjusted_mantissa answer; - answer.mantissa = w << hilz; - int bias = binary::mantissa_explicit_bits() - binary::minimum_exponent(); - answer.power2 = int32_t(detail::power(int32_t(q)) + bias - hilz - lz - 62 + invalid_am_bias); - return answer; -} - -// w * 10 ** q, without rounding the representation up. -// the power2 in the exponent will be adjusted by invalid_am_bias. -template -fastfloat_really_inline -adjusted_mantissa compute_error(int64_t q, uint64_t w) noexcept { - int lz = leading_zeroes(w); - w <<= lz; - value128 product = compute_product_approximation(q, w); - return compute_error_scaled(q, product.high, lz); -} - -// w * 10 ** q -// The returned value should be a valid ieee64 number that simply need to be packed. -// However, in some very rare cases, the computation will fail. In such cases, we -// return an adjusted_mantissa with a negative power of 2: the caller should recompute -// in such cases. -template -fastfloat_really_inline -adjusted_mantissa compute_float(int64_t q, uint64_t w) noexcept { - adjusted_mantissa answer; - if ((w == 0) || (q < binary::smallest_power_of_ten())) { - answer.power2 = 0; - answer.mantissa = 0; - // result should be zero - return answer; - } - if (q > binary::largest_power_of_ten()) { - // we want to get infinity: - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } - // At this point in time q is in [powers::smallest_power_of_five, powers::largest_power_of_five]. - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(w); - w <<= lz; - - // The required precision is binary::mantissa_explicit_bits() + 3 because - // 1. We need the implicit bit - // 2. We need an extra bit for rounding purposes - // 3. We might lose a bit due to the "upperbit" routine (result too small, requiring a shift) - - value128 product = compute_product_approximation(q, w); - if(product.low == 0xFFFFFFFFFFFFFFFF) { // could guard it further - // In some very rare cases, this could happen, in which case we might need a more accurate - // computation that what we can provide cheaply. This is very, very unlikely. - // - const bool inside_safe_exponent = (q >= -27) && (q <= 55); // always good because 5**q <2**128 when q>=0, - // and otherwise, for q<0, we have 5**-q<2**64 and the 128-bit reciprocal allows for exact computation. - if(!inside_safe_exponent) { - return compute_error_scaled(q, product.high, lz); - } - } - // The "compute_product_approximation" function can be slightly slower than a branchless approach: - // value128 product = compute_product(q, w); - // but in practice, we can win big with the compute_product_approximation if its additional branch - // is easily predicted. Which is best is data specific. - int upperbit = int(product.high >> 63); - - answer.mantissa = product.high >> (upperbit + 64 - binary::mantissa_explicit_bits() - 3); - - answer.power2 = int32_t(detail::power(int32_t(q)) + upperbit - lz - binary::minimum_exponent()); - if (answer.power2 <= 0) { // we have a subnormal? - // Here have that answer.power2 <= 0 so -answer.power2 >= 0 - if(-answer.power2 + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - answer.power2 = 0; - answer.mantissa = 0; - // result should be zero - return answer; - } - // next line is safe because -answer.power2 + 1 < 64 - answer.mantissa >>= -answer.power2 + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - answer.mantissa += (answer.mantissa & 1); // round up - answer.mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - answer.power2 = (answer.mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) ? 0 : 1; - return answer; - } - - // usually, we round *up*, but if we fall right in between and and we have an - // even basis, we need to round down - // We are only concerned with the cases where 5**q fits in single 64-bit word. - if ((product.low <= 1) && (q >= binary::min_exponent_round_to_even()) && (q <= binary::max_exponent_round_to_even()) && - ((answer.mantissa & 3) == 1) ) { // we may fall between two floats! - // To be in-between two floats we need that in doing - // answer.mantissa = product.high >> (upperbit + 64 - binary::mantissa_explicit_bits() - 3); - // ... we dropped out only zeroes. But if this happened, then we can go back!!! - if((answer.mantissa << (upperbit + 64 - binary::mantissa_explicit_bits() - 3)) == product.high) { - answer.mantissa &= ~uint64_t(1); // flip it so that we do not round up - } - } - - answer.mantissa += (answer.mantissa & 1); // round up - answer.mantissa >>= 1; - if (answer.mantissa >= (uint64_t(2) << binary::mantissa_explicit_bits())) { - answer.mantissa = (uint64_t(1) << binary::mantissa_explicit_bits()); - answer.power2++; // undo previous addition - } - - answer.mantissa &= ~(uint64_t(1) << binary::mantissa_explicit_bits()); - if (answer.power2 >= binary::infinite_power()) { // infinity - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - } - return answer; -} - -} // namespace fast_float - -#endif - - -#ifndef FASTFLOAT_BIGINT_H -#define FASTFLOAT_BIGINT_H - -#include -#include -#include -#include - - -namespace fast_float { - -// the limb width: we want efficient multiplication of double the bits in -// limb, or for 64-bit limbs, at least 64-bit multiplication where we can -// extract the high and low parts efficiently. this is every 64-bit -// architecture except for sparc, which emulates 128-bit multiplication. -// we might have platforms where `CHAR_BIT` is not 8, so let's avoid -// doing `8 * sizeof(limb)`. -#if defined(FASTFLOAT_64BIT) && !defined(__sparc) -#define FASTFLOAT_64BIT_LIMB -typedef uint64_t limb; -constexpr size_t limb_bits = 64; -#else -#define FASTFLOAT_32BIT_LIMB -typedef uint32_t limb; -constexpr size_t limb_bits = 32; -#endif - -typedef span limb_span; - -// number of bits in a bigint. this needs to be at least the number -// of bits required to store the largest bigint, which is -// `log2(10**(digits + max_exp))`, or `log2(10**(767 + 342))`, or -// ~3600 bits, so we round to 4000. -constexpr size_t bigint_bits = 4000; -constexpr size_t bigint_limbs = bigint_bits / limb_bits; - -// vector-like type that is allocated on the stack. the entire -// buffer is pre-allocated, and only the length changes. -template -struct stackvec { - limb data[size]; - // we never need more than 150 limbs - uint16_t length{0}; - - stackvec() = default; - stackvec(const stackvec &) = delete; - stackvec &operator=(const stackvec &) = delete; - stackvec(stackvec &&) = delete; - stackvec &operator=(stackvec &&other) = delete; - - // create stack vector from existing limb span. - stackvec(limb_span s) { - FASTFLOAT_ASSERT(try_extend(s)); - } - - limb& operator[](size_t index) noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - return data[index]; - } - const limb& operator[](size_t index) const noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - return data[index]; - } - // index from the end of the container - const limb& rindex(size_t index) const noexcept { - FASTFLOAT_DEBUG_ASSERT(index < length); - size_t rindex = length - index - 1; - return data[rindex]; - } - - // set the length, without bounds checking. - void set_len(size_t len) noexcept { - length = uint16_t(len); - } - constexpr size_t len() const noexcept { - return length; - } - constexpr bool is_empty() const noexcept { - return length == 0; - } - constexpr size_t capacity() const noexcept { - return size; - } - // append item to vector, without bounds checking - void push_unchecked(limb value) noexcept { - data[length] = value; - length++; - } - // append item to vector, returning if item was added - bool try_push(limb value) noexcept { - if (len() < capacity()) { - push_unchecked(value); - return true; - } else { - return false; - } - } - // add items to the vector, from a span, without bounds checking - void extend_unchecked(limb_span s) noexcept { - limb* ptr = data + length; - ::memcpy((void*)ptr, (const void*)s.ptr, sizeof(limb) * s.len()); - set_len(len() + s.len()); - } - // try to add items to the vector, returning if items were added - bool try_extend(limb_span s) noexcept { - if (len() + s.len() <= capacity()) { - extend_unchecked(s); - return true; - } else { - return false; - } - } - // resize the vector, without bounds checking - // if the new size is longer than the vector, assign value to each - // appended item. - void resize_unchecked(size_t new_len, limb value) noexcept { - if (new_len > len()) { - size_t count = new_len - len(); - limb* first = data + len(); - limb* last = first + count; - ::std::fill(first, last, value); - set_len(new_len); - } else { - set_len(new_len); - } - } - // try to resize the vector, returning if the vector was resized. - bool try_resize(size_t new_len, limb value) noexcept { - if (new_len > capacity()) { - return false; - } else { - resize_unchecked(new_len, value); - return true; - } - } - // check if any limbs are non-zero after the given index. - // this needs to be done in reverse order, since the index - // is relative to the most significant limbs. - bool nonzero(size_t index) const noexcept { - while (index < len()) { - if (rindex(index) != 0) { - return true; - } - index++; - } - return false; - } - // normalize the big integer, so most-significant zero limbs are removed. - void normalize() noexcept { - while (len() > 0 && rindex(0) == 0) { - length--; - } - } -}; - -fastfloat_really_inline -uint64_t empty_hi64(bool& truncated) noexcept { - truncated = false; - return 0; -} - -fastfloat_really_inline -uint64_t uint64_hi64(uint64_t r0, bool& truncated) noexcept { - truncated = false; - int shl = leading_zeroes(r0); - return r0 << shl; -} - -fastfloat_really_inline -uint64_t uint64_hi64(uint64_t r0, uint64_t r1, bool& truncated) noexcept { - int shl = leading_zeroes(r0); - if (shl == 0) { - truncated = r1 != 0; - return r0; - } else { - int shr = 64 - shl; - truncated = (r1 << shl) != 0; - return (r0 << shl) | (r1 >> shr); - } -} - -fastfloat_really_inline -uint64_t uint32_hi64(uint32_t r0, bool& truncated) noexcept { - return uint64_hi64(r0, truncated); -} - -fastfloat_really_inline -uint64_t uint32_hi64(uint32_t r0, uint32_t r1, bool& truncated) noexcept { - uint64_t x0 = r0; - uint64_t x1 = r1; - return uint64_hi64((x0 << 32) | x1, truncated); -} - -fastfloat_really_inline -uint64_t uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool& truncated) noexcept { - uint64_t x0 = r0; - uint64_t x1 = r1; - uint64_t x2 = r2; - return uint64_hi64(x0, (x1 << 32) | x2, truncated); -} - -// add two small integers, checking for overflow. -// we want an efficient operation. for msvc, where -// we don't have built-in intrinsics, this is still -// pretty fast. -fastfloat_really_inline -limb scalar_add(limb x, limb y, bool& overflow) noexcept { - limb z; - -// gcc and clang -#if defined(__has_builtin) - #if __has_builtin(__builtin_add_overflow) - overflow = __builtin_add_overflow(x, y, &z); - return z; - #endif -#endif - - // generic, this still optimizes correctly on MSVC. - z = x + y; - overflow = z < x; - return z; -} - -// multiply two small integers, getting both the high and low bits. -fastfloat_really_inline -limb scalar_mul(limb x, limb y, limb& carry) noexcept { -#ifdef FASTFLOAT_64BIT_LIMB - #if defined(__SIZEOF_INT128__) - // GCC and clang both define it as an extension. - __uint128_t z = __uint128_t(x) * __uint128_t(y) + __uint128_t(carry); - carry = limb(z >> limb_bits); - return limb(z); - #else - // fallback, no native 128-bit integer multiplication with carry. - // on msvc, this optimizes identically, somehow. - value128 z = full_multiplication(x, y); - bool overflow; - z.low = scalar_add(z.low, carry, overflow); - z.high += uint64_t(overflow); // cannot overflow - carry = z.high; - return z.low; - #endif -#else - uint64_t z = uint64_t(x) * uint64_t(y) + uint64_t(carry); - carry = limb(z >> limb_bits); - return limb(z); -#endif -} - -// add scalar value to bigint starting from offset. -// used in grade school multiplication -template -inline bool small_add_from(stackvec& vec, limb y, size_t start) noexcept { - size_t index = start; - limb carry = y; - bool overflow; - while (carry != 0 && index < vec.len()) { - vec[index] = scalar_add(vec[index], carry, overflow); - carry = limb(overflow); - index += 1; - } - if (carry != 0) { - FASTFLOAT_TRY(vec.try_push(carry)); - } - return true; -} - -// add scalar value to bigint. -template -fastfloat_really_inline bool small_add(stackvec& vec, limb y) noexcept { - return small_add_from(vec, y, 0); -} - -// multiply bigint by scalar value. -template -inline bool small_mul(stackvec& vec, limb y) noexcept { - limb carry = 0; - for (size_t index = 0; index < vec.len(); index++) { - vec[index] = scalar_mul(vec[index], y, carry); - } - if (carry != 0) { - FASTFLOAT_TRY(vec.try_push(carry)); - } - return true; -} - -// add bigint to bigint starting from index. -// used in grade school multiplication -template -bool large_add_from(stackvec& x, limb_span y, size_t start) noexcept { - // the effective x buffer is from `xstart..x.len()`, so exit early - // if we can't get that current range. - if (x.len() < start || y.len() > x.len() - start) { - FASTFLOAT_TRY(x.try_resize(y.len() + start, 0)); - } - - bool carry = false; - for (size_t index = 0; index < y.len(); index++) { - limb xi = x[index + start]; - limb yi = y[index]; - bool c1 = false; - bool c2 = false; - xi = scalar_add(xi, yi, c1); - if (carry) { - xi = scalar_add(xi, 1, c2); - } - x[index + start] = xi; - carry = c1 | c2; - } - - // handle overflow - if (carry) { - FASTFLOAT_TRY(small_add_from(x, 1, y.len() + start)); - } - return true; -} - -// add bigint to bigint. -template -fastfloat_really_inline bool large_add_from(stackvec& x, limb_span y) noexcept { - return large_add_from(x, y, 0); -} - -// grade-school multiplication algorithm -template -bool long_mul(stackvec& x, limb_span y) noexcept { - limb_span xs = limb_span(x.data, x.len()); - stackvec z(xs); - limb_span zs = limb_span(z.data, z.len()); - - if (y.len() != 0) { - limb y0 = y[0]; - FASTFLOAT_TRY(small_mul(x, y0)); - for (size_t index = 1; index < y.len(); index++) { - limb yi = y[index]; - stackvec zi; - if (yi != 0) { - // re-use the same buffer throughout - zi.set_len(0); - FASTFLOAT_TRY(zi.try_extend(zs)); - FASTFLOAT_TRY(small_mul(zi, yi)); - limb_span zis = limb_span(zi.data, zi.len()); - FASTFLOAT_TRY(large_add_from(x, zis, index)); - } - } - } - - x.normalize(); - return true; -} - -// grade-school multiplication algorithm -template -bool large_mul(stackvec& x, limb_span y) noexcept { - if (y.len() == 1) { - FASTFLOAT_TRY(small_mul(x, y[0])); - } else { - FASTFLOAT_TRY(long_mul(x, y)); - } - return true; -} - -// big integer type. implements a small subset of big integer -// arithmetic, using simple algorithms since asymptotically -// faster algorithms are slower for a small number of limbs. -// all operations assume the big-integer is normalized. -struct bigint { - // storage of the limbs, in little-endian order. - stackvec vec; - - bigint(): vec() {} - bigint(const bigint &) = delete; - bigint &operator=(const bigint &) = delete; - bigint(bigint &&) = delete; - bigint &operator=(bigint &&other) = delete; - - bigint(uint64_t value): vec() { -#ifdef FASTFLOAT_64BIT_LIMB - vec.push_unchecked(value); -#else - vec.push_unchecked(uint32_t(value)); - vec.push_unchecked(uint32_t(value >> 32)); -#endif - vec.normalize(); - } - - // get the high 64 bits from the vector, and if bits were truncated. - // this is to get the significant digits for the float. - uint64_t hi64(bool& truncated) const noexcept { -#ifdef FASTFLOAT_64BIT_LIMB - if (vec.len() == 0) { - return empty_hi64(truncated); - } else if (vec.len() == 1) { - return uint64_hi64(vec.rindex(0), truncated); - } else { - uint64_t result = uint64_hi64(vec.rindex(0), vec.rindex(1), truncated); - truncated |= vec.nonzero(2); - return result; - } -#else - if (vec.len() == 0) { - return empty_hi64(truncated); - } else if (vec.len() == 1) { - return uint32_hi64(vec.rindex(0), truncated); - } else if (vec.len() == 2) { - return uint32_hi64(vec.rindex(0), vec.rindex(1), truncated); - } else { - uint64_t result = uint32_hi64(vec.rindex(0), vec.rindex(1), vec.rindex(2), truncated); - truncated |= vec.nonzero(3); - return result; - } -#endif - } - - // compare two big integers, returning the large value. - // assumes both are normalized. if the return value is - // negative, other is larger, if the return value is - // positive, this is larger, otherwise they are equal. - // the limbs are stored in little-endian order, so we - // must compare the limbs in ever order. - int compare(const bigint& other) const noexcept { - if (vec.len() > other.vec.len()) { - return 1; - } else if (vec.len() < other.vec.len()) { - return -1; - } else { - for (size_t index = vec.len(); index > 0; index--) { - limb xi = vec[index - 1]; - limb yi = other.vec[index - 1]; - if (xi > yi) { - return 1; - } else if (xi < yi) { - return -1; - } - } - return 0; - } - } - - // shift left each limb n bits, carrying over to the new limb - // returns true if we were able to shift all the digits. - bool shl_bits(size_t n) noexcept { - // Internally, for each item, we shift left by n, and add the previous - // right shifted limb-bits. - // For example, we transform (for u8) shifted left 2, to: - // b10100100 b01000010 - // b10 b10010001 b00001000 - FASTFLOAT_DEBUG_ASSERT(n != 0); - FASTFLOAT_DEBUG_ASSERT(n < sizeof(limb) * 8); - - size_t shl = n; - size_t shr = limb_bits - shl; - limb prev = 0; - for (size_t index = 0; index < vec.len(); index++) { - limb xi = vec[index]; - vec[index] = (xi << shl) | (prev >> shr); - prev = xi; - } - - limb carry = prev >> shr; - if (carry != 0) { - return vec.try_push(carry); - } - return true; - } - - // move the limbs left by `n` limbs. - bool shl_limbs(size_t n) noexcept { - FASTFLOAT_DEBUG_ASSERT(n != 0); - if (n + vec.len() > vec.capacity()) { - return false; - } else if (!vec.is_empty()) { - // move limbs - limb* dst = vec.data + n; - const limb* src = vec.data; - ::memmove(dst, src, sizeof(limb) * vec.len()); - // fill in empty limbs - limb* first = vec.data; - limb* last = first + n; - ::std::fill(first, last, 0); - vec.set_len(n + vec.len()); - return true; - } else { - return true; - } - } - - // move the limbs left by `n` bits. - bool shl(size_t n) noexcept { - size_t rem = n % limb_bits; - size_t div = n / limb_bits; - if (rem != 0) { - FASTFLOAT_TRY(shl_bits(rem)); - } - if (div != 0) { - FASTFLOAT_TRY(shl_limbs(div)); - } - return true; - } - - // get the number of leading zeros in the bigint. - int ctlz() const noexcept { - if (vec.is_empty()) { - return 0; - } else { -#ifdef FASTFLOAT_64BIT_LIMB - return leading_zeroes(vec.rindex(0)); -#else - // no use defining a specialized leading_zeroes for a 32-bit type. - uint64_t r0 = vec.rindex(0); - return leading_zeroes(r0 << 32); -#endif - } - } - - // get the number of bits in the bigint. - int bit_length() const noexcept { - int lz = ctlz(); - return int(limb_bits * vec.len()) - lz; - } - - bool mul(limb y) noexcept { - return small_mul(vec, y); - } - - bool add(limb y) noexcept { - return small_add(vec, y); - } - - // multiply as if by 2 raised to a power. - bool pow2(uint32_t exp) noexcept { - return shl(exp); - } - - // multiply as if by 5 raised to a power. - bool pow5(uint32_t exp) noexcept { - // multiply by a power of 5 - static constexpr uint32_t large_step = 135; - static constexpr uint64_t small_power_of_5[] = { - 1UL, 5UL, 25UL, 125UL, 625UL, 3125UL, 15625UL, 78125UL, 390625UL, - 1953125UL, 9765625UL, 48828125UL, 244140625UL, 1220703125UL, - 6103515625UL, 30517578125UL, 152587890625UL, 762939453125UL, - 3814697265625UL, 19073486328125UL, 95367431640625UL, 476837158203125UL, - 2384185791015625UL, 11920928955078125UL, 59604644775390625UL, - 298023223876953125UL, 1490116119384765625UL, 7450580596923828125UL, - }; -#ifdef FASTFLOAT_64BIT_LIMB - constexpr static limb large_power_of_5[] = { - 1414648277510068013UL, 9180637584431281687UL, 4539964771860779200UL, - 10482974169319127550UL, 198276706040285095UL}; -#else - constexpr static limb large_power_of_5[] = { - 4279965485U, 329373468U, 4020270615U, 2137533757U, 4287402176U, - 1057042919U, 1071430142U, 2440757623U, 381945767U, 46164893U}; -#endif - size_t large_length = sizeof(large_power_of_5) / sizeof(limb); - limb_span large = limb_span(large_power_of_5, large_length); - while (exp >= large_step) { - FASTFLOAT_TRY(large_mul(vec, large)); - exp -= large_step; - } -#ifdef FASTFLOAT_64BIT_LIMB - uint32_t small_step = 27; - limb max_native = 7450580596923828125UL; -#else - uint32_t small_step = 13; - limb max_native = 1220703125U; -#endif - while (exp >= small_step) { - FASTFLOAT_TRY(small_mul(vec, max_native)); - exp -= small_step; - } - if (exp != 0) { - FASTFLOAT_TRY(small_mul(vec, limb(small_power_of_5[exp]))); - } - - return true; - } - - // multiply as if by 10 raised to a power. - bool pow10(uint32_t exp) noexcept { - FASTFLOAT_TRY(pow5(exp)); - return pow2(exp); - } -}; - -} // namespace fast_float - -#endif - - -#ifndef FASTFLOAT_ASCII_NUMBER_H -#define FASTFLOAT_ASCII_NUMBER_H - -#include -#include -#include -#include - - -namespace fast_float { - -// Next function can be micro-optimized, but compilers are entirely -// able to optimize it well. -fastfloat_really_inline bool is_integer(char c) noexcept { return c >= '0' && c <= '9'; } - -fastfloat_really_inline uint64_t byteswap(uint64_t val) { - return (val & 0xFF00000000000000) >> 56 - | (val & 0x00FF000000000000) >> 40 - | (val & 0x0000FF0000000000) >> 24 - | (val & 0x000000FF00000000) >> 8 - | (val & 0x00000000FF000000) << 8 - | (val & 0x0000000000FF0000) << 24 - | (val & 0x000000000000FF00) << 40 - | (val & 0x00000000000000FF) << 56; -} - -fastfloat_really_inline uint64_t read_u64(const char *chars) { - uint64_t val; - ::memcpy(&val, chars, sizeof(uint64_t)); -#if FASTFLOAT_IS_BIG_ENDIAN == 1 - // Need to read as-if the number was in little-endian order. - val = byteswap(val); -#endif - return val; -} - -fastfloat_really_inline void write_u64(uint8_t *chars, uint64_t val) { -#if FASTFLOAT_IS_BIG_ENDIAN == 1 - // Need to read as-if the number was in little-endian order. - val = byteswap(val); -#endif - ::memcpy(chars, &val, sizeof(uint64_t)); -} - -// credit @aqrit -fastfloat_really_inline uint32_t parse_eight_digits_unrolled(uint64_t val) { - const uint64_t mask = 0x000000FF000000FF; - const uint64_t mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32) - const uint64_t mul2 = 0x0000271000000001; // 1 + (10000ULL << 32) - val -= 0x3030303030303030; - val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8; - val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; - return uint32_t(val); -} - -fastfloat_really_inline uint32_t parse_eight_digits_unrolled(const char *chars) noexcept { - return parse_eight_digits_unrolled(read_u64(chars)); -} - -// credit @aqrit -fastfloat_really_inline bool is_made_of_eight_digits_fast(uint64_t val) noexcept { - return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) & - 0x8080808080808080)); -} - -fastfloat_really_inline bool is_made_of_eight_digits_fast(const char *chars) noexcept { - return is_made_of_eight_digits_fast(read_u64(chars)); -} - -typedef span byte_span; - -struct parsed_number_string { - int64_t exponent{0}; - uint64_t mantissa{0}; - const char *lastmatch{nullptr}; - bool negative{false}; - bool valid{false}; - bool too_many_digits{false}; - // contains the range of the significant digits - byte_span integer{}; // non-nullable - byte_span fraction{}; // nullable -}; - -// Assuming that you use no more than 19 digits, this will -// parse an ASCII string. -fastfloat_really_inline -parsed_number_string parse_number_string(const char *p, const char *pend, parse_options options) noexcept { - const chars_format fmt = options.format; - const char decimal_point = options.decimal_point; - - parsed_number_string answer; - answer.valid = false; - answer.too_many_digits = false; - answer.negative = (*p == '-'); - if (*p == '-') { // C++17 20.19.3.(7.1) explicitly forbids '+' sign here - ++p; - if (p == pend) { - return answer; - } - if (!is_integer(*p) && (*p != decimal_point)) { // a sign must be followed by an integer or the dot - return answer; - } - } - const char *const start_digits = p; - - uint64_t i = 0; // an unsigned int avoids signed overflows (which are bad) - - while ((std::distance(p, pend) >= 8) && is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok - p += 8; - } - while ((p != pend) && is_integer(*p)) { - // a multiplication by 10 is cheaper than an arbitrary integer - // multiplication - i = 10 * i + - uint64_t(*p - '0'); // might overflow, we will handle the overflow later - ++p; - } - const char *const end_of_integer_part = p; - int64_t digit_count = int64_t(end_of_integer_part - start_digits); - answer.integer = byte_span(start_digits, size_t(digit_count)); - int64_t exponent = 0; - if ((p != pend) && (*p == decimal_point)) { - ++p; - const char* before = p; - // can occur at most twice without overflowing, but let it occur more, since - // for integers with many digits, digit parsing is the primary bottleneck. - while ((std::distance(p, pend) >= 8) && is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok - p += 8; - } - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - ++p; - i = i * 10 + digit; // in rare cases, this will overflow, but that's ok - } - exponent = before - p; - answer.fraction = byte_span(before, size_t(p - before)); - digit_count -= exponent; - } - // we must have encountered at least one integer! - if (digit_count == 0) { - return answer; - } - int64_t exp_number = 0; // explicit exponential part - if ((fmt & chars_format::scientific) && (p != pend) && (('e' == *p) || ('E' == *p))) { - const char * location_of_e = p; - ++p; - bool neg_exp = false; - if ((p != pend) && ('-' == *p)) { - neg_exp = true; - ++p; - } else if ((p != pend) && ('+' == *p)) { // '+' on exponent is allowed by C++17 20.19.3.(7.1) - ++p; - } - if ((p == pend) || !is_integer(*p)) { - if(!(fmt & chars_format::fixed)) { - // We are in error. - return answer; - } - // Otherwise, we will be ignoring the 'e'. - p = location_of_e; - } else { - while ((p != pend) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - if (exp_number < 0x10000000) { - exp_number = 10 * exp_number + digit; - } - ++p; - } - if(neg_exp) { exp_number = - exp_number; } - exponent += exp_number; - } - } else { - // If it scientific and not fixed, we have to bail out. - if((fmt & chars_format::scientific) && !(fmt & chars_format::fixed)) { return answer; } - } - answer.lastmatch = p; - answer.valid = true; - - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon. - // - // We can deal with up to 19 digits. - if (digit_count > 19) { // this is uncommon - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - // We need to be mindful of the case where we only have zeroes... - // E.g., 0.000000000...000. - const char *start = start_digits; - while ((start != pend) && (*start == '0' || *start == decimal_point)) { - if(*start == '0') { digit_count --; } - start++; - } - if (digit_count > 19) { - answer.too_many_digits = true; - // Let us start again, this time, avoiding overflows. - // We don't need to check if is_integer, since we use the - // pre-tokenized spans from above. - i = 0; - p = answer.integer.ptr; - const char* int_end = p + answer.integer.len(); - const uint64_t minimal_nineteen_digit_integer{1000000000000000000}; - while((i < minimal_nineteen_digit_integer) && (p != int_end)) { - i = i * 10 + uint64_t(*p - '0'); - ++p; - } - if (i >= minimal_nineteen_digit_integer) { // We have a big integers - exponent = end_of_integer_part - p + exp_number; - } else { // We have a value with a fractional component. - p = answer.fraction.ptr; - const char* frac_end = p + answer.fraction.len(); - while((i < minimal_nineteen_digit_integer) && (p != frac_end)) { - i = i * 10 + uint64_t(*p - '0'); - ++p; - } - exponent = answer.fraction.ptr - p + exp_number; - } - // We have now corrected both exponent and i, to a truncated value - } - } - answer.exponent = exponent; - answer.mantissa = i; - return answer; -} - -} // namespace fast_float - -#endif - - -#ifndef FASTFLOAT_DIGIT_COMPARISON_H -#define FASTFLOAT_DIGIT_COMPARISON_H - -#include -#include -#include -#include - - -namespace fast_float { - -// 1e0 to 1e19 -constexpr static uint64_t powers_of_ten_uint64[] = { - 1UL, 10UL, 100UL, 1000UL, 10000UL, 100000UL, 1000000UL, 10000000UL, 100000000UL, - 1000000000UL, 10000000000UL, 100000000000UL, 1000000000000UL, 10000000000000UL, - 100000000000000UL, 1000000000000000UL, 10000000000000000UL, 100000000000000000UL, - 1000000000000000000UL, 10000000000000000000UL}; - -// calculate the exponent, in scientific notation, of the number. -// this algorithm is not even close to optimized, but it has no practical -// effect on performance: in order to have a faster algorithm, we'd need -// to slow down performance for faster algorithms, and this is still fast. -fastfloat_really_inline int32_t scientific_exponent(parsed_number_string& num) noexcept { - uint64_t mantissa = num.mantissa; - int32_t exponent = int32_t(num.exponent); - while (mantissa >= 10000) { - mantissa /= 10000; - exponent += 4; - } - while (mantissa >= 100) { - mantissa /= 100; - exponent += 2; - } - while (mantissa >= 10) { - mantissa /= 10; - exponent += 1; - } - return exponent; -} - -// this converts a native floating-point number to an extended-precision float. -template -fastfloat_really_inline adjusted_mantissa to_extended(T value) noexcept { - adjusted_mantissa am; - int32_t bias = binary_format::mantissa_explicit_bits() - binary_format::minimum_exponent(); - if (std::is_same::value) { - constexpr uint32_t exponent_mask = 0x7F800000; - constexpr uint32_t mantissa_mask = 0x007FFFFF; - constexpr uint64_t hidden_bit_mask = 0x00800000; - uint32_t bits; - ::memcpy(&bits, &value, sizeof(T)); - if ((bits & exponent_mask) == 0) { - // denormal - am.power2 = 1 - bias; - am.mantissa = bits & mantissa_mask; - } else { - // normal - am.power2 = int32_t((bits & exponent_mask) >> binary_format::mantissa_explicit_bits()); - am.power2 -= bias; - am.mantissa = (bits & mantissa_mask) | hidden_bit_mask; - } - } else { - constexpr uint64_t exponent_mask = 0x7FF0000000000000; - constexpr uint64_t mantissa_mask = 0x000FFFFFFFFFFFFF; - constexpr uint64_t hidden_bit_mask = 0x0010000000000000; - uint64_t bits; - ::memcpy(&bits, &value, sizeof(T)); - if ((bits & exponent_mask) == 0) { - // denormal - am.power2 = 1 - bias; - am.mantissa = bits & mantissa_mask; - } else { - // normal - am.power2 = int32_t((bits & exponent_mask) >> binary_format::mantissa_explicit_bits()); - am.power2 -= bias; - am.mantissa = (bits & mantissa_mask) | hidden_bit_mask; - } - } - - return am; -} - -// get the extended precision value of the halfway point between b and b+u. -// we are given a native float that represents b, so we need to adjust it -// halfway between b and b+u. -template -fastfloat_really_inline adjusted_mantissa to_extended_halfway(T value) noexcept { - adjusted_mantissa am = to_extended(value); - am.mantissa <<= 1; - am.mantissa += 1; - am.power2 -= 1; - return am; -} - -// round an extended-precision float to the nearest machine float. -template -fastfloat_really_inline void round(adjusted_mantissa& am, callback cb) noexcept { - int32_t mantissa_shift = 64 - binary_format::mantissa_explicit_bits() - 1; - if (-am.power2 >= mantissa_shift) { - // have a denormal float - int32_t shift = -am.power2 + 1; - cb(am, std::min(shift, 64)); - // check for round-up: if rounding-nearest carried us to the hidden bit. - am.power2 = (am.mantissa < (uint64_t(1) << binary_format::mantissa_explicit_bits())) ? 0 : 1; - return; - } - - // have a normal float, use the default shift. - cb(am, mantissa_shift); - - // check for carry - if (am.mantissa >= (uint64_t(2) << binary_format::mantissa_explicit_bits())) { - am.mantissa = (uint64_t(1) << binary_format::mantissa_explicit_bits()); - am.power2++; - } - - // check for infinite: we could have carried to an infinite power - am.mantissa &= ~(uint64_t(1) << binary_format::mantissa_explicit_bits()); - if (am.power2 >= binary_format::infinite_power()) { - am.power2 = binary_format::infinite_power(); - am.mantissa = 0; - } -} - -template -fastfloat_really_inline -void round_nearest_tie_even(adjusted_mantissa& am, int32_t shift, callback cb) noexcept { - uint64_t mask; - uint64_t halfway; - if (shift == 64) { - mask = UINT64_MAX; - } else { - mask = (uint64_t(1) << shift) - 1; - } - if (shift == 0) { - halfway = 0; - } else { - halfway = uint64_t(1) << (shift - 1); - } - uint64_t truncated_bits = am.mantissa & mask; - uint64_t is_above = truncated_bits > halfway; - uint64_t is_halfway = truncated_bits == halfway; - - // shift digits into position - if (shift == 64) { - am.mantissa = 0; - } else { - am.mantissa >>= shift; - } - am.power2 += shift; - - bool is_odd = (am.mantissa & 1) == 1; - am.mantissa += uint64_t(cb(is_odd, is_halfway, is_above)); -} - -fastfloat_really_inline void round_down(adjusted_mantissa& am, int32_t shift) noexcept { - if (shift == 64) { - am.mantissa = 0; - } else { - am.mantissa >>= shift; - } - am.power2 += shift; -} - -fastfloat_really_inline void skip_zeros(const char*& first, const char* last) noexcept { - uint64_t val; - while (std::distance(first, last) >= 8) { - ::memcpy(&val, first, sizeof(uint64_t)); - if (val != 0x3030303030303030) { - break; - } - first += 8; - } - while (first != last) { - if (*first != '0') { - break; - } - first++; - } -} - -// determine if any non-zero digits were truncated. -// all characters must be valid digits. -fastfloat_really_inline bool is_truncated(const char* first, const char* last) noexcept { - // do 8-bit optimizations, can just compare to 8 literal 0s. - uint64_t val; - while (std::distance(first, last) >= 8) { - ::memcpy(&val, first, sizeof(uint64_t)); - if (val != 0x3030303030303030) { - return true; - } - first += 8; - } - while (first != last) { - if (*first != '0') { - return true; - } - first++; - } - return false; -} - -fastfloat_really_inline bool is_truncated(byte_span s) noexcept { - return is_truncated(s.ptr, s.ptr + s.len()); -} - -fastfloat_really_inline -void parse_eight_digits(const char*& p, limb& value, size_t& counter, size_t& count) noexcept { - value = value * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - counter += 8; - count += 8; -} - -fastfloat_really_inline -void parse_one_digit(const char*& p, limb& value, size_t& counter, size_t& count) noexcept { - value = value * 10 + limb(*p - '0'); - p++; - counter++; - count++; -} - -fastfloat_really_inline -void add_native(bigint& big, limb power, limb value) noexcept { - big.mul(power); - big.add(value); -} - -fastfloat_really_inline void round_up_bigint(bigint& big, size_t& count) noexcept { - // need to round-up the digits, but need to avoid rounding - // ....9999 to ...10000, which could cause a false halfway point. - add_native(big, 10, 1); - count++; -} - -// parse the significant digits into a big integer -inline void parse_mantissa(bigint& result, parsed_number_string& num, size_t max_digits, size_t& digits) noexcept { - // try to minimize the number of big integer and scalar multiplication. - // therefore, try to parse 8 digits at a time, and multiply by the largest - // scalar value (9 or 19 digits) for each step. - size_t counter = 0; - digits = 0; - limb value = 0; -#ifdef FASTFLOAT_64BIT_LIMB - size_t step = 19; -#else - size_t step = 9; -#endif - - // process all integer digits. - const char* p = num.integer.ptr; - const char* pend = p + num.integer.len(); - skip_zeros(p, pend); - // process all digits, in increments of step per loop - while (p != pend) { - while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && (max_digits - digits >= 8)) { - parse_eight_digits(p, value, counter, digits); - } - while (counter < step && p != pend && digits < max_digits) { - parse_one_digit(p, value, counter, digits); - } - if (digits == max_digits) { - // add the temporary value, then check if we've truncated any digits - add_native(result, limb(powers_of_ten_uint64[counter]), value); - bool truncated = is_truncated(p, pend); - if (num.fraction.ptr != nullptr) { - truncated |= is_truncated(num.fraction); - } - if (truncated) { - round_up_bigint(result, digits); - } - return; - } else { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - counter = 0; - value = 0; - } - } - - // add our fraction digits, if they're available. - if (num.fraction.ptr != nullptr) { - p = num.fraction.ptr; - pend = p + num.fraction.len(); - if (digits == 0) { - skip_zeros(p, pend); - } - // process all digits, in increments of step per loop - while (p != pend) { - while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && (max_digits - digits >= 8)) { - parse_eight_digits(p, value, counter, digits); - } - while (counter < step && p != pend && digits < max_digits) { - parse_one_digit(p, value, counter, digits); - } - if (digits == max_digits) { - // add the temporary value, then check if we've truncated any digits - add_native(result, limb(powers_of_ten_uint64[counter]), value); - bool truncated = is_truncated(p, pend); - if (truncated) { - round_up_bigint(result, digits); - } - return; - } else { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - counter = 0; - value = 0; - } - } - } - - if (counter != 0) { - add_native(result, limb(powers_of_ten_uint64[counter]), value); - } -} - -template -inline adjusted_mantissa positive_digit_comp(bigint& bigmant, int32_t exponent) noexcept { - FASTFLOAT_ASSERT(bigmant.pow10(uint32_t(exponent))); - adjusted_mantissa answer; - bool truncated; - answer.mantissa = bigmant.hi64(truncated); - int bias = binary_format::mantissa_explicit_bits() - binary_format::minimum_exponent(); - answer.power2 = bigmant.bit_length() - 64 + bias; - - round(answer, [truncated](adjusted_mantissa& a, int32_t shift) { - round_nearest_tie_even(a, shift, [truncated](bool is_odd, bool is_halfway, bool is_above) -> bool { - return is_above || (is_halfway && truncated) || (is_odd && is_halfway); - }); - }); - - return answer; -} - -// the scaling here is quite simple: we have, for the real digits `m * 10^e`, -// and for the theoretical digits `n * 2^f`. Since `e` is always negative, -// to scale them identically, we do `n * 2^f * 5^-f`, so we now have `m * 2^e`. -// we then need to scale by `2^(f- e)`, and then the two significant digits -// are of the same magnitude. -template -inline adjusted_mantissa negative_digit_comp(bigint& bigmant, adjusted_mantissa am, int32_t exponent) noexcept { - bigint& real_digits = bigmant; - int32_t real_exp = exponent; - - // get the value of `b`, rounded down, and get a bigint representation of b+h - adjusted_mantissa am_b = am; - // gcc7 buf: use a lambda to remove the noexcept qualifier bug with -Wnoexcept-type. - round(am_b, [](adjusted_mantissa&a, int32_t shift) { round_down(a, shift); }); - T b; - to_float(false, am_b, b); - adjusted_mantissa theor = to_extended_halfway(b); - bigint theor_digits(theor.mantissa); - int32_t theor_exp = theor.power2; - - // scale real digits and theor digits to be same power. - int32_t pow2_exp = theor_exp - real_exp; - uint32_t pow5_exp = uint32_t(-real_exp); - if (pow5_exp != 0) { - FASTFLOAT_ASSERT(theor_digits.pow5(pow5_exp)); - } - if (pow2_exp > 0) { - FASTFLOAT_ASSERT(theor_digits.pow2(uint32_t(pow2_exp))); - } else if (pow2_exp < 0) { - FASTFLOAT_ASSERT(real_digits.pow2(uint32_t(-pow2_exp))); - } - - // compare digits, and use it to director rounding - int ord = real_digits.compare(theor_digits); - adjusted_mantissa answer = am; - round(answer, [ord](adjusted_mantissa& a, int32_t shift) { - round_nearest_tie_even(a, shift, [ord](bool is_odd, bool _, bool __) -> bool { - (void)_; // not needed, since we've done our comparison - (void)__; // not needed, since we've done our comparison - if (ord > 0) { - return true; - } else if (ord < 0) { - return false; - } else { - return is_odd; - } - }); - }); - - return answer; -} - -// parse the significant digits as a big integer to unambiguously round the -// the significant digits. here, we are trying to determine how to round -// an extended float representation close to `b+h`, halfway between `b` -// (the float rounded-down) and `b+u`, the next positive float. this -// algorithm is always correct, and uses one of two approaches. when -// the exponent is positive relative to the significant digits (such as -// 1234), we create a big-integer representation, get the high 64-bits, -// determine if any lower bits are truncated, and use that to direct -// rounding. in case of a negative exponent relative to the significant -// digits (such as 1.2345), we create a theoretical representation of -// `b` as a big-integer type, scaled to the same binary exponent as -// the actual digits. we then compare the big integer representations -// of both, and use that to direct rounding. -template -inline adjusted_mantissa digit_comp(parsed_number_string& num, adjusted_mantissa am) noexcept { - // remove the invalid exponent bias - am.power2 -= invalid_am_bias; - - int32_t sci_exp = scientific_exponent(num); - size_t max_digits = binary_format::max_digits(); - size_t digits = 0; - bigint bigmant; - parse_mantissa(bigmant, num, max_digits, digits); - // can't underflow, since digits is at most max_digits. - int32_t exponent = sci_exp + 1 - int32_t(digits); - if (exponent >= 0) { - return positive_digit_comp(bigmant, exponent); - } else { - return negative_digit_comp(bigmant, am, exponent); - } -} - -} // namespace fast_float - -#endif - - -#ifndef FASTFLOAT_PARSE_NUMBER_H -#define FASTFLOAT_PARSE_NUMBER_H - - -#include -#include -#include -#include - -namespace fast_float { - - -namespace detail { -/** - * Special case +inf, -inf, nan, infinity, -infinity. - * The case comparisons could be made much faster given that we know that the - * strings a null-free and fixed. - **/ -template -from_chars_result parse_infnan(const char *first, const char *last, T &value) noexcept { - from_chars_result answer; - answer.ptr = first; - answer.ec = std::errc(); // be optimistic - bool minusSign = false; - if (*first == '-') { // assume first < last, so dereference without checks; C++17 20.19.3.(7.1) explicitly forbids '+' here - minusSign = true; - ++first; - } - if (last - first >= 3) { - if (fastfloat_strncasecmp(first, "nan", 3)) { - answer.ptr = (first += 3); - value = minusSign ? -std::numeric_limits::quiet_NaN() : std::numeric_limits::quiet_NaN(); - // Check for possible nan(n-char-seq-opt), C++17 20.19.3.7, C11 7.20.1.3.3. At least MSVC produces nan(ind) and nan(snan). - if(first != last && *first == '(') { - for(const char* ptr = first + 1; ptr != last; ++ptr) { - if (*ptr == ')') { - answer.ptr = ptr + 1; // valid nan(n-char-seq-opt) - break; - } - else if(!(('a' <= *ptr && *ptr <= 'z') || ('A' <= *ptr && *ptr <= 'Z') || ('0' <= *ptr && *ptr <= '9') || *ptr == '_')) - break; // forbidden char, not nan(n-char-seq-opt) - } - } - return answer; - } - if (fastfloat_strncasecmp(first, "inf", 3)) { - if ((last - first >= 8) && fastfloat_strncasecmp(first + 3, "inity", 5)) { - answer.ptr = first + 8; - } else { - answer.ptr = first + 3; - } - value = minusSign ? -std::numeric_limits::infinity() : std::numeric_limits::infinity(); - return answer; - } - } - answer.ec = std::errc::invalid_argument; - return answer; -} - -} // namespace detail - -template -from_chars_result from_chars(const char *first, const char *last, - T &value, chars_format fmt /*= chars_format::general*/) noexcept { - return from_chars_advanced(first, last, value, parse_options{fmt}); -} - -template -from_chars_result from_chars_advanced(const char *first, const char *last, - T &value, parse_options options) noexcept { - - static_assert (std::is_same::value || std::is_same::value, "only float and double are supported"); - - - from_chars_result answer; - if (first == last) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; - return answer; - } - parsed_number_string pns = parse_number_string(first, last, options); - if (!pns.valid) { - return detail::parse_infnan(first, last, value); - } - answer.ec = std::errc(); // be optimistic - answer.ptr = pns.lastmatch; - // Next is Clinger's fast path. - if (binary_format::min_exponent_fast_path() <= pns.exponent && pns.exponent <= binary_format::max_exponent_fast_path() && pns.mantissa <=binary_format::max_mantissa_fast_path() && !pns.too_many_digits) { - value = T(pns.mantissa); - if (pns.exponent < 0) { value = value / binary_format::exact_power_of_ten(-pns.exponent); } - else { value = value * binary_format::exact_power_of_ten(pns.exponent); } - if (pns.negative) { value = -value; } - return answer; - } - adjusted_mantissa am = compute_float>(pns.exponent, pns.mantissa); - if(pns.too_many_digits && am.power2 >= 0) { - if(am != compute_float>(pns.exponent, pns.mantissa + 1)) { - am = compute_error>(pns.exponent, pns.mantissa); - } - } - // If we called compute_float>(pns.exponent, pns.mantissa) and we have an invalid power (am.power2 < 0), - // then we need to go the long way around again. This is very uncommon. - if(am.power2 < 0) { am = digit_comp(pns, am); } - to_float(pns.negative, am, value); - return answer; -} - -} // namespace fast_float - -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/rng/rng.hpp b/thirdparty/ryml/ext/c4core/src/c4/ext/rng/rng.hpp deleted file mode 100644 index 63af09008..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/rng/rng.hpp +++ /dev/null @@ -1,192 +0,0 @@ -/* Copyright (c) 2018 Arvid Gerstmann. - * - * https://arvid.io/2018/07/02/better-cxx-prng/ - * - * This code is licensed under MIT license. */ -#ifndef AG_RANDOM_H -#define AG_RANDOM_H - -#include -#include - - -namespace c4 { -namespace rng { - - -class splitmix -{ -public: - using result_type = uint32_t; - static constexpr result_type (min)() { return 0; } - static constexpr result_type (max)() { return UINT32_MAX; } - friend bool operator==(splitmix const &, splitmix const &); - friend bool operator!=(splitmix const &, splitmix const &); - - splitmix() : m_seed(1) {} - explicit splitmix(uint64_t s) : m_seed(s) {} - explicit splitmix(std::random_device &rd) - { - seed(rd); - } - - void seed(uint64_t s) { m_seed = s; } - void seed(std::random_device &rd) - { - m_seed = uint64_t(rd()) << 31 | uint64_t(rd()); - } - - result_type operator()() - { - uint64_t z = (m_seed += UINT64_C(0x9E3779B97F4A7C15)); - z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB); - return result_type((z ^ (z >> 31)) >> 31); - } - - void discard(unsigned long long n) - { - for (unsigned long long i = 0; i < n; ++i) - operator()(); - } - -private: - uint64_t m_seed; -}; - -inline bool operator==(splitmix const &lhs, splitmix const &rhs) -{ - return lhs.m_seed == rhs.m_seed; -} -inline bool operator!=(splitmix const &lhs, splitmix const &rhs) -{ - return lhs.m_seed != rhs.m_seed; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -class xorshift -{ -public: - using result_type = uint32_t; - static constexpr result_type (min)() { return 0; } - static constexpr result_type (max)() { return UINT32_MAX; } - friend bool operator==(xorshift const &, xorshift const &); - friend bool operator!=(xorshift const &, xorshift const &); - - xorshift() : m_seed(0xc1f651c67c62c6e0ull) {} - explicit xorshift(std::random_device &rd) - { - seed(rd); - } - - void seed(uint64_t s) { m_seed = s; } - void seed(std::random_device &rd) - { - m_seed = uint64_t(rd()) << 31 | uint64_t(rd()); - } - - result_type operator()() - { - uint64_t result = m_seed * 0xd989bcacc137dcd5ull; - m_seed ^= m_seed >> 11; - m_seed ^= m_seed << 31; - m_seed ^= m_seed >> 18; - return uint32_t(result >> 32ull); - } - - void discard(unsigned long long n) - { - for (unsigned long long i = 0; i < n; ++i) - operator()(); - } - -private: - uint64_t m_seed; -}; - -inline bool operator==(xorshift const &lhs, xorshift const &rhs) -{ - return lhs.m_seed == rhs.m_seed; -} -inline bool operator!=(xorshift const &lhs, xorshift const &rhs) -{ - return lhs.m_seed != rhs.m_seed; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -class pcg -{ -public: - using result_type = uint32_t; - static constexpr result_type (min)() { return 0; } - static constexpr result_type (max)() { return UINT32_MAX; } - friend bool operator==(pcg const &, pcg const &); - friend bool operator!=(pcg const &, pcg const &); - - pcg() - : m_state(0x853c49e6748fea9bULL) - , m_inc(0xda3e39cb94b95bdbULL) - {} - explicit pcg(uint64_t s) { m_state = s; m_inc = m_state << 1; } - explicit pcg(std::random_device &rd) - { - seed(rd); - } - - void seed(uint64_t s) { m_state = s; } - void seed(std::random_device &rd) - { - uint64_t s0 = uint64_t(rd()) << 31 | uint64_t(rd()); - uint64_t s1 = uint64_t(rd()) << 31 | uint64_t(rd()); - - m_state = 0; - m_inc = (s1 << 1) | 1; - (void)operator()(); - m_state += s0; - (void)operator()(); - } - - result_type operator()() - { - uint64_t oldstate = m_state; - m_state = oldstate * 6364136223846793005ULL + m_inc; - uint32_t xorshifted = uint32_t(((oldstate >> 18u) ^ oldstate) >> 27u); - //int rot = oldstate >> 59u; // the original. error? - int64_t rot = (int64_t)oldstate >> 59u; // error? - return (xorshifted >> rot) | (xorshifted << ((uint64_t)(-rot) & 31)); - } - - void discard(unsigned long long n) - { - for (unsigned long long i = 0; i < n; ++i) - operator()(); - } - -private: - uint64_t m_state; - uint64_t m_inc; -}; - -inline bool operator==(pcg const &lhs, pcg const &rhs) -{ - return lhs.m_state == rhs.m_state - && lhs.m_inc == rhs.m_inc; -} -inline bool operator!=(pcg const &lhs, pcg const &rhs) -{ - return lhs.m_state != rhs.m_state - || lhs.m_inc != rhs.m_inc; -} - -} // namespace rng -} // namespace c4 - -#endif /* AG_RANDOM_H */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/sg14/README.md b/thirdparty/ryml/ext/c4core/src/c4/ext/sg14/README.md deleted file mode 100644 index 11d0f94a0..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/sg14/README.md +++ /dev/null @@ -1 +0,0 @@ -https://github.com/WG21-SG14/SG14/blob/master/SG14/inplace_function.h diff --git a/thirdparty/ryml/ext/c4core/src/c4/ext/sg14/inplace_function.h b/thirdparty/ryml/ext/c4core/src/c4/ext/sg14/inplace_function.h deleted file mode 100644 index af3e83054..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/ext/sg14/inplace_function.h +++ /dev/null @@ -1,347 +0,0 @@ -/* - * Boost Software License - Version 1.0 - August 17th, 2003 - * - * Permission is hereby granted, free of charge, to any person or organization - * obtaining a copy of the software and accompanying documentation covered by - * this license (the "Software") to use, reproduce, display, distribute, - * execute, and transmit the Software, and to prepare derivative works of the - * Software, and to permit third-parties to whom the Software is furnished to - * do so, all subject to the following: - * - * The copyright notices in the Software and this entire statement, including - * the above license grant, this restriction and the following disclaimer, - * must be included in all copies of the Software, in whole or in part, and - * all derivative works of the Software, unless such copies or derivative - * works are solely in the form of machine-executable object code generated by - * a source language processor. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT - * SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE - * FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, - * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - * DEALINGS IN THE SOFTWARE. - */ -#ifndef _C4_EXT_SG14_INPLACE_FUNCTION_H_ -#define _C4_EXT_SG14_INPLACE_FUNCTION_H_ - -#include -#include -#include - -namespace stdext { - -namespace inplace_function_detail { - -static constexpr size_t InplaceFunctionDefaultCapacity = 32; - -#if defined(__GLIBCXX__) // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61458 -template -union aligned_storage_helper { - struct double1 { double a; }; - struct double4 { double a[4]; }; - template using maybe = typename std::conditional<(Cap >= sizeof(T)), T, char>::type; - char real_data[Cap]; - maybe a; - maybe b; - maybe c; - maybe d; - maybe e; - maybe f; - maybe g; - maybe h; -}; - -template>::value> -struct aligned_storage { - using type = typename std::aligned_storage::type; -}; -#else -using std::aligned_storage; -#endif - -template struct wrapper -{ - using type = T; -}; - -template struct vtable -{ - using storage_ptr_t = void*; - - using invoke_ptr_t = R(*)(storage_ptr_t, Args&&...); - using process_ptr_t = void(*)(storage_ptr_t, storage_ptr_t); - using destructor_ptr_t = void(*)(storage_ptr_t); - - const invoke_ptr_t invoke_ptr; - const process_ptr_t copy_ptr; - const process_ptr_t move_ptr; - const destructor_ptr_t destructor_ptr; - - explicit constexpr vtable() noexcept : - invoke_ptr{ [](storage_ptr_t, Args&&...) -> R - { throw std::bad_function_call(); } - }, - copy_ptr{ [](storage_ptr_t, storage_ptr_t) noexcept -> void {} }, - move_ptr{ [](storage_ptr_t, storage_ptr_t) noexcept -> void {} }, - destructor_ptr{ [](storage_ptr_t) noexcept -> void {} } - {} - - template explicit constexpr vtable(wrapper) noexcept : - invoke_ptr{ [](storage_ptr_t storage_ptr, Args&&... args) - noexcept(noexcept(std::declval()(args...))) -> R - { return (*static_cast(storage_ptr))( - std::forward(args)... - ); } - }, - copy_ptr{ [](storage_ptr_t dst_ptr, storage_ptr_t src_ptr) - noexcept(std::is_nothrow_copy_constructible::value) -> void - { new (dst_ptr) C{ (*static_cast(src_ptr)) }; } - }, - move_ptr{ [](storage_ptr_t dst_ptr, storage_ptr_t src_ptr) - noexcept(std::is_nothrow_move_constructible::value) -> void - { new (dst_ptr) C{ std::move(*static_cast(src_ptr)) }; } - }, - destructor_ptr{ [](storage_ptr_t storage_ptr) - noexcept -> void - { static_cast(storage_ptr)->~C(); } - } - {} - - vtable(const vtable&) = delete; - vtable(vtable&&) = delete; - - vtable& operator= (const vtable&) = delete; - vtable& operator= (vtable&&) = delete; - - ~vtable() = default; -}; - -template -struct is_valid_inplace_dst : std::true_type -{ - static_assert(DstCap >= SrcCap, - "Can't squeeze larger inplace_function into a smaller one" - ); - - static_assert(DstAlign % SrcAlign == 0, - "Incompatible inplace_function alignments" - ); -}; - -} // namespace inplace_function_detail - -template< - typename Signature, - size_t Capacity = inplace_function_detail::InplaceFunctionDefaultCapacity, - size_t Alignment = std::alignment_of::type>::value -> -class inplace_function; // unspecified - -template< - typename R, - typename... Args, - size_t Capacity, - size_t Alignment -> -class inplace_function -{ - static const constexpr inplace_function_detail::vtable empty_vtable{}; -public: - using capacity = std::integral_constant; - using alignment = std::integral_constant; - - using storage_t = typename inplace_function_detail::aligned_storage::type; - using vtable_t = inplace_function_detail::vtable; - using vtable_ptr_t = const vtable_t*; - - template friend class inplace_function; - - inplace_function() noexcept : - vtable_ptr_{std::addressof(empty_vtable)} - {} - - template< - typename T, - typename C = typename std::decay::type, - typename = typename std::enable_if< - !(std::is_same::value - || std::is_convertible::value) - >::type - > - inplace_function(T&& closure) - { -#if __cplusplus >= 201703L - static_assert(std::is_invocable_r::value, - "inplace_function cannot be constructed from non-callable type" - ); -#endif - static_assert(std::is_copy_constructible::value, - "inplace_function cannot be constructed from non-copyable type" - ); - - static_assert(sizeof(C) <= Capacity, - "inplace_function cannot be constructed from object with this (large) size" - ); - - static_assert(Alignment % std::alignment_of::value == 0, - "inplace_function cannot be constructed from object with this (large) alignment" - ); - - static const vtable_t vt{inplace_function_detail::wrapper{}}; - vtable_ptr_ = std::addressof(vt); - - new (std::addressof(storage_)) C{std::forward(closure)}; - } - - inplace_function(std::nullptr_t) noexcept : - vtable_ptr_{std::addressof(empty_vtable)} - {} - - inplace_function(const inplace_function& other) : - vtable_ptr_{other.vtable_ptr_} - { - vtable_ptr_->copy_ptr( - std::addressof(storage_), - std::addressof(other.storage_) - ); - } - - inplace_function(inplace_function&& other) : - vtable_ptr_{other.vtable_ptr_} - { - vtable_ptr_->move_ptr( - std::addressof(storage_), - std::addressof(other.storage_) - ); - } - - inplace_function& operator= (std::nullptr_t) noexcept - { - vtable_ptr_->destructor_ptr(std::addressof(storage_)); - vtable_ptr_ = std::addressof(empty_vtable); - return *this; - } - - inplace_function& operator= (const inplace_function& other) - { - if(this != std::addressof(other)) - { - vtable_ptr_->destructor_ptr(std::addressof(storage_)); - - vtable_ptr_ = other.vtable_ptr_; - vtable_ptr_->copy_ptr( - std::addressof(storage_), - std::addressof(other.storage_) - ); - } - return *this; - } - - inplace_function& operator= (inplace_function&& other) - { - if(this != std::addressof(other)) - { - vtable_ptr_->destructor_ptr(std::addressof(storage_)); - - vtable_ptr_ = other.vtable_ptr_; - vtable_ptr_->move_ptr( - std::addressof(storage_), - std::addressof(other.storage_) - ); - } - return *this; - } - - ~inplace_function() - { - vtable_ptr_->destructor_ptr(std::addressof(storage_)); - } - - R operator() (Args... args) const - { - return vtable_ptr_->invoke_ptr( - std::addressof(storage_), - std::forward(args)... - ); - } - - constexpr bool operator== (std::nullptr_t) const noexcept - { - return !operator bool(); - } - - constexpr bool operator!= (std::nullptr_t) const noexcept - { - return operator bool(); - } - - explicit constexpr operator bool() const noexcept - { - return vtable_ptr_ != std::addressof(empty_vtable); - } - - template - operator inplace_function() const& - { - static_assert(inplace_function_detail::is_valid_inplace_dst< - Cap, Align, Capacity, Alignment - >::value, "conversion not allowed"); - - return {vtable_ptr_, vtable_ptr_->copy_ptr, std::addressof(storage_)}; - } - - template - operator inplace_function() && - { - static_assert(inplace_function_detail::is_valid_inplace_dst< - Cap, Align, Capacity, Alignment - >::value, "conversion not allowed"); - - return {vtable_ptr_, vtable_ptr_->move_ptr, std::addressof(storage_)}; - } - - void swap(inplace_function& other) - { - if (this == std::addressof(other)) return; - - storage_t tmp; - vtable_ptr_->move_ptr( - std::addressof(tmp), - std::addressof(storage_) - ); - vtable_ptr_->destructor_ptr(std::addressof(storage_)); - - other.vtable_ptr_->move_ptr( - std::addressof(storage_), - std::addressof(other.storage_) - ); - other.vtable_ptr_->destructor_ptr(std::addressof(other.storage_)); - - vtable_ptr_->move_ptr( - std::addressof(other.storage_), - std::addressof(tmp) - ); - vtable_ptr_->destructor_ptr(std::addressof(tmp)); - - std::swap(vtable_ptr_, other.vtable_ptr_); - } - -private: - vtable_ptr_t vtable_ptr_; - mutable storage_t storage_; - - inplace_function( - vtable_ptr_t vtable_ptr, - typename vtable_t::process_ptr_t process_ptr, - typename vtable_t::storage_ptr_t storage_ptr - ) : vtable_ptr_{vtable_ptr} - { - process_ptr(std::addressof(storage_), storage_ptr); - } -}; - -} // namespace stdext - -#endif /* _C4_EXT_SG14_INPLACE_FUNCTION_H_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/format.cpp b/thirdparty/ryml/ext/c4core/src/c4/format.cpp deleted file mode 100644 index cc058c311..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/format.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "c4/format.hpp" - -#include // for std::align - -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wformat-nonliteral" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wformat-nonliteral" -#endif - -namespace c4 { - - -size_t to_chars(substr buf, fmt::const_raw_wrapper r) -{ - void * vptr = buf.str; - size_t space = buf.len; - auto ptr = (decltype(buf.str)) std::align(r.alignment, r.len, vptr, space); - if(ptr == nullptr) - { - // if it was not possible to align, return a conservative estimate - // of the required space - return r.alignment + r.len; - } - C4_CHECK(ptr >= buf.begin() && ptr <= buf.end()); - size_t sz = static_cast(ptr - buf.str) + r.len; - if(sz <= buf.len) - { - memcpy(ptr, r.buf, r.len); - } - return sz; -} - - -bool from_chars(csubstr buf, fmt::raw_wrapper *r) -{ - void * vptr = (void*)buf.str; - size_t space = buf.len; - auto ptr = (decltype(buf.str)) std::align(r->alignment, r->len, vptr, space); - C4_CHECK(ptr != nullptr); - C4_CHECK(ptr >= buf.begin() && ptr <= buf.end()); - //size_t dim = (ptr - buf.str) + r->len; - memcpy(r->buf, ptr, r->len); - return true; -} - - -} // namespace c4 - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/ext/c4core/src/c4/format.hpp b/thirdparty/ryml/ext/c4core/src/c4/format.hpp deleted file mode 100644 index 589bd9842..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/format.hpp +++ /dev/null @@ -1,900 +0,0 @@ -#ifndef _C4_FORMAT_HPP_ -#define _C4_FORMAT_HPP_ - -/** @file format.hpp provides type-safe facilities for formatting arguments - * to string buffers */ - -#include "c4/charconv.hpp" -#include "c4/blob.hpp" - - -#ifdef _MSC_VER -# pragma warning(push) -# if C4_MSVC_VERSION != C4_MSVC_VERSION_2017 -# pragma warning(disable: 4800) // forcing value to bool 'true' or 'false' (performance warning) -# endif -# pragma warning(disable: 4996) // snprintf/scanf: this function or variable may be unsafe -#elif defined(__clang__) -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wuseless-cast" -#endif - -namespace c4 { - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// formatting truthy types as booleans - -namespace fmt { - -/** write a variable as an alphabetic boolean, ie as either true or false - * @param strict_read */ -template -struct boolalpha_ -{ - boolalpha_(T val_, bool strict_read_=false) : val(val_ ? true : false), strict_read(strict_read_) {} - bool val; - bool strict_read; -}; - -template -boolalpha_ boolalpha(T const& val, bool strict_read=false) -{ - return boolalpha_(val, strict_read); -} - -} // namespace fmt - -/** write a variable as an alphabetic boolean, ie as either true or false */ -template -inline size_t to_chars(substr buf, fmt::boolalpha_ fmt) -{ - return to_chars(buf, fmt.val ? "true" : "false"); -} - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// formatting integral types - -namespace fmt { - -/** format an integral type with a custom radix */ -template -struct integral_ -{ - T val; - T radix; - C4_ALWAYS_INLINE integral_(T val_, T radix_) : val(val_), radix(radix_) {} -}; - -/** format an integral type with a custom radix, and pad with zeroes on the left */ -template -struct integral_padded_ -{ - T val; - T radix; - size_t num_digits; - C4_ALWAYS_INLINE integral_padded_(T val_, T radix_, size_t nd) : val(val_), radix(radix_), num_digits(nd) {} -}; - -/** format an integral type with a custom radix */ -template -C4_ALWAYS_INLINE integral_ integral(T val, T radix=10) -{ - return integral_(val, radix); -} -/** format an integral type with a custom radix */ -template -C4_ALWAYS_INLINE integral_ integral(T const* val, T radix=10) -{ - return integral_(reinterpret_cast(val), static_cast(radix)); -} -/** format an integral type with a custom radix */ -template -C4_ALWAYS_INLINE integral_ integral(std::nullptr_t, T radix=10) -{ - return integral_(intptr_t(0), static_cast(radix)); -} -/** pad the argument with zeroes on the left, with decimal radix */ -template -C4_ALWAYS_INLINE integral_padded_ zpad(T val, size_t num_digits) -{ - return integral_padded_(val, T(10), num_digits); -} -/** pad the argument with zeroes on the left */ -template -C4_ALWAYS_INLINE integral_padded_ zpad(integral_ val, size_t num_digits) -{ - return integral_padded_(val.val, val.radix, num_digits); -} -/** pad the argument with zeroes on the left */ -C4_ALWAYS_INLINE integral_padded_ zpad(std::nullptr_t, size_t num_digits) -{ - return integral_padded_(0, 16, num_digits); -} -/** pad the argument with zeroes on the left */ -template -C4_ALWAYS_INLINE integral_padded_ zpad(T const* val, size_t num_digits) -{ - return integral_padded_(reinterpret_cast(val), 16, num_digits); -} -template -C4_ALWAYS_INLINE integral_padded_ zpad(T * val, size_t num_digits) -{ - return integral_padded_(reinterpret_cast(val), 16, num_digits); -} - - -/** format the pointer as an hexadecimal value */ -template -inline integral_ hex(T * v) -{ - return integral_(reinterpret_cast(v), intptr_t(16)); -} -/** format the pointer as an hexadecimal value */ -template -inline integral_ hex(T const* v) -{ - return integral_(reinterpret_cast(v), intptr_t(16)); -} -/** format null as an hexadecimal value - * @overload hex */ -inline integral_ hex(std::nullptr_t) -{ - return integral_(0, intptr_t(16)); -} -/** format the integral_ argument as an hexadecimal value - * @overload hex */ -template -inline integral_ hex(T v) -{ - return integral_(v, T(16)); -} - -/** format the pointer as an octal value */ -template -inline integral_ oct(T const* v) -{ - return integral_(reinterpret_cast(v), intptr_t(8)); -} -/** format the pointer as an octal value */ -template -inline integral_ oct(T * v) -{ - return integral_(reinterpret_cast(v), intptr_t(8)); -} -/** format null as an octal value */ -inline integral_ oct(std::nullptr_t) -{ - return integral_(intptr_t(0), intptr_t(8)); -} -/** format the integral_ argument as an octal value */ -template -inline integral_ oct(T v) -{ - return integral_(v, T(8)); -} - -/** format the pointer as a binary 0-1 value - * @see c4::raw() if you want to use a binary memcpy instead of 0-1 formatting */ -template -inline integral_ bin(T const* v) -{ - return integral_(reinterpret_cast(v), intptr_t(2)); -} -/** format the pointer as a binary 0-1 value - * @see c4::raw() if you want to use a binary memcpy instead of 0-1 formatting */ -template -inline integral_ bin(T * v) -{ - return integral_(reinterpret_cast(v), intptr_t(2)); -} -/** format null as a binary 0-1 value - * @see c4::raw() if you want to use a binary memcpy instead of 0-1 formatting */ -inline integral_ bin(std::nullptr_t) -{ - return integral_(intptr_t(0), intptr_t(2)); -} -/** format the integral_ argument as a binary 0-1 value - * @see c4::raw() if you want to use a raw memcpy-based binary dump instead of 0-1 formatting */ -template -inline integral_ bin(T v) -{ - return integral_(v, T(2)); -} - - -template -struct overflow_checked_ -{ - static_assert(std::is_integral::value, "range checking only for integral types"); - C4_ALWAYS_INLINE overflow_checked_(T &val_) : val(&val_) {} - T *val; -}; -template -C4_ALWAYS_INLINE overflow_checked_ overflow_checked(T &val) -{ - return overflow_checked_(val); -} - -} // namespace fmt - -/** format an integral_ signed type */ -template -C4_ALWAYS_INLINE -typename std::enable_if::value, size_t>::type -to_chars(substr buf, fmt::integral_ fmt) -{ - return itoa(buf, fmt.val, fmt.radix); -} -/** format an integral_ signed type, pad with zeroes */ -template -C4_ALWAYS_INLINE -typename std::enable_if::value, size_t>::type -to_chars(substr buf, fmt::integral_padded_ fmt) -{ - return itoa(buf, fmt.val, fmt.radix, fmt.num_digits); -} - -/** format an integral_ unsigned type */ -template -C4_ALWAYS_INLINE -typename std::enable_if::value, size_t>::type -to_chars(substr buf, fmt::integral_ fmt) -{ - return utoa(buf, fmt.val, fmt.radix); -} -/** format an integral_ unsigned type, pad with zeroes */ -template -C4_ALWAYS_INLINE -typename std::enable_if::value, size_t>::type -to_chars(substr buf, fmt::integral_padded_ fmt) -{ - return utoa(buf, fmt.val, fmt.radix, fmt.num_digits); -} - -template -C4_ALWAYS_INLINE bool from_chars(csubstr s, fmt::overflow_checked_ wrapper) -{ - if(C4_LIKELY(!overflows(s))) - return atox(s, wrapper.val); - return false; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// formatting real types - -namespace fmt { - -template -struct real_ -{ - T val; - int precision; - RealFormat_e fmt; - real_(T v, int prec=-1, RealFormat_e f=FTOA_FLOAT) : val(v), precision(prec), fmt(f) {} -}; - -template -real_ real(T val, int precision, RealFormat_e fmt=FTOA_FLOAT) -{ - return real_(val, precision, fmt); -} - -} // namespace fmt - -inline size_t to_chars(substr buf, fmt::real_< float> fmt) { return ftoa(buf, fmt.val, fmt.precision, fmt.fmt); } -inline size_t to_chars(substr buf, fmt::real_ fmt) { return dtoa(buf, fmt.val, fmt.precision, fmt.fmt); } - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// writing raw binary data - -namespace fmt { - -/** @see blob_ */ -template -struct raw_wrapper_ : public blob_ -{ - size_t alignment; - - C4_ALWAYS_INLINE raw_wrapper_(blob_ data, size_t alignment_) noexcept - : - blob_(data), - alignment(alignment_) - { - C4_ASSERT_MSG(alignment > 0 && (alignment & (alignment - 1)) == 0, "alignment must be a power of two"); - } -}; - -using const_raw_wrapper = raw_wrapper_; -using raw_wrapper = raw_wrapper_; - -/** mark a variable to be written in raw binary format, using memcpy - * @see blob_ */ -inline const_raw_wrapper craw(cblob data, size_t alignment=alignof(max_align_t)) -{ - return const_raw_wrapper(data, alignment); -} -/** mark a variable to be written in raw binary format, using memcpy - * @see blob_ */ -inline const_raw_wrapper raw(cblob data, size_t alignment=alignof(max_align_t)) -{ - return const_raw_wrapper(data, alignment); -} -/** mark a variable to be written in raw binary format, using memcpy - * @see blob_ */ -template -inline const_raw_wrapper craw(T const& C4_RESTRICT data, size_t alignment=alignof(T)) -{ - return const_raw_wrapper(cblob(data), alignment); -} -/** mark a variable to be written in raw binary format, using memcpy - * @see blob_ */ -template -inline const_raw_wrapper raw(T const& C4_RESTRICT data, size_t alignment=alignof(T)) -{ - return const_raw_wrapper(cblob(data), alignment); -} - -/** mark a variable to be read in raw binary format, using memcpy */ -inline raw_wrapper raw(blob data, size_t alignment=alignof(max_align_t)) -{ - return raw_wrapper(data, alignment); -} -/** mark a variable to be read in raw binary format, using memcpy */ -template -inline raw_wrapper raw(T & C4_RESTRICT data, size_t alignment=alignof(T)) -{ - return raw_wrapper(blob(data), alignment); -} - -} // namespace fmt - - -/** write a variable in raw binary format, using memcpy */ -C4CORE_EXPORT size_t to_chars(substr buf, fmt::const_raw_wrapper r); - -/** read a variable in raw binary format, using memcpy */ -C4CORE_EXPORT bool from_chars(csubstr buf, fmt::raw_wrapper *r); -/** read a variable in raw binary format, using memcpy */ -inline bool from_chars(csubstr buf, fmt::raw_wrapper r) -{ - return from_chars(buf, &r); -} - -/** read a variable in raw binary format, using memcpy */ -inline size_t from_chars_first(csubstr buf, fmt::raw_wrapper *r) -{ - return from_chars(buf, r); -} -/** read a variable in raw binary format, using memcpy */ -inline size_t from_chars_first(csubstr buf, fmt::raw_wrapper r) -{ - return from_chars(buf, &r); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// formatting aligned to left/right - -namespace fmt { - -template -struct left_ -{ - T val; - size_t width; - char pad; - left_(T v, size_t w, char p) : val(v), width(w), pad(p) {} -}; - -template -struct right_ -{ - T val; - size_t width; - char pad; - right_(T v, size_t w, char p) : val(v), width(w), pad(p) {} -}; - -/** mark an argument to be aligned left */ -template -left_ left(T val, size_t width, char padchar=' ') -{ - return left_(val, width, padchar); -} - -/** mark an argument to be aligned right */ -template -right_ right(T val, size_t width, char padchar=' ') -{ - return right_(val, width, padchar); -} - -} // namespace fmt - - -template -size_t to_chars(substr buf, fmt::left_ const& C4_RESTRICT align) -{ - size_t ret = to_chars(buf, align.val); - if(ret >= buf.len || ret >= align.width) - return ret > align.width ? ret : align.width; - buf.first(align.width).sub(ret).fill(align.pad); - to_chars(buf, align.val); - return align.width; -} - -template -size_t to_chars(substr buf, fmt::right_ const& C4_RESTRICT align) -{ - size_t ret = to_chars(buf, align.val); - if(ret >= buf.len || ret >= align.width) - return ret > align.width ? ret : align.width; - size_t rem = static_cast(align.width - ret); - buf.first(rem).fill(align.pad); - to_chars(buf.sub(rem), align.val); - return align.width; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/// @cond dev -// terminates the variadic recursion -inline size_t cat(substr /*buf*/) -{ - return 0; -} -/// @endcond - - -/** serialize the arguments, concatenating them to the given fixed-size buffer. - * The buffer size is strictly respected: no writes will occur beyond its end. - * @return the number of characters needed to write all the arguments into the buffer. - * @see c4::catrs() if instead of a fixed-size buffer, a resizeable container is desired - * @see c4::uncat() for the inverse function - * @see c4::catsep() if a separator between each argument is to be used - * @see c4::format() if a format string is desired */ -template -size_t cat(substr buf, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - size_t num = to_chars(buf, a); - buf = buf.len >= num ? buf.sub(num) : substr{}; - num += cat(buf, more...); - return num; -} - -/** like c4::cat() but return a substr instead of a size */ -template -substr cat_sub(substr buf, Args && ...args) -{ - size_t sz = cat(buf, std::forward(args)...); - C4_CHECK(sz <= buf.len); - return {buf.str, sz <= buf.len ? sz : buf.len}; -} - - -//----------------------------------------------------------------------------- - -/// @cond dev -// terminates the variadic recursion -inline size_t uncat(csubstr /*buf*/) -{ - return 0; -} -/// @endcond - - -/** deserialize the arguments from the given buffer. - * - * @return the number of characters read from the buffer, or csubstr::npos - * if a conversion was not successful. - * @see c4::cat(). c4::uncat() is the inverse of c4::cat(). */ -template -size_t uncat(csubstr buf, Arg & C4_RESTRICT a, Args & C4_RESTRICT ...more) -{ - size_t out = from_chars_first(buf, &a); - if(C4_UNLIKELY(out == csubstr::npos)) - return csubstr::npos; - buf = buf.len >= out ? buf.sub(out) : substr{}; - size_t num = uncat(buf, more...); - if(C4_UNLIKELY(num == csubstr::npos)) - return csubstr::npos; - return out + num; -} - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace detail { - -template -inline size_t catsep_more(substr /*buf*/, Sep const& C4_RESTRICT /*sep*/) -{ - return 0; -} - -template -size_t catsep_more(substr buf, Sep const& C4_RESTRICT sep, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - size_t ret = to_chars(buf, sep), num = ret; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = to_chars(buf, a); - num += ret; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = catsep_more(buf, sep, more...); - num += ret; - return num; -} - -template -inline size_t uncatsep_more(csubstr /*buf*/, Sep & /*sep*/) -{ - return 0; -} - -template -size_t uncatsep_more(csubstr buf, Sep & C4_RESTRICT sep, Arg & C4_RESTRICT a, Args & C4_RESTRICT ...more) -{ - size_t ret = from_chars_first(buf, &sep), num = ret; - if(C4_UNLIKELY(ret == csubstr::npos)) - return csubstr::npos; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = from_chars_first(buf, &a); - if(C4_UNLIKELY(ret == csubstr::npos)) - return csubstr::npos; - num += ret; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = uncatsep_more(buf, sep, more...); - if(C4_UNLIKELY(ret == csubstr::npos)) - return csubstr::npos; - num += ret; - return num; -} - -} // namespace detail - - -/** serialize the arguments, concatenating them to the given fixed-size - * buffer, using a separator between each argument. - * The buffer size is strictly respected: no writes will occur beyond its end. - * @return the number of characters needed to write all the arguments into the buffer. - * @see c4::catseprs() if instead of a fixed-size buffer, a resizeable container is desired - * @see c4::uncatsep() for the inverse function (ie, reading instead of writing) - * @see c4::cat() if no separator is needed - * @see c4::format() if a format string is desired */ -template -size_t catsep(substr buf, Sep const& C4_RESTRICT sep, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - size_t num = to_chars(buf, a); - buf = buf.len >= num ? buf.sub(num) : substr{}; - num += detail::catsep_more(buf, sep, more...); - return num; -} - -/** like c4::catsep() but return a substr instead of a size - * @see c4::catsep(). c4::uncatsep() is the inverse of c4::catsep(). */ -template -substr catsep_sub(substr buf, Args && ...args) -{ - size_t sz = catsep(buf, std::forward(args)...); - C4_CHECK(sz <= buf.len); - return {buf.str, sz <= buf.len ? sz : buf.len}; -} - -/** deserialize the arguments from the given buffer, using a separator. - * - * @return the number of characters read from the buffer, or csubstr::npos - * if a conversion was not successful - * @see c4::catsep(). c4::uncatsep() is the inverse of c4::catsep(). */ -template -size_t uncatsep(csubstr buf, Sep & C4_RESTRICT sep, Arg & C4_RESTRICT a, Args & C4_RESTRICT ...more) -{ - size_t ret = from_chars_first(buf, &a), num = ret; - if(C4_UNLIKELY(ret == csubstr::npos)) - return csubstr::npos; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = detail::uncatsep_more(buf, sep, more...); - if(C4_UNLIKELY(ret == csubstr::npos)) - return csubstr::npos; - num += ret; - return num; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/// @cond dev -// terminates the variadic recursion -inline size_t format(substr buf, csubstr fmt) -{ - return to_chars(buf, fmt); -} -/// @endcond - - -/** using a format string, serialize the arguments into the given - * fixed-size buffer. - * The buffer size is strictly respected: no writes will occur beyond its end. - * In the format string, each argument is marked with a compact - * curly-bracket pair: {}. Arguments beyond the last curly bracket pair - * are silently ignored. For example: - * @code{.cpp} - * c4::format(buf, "the {} drank {} {}", "partier", 5, "beers"); // the partier drank 5 beers - * c4::format(buf, "the {} drank {} {}", "programmer", 6, "coffees"); // the programmer drank 6 coffees - * @endcode - * @return the number of characters needed to write into the buffer. - * @see c4::formatrs() if instead of a fixed-size buffer, a resizeable container is desired - * @see c4::unformat() for the inverse function - * @see c4::cat() if no format or separator is needed - * @see c4::catsep() if no format is needed, but a separator must be used */ -template -size_t format(substr buf, csubstr fmt, Arg const& C4_RESTRICT a, Args const& C4_RESTRICT ...more) -{ - size_t pos = fmt.find("{}"); // @todo use _find_fmt() - if(C4_UNLIKELY(pos == csubstr::npos)) - return to_chars(buf, fmt); - size_t num = to_chars(buf, fmt.sub(0, pos)); - size_t out = num; - buf = buf.len >= num ? buf.sub(num) : substr{}; - num = to_chars(buf, a); - out += num; - buf = buf.len >= num ? buf.sub(num) : substr{}; - num = format(buf, fmt.sub(pos + 2), more...); - out += num; - return out; -} - -/** like c4::format() but return a substr instead of a size - * @see c4::format() - * @see c4::catsep(). uncatsep() is the inverse of catsep(). */ -template -substr format_sub(substr buf, csubstr fmt, Args const& C4_RESTRICT ...args) -{ - size_t sz = c4::format(buf, fmt, args...); - C4_CHECK(sz <= buf.len); - return {buf.str, sz <= buf.len ? sz : buf.len}; -} - - -//----------------------------------------------------------------------------- - -/// @cond dev -// terminates the variadic recursion -inline size_t unformat(csubstr /*buf*/, csubstr fmt) -{ - return fmt.len; -} -/// @endcond - - -/** using a format string, deserialize the arguments from the given - * buffer. - * @return the number of characters read from the buffer, or npos if a conversion failed. - * @see c4::format(). c4::unformat() is the inverse function to format(). */ -template -size_t unformat(csubstr buf, csubstr fmt, Arg & C4_RESTRICT a, Args & C4_RESTRICT ...more) -{ - const size_t pos = fmt.find("{}"); - if(C4_UNLIKELY(pos == csubstr::npos)) - return unformat(buf, fmt); - size_t num = pos; - size_t out = num; - buf = buf.len >= num ? buf.sub(num) : substr{}; - num = from_chars_first(buf, &a); - if(C4_UNLIKELY(num == csubstr::npos)) - return csubstr::npos; - out += num; - buf = buf.len >= num ? buf.sub(num) : substr{}; - num = unformat(buf, fmt.sub(pos + 2), more...); - if(C4_UNLIKELY(num == csubstr::npos)) - return csubstr::npos; - out += num; - return out; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** a tag type for marking append to container - * @see c4::catrs() */ -struct append_t {}; - -/** a tag variable - * @see c4::catrs() */ -constexpr const append_t append = {}; - - -//----------------------------------------------------------------------------- - -/** like c4::cat(), but receives a container, and resizes it as needed to contain - * the result. The container is overwritten. To append to it, use the append - * overload. - * @see c4::cat() */ -template -inline void catrs(CharOwningContainer * C4_RESTRICT cont, Args const& C4_RESTRICT ...args) -{ -retry: - substr buf = to_substr(*cont); - size_t ret = cat(buf, args...); - cont->resize(ret); - if(ret > buf.len) - goto retry; -} - -/** like c4::cat(), but creates and returns a new container sized as needed to contain - * the result. - * @see c4::cat() */ -template -inline CharOwningContainer catrs(Args const& C4_RESTRICT ...args) -{ - CharOwningContainer cont; - catrs(&cont, args...); - return cont; -} - -/** like c4::cat(), but receives a container, and appends to it instead of - * overwriting it. The container is resized as needed to contain the result. - * @return the region newly appended to the original container - * @see c4::cat() - * @see c4::catrs() */ -template -inline csubstr catrs(append_t, CharOwningContainer * C4_RESTRICT cont, Args const& C4_RESTRICT ...args) -{ - const size_t pos = cont->size(); -retry: - substr buf = to_substr(*cont).sub(pos); - size_t ret = cat(buf, args...); - cont->resize(pos + ret); - if(ret > buf.len) - goto retry; - return to_csubstr(*cont).range(pos, cont->size()); -} - - -//----------------------------------------------------------------------------- - -/// @cond dev -// terminates the recursion -template -inline void catseprs(CharOwningContainer * C4_RESTRICT, Sep const& C4_RESTRICT) -{ - return; -} -/// @end cond - - -/** like c4::catsep(), but receives a container, and resizes it as needed to contain the result. - * The container is overwritten. To append to the container use the append overload. - * @see c4::catsep() */ -template -inline void catseprs(CharOwningContainer * C4_RESTRICT cont, Sep const& C4_RESTRICT sep, Args const& C4_RESTRICT ...args) -{ -retry: - substr buf = to_substr(*cont); - size_t ret = catsep(buf, sep, args...); - cont->resize(ret); - if(ret > buf.len) - goto retry; -} - -/** like c4::catsep(), but create a new container with the result. - * @return the requested container */ -template -inline CharOwningContainer catseprs(Sep const& C4_RESTRICT sep, Args const& C4_RESTRICT ...args) -{ - CharOwningContainer cont; - catseprs(&cont, sep, args...); - return cont; -} - - -/// @cond dev -// terminates the recursion -template -inline csubstr catseprs(append_t, CharOwningContainer * C4_RESTRICT, Sep const& C4_RESTRICT) -{ - csubstr s; - return s; -} -/// @endcond - -/** like catsep(), but receives a container, and appends the arguments, resizing the - * container as needed to contain the result. The buffer is appended to. - * @return a csubstr of the appended part - * @ingroup formatting_functions */ -template -inline csubstr catseprs(append_t, CharOwningContainer * C4_RESTRICT cont, Sep const& C4_RESTRICT sep, Args const& C4_RESTRICT ...args) -{ - const size_t pos = cont->size(); -retry: - substr buf = to_substr(*cont).sub(pos); - size_t ret = catsep(buf, sep, args...); - cont->resize(pos + ret); - if(ret > buf.len) - goto retry; - return to_csubstr(*cont).range(pos, cont->size()); -} - - -//----------------------------------------------------------------------------- - -/** like c4::format(), but receives a container, and resizes it as needed - * to contain the result. The container is overwritten. To append to - * the container use the append overload. - * @see c4::format() */ -template -inline void formatrs(CharOwningContainer * C4_RESTRICT cont, csubstr fmt, Args const& C4_RESTRICT ...args) -{ -retry: - substr buf = to_substr(*cont); - size_t ret = format(buf, fmt, args...); - cont->resize(ret); - if(ret > buf.len) - goto retry; -} - -/** like c4::format(), but create a new container with the result. - * @return the requested container */ -template -inline CharOwningContainer formatrs(csubstr fmt, Args const& C4_RESTRICT ...args) -{ - CharOwningContainer cont; - formatrs(&cont, fmt, args...); - return cont; -} - -/** like format(), but receives a container, and appends the - * arguments, resizing the container as needed to contain the - * result. The buffer is appended to. - * @return the region newly appended to the original container - * @ingroup formatting_functions */ -template -inline csubstr formatrs(append_t, CharOwningContainer * C4_RESTRICT cont, csubstr fmt, Args const& C4_RESTRICT ...args) -{ - const size_t pos = cont->size(); -retry: - substr buf = to_substr(*cont).sub(pos); - size_t ret = format(buf, fmt, args...); - cont->resize(pos + ret); - if(ret > buf.len) - goto retry; - return to_csubstr(*cont).range(pos, cont->size()); -} - -} // namespace c4 - -#ifdef _MSC_VER -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -#endif /* _C4_FORMAT_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/hash.hpp b/thirdparty/ryml/ext/c4core/src/c4/hash.hpp deleted file mode 100644 index 635bc1544..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/hash.hpp +++ /dev/null @@ -1,95 +0,0 @@ -#ifndef _C4_HASH_HPP_ -#define _C4_HASH_HPP_ - -#include "c4/config.hpp" -#include - -/** @file hash.hpp */ - -/** @defgroup hash Hash utils - * @see http://aras-p.info/blog/2016/08/02/Hash-Functions-all-the-way-down/ */ - -namespace c4 { - -namespace detail { - -/** @internal - * @ingroup hash - * @see this was taken a great answer in stackoverflow: - * https://stackoverflow.com/a/34597785/5875572 - * @see http://aras-p.info/blog/2016/08/02/Hash-Functions-all-the-way-down/ */ -template -class basic_fnv1a final -{ - - static_assert(std::is_unsigned::value, "need unsigned integer"); - -public: - - using result_type = ResultT; - -private: - - result_type state_ {}; - -public: - - C4_CONSTEXPR14 basic_fnv1a() noexcept : state_ {OffsetBasis} {} - - C4_CONSTEXPR14 void update(const void *const data, const size_t size) noexcept - { - auto cdata = static_cast(data); - auto acc = this->state_; - for(size_t i = 0; i < size; ++i) - { - const auto next = size_t(cdata[i]); - acc = (acc ^ next) * Prime; - } - this->state_ = acc; - } - - C4_CONSTEXPR14 result_type digest() const noexcept - { - return this->state_; - } - -}; - -using fnv1a_32 = basic_fnv1a; -using fnv1a_64 = basic_fnv1a; - -template struct fnv1a; -template<> struct fnv1a<32> { using type = fnv1a_32; }; -template<> struct fnv1a<64> { using type = fnv1a_64; }; - -} // namespace detail - - -/** @ingroup hash */ -template -using fnv1a_t = typename detail::fnv1a::type; - - -/** @ingroup hash */ -C4_CONSTEXPR14 inline size_t hash_bytes(const void *const data, const size_t size) noexcept -{ - fnv1a_t fn{}; - fn.update(data, size); - return fn.digest(); -} - -/** - * @overload hash_bytes - * @ingroup hash */ -template -C4_CONSTEXPR14 inline size_t hash_bytes(const char (&str)[N]) noexcept -{ - fnv1a_t fn{}; - fn.update(str, N); - return fn.digest(); -} - -} // namespace c4 - - -#endif // _C4_HASH_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/language.cpp b/thirdparty/ryml/ext/c4core/src/c4/language.cpp deleted file mode 100644 index d1b569741..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/language.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "c4/language.hpp" - -namespace c4 { -namespace detail { - -#ifndef __GNUC__ -void use_char_pointer(char const volatile* v) -{ - C4_UNUSED(v); -} -#else -void foo() {} // to avoid empty file warning from the linker -#endif - -} // namespace detail -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/src/c4/language.hpp b/thirdparty/ryml/ext/c4core/src/c4/language.hpp deleted file mode 100644 index 90ff9bab5..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/language.hpp +++ /dev/null @@ -1,275 +0,0 @@ -#ifndef _C4_LANGUAGE_HPP_ -#define _C4_LANGUAGE_HPP_ - -/** @file language.hpp Provides language standard information macros and - * compiler agnostic utility macros: namespace facilities, function attributes, - * variable attributes, etc. - * @ingroup basic_headers */ - -#include "c4/preprocessor.hpp" -#include "c4/compiler.hpp" - -/* Detect C++ standard. - * @see http://stackoverflow.com/a/7132549/5875572 */ -#ifndef C4_CPP -# ifdef _MSC_VER -# if _MSC_VER >= 1910 // >VS2015: VS2017, VS2019 -# if (!defined(_MSVC_LANG)) -# error _MSVC not defined -# endif -# if _MSVC_LANG >= 201705L -# define C4_CPP 20 -# define C4_CPP20 -# elif _MSVC_LANG == 201703L -# define C4_CPP 17 -# define C4_CPP17 -# elif _MSVC_LANG >= 201402L -# define C4_CPP 14 -# define C4_CPP14 -# elif _MSVC_LANG >= 201103L -# define C4_CPP 11 -# define C4_CPP11 -# else -# error C++ lesser than C++11 not supported -# endif -# else -# if _MSC_VER == 1900 -# define C4_CPP 14 // VS2015 is c++14 https://devblogs.microsoft.com/cppblog/c111417-features-in-vs-2015-rtm/ -# define C4_CPP14 -# elif _MSC_VER == 1800 // VS2013 -# define C4_CPP 11 -# define C4_CPP11 -# else -# error C++ lesser than C++11 not supported -# endif -# endif -# elif defined(__INTEL_COMPILER) // https://software.intel.com/en-us/node/524490 -# ifdef __INTEL_CXX20_MODE__ // not sure about this -# define C4_CPP 20 -# define C4_CPP20 -# elif defined __INTEL_CXX17_MODE__ // not sure about this -# define C4_CPP 17 -# define C4_CPP17 -# elif defined __INTEL_CXX14_MODE__ // not sure about this -# define C4_CPP 14 -# define C4_CPP14 -# elif defined __INTEL_CXX11_MODE__ -# define C4_CPP 11 -# define C4_CPP11 -# else -# error C++ lesser than C++11 not supported -# endif -# else -# ifndef __cplusplus -# error __cplusplus is not defined? -# endif -# if __cplusplus == 1 -# error cannot handle __cplusplus==1 -# elif __cplusplus >= 201709L -# define C4_CPP 20 -# define C4_CPP20 -# elif __cplusplus >= 201703L -# define C4_CPP 17 -# define C4_CPP17 -# elif __cplusplus >= 201402L -# define C4_CPP 14 -# define C4_CPP14 -# elif __cplusplus >= 201103L -# define C4_CPP 11 -# define C4_CPP11 -# elif __cplusplus >= 199711L -# error C++ lesser than C++11 not supported -# endif -# endif -#else -# ifdef C4_CPP == 20 -# define C4_CPP20 -# elif C4_CPP == 17 -# define C4_CPP17 -# elif C4_CPP == 14 -# define C4_CPP14 -# elif C4_CPP == 11 -# define C4_CPP11 -# elif C4_CPP == 98 -# define C4_CPP98 -# error C++ lesser than C++11 not supported -# else -# error C4_CPP must be one of 20, 17, 14, 11, 98 -# endif -#endif - -#ifdef C4_CPP20 -# define C4_CPP17 -# define C4_CPP14 -# define C4_CPP11 -#elif defined(C4_CPP17) -# define C4_CPP14 -# define C4_CPP11 -#elif defined(C4_CPP14) -# define C4_CPP11 -#endif - -/** lifted from this answer: http://stackoverflow.com/a/20170989/5875572 */ -#ifndef _MSC_VER -# if __cplusplus < 201103 -# define C4_CONSTEXPR11 -# define C4_CONSTEXPR14 -//# define C4_NOEXCEPT -# elif __cplusplus == 201103 -# define C4_CONSTEXPR11 constexpr -# define C4_CONSTEXPR14 -//# define C4_NOEXCEPT noexcept -# else -# define C4_CONSTEXPR11 constexpr -# define C4_CONSTEXPR14 constexpr -//# define C4_NOEXCEPT noexcept -# endif -#else // _MSC_VER -# if _MSC_VER < 1900 -# define C4_CONSTEXPR11 -# define C4_CONSTEXPR14 -//# define C4_NOEXCEPT -# elif _MSC_VER < 2000 -# define C4_CONSTEXPR11 constexpr -# define C4_CONSTEXPR14 -//# define C4_NOEXCEPT noexcept -# else -# define C4_CONSTEXPR11 constexpr -# define C4_CONSTEXPR14 constexpr -//# define C4_NOEXCEPT noexcept -# endif -#endif // _MSC_VER - - -#if C4_CPP < 17 -#define C4_IF_CONSTEXPR -#define C4_INLINE_CONSTEXPR constexpr -#else -#define C4_IF_CONSTEXPR constexpr -#define C4_INLINE_CONSTEXPR inline constexpr -#endif - - -//------------------------------------------------------------ - -#define _C4_BEGIN_NAMESPACE(ns) namespace ns { -#define _C4_END_NAMESPACE(ns) } - -// MSVC cant handle the C4_FOR_EACH macro... need to fix this -//#define C4_BEGIN_NAMESPACE(...) C4_FOR_EACH_SEP(_C4_BEGIN_NAMESPACE, , __VA_ARGS__) -//#define C4_END_NAMESPACE(...) C4_FOR_EACH_SEP(_C4_END_NAMESPACE, , __VA_ARGS__) -#define C4_BEGIN_NAMESPACE(ns) namespace ns { -#define C4_END_NAMESPACE(ns) } - -#define C4_BEGIN_HIDDEN_NAMESPACE namespace /*hidden*/ { -#define C4_END_HIDDEN_NAMESPACE } /* namespace hidden */ - -//------------------------------------------------------------ - -#ifndef C4_API -# if defined(_MSC_VER) -# if defined(C4_EXPORT) -# define C4_API __declspec(dllexport) -# elif defined(C4_IMPORT) -# define C4_API __declspec(dllimport) -# else -# define C4_API -# endif -# else -# define C4_API -# endif -#endif - -#ifndef _MSC_VER ///< @todo assuming gcc-like compiler. check it is actually so. -/** for function attributes in GCC, - * @see https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#Common-Function-Attributes */ -/** for __builtin functions in GCC, - * @see https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html */ -# define C4_RESTRICT __restrict__ -# define C4_RESTRICT_FN __attribute__((restrict)) -# define C4_NO_INLINE __attribute__((noinline)) -# define C4_ALWAYS_INLINE inline __attribute__((always_inline)) -# define C4_CONST __attribute__((const)) -# define C4_PURE __attribute__((pure)) -/** force inlining of every callee function */ -# define C4_FLATTEN __atribute__((flatten)) -/** mark a function as hot, ie as having a visible impact in CPU time - * thus making it more likely to inline, etc - * @see http://stackoverflow.com/questions/15028990/semantics-of-gcc-hot-attribute */ -# define C4_HOT __attribute__((hot)) -/** mark a function as cold, ie as NOT having a visible impact in CPU time - * @see http://stackoverflow.com/questions/15028990/semantics-of-gcc-hot-attribute */ -# define C4_COLD __attribute__((cold)) -# define C4_EXPECT(x, y) __builtin_expect(x, y) ///< @see https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html -# define C4_LIKELY(x) __builtin_expect(x, 1) -# define C4_UNLIKELY(x) __builtin_expect(x, 0) -# define C4_UNREACHABLE() __builtin_unreachable() -# define C4_ATTR_FORMAT(...) //__attribute__((format (__VA_ARGS__))) ///< @see https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#Common-Function-Attributes -# define C4_NORETURN __attribute__((noreturn)) -#else -# define C4_RESTRICT __restrict -# define C4_RESTRICT_FN __declspec(restrict) -# define C4_NO_INLINE __declspec(noinline) -# define C4_ALWAYS_INLINE inline __forceinline -/** these are not available in VS AFAIK */ -# define C4_CONST -# define C4_PURE -# define C4_FLATTEN -# define C4_HOT /** @todo */ -# define C4_COLD /** @todo */ -# define C4_EXPECT(x, y) x /** @todo */ -# define C4_LIKELY(x) x /** @todo */ -# define C4_UNLIKELY(x) x /** @todo */ -# define C4_UNREACHABLE() /** @todo */ -# define C4_ATTR_FORMAT(...) /** */ -# define C4_NORETURN /** @todo */ -#endif - -#ifndef _MSC_VER -# define C4_FUNC __FUNCTION__ -# define C4_PRETTY_FUNC __PRETTY_FUNCTION__ -#else /// @todo assuming gcc-like compiler. check it is actually so. -# define C4_FUNC __FUNCTION__ -# define C4_PRETTY_FUNC __FUNCSIG__ -#endif - -/** prevent compiler warnings about a specific var being unused */ -#define C4_UNUSED(var) (void)var - -#if C4_CPP >= 17 -#define C4_STATIC_ASSERT(cond) static_assert(cond) -#else -#define C4_STATIC_ASSERT(cond) static_assert((cond), #cond) -#endif -#define C4_STATIC_ASSERT_MSG(cond, msg) static_assert((cond), #cond ": " msg) - -/** @def C4_DONT_OPTIMIZE idea lifted from GoogleBenchmark. - * @see https://github.com/google/benchmark/blob/master/include/benchmark/benchmark_api.h */ -namespace c4 { -namespace detail { -#ifdef __GNUC__ -# define C4_DONT_OPTIMIZE(var) c4::detail::dont_optimize(var) -template< class T > -C4_ALWAYS_INLINE void dont_optimize(T const& value) { asm volatile("" : : "g"(value) : "memory"); } -#else -# define C4_DONT_OPTIMIZE(var) c4::detail::use_char_pointer(reinterpret_cast< const char* >(&var)) -void use_char_pointer(char const volatile*); -#endif -} // namespace detail -} // namespace c4 - -/** @def C4_KEEP_EMPTY_LOOP prevent an empty loop from being optimized out. - * @see http://stackoverflow.com/a/7084193/5875572 */ -#ifndef _MSC_VER -# define C4_KEEP_EMPTY_LOOP { asm(""); } -#else -# define C4_KEEP_EMPTY_LOOP { char c; C4_DONT_OPTIMIZE(c); } -#endif - -/** @def C4_VA_LIST_REUSE_MUST_COPY - * @todo I strongly suspect that this is actually only in UNIX platforms. revisit this. */ -#ifdef __GNUC__ -# define C4_VA_LIST_REUSE_MUST_COPY -#endif - -#endif /* _C4_LANGUAGE_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/memory_resource.cpp b/thirdparty/ryml/ext/c4core/src/c4/memory_resource.cpp deleted file mode 100644 index 647d26736..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/memory_resource.cpp +++ /dev/null @@ -1,338 +0,0 @@ -#include "c4/memory_resource.hpp" -#include "c4/memory_util.hpp" - -#include -#include -#if defined(C4_POSIX) || defined(C4_IOS) || defined(C4_MACOS) || defined(C4_ARM) -# include -#endif -#if defined(C4_ARM) -# include -#endif - -#include - -namespace c4 { - -namespace detail { - - -#ifdef C4_NO_ALLOC_DEFAULTS -aalloc_pfn s_aalloc = nullptr; -free_pfn s_afree = nullptr; -arealloc_pfn s_arealloc = nullptr; -#else - - -void afree_impl(void *ptr) -{ -#if defined(C4_WIN) || defined(C4_XBOX) - ::_aligned_free(ptr); -#else - ::free(ptr); -#endif -} - - -void* aalloc_impl(size_t size, size_t alignment) -{ - void *mem; -#if defined(C4_WIN) || defined(C4_XBOX) - mem = ::_aligned_malloc(size, alignment); - C4_CHECK(mem != nullptr || size == 0); -#elif defined(C4_ARM) - // https://stackoverflow.com/questions/53614538/undefined-reference-to-posix-memalign-in-arm-gcc - // https://electronics.stackexchange.com/questions/467382/e2-studio-undefined-reference-to-posix-memalign/467753 - mem = memalign(alignment, size); - C4_CHECK(mem != nullptr || size == 0); -#elif defined(C4_POSIX) || defined(C4_IOS) || defined(C4_MACOS) - // NOTE: alignment needs to be sized in multiples of sizeof(void*) - size_t amult = alignment; - if(C4_UNLIKELY(alignment < sizeof(void*))) - { - amult = sizeof(void*); - } - int ret = ::posix_memalign(&mem, amult, size); - if(C4_UNLIKELY(ret)) - { - if(ret == EINVAL) - { - C4_ERROR("The alignment argument %zu was not a power of two, " - "or was not a multiple of sizeof(void*)", alignment); - } - else if(ret == ENOMEM) - { - C4_ERROR("There was insufficient memory to fulfill the " - "allocation request of %zu bytes (alignment=%lu)", size, size); - } - return nullptr; - } -#else - C4_NOT_IMPLEMENTED_MSG("need to implement an aligned allocation for this platform"); -#endif - C4_ASSERT_MSG((uintptr_t(mem) & (alignment-1)) == 0, "address %p is not aligned to %zu boundary", mem, alignment); - return mem; -} - - -void* arealloc_impl(void* ptr, size_t oldsz, size_t newsz, size_t alignment) -{ - /** @todo make this more efficient - * @see https://stackoverflow.com/questions/9078259/does-realloc-keep-the-memory-alignment-of-posix-memalign - * @see look for qReallocAligned() in http://code.qt.io/cgit/qt/qtbase.git/tree/src/corelib/global/qmalloc.cpp - */ - void *tmp = aalloc(newsz, alignment); - size_t min = newsz < oldsz ? newsz : oldsz; - if(mem_overlaps(ptr, tmp, oldsz, newsz)) - { - ::memmove(tmp, ptr, min); - } - else - { - ::memcpy(tmp, ptr, min); - } - afree(ptr); - return tmp; -} - -aalloc_pfn s_aalloc = aalloc_impl; -afree_pfn s_afree = afree_impl; -arealloc_pfn s_arealloc = arealloc_impl; - -#endif // C4_NO_ALLOC_DEFAULTS - -} // namespace detail - - -aalloc_pfn get_aalloc() -{ - return detail::s_aalloc; -} -void set_aalloc(aalloc_pfn fn) -{ - detail::s_aalloc = fn; -} - -afree_pfn get_afree() -{ - return detail::s_afree; -} -void set_afree(afree_pfn fn) -{ - detail::s_afree = fn; -} - -arealloc_pfn get_arealloc() -{ - return detail::s_arealloc; -} -void set_arealloc(arealloc_pfn fn) -{ - detail::s_arealloc = fn; -} - - -void* aalloc(size_t sz, size_t alignment) -{ - C4_ASSERT_MSG(c4::get_aalloc() != nullptr, "did you forget to call set_aalloc()?"); - auto fn = c4::get_aalloc(); - void* ptr = fn(sz, alignment); - return ptr; -} - -void afree(void* ptr) -{ - C4_ASSERT_MSG(c4::get_afree() != nullptr, "did you forget to call set_afree()?"); - auto fn = c4::get_afree(); - fn(ptr); -} - -void* arealloc(void *ptr, size_t oldsz, size_t newsz, size_t alignment) -{ - C4_ASSERT_MSG(c4::get_arealloc() != nullptr, "did you forget to call set_arealloc()?"); - auto fn = c4::get_arealloc(); - void* nptr = fn(ptr, oldsz, newsz, alignment); - return nptr; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void detail::_MemoryResourceSingleChunk::release() -{ - if(m_mem && m_owner) - { - impl_type::deallocate(m_mem, m_size); - } - m_mem = nullptr; - m_size = 0; - m_owner = false; - m_pos = 0; -} - -void detail::_MemoryResourceSingleChunk::acquire(size_t sz) -{ - clear(); - m_owner = true; - m_mem = (char*) impl_type::allocate(sz, alignof(max_align_t)); - m_size = sz; - m_pos = 0; -} - -void detail::_MemoryResourceSingleChunk::acquire(void *mem, size_t sz) -{ - clear(); - m_owner = false; - m_mem = (char*) mem; - m_size = sz; - m_pos = 0; -} - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void* MemoryResourceLinear::do_allocate(size_t sz, size_t alignment, void *hint) -{ - C4_UNUSED(hint); - if(sz == 0) return nullptr; - // make sure there's enough room to allocate - if(m_pos + sz > m_size) - { - C4_ERROR("out of memory"); - return nullptr; - } - void *mem = m_mem + m_pos; - size_t space = m_size - m_pos; - if(std::align(alignment, sz, mem, space)) - { - C4_ASSERT(m_pos <= m_size); - C4_ASSERT(m_size - m_pos >= space); - m_pos += (m_size - m_pos) - space; - m_pos += sz; - C4_ASSERT(m_pos <= m_size); - } - else - { - C4_ERROR("could not align memory"); - mem = nullptr; - } - return mem; -} - -void MemoryResourceLinear::do_deallocate(void* ptr, size_t sz, size_t alignment) -{ - C4_UNUSED(ptr); - C4_UNUSED(sz); - C4_UNUSED(alignment); - // nothing to do!! -} - -void* MemoryResourceLinear::do_reallocate(void* ptr, size_t oldsz, size_t newsz, size_t alignment) -{ - if(newsz == oldsz) return ptr; - // is ptr the most recently allocated (MRA) block? - char *cptr = (char*)ptr; - bool same_pos = (m_mem + m_pos == cptr + oldsz); - // no need to get more memory when shrinking - if(newsz < oldsz) - { - // if this is the MRA, we can safely shrink the position - if(same_pos) - { - m_pos -= oldsz - newsz; - } - return ptr; - } - // we're growing the block, and it fits in size - else if(same_pos && cptr + newsz <= m_mem + m_size) - { - // if this is the MRA, we can safely shrink the position - m_pos += newsz - oldsz; - return ptr; - } - // we're growing the block or it doesn't fit - - // delegate any of these situations to do_deallocate() - return do_allocate(newsz, alignment, ptr); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** @todo add a free list allocator. A good candidate because of its - * small size is TLSF. - * - * @see https://github.com/mattconte/tlsf - * - * Comparisons: - * - * @see https://www.researchgate.net/publication/262375150_A_Comparative_Study_on_Memory_Allocators_in_Multicore_and_Multithreaded_Applications_-_SBESC_2011_-_Presentation_Slides - * @see http://webkit.sed.hu/blog/20100324/war-allocators-tlsf-action - * @see https://github.com/emeryberger/Malloc-Implementations/tree/master/allocators - * - * */ - -} // namespace c4 - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -#ifdef C4_REDEFINE_CPPNEW -#include -void* operator new(size_t size) -{ - auto *mr = ::c4::get_memory_resource(); - return mr->allocate(size); -} -void operator delete(void *p) noexcept -{ - C4_NEVER_REACH(); -} -void operator delete(void *p, size_t size) -{ - auto *mr = ::c4::get_memory_resource(); - mr->deallocate(p, size); -} -void* operator new[](size_t size) -{ - return operator new(size); -} -void operator delete[](void *p) noexcept -{ - operator delete(p); -} -void operator delete[](void *p, size_t size) -{ - operator delete(p, size); -} -void* operator new(size_t size, std::nothrow_t) -{ - return operator new(size); -} -void operator delete(void *p, std::nothrow_t) -{ - operator delete(p); -} -void operator delete(void *p, size_t size, std::nothrow_t) -{ - operator delete(p, size); -} -void* operator new[](size_t size, std::nothrow_t) -{ - return operator new(size); -} -void operator delete[](void *p, std::nothrow_t) -{ - operator delete(p); -} -void operator delete[](void *p, size_t, std::nothrow_t) -{ - operator delete(p, size); -} -#endif // C4_REDEFINE_CPPNEW diff --git a/thirdparty/ryml/ext/c4core/src/c4/memory_resource.hpp b/thirdparty/ryml/ext/c4core/src/c4/memory_resource.hpp deleted file mode 100644 index 0818f5874..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/memory_resource.hpp +++ /dev/null @@ -1,568 +0,0 @@ -#ifndef _C4_MEMORY_RESOURCE_HPP_ -#define _C4_MEMORY_RESOURCE_HPP_ - -/** @file memory_resource.hpp Provides facilities to allocate typeless - * memory, via the memory resource model consecrated with C++17. */ - -/** @defgroup memory memory utilities */ - -/** @defgroup raw_memory_alloc Raw memory allocation - * @ingroup memory - */ - -/** @defgroup memory_resources Memory resources - * @ingroup memory - */ - -#include "c4/config.hpp" -#include "c4/error.hpp" - -namespace c4 { - -// need these forward decls here -struct MemoryResource; -struct MemoryResourceMalloc; -struct MemoryResourceStack; -MemoryResourceMalloc* get_memory_resource_malloc(); -MemoryResourceStack* get_memory_resource_stack(); -namespace detail { MemoryResource*& get_memory_resource(); } - - -// c-style allocation --------------------------------------------------------- - -// this API provides aligned allocation functions. -// These functions forward the call to a user-modifiable function. - - -// aligned allocation. - -/** Aligned allocation. Merely calls the current get_aalloc() function. - * @see get_aalloc() - * @ingroup raw_memory_alloc */ -void* aalloc(size_t sz, size_t alignment); - -/** Aligned free. Merely calls the current get_afree() function. - * @see get_afree() - * @ingroup raw_memory_alloc */ -void afree(void* ptr); - -/** Aligned reallocation. Merely calls the current get_arealloc() function. - * @see get_arealloc() - * @ingroup raw_memory_alloc */ -void* arealloc(void* ptr, size_t oldsz, size_t newsz, size_t alignment); - - -// allocation setup facilities. - -/** Function pointer type for aligned allocation - * @see set_aalloc() - * @ingroup raw_memory_alloc */ -using aalloc_pfn = void* (*)(size_t size, size_t alignment); - -/** Function pointer type for aligned deallocation - * @see set_afree() - * @ingroup raw_memory_alloc */ -using afree_pfn = void (*)(void *ptr); - -/** Function pointer type for aligned reallocation - * @see set_arealloc() - * @ingroup raw_memory_alloc */ -using arealloc_pfn = void* (*)(void *ptr, size_t oldsz, size_t newsz, size_t alignment); - - -// allocation function pointer setters/getters - -/** Set the global aligned allocation function. - * @see aalloc() - * @see get_aalloc() - * @ingroup raw_memory_alloc */ -void set_aalloc(aalloc_pfn fn); - -/** Set the global aligned deallocation function. - * @see afree() - * @see get_afree() - * @ingroup raw_memory_alloc */ -void set_afree(afree_pfn fn); - -/** Set the global aligned reallocation function. - * @see arealloc() - * @see get_arealloc() - * @ingroup raw_memory_alloc */ -void set_arealloc(arealloc_pfn fn); - - -/** Get the global aligned reallocation function. - * @see arealloc() - * @ingroup raw_memory_alloc */ -aalloc_pfn get_aalloc(); - -/** Get the global aligned deallocation function. - * @see afree() - * @ingroup raw_memory_alloc */ -afree_pfn get_afree(); - -/** Get the global aligned reallocation function. - * @see arealloc() - * @ingroup raw_memory_alloc */ -arealloc_pfn get_arealloc(); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// c++-style allocation ------------------------------------------------------- - -/** C++17-style memory_resource base class. See http://en.cppreference.com/w/cpp/experimental/memory_resource - * @ingroup memory_resources */ -struct MemoryResource -{ - const char *name = nullptr; - virtual ~MemoryResource() {} - - void* allocate(size_t sz, size_t alignment=alignof(max_align_t), void *hint=nullptr) - { - void *mem = this->do_allocate(sz, alignment, hint); - C4_CHECK_MSG(mem != nullptr, "could not allocate %lu bytes", sz); - return mem; - } - - void* reallocate(void* ptr, size_t oldsz, size_t newsz, size_t alignment=alignof(max_align_t)) - { - void *mem = this->do_reallocate(ptr, oldsz, newsz, alignment); - C4_CHECK_MSG(mem != nullptr, "could not reallocate from %lu to %lu bytes", oldsz, newsz); - return mem; - } - - void deallocate(void* ptr, size_t sz, size_t alignment=alignof(max_align_t)) - { - this->do_deallocate(ptr, sz, alignment); - } - -protected: - - virtual void* do_allocate(size_t sz, size_t alignment, void* hint) = 0; - virtual void* do_reallocate(void* ptr, size_t oldsz, size_t newsz, size_t alignment) = 0; - virtual void do_deallocate(void* ptr, size_t sz, size_t alignment) = 0; - -}; - -/** get the current global memory resource. To avoid static initialization - * order problems, this is implemented using a function call to ensure - * that it is available when first used. - * @ingroup memory_resources */ -C4_ALWAYS_INLINE MemoryResource* get_memory_resource() -{ - return detail::get_memory_resource(); -} - -/** set the global memory resource - * @ingroup memory_resources */ -C4_ALWAYS_INLINE void set_memory_resource(MemoryResource* mr) -{ - C4_ASSERT(mr != nullptr); - detail::get_memory_resource() = mr; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** A c4::aalloc-based memory resource. Thread-safe if the implementation - * called by c4::aalloc() is safe. - * @ingroup memory_resources */ -struct MemoryResourceMalloc : public MemoryResource -{ - - MemoryResourceMalloc() { name = "malloc"; } - virtual ~MemoryResourceMalloc() override {} - -protected: - - virtual void* do_allocate(size_t sz, size_t alignment, void *hint) override - { - C4_UNUSED(hint); - return c4::aalloc(sz, alignment); - } - - virtual void do_deallocate(void* ptr, size_t sz, size_t alignment) override - { - C4_UNUSED(sz); - C4_UNUSED(alignment); - c4::afree(ptr); - } - - virtual void* do_reallocate(void* ptr, size_t oldsz, size_t newsz, size_t alignment) override - { - return c4::arealloc(ptr, oldsz, newsz, alignment); - } - -}; - -/** returns a malloc-based memory resource - * @ingroup memory_resources */ -C4_ALWAYS_INLINE MemoryResourceMalloc* get_memory_resource_malloc() -{ - /** @todo use a nifty counter: - * https://en.wikibooks.org/wiki/More_C%2B%2B_Idioms/Nifty_Counter */ - static MemoryResourceMalloc mr; - return &mr; -} - -namespace detail { -C4_ALWAYS_INLINE MemoryResource* & get_memory_resource() -{ - /** @todo use a nifty counter: - * https://en.wikibooks.org/wiki/More_C%2B%2B_Idioms/Nifty_Counter */ - thread_local static MemoryResource* mr = get_memory_resource_malloc(); - return mr; -} -} // namespace detail - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace detail { - -/** Allows a memory resource to obtain its memory from another memory resource. - * @ingroup memory_resources */ -struct DerivedMemoryResource : public MemoryResource -{ -public: - - DerivedMemoryResource(MemoryResource *mr_=nullptr) : m_local(mr_ ? mr_ : get_memory_resource()) {} - -private: - - MemoryResource *m_local; - -protected: - - virtual void* do_allocate(size_t sz, size_t alignment, void* hint) override - { - return m_local->allocate(sz, alignment, hint); - } - - virtual void* do_reallocate(void* ptr, size_t oldsz, size_t newsz, size_t alignment) override - { - return m_local->reallocate(ptr, oldsz, newsz, alignment); - } - - virtual void do_deallocate(void* ptr, size_t sz, size_t alignment) override - { - return m_local->deallocate(ptr, sz, alignment); - } -}; - -/** Provides common facilities for memory resource consisting of a single memory block - * @ingroup memory_resources */ -struct _MemoryResourceSingleChunk : public DerivedMemoryResource -{ - - C4_NO_COPY_OR_MOVE(_MemoryResourceSingleChunk); - - using impl_type = DerivedMemoryResource; - -public: - - _MemoryResourceSingleChunk(MemoryResource *impl=nullptr) : DerivedMemoryResource(impl) { name = "linear_malloc"; } - - /** initialize with owned memory, allocated from the given (or the global) memory resource */ - _MemoryResourceSingleChunk(size_t sz, MemoryResource *impl=nullptr) : _MemoryResourceSingleChunk(impl) { acquire(sz); } - /** initialize with borrowed memory */ - _MemoryResourceSingleChunk(void *mem, size_t sz) : _MemoryResourceSingleChunk() { acquire(mem, sz); } - - virtual ~_MemoryResourceSingleChunk() override { release(); } - -public: - - void const* mem() const { return m_mem; } - - size_t capacity() const { return m_size; } - size_t size() const { return m_pos; } - size_t slack() const { C4_ASSERT(m_size >= m_pos); return m_size - m_pos; } - -public: - - char *m_mem{nullptr}; - size_t m_size{0}; - size_t m_pos{0}; - bool m_owner; - -public: - - /** set the internal pointer to the beginning of the linear buffer */ - void clear() { m_pos = 0; } - - /** initialize with owned memory, allocated from the global memory resource */ - void acquire(size_t sz); - /** initialize with borrowed memory */ - void acquire(void *mem, size_t sz); - /** release the memory */ - void release(); - -}; - -} // namespace detail - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** provides a linear memory resource. Allocates incrementally from a linear - * buffer, without ever deallocating. Deallocations are a no-op, and the - * memory is freed only when the resource is release()d. The memory used by - * this object can be either owned or borrowed. When borrowed, no calls to - * malloc/free take place. - * - * @ingroup memory_resources */ -struct MemoryResourceLinear : public detail::_MemoryResourceSingleChunk -{ - - C4_NO_COPY_OR_MOVE(MemoryResourceLinear); - -public: - - using detail::_MemoryResourceSingleChunk::_MemoryResourceSingleChunk; - -protected: - - virtual void* do_allocate(size_t sz, size_t alignment, void *hint) override; - virtual void do_deallocate(void* ptr, size_t sz, size_t alignment) override; - virtual void* do_reallocate(void* ptr, size_t oldsz, size_t newsz, size_t alignment) override; -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** provides a stack-type malloc-based memory resource. - * @ingroup memory_resources */ -struct MemoryResourceStack : public detail::_MemoryResourceSingleChunk -{ - - C4_NO_COPY_OR_MOVE(MemoryResourceStack); - -public: - - using detail::_MemoryResourceSingleChunk::_MemoryResourceSingleChunk; - -protected: - - virtual void* do_allocate(size_t sz, size_t alignment, void *hint) override; - virtual void do_deallocate(void* ptr, size_t sz, size_t alignment) override; - virtual void* do_reallocate(void* ptr, size_t oldsz, size_t newsz, size_t alignment) override; -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** provides a linear array-based memory resource. - * @see MemoryResourceLinear - * @ingroup memory_resources */ -template -struct MemoryResourceLinearArr : public MemoryResourceLinear -{ - #ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable: 4324) // structure was padded due to alignment specifier - #endif - alignas(alignof(max_align_t)) char m_arr[N]; - #ifdef _MSC_VER - #pragma warning(pop) - #endif - MemoryResourceLinearArr() : MemoryResourceLinear(m_arr, N) { name = "linear_arr"; } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -struct AllocationCounts -{ - struct Item - { - ssize_t allocs; - ssize_t size; - - void add(size_t sz) - { - ++allocs; - size += static_cast(sz); - } - void rem(size_t sz) - { - --allocs; - size -= static_cast(sz); - } - Item max(Item const& that) const - { - Item r(*this); - r.allocs = r.allocs > that.allocs ? r.allocs : that.allocs; - r.size = r.size > that.size ? r.size : that.size; - return r; - } - }; - - Item curr = {0, 0}; - Item total = {0, 0}; - Item max = {0, 0}; - - void clear_counts() - { - curr = {0, 0}; - total = {0, 0}; - max = {0, 0}; - } - - void update(AllocationCounts const& that) - { - curr.allocs += that.curr.allocs; - curr.size += that.curr.size; - total.allocs += that.total.allocs; - total.size += that.total.size; - max.allocs += that.max.allocs; - max.size += that.max.size; - } - - void add_counts(void* ptr, size_t sz) - { - if(ptr == nullptr) return; - curr.add(sz); - total.add(sz); - max = max.max(curr); - } - - void rem_counts(void *ptr, size_t sz) - { - if(ptr == nullptr) return; - curr.rem(sz); - } - - AllocationCounts operator- (AllocationCounts const& that) const - { - AllocationCounts r(*this); - r.curr.allocs -= that.curr.allocs; - r.curr.size -= that.curr.size; - r.total.allocs -= that.total.allocs; - r.total.size -= that.total.size; - r.max.allocs -= that.max.allocs; - r.max.size -= that.max.size; - return r; - } - - AllocationCounts operator+ (AllocationCounts const& that) const - { - AllocationCounts r(*this); - r.curr.allocs += that.curr.allocs; - r.curr.size += that.curr.size; - r.total.allocs += that.total.allocs; - r.total.size += that.total.size; - r.max.allocs += that.max.allocs; - r.max.size += that.max.size; - return r; - } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** a MemoryResource which latches onto another MemoryResource - * and counts allocations and sizes. - * @ingroup memory_resources */ -class MemoryResourceCounts : public MemoryResource -{ -public: - - MemoryResourceCounts() : m_resource(get_memory_resource()) - { - C4_ASSERT(m_resource != this); - name = "MemoryResourceCounts"; - } - MemoryResourceCounts(MemoryResource *res) : m_resource(res) - { - C4_ASSERT(m_resource != this); - name = "MemoryResourceCounts"; - } - - MemoryResource *resource() { return m_resource; } - AllocationCounts const& counts() const { return m_counts; } - -protected: - - MemoryResource *m_resource; - AllocationCounts m_counts; - -protected: - - virtual void* do_allocate(size_t sz, size_t alignment, void * /*hint*/) override - { - void *ptr = m_resource->allocate(sz, alignment); - m_counts.add_counts(ptr, sz); - return ptr; - } - - virtual void do_deallocate(void* ptr, size_t sz, size_t alignment) override - { - m_counts.rem_counts(ptr, sz); - m_resource->deallocate(ptr, sz, alignment); - } - - virtual void* do_reallocate(void* ptr, size_t oldsz, size_t newsz, size_t alignment) override - { - m_counts.rem_counts(ptr, oldsz); - void* nptr = m_resource->reallocate(ptr, oldsz, newsz, alignment); - m_counts.add_counts(nptr, newsz); - return nptr; - } - -}; - -//----------------------------------------------------------------------------- -/** RAII class which binds a memory resource with a scope duration. - * @ingroup memory_resources */ -struct ScopedMemoryResource -{ - MemoryResource *m_original; - - ScopedMemoryResource(MemoryResource *r) - : - m_original(get_memory_resource()) - { - set_memory_resource(r); - } - - ~ScopedMemoryResource() - { - set_memory_resource(m_original); - } -}; - -//----------------------------------------------------------------------------- -/** RAII class which counts allocations and frees inside a scope. Can - * optionally set also the memory resource to be used. - * @ingroup memory_resources */ -struct ScopedMemoryResourceCounts -{ - MemoryResourceCounts mr; - - ScopedMemoryResourceCounts() : mr() - { - set_memory_resource(&mr); - } - ScopedMemoryResourceCounts(MemoryResource *m) : mr(m) - { - set_memory_resource(&mr); - } - ~ScopedMemoryResourceCounts() - { - set_memory_resource(mr.resource()); - } -}; - -} // namespace c4 - -#endif /* _C4_MEMORY_RESOURCE_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/memory_util.cpp b/thirdparty/ryml/ext/c4core/src/c4/memory_util.cpp deleted file mode 100644 index 981c62bb1..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/memory_util.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "c4/memory_util.hpp" -#include "c4/error.hpp" - -namespace c4 { - -/** Fills 'dest' with the first 'pattern_size' bytes at 'pattern', 'num_times'. */ -void mem_repeat(void* dest, void const* pattern, size_t pattern_size, size_t num_times) -{ - if(C4_UNLIKELY(num_times == 0)) - return; - C4_ASSERT( ! mem_overlaps(dest, pattern, num_times*pattern_size, pattern_size)); - char *begin = (char*)dest; - char *end = begin + num_times * pattern_size; - // copy the pattern once - ::memcpy(begin, pattern, pattern_size); - // now copy from dest to itself, doubling up every time - size_t n = pattern_size; - while(begin + 2*n < end) - { - ::memcpy(begin + n, begin, n); - n <<= 1; // double n - } - // copy the missing part - if(begin + n < end) - { - ::memcpy(begin + n, begin, static_cast(end - (begin + n))); - } -} - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/src/c4/memory_util.hpp b/thirdparty/ryml/ext/c4core/src/c4/memory_util.hpp deleted file mode 100644 index 7dfb263ba..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/memory_util.hpp +++ /dev/null @@ -1,774 +0,0 @@ -#ifndef _C4_MEMORY_UTIL_HPP_ -#define _C4_MEMORY_UTIL_HPP_ - -#include "c4/config.hpp" -#include "c4/error.hpp" -#include "c4/compiler.hpp" -#include "c4/cpu.hpp" -#ifdef C4_MSVC -#include -#endif -#include - -#if (defined(__GNUC__) && __GNUC__ >= 10) || defined(__has_builtin) -#define _C4_USE_LSB_INTRINSIC(which) __has_builtin(which) -#define _C4_USE_MSB_INTRINSIC(which) __has_builtin(which) -#elif defined(C4_MSVC) -#define _C4_USE_LSB_INTRINSIC(which) true -#define _C4_USE_MSB_INTRINSIC(which) true -#else -// let's try our luck -#define _C4_USE_LSB_INTRINSIC(which) true -#define _C4_USE_MSB_INTRINSIC(which) true -#endif - - -/** @file memory_util.hpp Some memory utilities. */ - -namespace c4 { - -/** set the given memory to zero */ -C4_ALWAYS_INLINE void mem_zero(void* mem, size_t num_bytes) -{ - memset(mem, 0, num_bytes); -} -/** set the given memory to zero */ -template -C4_ALWAYS_INLINE void mem_zero(T* mem, size_t num_elms) -{ - memset(mem, 0, sizeof(T) * num_elms); -} -/** set the given memory to zero */ -template -C4_ALWAYS_INLINE void mem_zero(T* mem) -{ - memset(mem, 0, sizeof(T)); -} - -C4_ALWAYS_INLINE C4_CONST bool mem_overlaps(void const* a, void const* b, size_t sza, size_t szb) -{ - // thanks @timwynants - return (((const char*)b + szb) > a && b < ((const char*)a+sza)); -} - -void mem_repeat(void* dest, void const* pattern, size_t pattern_size, size_t num_times); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -C4_ALWAYS_INLINE C4_CONST bool is_aligned(T *ptr, uintptr_t alignment=alignof(T)) -{ - return (uintptr_t(ptr) & (alignment - uintptr_t(1))) == uintptr_t(0); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// least significant bit - -/** @name msb Compute the least significant bit - * @note the input value must be nonzero - * @note the input type must be unsigned - */ -/** @{ */ - -// https://graphics.stanford.edu/~seander/bithacks.html#ZerosOnRightLinear -#define _c4_lsb_fallback \ - unsigned c = 0; \ - v = (v ^ (v - 1)) >> 1; /* Set v's trailing 0s to 1s and zero rest */ \ - for(; v; ++c) \ - v >>= 1; \ - return (unsigned) c - -// u8 -template -C4_CONSTEXPR14 -auto lsb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_LSB_INTRINSIC(__builtin_ctz) - // upcast to use the intrinsic, it's cheaper. - #ifdef C4_MSVC - #if !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanForward(&bit, (unsigned long)v); - return bit; - #else - _c4_lsb_fallback; - #endif - #else - return (unsigned)__builtin_ctz((unsigned)v); - #endif - #else - _c4_lsb_fallback; - #endif -} - -// u16 -template -C4_CONSTEXPR14 -auto lsb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_LSB_INTRINSIC(__builtin_ctz) - // upcast to use the intrinsic, it's cheaper. - // Then remember that the upcast makes it to 31bits - #ifdef C4_MSVC - #if !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanForward(&bit, (unsigned long)v); - return bit; - #else - _c4_lsb_fallback; - #endif - #else - return (unsigned)__builtin_ctz((unsigned)v); - #endif - #else - _c4_lsb_fallback; - #endif -} - -// u32 -template -C4_CONSTEXPR14 -auto lsb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_LSB_INTRINSIC(__builtin_ctz) - #ifdef C4_MSVC - #if !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanForward(&bit, v); - return bit; - #else - _c4_lsb_fallback; - #endif - #else - return (unsigned)__builtin_ctz((unsigned)v); - #endif - #else - _c4_lsb_fallback; - #endif -} - -// u64 in 64bits -template -C4_CONSTEXPR14 -auto lsb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_LSB_INTRINSIC(__builtin_ctzl) - #if defined(C4_MSVC) - #if !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanForward64(&bit, v); - return bit; - #else - _c4_lsb_fallback; - #endif - #else - return (unsigned)__builtin_ctzl((unsigned long)v); - #endif - #else - _c4_lsb_fallback; - #endif -} - -// u64 in 32bits -template -C4_CONSTEXPR14 -auto lsb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_LSB_INTRINSIC(__builtin_ctzll) - #if defined(C4_MSVC) - #if !defined(C4_CPU_X86) && !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanForward64(&bit, v); - return bit; - #else - _c4_lsb_fallback; - #endif - #else - return (unsigned)__builtin_ctzll((unsigned long long)v); - #endif - #else - _c4_lsb_fallback; - #endif -} - -#undef _c4_lsb_fallback - -/** @} */ - - -namespace detail { -template struct _lsb11; -template -struct _lsb11 -{ - enum : unsigned { num = _lsb11>1), num_bits+I(1), (((val>>1)&I(1))!=I(0))>::num }; -}; -template -struct _lsb11 -{ - enum : unsigned { num = num_bits }; -}; -} // namespace detail - - -/** TMP version of lsb(); this needs to be implemented with template - * meta-programming because C++11 cannot use a constexpr function with - * local variables - * @see lsb */ -template -struct lsb11 -{ - static_assert(number != 0, "lsb: number must be nonzero"); - enum : unsigned { value = detail::_lsb11::num}; -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// most significant bit - - -/** @name msb Compute the most significant bit - * @note the input value must be nonzero - * @note the input type must be unsigned - */ -/** @{ */ - - -#define _c4_msb8_fallback \ - unsigned n = 0; \ - if(v & I(0xf0)) v >>= 4, n |= I(4); \ - if(v & I(0x0c)) v >>= 2, n |= I(2); \ - if(v & I(0x02)) v >>= 1, n |= I(1); \ - return n - -#define _c4_msb16_fallback \ - unsigned n = 0; \ - if(v & I(0xff00)) v >>= 8, n |= I(8); \ - if(v & I(0x00f0)) v >>= 4, n |= I(4); \ - if(v & I(0x000c)) v >>= 2, n |= I(2); \ - if(v & I(0x0002)) v >>= 1, n |= I(1); \ - return n - -#define _c4_msb32_fallback \ - unsigned n = 0; \ - if(v & I(0xffff0000)) v >>= 16, n |= 16; \ - if(v & I(0x0000ff00)) v >>= 8, n |= 8; \ - if(v & I(0x000000f0)) v >>= 4, n |= 4; \ - if(v & I(0x0000000c)) v >>= 2, n |= 2; \ - if(v & I(0x00000002)) v >>= 1, n |= 1; \ - return n - -#define _c4_msb64_fallback \ - unsigned n = 0; \ - if(v & I(0xffffffff00000000)) v >>= 32, n |= I(32); \ - if(v & I(0x00000000ffff0000)) v >>= 16, n |= I(16); \ - if(v & I(0x000000000000ff00)) v >>= 8, n |= I(8); \ - if(v & I(0x00000000000000f0)) v >>= 4, n |= I(4); \ - if(v & I(0x000000000000000c)) v >>= 2, n |= I(2); \ - if(v & I(0x0000000000000002)) v >>= 1, n |= I(1); \ - return n - - -// u8 -template -C4_CONSTEXPR14 -auto msb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_MSB_INTRINSIC(__builtin_clz) - // upcast to use the intrinsic, it's cheaper. - // Then remember that the upcast makes it to 31bits - #ifdef C4_MSVC - #if !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanReverse(&bit, (unsigned long)v); - return bit; - #else - _c4_msb8_fallback; - #endif - #else - return 31u - (unsigned)__builtin_clz((unsigned)v); - #endif - #else - _c4_msb8_fallback; - #endif -} - -// u16 -template -C4_CONSTEXPR14 -auto msb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_MSB_INTRINSIC(__builtin_clz) - // upcast to use the intrinsic, it's cheaper. - // Then remember that the upcast makes it to 31bits - #ifdef C4_MSVC - #if !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanReverse(&bit, (unsigned long)v); - return bit; - #else - _c4_msb16_fallback; - #endif - #else - return 31u - (unsigned)__builtin_clz((unsigned)v); - #endif - #else - _c4_msb16_fallback; - #endif -} - -// u32 -template -C4_CONSTEXPR14 -auto msb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_MSB_INTRINSIC(__builtin_clz) - #ifdef C4_MSVC - #if !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanReverse(&bit, v); - return bit; - #else - _c4_msb32_fallback; - #endif - #else - return 31u - (unsigned)__builtin_clz((unsigned)v); - #endif - #else - _c4_msb32_fallback; - #endif -} - -// u64 in 64bits -template -C4_CONSTEXPR14 -auto msb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_MSB_INTRINSIC(__builtin_clzl) - #ifdef C4_MSVC - #if !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanReverse64(&bit, v); - return bit; - #else - _c4_msb64_fallback; - #endif - #else - return 63u - (unsigned)__builtin_clzl((unsigned long)v); - #endif - #else - _c4_msb64_fallback; - #endif -} - -// u64 in 32bits -template -C4_CONSTEXPR14 -auto msb(I v) noexcept - -> typename std::enable_if::type -{ - C4_STATIC_ASSERT(std::is_unsigned::value); - C4_ASSERT(v != 0); - #if _C4_USE_MSB_INTRINSIC(__builtin_clzll) - #ifdef C4_MSVC - #if !defined(C4_CPU_X86) && !defined(C4_CPU_ARM64) && !defined(C4_CPU_ARM) - unsigned long bit; - _BitScanReverse64(&bit, v); - return bit; - #else - _c4_msb64_fallback; - #endif - #else - return 63u - (unsigned)__builtin_clzll((unsigned long long)v); - #endif - #else - _c4_msb64_fallback; - #endif -} - -#undef _c4_msb8_fallback -#undef _c4_msb16_fallback -#undef _c4_msb32_fallback -#undef _c4_msb64_fallback - -/** @} */ - - -namespace detail { -template struct _msb11; -template -struct _msb11< I, val, num_bits, false> -{ - enum : unsigned { num = _msb11>1), num_bits+I(1), ((val>>1)==I(0))>::num }; -}; -template -struct _msb11 -{ - static_assert(val == 0, "bad implementation"); - enum : unsigned { num = (unsigned)(num_bits-1) }; -}; -} // namespace detail - - -/** TMP version of msb(); this needs to be implemented with template - * meta-programming because C++11 cannot use a constexpr function with - * local variables - * @see msb */ -template -struct msb11 -{ - enum : unsigned { value = detail::_msb11::num }; -}; - - - -#undef _C4_USE_LSB_INTRINSIC -#undef _C4_USE_MSB_INTRINSIC - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// there is an implicit conversion below; it happens when E or B are -// narrower than int, and thus any operation will upcast the result to -// int, and then downcast to assign -C4_SUPPRESS_WARNING_GCC_CLANG_WITH_PUSH("-Wconversion") - -/** integer power; this function is constexpr-14 because of the local - * variables */ -template -C4_CONSTEXPR14 C4_CONST auto ipow(B base, E exponent) noexcept -> typename std::enable_if::value, B>::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - B r = B(1); - if(exponent >= 0) - { - for(E e = 0; e < exponent; ++e) - r *= base; - } - else - { - exponent *= E(-1); - for(E e = 0; e < exponent; ++e) - r /= base; - } - return r; -} - -/** integer power; this function is constexpr-14 because of the local - * variables */ -template -C4_CONSTEXPR14 C4_CONST auto ipow(E exponent) noexcept -> typename std::enable_if::value, B>::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - B r = B(1); - if(exponent >= 0) - { - for(E e = 0; e < exponent; ++e) - r *= base; - } - else - { - exponent *= E(-1); - for(E e = 0; e < exponent; ++e) - r /= base; - } - return r; -} - -/** integer power; this function is constexpr-14 because of the local - * variables */ -template -C4_CONSTEXPR14 C4_CONST auto ipow(E exponent) noexcept -> typename std::enable_if::value, B>::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - B r = B(1); - B bbase = B(base); - if(exponent >= 0) - { - for(E e = 0; e < exponent; ++e) - r *= bbase; - } - else - { - exponent *= E(-1); - for(E e = 0; e < exponent; ++e) - r /= bbase; - } - return r; -} - -/** integer power; this function is constexpr-14 because of the local - * variables */ -template -C4_CONSTEXPR14 C4_CONST auto ipow(B base, E exponent) noexcept -> typename std::enable_if::value, B>::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - B r = B(1); - for(E e = 0; e < exponent; ++e) - r *= base; - return r; -} - -/** integer power; this function is constexpr-14 because of the local - * variables */ -template -C4_CONSTEXPR14 C4_CONST auto ipow(E exponent) noexcept -> typename std::enable_if::value, B>::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - B r = B(1); - for(E e = 0; e < exponent; ++e) - r *= base; - return r; -} -/** integer power; this function is constexpr-14 because of the local - * variables */ -template -C4_CONSTEXPR14 C4_CONST auto ipow(E exponent) noexcept -> typename std::enable_if::value, B>::type -{ - C4_STATIC_ASSERT(std::is_integral::value); - B r = B(1); - B bbase = B(base); - for(E e = 0; e < exponent; ++e) - r *= bbase; - return r; -} - -C4_SUPPRESS_WARNING_GCC_CLANG_POP - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** return a mask with all bits set [first_bit,last_bit[; this function - * is constexpr-14 because of the local variables */ -template -C4_CONSTEXPR14 I contiguous_mask(I first_bit, I last_bit) -{ - I r = 0; - for(I i = first_bit; i < last_bit; ++i) - { - r |= (I(1) << i); - } - return r; -} - - -namespace detail { - -template -struct _ctgmsk11; - -template -struct _ctgmsk11< I, val, first, last, true> -{ - enum : I { value = _ctgmsk11::value }; -}; - -template -struct _ctgmsk11< I, val, first, last, false> -{ - enum : I { value = val }; -}; - -} // namespace detail - - -/** TMP version of contiguous_mask(); this needs to be implemented with template - * meta-programming because C++11 cannot use a constexpr function with - * local variables - * @see contiguous_mask */ -template -struct contiguous_mask11 -{ - enum : I { value = detail::_ctgmsk11::value }; -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** use Empty Base Class Optimization to reduce the size of a pair of - * potentially empty types*/ - -namespace detail { -typedef enum { - tpc_same, - tpc_same_empty, - tpc_both_empty, - tpc_first_empty, - tpc_second_empty, - tpc_general -} TightPairCase_e; - -template -constexpr TightPairCase_e tpc_which_case() -{ - return std::is_same::value ? - std::is_empty::value ? - tpc_same_empty - : - tpc_same - : - std::is_empty::value && std::is_empty::value ? - tpc_both_empty - : - std::is_empty::value ? - tpc_first_empty - : - std::is_empty::value ? - tpc_second_empty - : - tpc_general - ; -} - -template -struct tight_pair -{ -private: - - First m_first; - Second m_second; - -public: - - using first_type = First; - using second_type = Second; - - tight_pair() : m_first(), m_second() {} - tight_pair(First const& f, Second const& s) : m_first(f), m_second(s) {} - - C4_ALWAYS_INLINE C4_CONSTEXPR14 First & first () { return m_first; } - C4_ALWAYS_INLINE C4_CONSTEXPR14 First const& first () const { return m_first; } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second & second() { return m_second; } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second const& second() const { return m_second; } -}; - -template -struct tight_pair : public First -{ - static_assert(std::is_same::value, "bad implementation"); - - using first_type = First; - using second_type = Second; - - tight_pair() : First() {} - tight_pair(First const& f, Second const& /*s*/) : First(f) {} - - C4_ALWAYS_INLINE C4_CONSTEXPR14 First & first () { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 First const& first () const { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second & second() { return reinterpret_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second const& second() const { return reinterpret_cast(*this); } -}; - -template -struct tight_pair : public First, public Second -{ - using first_type = First; - using second_type = Second; - - tight_pair() : First(), Second() {} - tight_pair(First const& f, Second const& s) : First(f), Second(s) {} - - C4_ALWAYS_INLINE C4_CONSTEXPR14 First & first () { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 First const& first () const { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second & second() { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second const& second() const { return static_cast(*this); } -}; - -template -struct tight_pair : public First -{ - Second m_second; - - using first_type = First; - using second_type = Second; - - tight_pair() : First() {} - tight_pair(First const& f, Second const& s) : First(f), m_second(s) {} - - C4_ALWAYS_INLINE C4_CONSTEXPR14 First & first () { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 First const& first () const { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second & second() { return m_second; } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second const& second() const { return m_second; } -}; - -template -struct tight_pair : public First -{ - Second m_second; - - using first_type = First; - using second_type = Second; - - tight_pair() : First(), m_second() {} - tight_pair(First const& f, Second const& s) : First(f), m_second(s) {} - - C4_ALWAYS_INLINE C4_CONSTEXPR14 First & first () { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 First const& first () const { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second & second() { return m_second; } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second const& second() const { return m_second; } -}; - -template -struct tight_pair : public Second -{ - First m_first; - - using first_type = First; - using second_type = Second; - - tight_pair() : Second(), m_first() {} - tight_pair(First const& f, Second const& s) : Second(s), m_first(f) {} - - C4_ALWAYS_INLINE C4_CONSTEXPR14 First & first () { return m_first; } - C4_ALWAYS_INLINE C4_CONSTEXPR14 First const& first () const { return m_first; } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second & second() { return static_cast(*this); } - C4_ALWAYS_INLINE C4_CONSTEXPR14 Second const& second() const { return static_cast(*this); } -}; - -} // namespace detail - -template -using tight_pair = detail::tight_pair()>; - -} // namespace c4 - -#endif /* _C4_MEMORY_UTIL_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/platform.hpp b/thirdparty/ryml/ext/c4core/src/c4/platform.hpp deleted file mode 100644 index 8f9a61f0f..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/platform.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef _C4_PLATFORM_HPP_ -#define _C4_PLATFORM_HPP_ - -/** @file platform.hpp Provides platform information macros - * @ingroup basic_headers */ - -// see also https://sourceforge.net/p/predef/wiki/OperatingSystems/ - -#if defined(_WIN64) -# define C4_WIN -# define C4_WIN64 -#elif defined(_WIN32) -# define C4_WIN -# define C4_WIN32 -#elif defined(__ANDROID__) -# define C4_ANDROID -#elif defined(__APPLE__) -# include "TargetConditionals.h" -# if TARGET_OS_IPHONE || TARGET_IPHONE_SIMULATOR -# define C4_IOS -# elif TARGET_OS_MAC || TARGET_OS_OSX -# define C4_MACOS -# else -# error "Unknown Apple platform" -# endif -#elif defined(__linux__) || defined(__linux) -# define C4_UNIX -# define C4_LINUX -#elif defined(__unix__) || defined(__unix) -# define C4_UNIX -#elif defined(__arm__) || defined(__aarch64__) -# define C4_ARM -#elif defined(SWIG) -# define C4_SWIG -#else -# error "unknown platform" -#endif - -#if defined(__posix) || defined(C4_UNIX) || defined(C4_LINUX) -# define C4_POSIX -#endif - - -#endif /* _C4_PLATFORM_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/preprocessor.hpp b/thirdparty/ryml/ext/c4core/src/c4/preprocessor.hpp deleted file mode 100644 index a55482540..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/preprocessor.hpp +++ /dev/null @@ -1,123 +0,0 @@ -#ifndef _C4_PREPROCESSOR_HPP_ -#define _C4_PREPROCESSOR_HPP_ - -/** @file preprocessor.hpp Contains basic macros and preprocessor utilities. - * @ingroup basic_headers */ - -#ifdef __clang__ - /* NOTE: using , ## __VA_ARGS__ to deal with zero-args calls to - * variadic macros is not portable, but works in clang, gcc, msvc, icc. - * clang requires switching off compiler warnings for pedantic mode. - * @see http://stackoverflow.com/questions/32047685/variadic-macro-without-arguments */ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" // warning: token pasting of ',' and __VA_ARGS__ is a GNU extension -#elif defined(__GNUC__) - /* GCC also issues a warning for zero-args calls to variadic macros. - * This warning is switched on with -pedantic and apparently there is no - * easy way to turn it off as with clang. But marking this as a system - * header works. - * @see https://gcc.gnu.org/onlinedocs/cpp/System-Headers.html - * @see http://stackoverflow.com/questions/35587137/ */ -# pragma GCC system_header -#endif - -#define C4_WIDEN(str) L"" str - -#define C4_COUNTOF(arr) (sizeof(arr)/sizeof((arr)[0])) - -#define C4_EXPAND(arg) arg - -/** useful in some macro calls with template arguments */ -#define C4_COMMA , -/** useful in some macro calls with template arguments - * @see C4_COMMA */ -#define C4_COMMA_X C4_COMMA - -/** expand and quote */ -#define C4_XQUOTE(arg) _C4_XQUOTE(arg) -#define _C4_XQUOTE(arg) C4_QUOTE(arg) -#define C4_QUOTE(arg) #arg - -/** expand and concatenate */ -#define C4_XCAT(arg1, arg2) _C4_XCAT(arg1, arg2) -#define _C4_XCAT(arg1, arg2) C4_CAT(arg1, arg2) -#define C4_CAT(arg1, arg2) arg1##arg2 - -#define C4_VERSION_CAT(major, minor, patch) ((major)*10000 + (minor)*100 + (patch)) - -/** A preprocessor foreach. Spectacular trick taken from: - * http://stackoverflow.com/a/1872506/5875572 - * The first argument is for a macro receiving a single argument, - * which will be called with every subsequent argument. There is - * currently a limit of 32 arguments, and at least 1 must be provided. - * -Example: -@code{.cpp} -struct Example { - int a; - int b; - int c; -}; -// define a one-arg macro to be called -#define PRN_STRUCT_OFFSETS(field) PRN_STRUCT_OFFSETS_(Example, field) -#define PRN_STRUCT_OFFSETS_(structure, field) printf(C4_XQUOTE(structure) ":" C4_XQUOTE(field)" - offset=%zu\n", offsetof(structure, field)); - -// now call the macro for a, b and c -C4_FOR_EACH(PRN_STRUCT_OFFSETS, a, b, c); -@endcode */ -#define C4_FOR_EACH(what, ...) C4_FOR_EACH_SEP(what, ;, __VA_ARGS__) - -/** same as C4_FOR_EACH(), but use a custom separator between statements. - * If a comma is needed as the separator, use the C4_COMMA macro. - * @see C4_FOR_EACH - * @see C4_COMMA - */ -#define C4_FOR_EACH_SEP(what, sep, ...) _C4_FOR_EACH_(_C4_FOR_EACH_NARG(__VA_ARGS__), what, sep, __VA_ARGS__) - -/// @cond dev - -#define _C4_FOR_EACH_01(what, sep, x) what(x) sep -#define _C4_FOR_EACH_02(what, sep, x, ...) what(x) sep _C4_FOR_EACH_01(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_03(what, sep, x, ...) what(x) sep _C4_FOR_EACH_02(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_04(what, sep, x, ...) what(x) sep _C4_FOR_EACH_03(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_05(what, sep, x, ...) what(x) sep _C4_FOR_EACH_04(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_06(what, sep, x, ...) what(x) sep _C4_FOR_EACH_05(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_07(what, sep, x, ...) what(x) sep _C4_FOR_EACH_06(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_08(what, sep, x, ...) what(x) sep _C4_FOR_EACH_07(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_09(what, sep, x, ...) what(x) sep _C4_FOR_EACH_08(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_10(what, sep, x, ...) what(x) sep _C4_FOR_EACH_09(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_11(what, sep, x, ...) what(x) sep _C4_FOR_EACH_10(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_12(what, sep, x, ...) what(x) sep _C4_FOR_EACH_11(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_13(what, sep, x, ...) what(x) sep _C4_FOR_EACH_12(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_14(what, sep, x, ...) what(x) sep _C4_FOR_EACH_13(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_15(what, sep, x, ...) what(x) sep _C4_FOR_EACH_14(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_16(what, sep, x, ...) what(x) sep _C4_FOR_EACH_15(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_17(what, sep, x, ...) what(x) sep _C4_FOR_EACH_16(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_18(what, sep, x, ...) what(x) sep _C4_FOR_EACH_17(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_19(what, sep, x, ...) what(x) sep _C4_FOR_EACH_18(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_20(what, sep, x, ...) what(x) sep _C4_FOR_EACH_19(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_21(what, sep, x, ...) what(x) sep _C4_FOR_EACH_20(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_22(what, sep, x, ...) what(x) sep _C4_FOR_EACH_21(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_23(what, sep, x, ...) what(x) sep _C4_FOR_EACH_22(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_24(what, sep, x, ...) what(x) sep _C4_FOR_EACH_23(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_25(what, sep, x, ...) what(x) sep _C4_FOR_EACH_24(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_26(what, sep, x, ...) what(x) sep _C4_FOR_EACH_25(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_27(what, sep, x, ...) what(x) sep _C4_FOR_EACH_26(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_28(what, sep, x, ...) what(x) sep _C4_FOR_EACH_27(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_29(what, sep, x, ...) what(x) sep _C4_FOR_EACH_28(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_30(what, sep, x, ...) what(x) sep _C4_FOR_EACH_29(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_31(what, sep, x, ...) what(x) sep _C4_FOR_EACH_30(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_32(what, sep, x, ...) what(x) sep _C4_FOR_EACH_31(what, sep, __VA_ARGS__) -#define _C4_FOR_EACH_NARG(...) _C4_FOR_EACH_NARG_(__VA_ARGS__, _C4_FOR_EACH_RSEQ_N()) -#define _C4_FOR_EACH_NARG_(...) _C4_FOR_EACH_ARG_N(__VA_ARGS__) -#define _C4_FOR_EACH_ARG_N(_01, _02, _03, _04, _05, _06, _07, _08, _09, _10, _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, _32, N, ...) N -#define _C4_FOR_EACH_RSEQ_N() 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 09, 08, 07, 06, 05, 04, 03, 02, 01 -#define _C4_FOR_EACH_(N, what, sep, ...) C4_XCAT(_C4_FOR_EACH_, N)(what, sep, __VA_ARGS__) - -/// @endcond - -#ifdef __clang__ -# pragma clang diagnostic pop -#endif - -#endif /* _C4_PREPROCESSOR_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/restrict.hpp b/thirdparty/ryml/ext/c4core/src/c4/restrict.hpp deleted file mode 100644 index a8a59301b..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/restrict.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef _C4_RESTRICT_HPP_ -#define _C4_RESTRICT_HPP_ - -/** @file restrict.hpp macros defining shorthand symbols for restricted - * pointers and references - * @see unrestrict.hpp - * @see restrict - */ - -/** @defgroup restrict Restrict utilities - * macros defining shorthand symbols for restricted - * pointers and references - * ```cpp - * void sum_arrays(size_t sz, float const* C4_RESTRICT a, float const *C4_RESTRICT b, float *result); - * float * C4_RESTRICT ptr; - * float & C4_RESTRICT ref = *ptr; - * float const* C4_RESTRICT cptr; - * float const& C4_RESTRICT cref = *cptr; - * - * // becomes this: - * #include - * void sum_arrays(size_t sz, float c$ a, float c$ b, float * result); - * float $ ptr; - * float $$ ref = *ptr; - * float c$ cptr; - * float c$$ cref = *cptr; - * ``` - * @ingroup types - * @{ */ - -/** @def \$ a restricted pointer */ -/** @def c\$ a restricted pointer to const data */ - -/** @def \$\$ a restricted reference */ -/** @def c\$\$ a restricted reference to const data */ - -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wdollar-in-identifier-extension" -#elif defined(__GNUC__) -#endif - -#define $ * C4_RESTRICT // a restricted pointer -#define c$ const* C4_RESTRICT // a restricted pointer to const data - -#define $$ & C4_RESTRICT // restricted reference -#define c$$ const& C4_RESTRICT // restricted reference to const data - -/** @} */ - -#endif /* _C4_RESTRICT_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/span.hpp b/thirdparty/ryml/ext/c4core/src/c4/span.hpp deleted file mode 100644 index cc6e463dc..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/span.hpp +++ /dev/null @@ -1,517 +0,0 @@ -#ifndef _C4_SPAN_HPP_ -#define _C4_SPAN_HPP_ - -/** @file span.hpp Provides span classes. */ - -#include "c4/config.hpp" -#include "c4/error.hpp" -#include "c4/szconv.hpp" - -#include - -namespace c4 { - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** a crtp base for implementing span classes - * - * A span is a non-owning range of elements contiguously stored in memory. - * Unlike STL's array_view, the span allows write-access to its members. - * - * To obtain subspans from a span, the following const member functions - * are available: - * - subspan(first, num) - * - range(first, last) - * - first(num) - * - last(num) - * - * A span can also be resized via the following non-const member functions: - * - resize(sz) - * - ltrim(num) - * - rtrim(num) - * - * @see span - * @see cspan - * @see spanrs - * @see cspanrs - * @see spanrsl - * @see cspanrsl - */ -template -class span_crtp -{ -// some utility defines, undefined at the end of this class -#define _c4this ((SpanImpl *)this) -#define _c4cthis ((SpanImpl const*)this) -#define _c4ptr ((SpanImpl *)this)->m_ptr -#define _c4cptr ((SpanImpl const*)this)->m_ptr -#define _c4sz ((SpanImpl *)this)->m_size -#define _c4csz ((SpanImpl const*)this)->m_size - -public: - - _c4_DEFINE_ARRAY_TYPES(T, I); - -public: - - C4_ALWAYS_INLINE constexpr I value_size() const noexcept { return sizeof(T); } - C4_ALWAYS_INLINE constexpr I elm_size () const noexcept { return sizeof(T); } - C4_ALWAYS_INLINE constexpr I type_size () const noexcept { return sizeof(T); } - C4_ALWAYS_INLINE I byte_size () const noexcept { return _c4csz*sizeof(T); } - - C4_ALWAYS_INLINE bool empty() const noexcept { return _c4csz == 0; } - C4_ALWAYS_INLINE I size() const noexcept { return _c4csz; } - //C4_ALWAYS_INLINE I capacity() const noexcept { return _c4sz; } // this must be defined by impl classes - - C4_ALWAYS_INLINE void clear() noexcept { _c4sz = 0; } - - C4_ALWAYS_INLINE T * data() noexcept { return _c4ptr; } - C4_ALWAYS_INLINE T const* data() const noexcept { return _c4cptr; } - - C4_ALWAYS_INLINE iterator begin() noexcept { return _c4ptr; } - C4_ALWAYS_INLINE const_iterator begin() const noexcept { return _c4cptr; } - C4_ALWAYS_INLINE const_iterator cbegin() const noexcept { return _c4cptr; } - - C4_ALWAYS_INLINE iterator end() noexcept { return _c4ptr + _c4sz; } - C4_ALWAYS_INLINE const_iterator end() const noexcept { return _c4cptr + _c4csz; } - C4_ALWAYS_INLINE const_iterator cend() const noexcept { return _c4cptr + _c4csz; } - - C4_ALWAYS_INLINE reverse_iterator rbegin() noexcept { return reverse_iterator(_c4ptr + _c4sz); } - C4_ALWAYS_INLINE const_reverse_iterator rbegin() const noexcept { return reverse_iterator(_c4cptr + _c4sz); } - C4_ALWAYS_INLINE const_reverse_iterator crbegin() const noexcept { return reverse_iterator(_c4cptr + _c4sz); } - - C4_ALWAYS_INLINE reverse_iterator rend() noexcept { return const_reverse_iterator(_c4ptr); } - C4_ALWAYS_INLINE const_reverse_iterator rend() const noexcept { return const_reverse_iterator(_c4cptr); } - C4_ALWAYS_INLINE const_reverse_iterator crend() const noexcept { return const_reverse_iterator(_c4cptr); } - - C4_ALWAYS_INLINE T & front() C4_NOEXCEPT_X { C4_XASSERT(!empty()); return _c4ptr [0]; } - C4_ALWAYS_INLINE T const& front() const C4_NOEXCEPT_X { C4_XASSERT(!empty()); return _c4cptr[0]; } - - C4_ALWAYS_INLINE T & back() C4_NOEXCEPT_X { C4_XASSERT(!empty()); return _c4ptr [_c4sz - 1]; } - C4_ALWAYS_INLINE T const& back() const C4_NOEXCEPT_X { C4_XASSERT(!empty()); return _c4cptr[_c4csz - 1]; } - - C4_ALWAYS_INLINE T & operator[] (I i) C4_NOEXCEPT_X { C4_XASSERT(i >= 0 && i < _c4sz ); return _c4ptr [i]; } - C4_ALWAYS_INLINE T const& operator[] (I i) const C4_NOEXCEPT_X { C4_XASSERT(i >= 0 && i < _c4csz); return _c4cptr[i]; } - - C4_ALWAYS_INLINE SpanImpl subspan(I first, I num) const C4_NOEXCEPT_X - { - C4_XASSERT((first >= 0 && first < _c4csz) || (first == _c4csz && num == 0)); - C4_XASSERT((first + num >= 0) && (first + num <= _c4csz)); - return _c4cthis->_select(_c4cptr + first, num); - } - C4_ALWAYS_INLINE SpanImpl subspan(I first) const C4_NOEXCEPT_X ///< goes up until the end of the span - { - C4_XASSERT(first >= 0 && first <= _c4csz); - return _c4cthis->_select(_c4cptr + first, _c4csz - first); - } - - C4_ALWAYS_INLINE SpanImpl range(I first, I last) const C4_NOEXCEPT_X ///< last element is NOT included - { - C4_XASSERT(((first >= 0) && (first < _c4csz)) || (first == _c4csz && first == last)); - C4_XASSERT((last >= 0) && (last <= _c4csz)); - C4_XASSERT(last >= first); - return _c4cthis->_select(_c4cptr + first, last - first); - } - C4_ALWAYS_INLINE SpanImpl range(I first) const C4_NOEXCEPT_X ///< goes up until the end of the span - { - C4_XASSERT(((first >= 0) && (first <= _c4csz))); - return _c4cthis->_select(_c4cptr + first, _c4csz - first); - } - - C4_ALWAYS_INLINE SpanImpl first(I num) const C4_NOEXCEPT_X ///< get the first num elements, starting at 0 - { - C4_XASSERT((num >= 0) && (num <= _c4csz)); - return _c4cthis->_select(_c4cptr, num); - } - C4_ALWAYS_INLINE SpanImpl last(I num) const C4_NOEXCEPT_X ///< get the last num elements, starting at size()-num - { - C4_XASSERT((num >= 0) && (num <= _c4csz)); - return _c4cthis->_select(_c4cptr + _c4csz - num, num); - } - - bool is_subspan(span_crtp const& ss) const noexcept - { - if(_c4cptr == nullptr) return false; - auto *b = begin(), *e = end(); - auto *ssb = ss.begin(), *sse = ss.end(); - if(ssb >= b && sse <= e) - { - return true; - } - else - { - return false; - } - } - - /** COMPLement Left: return the complement to the left of the beginning of the given subspan. - * If ss does not begin inside this, returns an empty substring. */ - SpanImpl compll(span_crtp const& ss) const C4_NOEXCEPT_X - { - auto ssb = ss.begin(); - auto b = begin(); - auto e = end(); - if(ssb >= b && ssb <= e) - { - return subspan(0, static_cast(ssb - b)); - } - else - { - return subspan(0, 0); - } - } - - /** COMPLement Right: return the complement to the right of the end of the given subspan. - * If ss does not end inside this, returns an empty substring. */ - SpanImpl complr(span_crtp const& ss) const C4_NOEXCEPT_X - { - auto sse = ss.end(); - auto b = begin(); - auto e = end(); - if(sse >= b && sse <= e) - { - return subspan(static_cast(sse - b), static_cast(e - sse)); - } - else - { - return subspan(0, 0); - } - } - - C4_ALWAYS_INLINE bool same_span(span_crtp const& that) const noexcept - { - return size() == that.size() && data() == that.data(); - } - template - C4_ALWAYS_INLINE bool same_span(span_crtp const& that) const C4_NOEXCEPT_X - { - I tsz = szconv(that.size()); // x-asserts that the size does not overflow - return size() == tsz && data() == that.data(); - } - -#undef _c4this -#undef _c4cthis -#undef _c4ptr -#undef _c4cptr -#undef _c4sz -#undef _c4csz -}; - -//----------------------------------------------------------------------------- -template -inline constexpr bool operator== -( - span_crtp const& l, - span_crtp const& r -) -{ -#if C4_CPP >= 14 - return std::equal(l.begin(), l.end(), r.begin(), r.end()); -#else - return l.same_span(r) || std::equal(l.begin(), l.end(), r.begin()); -#endif -} - -template -inline constexpr bool operator!= -( - span_crtp const& l, - span_crtp const& r -) -{ - return ! (l == r); -} - -//----------------------------------------------------------------------------- -template -inline constexpr bool operator< -( - span_crtp const& l, - span_crtp const& r -) -{ - return std::lexicographical_compare(l.begin(), l.end(), r.begin(), r.end()); -} - -template -inline constexpr bool operator<= -( - span_crtp const& l, - span_crtp const& r -) -{ - return ! (l > r); -} - -//----------------------------------------------------------------------------- -template -inline constexpr bool operator> -( - span_crtp const& l, - span_crtp const& r -) -{ - return r < l; -} - -//----------------------------------------------------------------------------- -template -inline constexpr bool operator>= -( - span_crtp const& l, - span_crtp const& r -) -{ - return ! (l < r); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** A non-owning span of elements contiguously stored in memory. */ -template -class span : public span_crtp> -{ - friend class span_crtp>; - - T * C4_RESTRICT m_ptr; - I m_size; - - C4_ALWAYS_INLINE span _select(T *p, I sz) const { return span(p, sz); } - -public: - - _c4_DEFINE_ARRAY_TYPES(T, I); - using NCT = typename std::remove_const::type; //!< NCT=non const type - using CT = typename std::add_const::type; //!< CT=const type - using const_type = span; - - /// convert automatically to span of const T - operator span () const { span s(m_ptr, m_size); return s; } - -public: - - C4_ALWAYS_INLINE C4_CONSTEXPR14 span() noexcept : m_ptr{nullptr}, m_size{0} {} - - span(span const&) = default; - span(span &&) = default; - - span& operator= (span const&) = default; - span& operator= (span &&) = default; - -public: - - /** @name Construction and assignment from same type */ - /** @{ */ - - template C4_ALWAYS_INLINE C4_CONSTEXPR14 span (T (&arr)[N]) noexcept : m_ptr{arr}, m_size{N} {} - template C4_ALWAYS_INLINE C4_CONSTEXPR14 void assign(T (&arr)[N]) noexcept { m_ptr = arr; m_size = N; } - - C4_ALWAYS_INLINE C4_CONSTEXPR14 span(T *p, I sz) noexcept : m_ptr{p}, m_size{sz} {} - C4_ALWAYS_INLINE C4_CONSTEXPR14 void assign(T *p, I sz) noexcept { m_ptr = p; m_size = sz; } - - C4_ALWAYS_INLINE C4_CONSTEXPR14 span (c4::aggregate_t, std::initializer_list il) noexcept : m_ptr{&*il.begin()}, m_size{il.size()} {} - C4_ALWAYS_INLINE C4_CONSTEXPR14 void assign(c4::aggregate_t, std::initializer_list il) noexcept { m_ptr = &*il.begin(); m_size = il.size(); } - - /** @} */ - -public: - - C4_ALWAYS_INLINE I capacity() const noexcept { return m_size; } - - C4_ALWAYS_INLINE void resize(I sz) C4_NOEXCEPT_A { C4_ASSERT(sz <= m_size); m_size = sz; } - C4_ALWAYS_INLINE void rtrim (I n ) C4_NOEXCEPT_A { C4_ASSERT(n >= 0 && n < m_size); m_size -= n; } - C4_ALWAYS_INLINE void ltrim (I n ) C4_NOEXCEPT_A { C4_ASSERT(n >= 0 && n < m_size); m_size -= n; m_ptr += n; } - -}; -template using cspan = span; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** A non-owning span resizeable up to a capacity. Subselection or resizing - * will keep the original provided it starts at begin(). If subselection or - * resizing change the pointer, then the original capacity information will - * be lost. - * - * Thus, resizing via resize() and ltrim() and subselecting via first() - * or any of subspan() or range() when starting from the beginning will keep - * the original capacity. OTOH, using last(), or any of subspan() or range() - * with an offset from the start will remove from capacity (shifting the - * pointer) by the corresponding offset. If this is undesired, then consider - * using spanrsl. - * - * @see spanrs for a span resizeable on the right - * @see spanrsl for a span resizeable on the right and left - */ - -template -class spanrs : public span_crtp> -{ - friend class span_crtp>; - - T * C4_RESTRICT m_ptr; - I m_size; - I m_capacity; - - C4_ALWAYS_INLINE spanrs _select(T *p, I sz) const noexcept - { - C4_ASSERT(p >= m_ptr); - size_t delta = static_cast(p - m_ptr); - C4_ASSERT(m_capacity >= delta); - return spanrs(p, sz, static_cast(m_capacity - delta)); - } - -public: - - _c4_DEFINE_ARRAY_TYPES(T, I); - using NCT = typename std::remove_const::type; //!< NCT=non const type - using CT = typename std::add_const::type; //!< CT=const type - using const_type = spanrs; - - /// convert automatically to span of T - C4_ALWAYS_INLINE operator span () const noexcept { return span(m_ptr, m_size); } - /// convert automatically to span of const T - //C4_ALWAYS_INLINE operator span () const noexcept { span s(m_ptr, m_size); return s; } - /// convert automatically to spanrs of const T - C4_ALWAYS_INLINE operator spanrs () const noexcept { spanrs s(m_ptr, m_size, m_capacity); return s; } - -public: - - C4_ALWAYS_INLINE spanrs() noexcept : m_ptr{nullptr}, m_size{0}, m_capacity{0} {} - - spanrs(spanrs const&) = default; - spanrs(spanrs &&) = default; - - spanrs& operator= (spanrs const&) = default; - spanrs& operator= (spanrs &&) = default; - -public: - - /** @name Construction and assignment from same type */ - /** @{ */ - - C4_ALWAYS_INLINE spanrs(T *p, I sz) noexcept : m_ptr{p}, m_size{sz}, m_capacity{sz} {} - /** @warning will reset the capacity to sz */ - C4_ALWAYS_INLINE void assign(T *p, I sz) noexcept { m_ptr = p; m_size = sz; m_capacity = sz; } - - C4_ALWAYS_INLINE spanrs(T *p, I sz, I cap) noexcept : m_ptr{p}, m_size{sz}, m_capacity{cap} {} - C4_ALWAYS_INLINE void assign(T *p, I sz, I cap) noexcept { m_ptr = p; m_size = sz; m_capacity = cap; } - - template C4_ALWAYS_INLINE spanrs(T (&arr)[N]) noexcept : m_ptr{arr}, m_size{N}, m_capacity{N} {} - template C4_ALWAYS_INLINE void assign(T (&arr)[N]) noexcept { m_ptr = arr; m_size = N; m_capacity = N; } - - C4_ALWAYS_INLINE spanrs(c4::aggregate_t, std::initializer_list il) noexcept : m_ptr{il.begin()}, m_size{il.size()}, m_capacity{il.size()} {} - C4_ALWAYS_INLINE void assign(c4::aggregate_t, std::initializer_list il) noexcept { m_ptr = il.begin(); m_size = il.size(); m_capacity = il.size(); } - - /** @} */ - -public: - - C4_ALWAYS_INLINE I capacity() const noexcept { return m_capacity; } - - C4_ALWAYS_INLINE void resize(I sz) C4_NOEXCEPT_A { C4_ASSERT(sz <= m_capacity); m_size = sz; } - C4_ALWAYS_INLINE void rtrim (I n ) C4_NOEXCEPT_A { C4_ASSERT(n >= 0 && n < m_size); m_size -= n; } - C4_ALWAYS_INLINE void ltrim (I n ) C4_NOEXCEPT_A { C4_ASSERT(n >= 0 && n < m_size); m_size -= n; m_ptr += n; m_capacity -= n; } - -}; -template using cspanrs = spanrs; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** A non-owning span which always retains the capacity of the original - * range it was taken from (though it may loose its original size). - * The resizing methods resize(), ltrim(), rtrim() as well - * as the subselection methods subspan(), range(), first() and last() can be - * used at will without loosing the original capacity; the full capacity span - * can always be recovered by calling original(). - */ -template -class spanrsl : public span_crtp> -{ - friend class span_crtp>; - - T *C4_RESTRICT m_ptr; ///< the current ptr. the original ptr is (m_ptr - m_offset). - I m_size; ///< the current size. the original size is unrecoverable. - I m_capacity; ///< the current capacity. the original capacity is (m_capacity + m_offset). - I m_offset; ///< the offset of the current m_ptr to the start of the original memory block. - - C4_ALWAYS_INLINE spanrsl _select(T *p, I sz) const noexcept - { - C4_ASSERT(p >= m_ptr); - I delta = static_cast(p - m_ptr); - C4_ASSERT(m_capacity >= delta); - return spanrsl(p, sz, static_cast(m_capacity - delta), m_offset + delta); - } - -public: - - _c4_DEFINE_ARRAY_TYPES(T, I); - using NCT = typename std::remove_const::type; //!< NCT=non const type - using CT = typename std::add_const::type; //!< CT=const type - using const_type = spanrsl; - - C4_ALWAYS_INLINE operator span () const noexcept { return span(m_ptr, m_size); } - C4_ALWAYS_INLINE operator spanrs () const noexcept { return spanrs(m_ptr, m_size, m_capacity); } - C4_ALWAYS_INLINE operator spanrsl () const noexcept { return spanrsl(m_ptr, m_size, m_capacity, m_offset); } - -public: - - C4_ALWAYS_INLINE spanrsl() noexcept : m_ptr{nullptr}, m_size{0}, m_capacity{0}, m_offset{0} {} - - spanrsl(spanrsl const&) = default; - spanrsl(spanrsl &&) = default; - - spanrsl& operator= (spanrsl const&) = default; - spanrsl& operator= (spanrsl &&) = default; - -public: - - C4_ALWAYS_INLINE spanrsl(T *p, I sz) noexcept : m_ptr{p}, m_size{sz}, m_capacity{sz}, m_offset{0} {} - C4_ALWAYS_INLINE void assign(T *p, I sz) noexcept { m_ptr = p; m_size = sz; m_capacity = sz; m_offset = 0; } - - C4_ALWAYS_INLINE spanrsl(T *p, I sz, I cap) noexcept : m_ptr{p}, m_size{sz}, m_capacity{cap}, m_offset{0} {} - C4_ALWAYS_INLINE void assign(T *p, I sz, I cap) noexcept { m_ptr = p; m_size = sz; m_capacity = cap; m_offset = 0; } - - C4_ALWAYS_INLINE spanrsl(T *p, I sz, I cap, I offs) noexcept : m_ptr{p}, m_size{sz}, m_capacity{cap}, m_offset{offs} {} - C4_ALWAYS_INLINE void assign(T *p, I sz, I cap, I offs) noexcept { m_ptr = p; m_size = sz; m_capacity = cap; m_offset = offs; } - - template C4_ALWAYS_INLINE spanrsl(T (&arr)[N]) noexcept : m_ptr{arr}, m_size{N}, m_capacity{N}, m_offset{0} {} - template C4_ALWAYS_INLINE void assign(T (&arr)[N]) noexcept { m_ptr = arr; m_size = N; m_capacity = N; m_offset = 0; } - - C4_ALWAYS_INLINE spanrsl(c4::aggregate_t, std::initializer_list il) noexcept : m_ptr{il.begin()}, m_size{il.size()}, m_capacity{il.size()}, m_offset{0} {} - C4_ALWAYS_INLINE void assign (c4::aggregate_t, std::initializer_list il) noexcept { m_ptr = il.begin(); m_size = il.size(); m_capacity = il.size(); m_offset = 0; } - -public: - - C4_ALWAYS_INLINE I offset() const noexcept { return m_offset; } - C4_ALWAYS_INLINE I capacity() const noexcept { return m_capacity; } - - C4_ALWAYS_INLINE void resize(I sz) C4_NOEXCEPT_A { C4_ASSERT(sz <= m_capacity); m_size = sz; } - C4_ALWAYS_INLINE void rtrim (I n ) C4_NOEXCEPT_A { C4_ASSERT(n >= 0 && n < m_size); m_size -= n; } - C4_ALWAYS_INLINE void ltrim (I n ) C4_NOEXCEPT_A { C4_ASSERT(n >= 0 && n < m_size); m_size -= n; m_ptr += n; m_offset += n; m_capacity -= n; } - - /** recover the original span as an spanrsl */ - C4_ALWAYS_INLINE spanrsl original() const - { - return spanrsl(m_ptr - m_offset, m_capacity + m_offset, m_capacity + m_offset, 0); - } - /** recover the original span as a different span type. Example: spanrs<...> orig = s.original(); */ - template class OtherSpanType> - C4_ALWAYS_INLINE OtherSpanType original() - { - return OtherSpanType(m_ptr - m_offset, m_capacity + m_offset); - } -}; -template using cspanrsl = spanrsl; - - -} // namespace c4 - - -#endif /* _C4_SPAN_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/std/std.hpp b/thirdparty/ryml/ext/c4core/src/c4/std/std.hpp deleted file mode 100644 index 3df8d1986..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/std/std.hpp +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef _C4_STD_STD_HPP_ -#define _C4_STD_STD_HPP_ - -/** @file std.hpp includes all c4-std interop files */ - -#include "c4/std/vector.hpp" -#include "c4/std/string.hpp" -#include "c4/std/tuple.hpp" - -#endif // _C4_STD_STD_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/std/std_fwd.hpp b/thirdparty/ryml/ext/c4core/src/c4/std/std_fwd.hpp deleted file mode 100644 index 8c42ce711..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/std/std_fwd.hpp +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef _C4_STD_STD_FWD_HPP_ -#define _C4_STD_STD_FWD_HPP_ - -/** @file std_fwd.hpp includes all c4-std interop fwd files */ - -#include "c4/std/vector_fwd.hpp" -#include "c4/std/string_fwd.hpp" -//#include "c4/std/tuple_fwd.hpp" - -#endif // _C4_STD_STD_FWD_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/std/string.hpp b/thirdparty/ryml/ext/c4core/src/c4/std/string.hpp deleted file mode 100644 index 5b9837ab4..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/std/string.hpp +++ /dev/null @@ -1,97 +0,0 @@ -#ifndef _C4_STD_STRING_HPP_ -#define _C4_STD_STRING_HPP_ - -/** @file string.hpp */ - -#ifndef C4CORE_SINGLE_HEADER -#include "c4/substr.hpp" -#endif - -#include - -namespace c4 { - -//----------------------------------------------------------------------------- - -/** get a writeable view to an existing std::string. - * When the string is empty, the returned view will be pointing - * at the character with value '\0', but the size will be zero. - * @see https://en.cppreference.com/w/cpp/string/basic_string/operator_at - */ -C4_ALWAYS_INLINE c4::substr to_substr(std::string &s) noexcept -{ - #if C4_CPP < 11 - #error this function will do undefined behavior - #endif - // since c++11 it is legal to call s[s.size()]. - return c4::substr(&s[0], s.size()); -} - -/** get a readonly view to an existing std::string. - * When the string is empty, the returned view will be pointing - * at the character with value '\0', but the size will be zero. - * @see https://en.cppreference.com/w/cpp/string/basic_string/operator_at - */ -C4_ALWAYS_INLINE c4::csubstr to_csubstr(std::string const& s) noexcept -{ - #if C4_CPP < 11 - #error this function will do undefined behavior - #endif - // since c++11 it is legal to call s[s.size()]. - return c4::csubstr(&s[0], s.size()); -} - -//----------------------------------------------------------------------------- - -C4_ALWAYS_INLINE bool operator== (c4::csubstr ss, std::string const& s) { return ss.compare(to_csubstr(s)) == 0; } -C4_ALWAYS_INLINE bool operator!= (c4::csubstr ss, std::string const& s) { return ss.compare(to_csubstr(s)) != 0; } -C4_ALWAYS_INLINE bool operator>= (c4::csubstr ss, std::string const& s) { return ss.compare(to_csubstr(s)) >= 0; } -C4_ALWAYS_INLINE bool operator> (c4::csubstr ss, std::string const& s) { return ss.compare(to_csubstr(s)) > 0; } -C4_ALWAYS_INLINE bool operator<= (c4::csubstr ss, std::string const& s) { return ss.compare(to_csubstr(s)) <= 0; } -C4_ALWAYS_INLINE bool operator< (c4::csubstr ss, std::string const& s) { return ss.compare(to_csubstr(s)) < 0; } - -C4_ALWAYS_INLINE bool operator== (std::string const& s, c4::csubstr ss) { return ss.compare(to_csubstr(s)) == 0; } -C4_ALWAYS_INLINE bool operator!= (std::string const& s, c4::csubstr ss) { return ss.compare(to_csubstr(s)) != 0; } -C4_ALWAYS_INLINE bool operator>= (std::string const& s, c4::csubstr ss) { return ss.compare(to_csubstr(s)) <= 0; } -C4_ALWAYS_INLINE bool operator> (std::string const& s, c4::csubstr ss) { return ss.compare(to_csubstr(s)) < 0; } -C4_ALWAYS_INLINE bool operator<= (std::string const& s, c4::csubstr ss) { return ss.compare(to_csubstr(s)) >= 0; } -C4_ALWAYS_INLINE bool operator< (std::string const& s, c4::csubstr ss) { return ss.compare(to_csubstr(s)) > 0; } - -//----------------------------------------------------------------------------- - -/** copy an std::string to a writeable string view */ -inline size_t to_chars(c4::substr buf, std::string const& s) -{ - C4_ASSERT(!buf.overlaps(to_csubstr(s))); - size_t len = buf.len < s.size() ? buf.len : s.size(); - // calling memcpy with null strings is undefined behavior - // and will wreak havoc in calling code's branches. - // see https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637 - if(len) - { - C4_ASSERT(s.data() != nullptr); - C4_ASSERT(buf.str != nullptr); - memcpy(buf.str, s.data(), len); - } - return s.size(); // return the number of needed chars -} - -/** copy a string view to an existing std::string */ -inline bool from_chars(c4::csubstr buf, std::string * s) -{ - s->resize(buf.len); - C4_ASSERT(!buf.overlaps(to_csubstr(*s))); - // calling memcpy with null strings is undefined behavior - // and will wreak havoc in calling code's branches. - // see https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637 - if(buf.len) - { - C4_ASSERT(buf.str != nullptr); - memcpy(&(*s)[0], buf.str, buf.len); - } - return true; -} - -} // namespace c4 - -#endif // _C4_STD_STRING_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/std/string_fwd.hpp b/thirdparty/ryml/ext/c4core/src/c4/std/string_fwd.hpp deleted file mode 100644 index 6f6f42146..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/std/string_fwd.hpp +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef _C4_STD_STRING_FWD_HPP_ -#define _C4_STD_STRING_FWD_HPP_ - -/** @file string_fwd.hpp */ - -#ifndef DOXYGEN - -#ifndef C4CORE_SINGLE_HEADER -#include "c4/substr_fwd.hpp" -#endif - -#include - -// forward declarations for std::string -#if defined(__GLIBCXX__) || defined(__GLIBCPP__) -#include // use the fwd header in glibcxx -#elif defined(_LIBCPP_VERSION) || defined(__APPLE_CC__) -#include // use the fwd header in stdlibc++ -#elif defined(_MSC_VER) -//! @todo is there a fwd header in msvc? -namespace std { -template struct char_traits; -template class allocator; -template class basic_string; -using string = basic_string, allocator>; -} /* namespace std */ -#else -#error "unknown standard library" -#endif - -namespace c4 { - -C4_ALWAYS_INLINE c4::substr to_substr(std::string &s) noexcept; -C4_ALWAYS_INLINE c4::csubstr to_csubstr(std::string const& s) noexcept; - -bool operator== (c4::csubstr ss, std::string const& s); -bool operator!= (c4::csubstr ss, std::string const& s); -bool operator>= (c4::csubstr ss, std::string const& s); -bool operator> (c4::csubstr ss, std::string const& s); -bool operator<= (c4::csubstr ss, std::string const& s); -bool operator< (c4::csubstr ss, std::string const& s); - -bool operator== (std::string const& s, c4::csubstr ss); -bool operator!= (std::string const& s, c4::csubstr ss); -bool operator>= (std::string const& s, c4::csubstr ss); -bool operator> (std::string const& s, c4::csubstr ss); -bool operator<= (std::string const& s, c4::csubstr ss); -bool operator< (std::string const& s, c4::csubstr ss); - -size_t to_chars(c4::substr buf, std::string const& s); -bool from_chars(c4::csubstr buf, std::string * s); - -} // namespace c4 - -#endif // DOXYGEN -#endif // _C4_STD_STRING_FWD_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/std/tuple.hpp b/thirdparty/ryml/ext/c4core/src/c4/std/tuple.hpp deleted file mode 100644 index 5edcd63da..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/std/tuple.hpp +++ /dev/null @@ -1,184 +0,0 @@ -#ifndef _C4_STD_TUPLE_HPP_ -#define _C4_STD_TUPLE_HPP_ - -/** @file tuple.hpp */ - -#ifndef C4CORE_SINGLE_HEADER -#include "c4/format.hpp" -#endif - -#include - -/** this is a work in progress */ -#undef C4_TUPLE_TO_CHARS - -namespace c4 { - -#ifdef C4_TUPLE_TO_CHARS -namespace detail { - -template< size_t Curr, class... Types > -struct tuple_helper -{ - static size_t do_cat(substr buf, std::tuple< Types... > const& tp) - { - size_t num = to_chars(buf, std::get(tp)); - buf = buf.len >= num ? buf.sub(num) : substr{}; - num += tuple_helper< Curr+1, Types... >::do_cat(buf, tp); - return num; - } - - static size_t do_uncat(csubstr buf, std::tuple< Types... > & tp) - { - size_t num = from_str_trim(buf, &std::get(tp)); - if(num == csubstr::npos) return csubstr::npos; - buf = buf.len >= num ? buf.sub(num) : substr{}; - num += tuple_helper< Curr+1, Types... >::do_uncat(buf, tp); - return num; - } - - template< class Sep > - static size_t do_catsep_more(substr buf, Sep const& sep, std::tuple< Types... > const& tp) - { - size_t ret = to_chars(buf, sep), num = ret; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = to_chars(buf, std::get(tp)); - num += ret; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = tuple_helper< Curr+1, Types... >::do_catsep_more(buf, sep, tp); - num += ret; - return num; - } - - template< class Sep > - static size_t do_uncatsep_more(csubstr buf, Sep & sep, std::tuple< Types... > & tp) - { - size_t ret = from_str_trim(buf, &sep), num = ret; - if(ret == csubstr::npos) return csubstr::npos; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = from_str_trim(buf, &std::get(tp)); - if(ret == csubstr::npos) return csubstr::npos; - num += ret; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = tuple_helper< Curr+1, Types... >::do_uncatsep_more(buf, sep, tp); - if(ret == csubstr::npos) return csubstr::npos; - num += ret; - return num; - } - - static size_t do_format(substr buf, csubstr fmt, std::tuple< Types... > const& tp) - { - auto pos = fmt.find("{}"); - if(pos != csubstr::npos) - { - size_t num = to_chars(buf, fmt.sub(0, pos)); - size_t out = num; - buf = buf.len >= num ? buf.sub(num) : substr{}; - num = to_chars(buf, std::get(tp)); - out += num; - buf = buf.len >= num ? buf.sub(num) : substr{}; - num = tuple_helper< Curr+1, Types... >::do_format(buf, fmt.sub(pos + 2), tp); - out += num; - return out; - } - else - { - return format(buf, fmt); - } - } - - static size_t do_unformat(csubstr buf, csubstr fmt, std::tuple< Types... > & tp) - { - auto pos = fmt.find("{}"); - if(pos != csubstr::npos) - { - size_t num = pos; - size_t out = num; - buf = buf.len >= num ? buf.sub(num) : substr{}; - num = from_str_trim(buf, &std::get(tp)); - out += num; - buf = buf.len >= num ? buf.sub(num) : substr{}; - num = tuple_helper< Curr+1, Types... >::do_unformat(buf, fmt.sub(pos + 2), tp); - out += num; - return out; - } - else - { - return tuple_helper< sizeof...(Types), Types... >::do_unformat(buf, fmt, tp); - } - } - -}; - -/** @todo VS compilation fails for this class */ -template< class... Types > -struct tuple_helper< sizeof...(Types), Types... > -{ - static size_t do_cat(substr /*buf*/, std::tuple const& /*tp*/) { return 0; } - static size_t do_uncat(csubstr /*buf*/, std::tuple & /*tp*/) { return 0; } - - template< class Sep > static size_t do_catsep_more(substr /*buf*/, Sep const& /*sep*/, std::tuple const& /*tp*/) { return 0; } - template< class Sep > static size_t do_uncatsep_more(csubstr /*buf*/, Sep & /*sep*/, std::tuple & /*tp*/) { return 0; } - - static size_t do_format(substr buf, csubstr fmt, std::tuple const& /*tp*/) - { - return to_chars(buf, fmt); - } - - static size_t do_unformat(csubstr buf, csubstr fmt, std::tuple const& /*tp*/) - { - return 0; - } -}; - -} // namespace detail - -template< class... Types > -inline size_t cat(substr buf, std::tuple< Types... > const& tp) -{ - return detail::tuple_helper< 0, Types... >::do_cat(buf, tp); -} - -template< class... Types > -inline size_t uncat(csubstr buf, std::tuple< Types... > & tp) -{ - return detail::tuple_helper< 0, Types... >::do_uncat(buf, tp); -} - -template< class Sep, class... Types > -inline size_t catsep(substr buf, Sep const& sep, std::tuple< Types... > const& tp) -{ - size_t num = to_chars(buf, std::cref(std::get<0>(tp))); - buf = buf.len >= num ? buf.sub(num) : substr{}; - num += detail::tuple_helper< 1, Types... >::do_catsep_more(buf, sep, tp); - return num; -} - -template< class Sep, class... Types > -inline size_t uncatsep(csubstr buf, Sep & sep, std::tuple< Types... > & tp) -{ - size_t ret = from_str_trim(buf, &std::get<0>(tp)), num = ret; - if(ret == csubstr::npos) return csubstr::npos; - buf = buf.len >= ret ? buf.sub(ret) : substr{}; - ret = detail::tuple_helper< 1, Types... >::do_uncatsep_more(buf, sep, tp); - if(ret == csubstr::npos) return csubstr::npos; - num += ret; - return num; -} - -template< class... Types > -inline size_t format(substr buf, csubstr fmt, std::tuple< Types... > const& tp) -{ - return detail::tuple_helper< 0, Types... >::do_format(buf, fmt, tp); -} - -template< class... Types > -inline size_t unformat(csubstr buf, csubstr fmt, std::tuple< Types... > & tp) -{ - return detail::tuple_helper< 0, Types... >::do_unformat(buf, fmt, tp); -} -#endif // C4_TUPLE_TO_CHARS - -} // namespace c4 - -#endif /* _C4_STD_TUPLE_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/std/vector.hpp b/thirdparty/ryml/ext/c4core/src/c4/std/vector.hpp deleted file mode 100644 index baaa24223..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/std/vector.hpp +++ /dev/null @@ -1,88 +0,0 @@ -#ifndef _C4_STD_VECTOR_HPP_ -#define _C4_STD_VECTOR_HPP_ - -/** @file vector.hpp provides conversion and comparison facilities - * from/between std::vector to c4::substr and c4::csubstr. - * @todo add to_span() and friends - */ - -#ifndef C4CORE_SINGLE_HEADER -#include "c4/substr.hpp" -#endif - -#include - -namespace c4 { - -//----------------------------------------------------------------------------- - -/** get a substr (writeable string view) of an existing std::vector */ -template -c4::substr to_substr(std::vector &vec) -{ - char *data = vec.empty() ? nullptr : vec.data(); // data() may or may not return a null pointer. - return c4::substr(data, vec.size()); -} - -/** get a csubstr (read-only string) view of an existing std::vector */ -template -c4::csubstr to_csubstr(std::vector const& vec) -{ - const char *data = vec.empty() ? nullptr : vec.data(); // data() may or may not return a null pointer. - return c4::csubstr(data, vec.size()); -} - -//----------------------------------------------------------------------------- -// comparisons between substrings and std::vector - -template C4_ALWAYS_INLINE bool operator!= (c4::csubstr ss, std::vector const& s) { return ss != to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator== (c4::csubstr ss, std::vector const& s) { return ss == to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator>= (c4::csubstr ss, std::vector const& s) { return ss >= to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator> (c4::csubstr ss, std::vector const& s) { return ss > to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator<= (c4::csubstr ss, std::vector const& s) { return ss <= to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator< (c4::csubstr ss, std::vector const& s) { return ss < to_csubstr(s); } - -template C4_ALWAYS_INLINE bool operator!= (std::vector const& s, c4::csubstr ss) { return ss != to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator== (std::vector const& s, c4::csubstr ss) { return ss == to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator>= (std::vector const& s, c4::csubstr ss) { return ss <= to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator> (std::vector const& s, c4::csubstr ss) { return ss < to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator<= (std::vector const& s, c4::csubstr ss) { return ss >= to_csubstr(s); } -template C4_ALWAYS_INLINE bool operator< (std::vector const& s, c4::csubstr ss) { return ss > to_csubstr(s); } - -//----------------------------------------------------------------------------- - -/** copy a std::vector to a writeable string view */ -template -inline size_t to_chars(c4::substr buf, std::vector const& s) -{ - C4_ASSERT(!buf.overlaps(to_csubstr(s))); - size_t len = buf.len < s.size() ? buf.len : s.size(); - // calling memcpy with null strings is undefined behavior - // and will wreak havoc in calling code's branches. - // see https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637 - if(len > 0) - { - memcpy(buf.str, s.data(), len); - } - return s.size(); // return the number of needed chars -} - -/** copy a string view to an existing std::vector */ -template -inline bool from_chars(c4::csubstr buf, std::vector * s) -{ - s->resize(buf.len); - C4_ASSERT(!buf.overlaps(to_csubstr(*s))); - // calling memcpy with null strings is undefined behavior - // and will wreak havoc in calling code's branches. - // see https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637 - if(buf.len > 0) - { - memcpy(&(*s)[0], buf.str, buf.len); - } - return true; -} - -} // namespace c4 - -#endif // _C4_STD_VECTOR_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/std/vector_fwd.hpp b/thirdparty/ryml/ext/c4core/src/c4/std/vector_fwd.hpp deleted file mode 100644 index a739b7d28..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/std/vector_fwd.hpp +++ /dev/null @@ -1,60 +0,0 @@ -#ifndef _C4_STD_VECTOR_FWD_HPP_ -#define _C4_STD_VECTOR_FWD_HPP_ - -/** @file vector_fwd.hpp */ - -#include - -// forward declarations for std::vector -#if defined(__GLIBCXX__) || defined(__GLIBCPP__) || defined(_MSC_VER) -#if defined(_MSC_VER) -__pragma(warning(push)) -__pragma(warning(disable : 4643)) -#endif -namespace std { -template class allocator; -template class vector; -} // namespace std -#if defined(_MSC_VER) -__pragma(warning(pop)) -#endif -#elif defined(_LIBCPP_ABI_NAMESPACE) -namespace std { -inline namespace _LIBCPP_ABI_NAMESPACE { -template class allocator; -template class vector; -} // namespace _LIBCPP_ABI_NAMESPACE -} // namespace std -#else -#error "unknown standard library" -#endif - -#ifndef C4CORE_SINGLE_HEADER -#include "c4/substr_fwd.hpp" -#endif - -namespace c4 { - -template c4::substr to_substr(std::vector &vec); -template c4::csubstr to_csubstr(std::vector const& vec); - -template bool operator!= (c4::csubstr ss, std::vector const& s); -template bool operator== (c4::csubstr ss, std::vector const& s); -template bool operator>= (c4::csubstr ss, std::vector const& s); -template bool operator> (c4::csubstr ss, std::vector const& s); -template bool operator<= (c4::csubstr ss, std::vector const& s); -template bool operator< (c4::csubstr ss, std::vector const& s); - -template bool operator!= (std::vector const& s, c4::csubstr ss); -template bool operator== (std::vector const& s, c4::csubstr ss); -template bool operator>= (std::vector const& s, c4::csubstr ss); -template bool operator> (std::vector const& s, c4::csubstr ss); -template bool operator<= (std::vector const& s, c4::csubstr ss); -template bool operator< (std::vector const& s, c4::csubstr ss); - -template size_t to_chars(c4::substr buf, std::vector const& s); -template bool from_chars(c4::csubstr buf, std::vector * s); - -} // namespace c4 - -#endif // _C4_STD_VECTOR_FWD_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/substr.hpp b/thirdparty/ryml/ext/c4core/src/c4/substr.hpp deleted file mode 100644 index ead46727e..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/substr.hpp +++ /dev/null @@ -1,2246 +0,0 @@ -#ifndef _C4_SUBSTR_HPP_ -#define _C4_SUBSTR_HPP_ - -/** @file substr.hpp read+write string views */ - -#include -#include -#include - -#include "c4/config.hpp" -#include "c4/error.hpp" -#include "c4/substr_fwd.hpp" - -#ifdef __clang__ -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wtype-limits" // disable warnings on size_t>=0, used heavily in assertions below. These assertions are a preparation step for providing the index type as a template parameter. -# pragma GCC diagnostic ignored "-Wuseless-cast" -#endif - - -namespace c4 { - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace detail { - -template -static inline void _do_reverse(C *C4_RESTRICT first, C *C4_RESTRICT last) -{ - while(last > first) - { - C tmp = *last; - *last-- = *first; - *first++ = tmp; - } -} - -} // namespace detail - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// utility macros to deuglify SFINAE code; undefined after the class. -// https://stackoverflow.com/questions/43051882/how-to-disable-a-class-member-funrtion-for-certain-template-types -#define C4_REQUIRE_RW(ret_type) \ - template \ - typename std::enable_if< ! std::is_const::value, ret_type>::type -// non-const-to-const -#define C4_NC2C(ty) \ - typename std::enable_if::value && ( ! std::is_const::value), ty>::type - - -/** a non-owning string-view, consisting of a character pointer - * and a length. - * - * @note The pointer is explicitly restricted. - * @note Because of a C++ limitation, there cannot coexist overloads for - * constructing from a char[N] and a char*; the latter will always be chosen - * by the compiler. To construct an object of this type, call to_substr() or - * to_csubstr(). For a more detailed explanation on why the overloads cannot - * coexist, see http://cplusplus.bordoon.com/specializeForCharacterArrays.html - * - * @see to_substr() - * @see to_csubstr() - */ -template -struct C4CORE_EXPORT basic_substring -{ -public: - - /** a restricted pointer to the first character of the substring */ - C * C4_RESTRICT str; - /** the length of the substring */ - size_t len; - -public: - - /** @name Types */ - /** @{ */ - - using CC = typename std::add_const::type; //!< CC=const char - using NCC_ = typename std::remove_const::type; //!< NCC_=non const char - - using ro_substr = basic_substring; - using rw_substr = basic_substring; - - using char_type = C; - using size_type = size_t; - - using iterator = C*; - using const_iterator = CC*; - - enum : size_t { npos = (size_t)-1, NONE = (size_t)-1 }; - - /// convert automatically to substring of const C - operator ro_substr () const { ro_substr s(str, len); return s; } - - /** @} */ - -public: - - /** @name Default construction and assignment */ - /** @{ */ - - constexpr basic_substring() : str(nullptr), len(0) {} - - constexpr basic_substring(basic_substring const&) = default; - constexpr basic_substring(basic_substring &&) = default; - constexpr basic_substring(std::nullptr_t) : str(nullptr), len(0) {} - - basic_substring& operator= (basic_substring const&) = default; - basic_substring& operator= (basic_substring &&) = default; - basic_substring& operator= (std::nullptr_t) { str = nullptr; len = 0; return *this; } - - /** @} */ - -public: - - /** @name Construction and assignment from characters with the same type */ - /** @{ */ - - //basic_substring(C *s_) : str(s_), len(s_ ? strlen(s_) : 0) {} - /** the overload for receiving a single C* pointer will always - * hide the array[N] overload. So it is disabled. If you want to - * construct a substr from a single pointer containing a C-style string, - * you can call c4::to_substr()/c4::to_csubstr(). - * @see c4::to_substr() - * @see c4::to_csubstr() */ - template - constexpr basic_substring(C (&s_)[N]) noexcept : str(s_), len(N-1) {} - basic_substring(C *s_, size_t len_) : str(s_), len(len_) { C4_ASSERT(str || !len_); } - basic_substring(C *beg_, C *end_) : str(beg_), len(static_cast(end_ - beg_)) { C4_ASSERT(end_ >= beg_); } - - //basic_substring& operator= (C *s_) { this->assign(s_); return *this; } - template - basic_substring& operator= (C (&s_)[N]) { this->assign(s_); return *this; } - - //void assign(C *s_) { str = (s_); len = (s_ ? strlen(s_) : 0); } - /** the overload for receiving a single C* pointer will always - * hide the array[N] overload. So it is disabled. If you want to - * construct a substr from a single pointer containing a C-style string, - * you can call c4::to_substr()/c4::to_csubstr(). - * @see c4::to_substr() - * @see c4::to_csubstr() */ - template - void assign(C (&s_)[N]) { str = (s_); len = (N-1); } - void assign(C *s_, size_t len_) { str = s_; len = len_; C4_ASSERT(str || !len_); } - void assign(C *beg_, C *end_) { C4_ASSERT(end_ >= beg_); str = (beg_); len = (end_ - beg_); } - - void clear() { str = nullptr; len = 0; } - - /** @} */ - -public: - - /** @name Construction from non-const characters */ - /** @{ */ - - // when the char type is const, allow construction and assignment from non-const chars - - /** only available when the char type is const */ - template explicit basic_substring(C4_NC2C(U) (&s_)[N]) { str = s_; len = N-1; } - /** only available when the char type is const */ - template< class U=NCC_> basic_substring(C4_NC2C(U) *s_, size_t len_) { str = s_; len = len_; } - /** only available when the char type is const */ - template< class U=NCC_> basic_substring(C4_NC2C(U) *beg_, C4_NC2C(U) *end_) { C4_ASSERT(end_ >= beg_); str = beg_; len = end_ - beg_; } - - /** only available when the char type is const */ - template void assign(C4_NC2C(U) (&s_)[N]) { str = s_; len = N-1; } - /** only available when the char type is const */ - template< class U=NCC_> void assign(C4_NC2C(U) *s_, size_t len_) { str = s_; len = len_; } - /** only available when the char type is const */ - template< class U=NCC_> void assign(C4_NC2C(U) *beg_, C4_NC2C(U) *end_) { C4_ASSERT(end_ >= beg_); str = beg_; len = end_ - beg_; } - - /** only available when the char type is const */ - template - basic_substring& operator=(C4_NC2C(U) (&s_)[N]) { str = s_; len = N-1; return *this; } - - /** @} */ - -public: - - /** @name Standard accessor methods */ - /** @{ */ - - C4_ALWAYS_INLINE C4_PURE bool has_str() const noexcept { return ! empty() && str[0] != C(0); } - C4_ALWAYS_INLINE C4_PURE bool empty() const noexcept { return (len == 0 || str == nullptr); } - C4_ALWAYS_INLINE C4_PURE bool not_empty() const noexcept { return (len != 0 && str != nullptr); } - C4_ALWAYS_INLINE C4_PURE size_t size() const noexcept { return len; } - - C4_ALWAYS_INLINE C4_PURE iterator begin() noexcept { return str; } - C4_ALWAYS_INLINE C4_PURE iterator end () noexcept { return str + len; } - - C4_ALWAYS_INLINE C4_PURE const_iterator begin() const noexcept { return str; } - C4_ALWAYS_INLINE C4_PURE const_iterator end () const noexcept { return str + len; } - - C4_ALWAYS_INLINE C4_PURE C * data() noexcept { return str; } - C4_ALWAYS_INLINE C4_PURE C const* data() const noexcept { return str; } - - C4_ALWAYS_INLINE C4_PURE C & operator[] (size_t i) noexcept { C4_ASSERT(i >= 0 && i < len); return str[i]; } - C4_ALWAYS_INLINE C4_PURE C const& operator[] (size_t i) const noexcept { C4_ASSERT(i >= 0 && i < len); return str[i]; } - - C4_ALWAYS_INLINE C4_PURE C & front() noexcept { C4_ASSERT(len > 0 && str != nullptr); return *str; } - C4_ALWAYS_INLINE C4_PURE C const& front() const noexcept { C4_ASSERT(len > 0 && str != nullptr); return *str; } - - C4_ALWAYS_INLINE C4_PURE C & back() noexcept { C4_ASSERT(len > 0 && str != nullptr); return *(str + len - 1); } - C4_ALWAYS_INLINE C4_PURE C const& back() const noexcept { C4_ASSERT(len > 0 && str != nullptr); return *(str + len - 1); } - - /** @} */ - -public: - - /** @name Comparison methods */ - /** @{ */ - - C4_PURE int compare(C const c) const noexcept - { - C4_XASSERT((str != nullptr) || len == 0); - if(C4_LIKELY(str != nullptr && len > 0)) - return (*str != c) ? *str - c : (static_cast(len) - 1); - else - return -1; - } - - C4_PURE int compare(const char *C4_RESTRICT that, size_t sz) const noexcept - { - C4_XASSERT(that || sz == 0); - C4_XASSERT(str || len == 0); - if(C4_LIKELY(str && that)) - { - { - const size_t min = len < sz ? len : sz; - for(size_t i = 0; i < min; ++i) - if(str[i] != that[i]) - return str[i] < that[i] ? -1 : 1; - } - if(len < sz) - return -1; - else if(len == sz) - return 0; - else - return 1; - } - else if(len == sz) - { - C4_XASSERT(len == 0 && sz == 0); - return 0; - } - return len < sz ? -1 : 1; - } - - C4_ALWAYS_INLINE C4_PURE int compare(ro_substr const that) const noexcept { return this->compare(that.str, that.len); } - - C4_ALWAYS_INLINE C4_PURE bool operator== (std::nullptr_t) const noexcept { return str == nullptr; } - C4_ALWAYS_INLINE C4_PURE bool operator!= (std::nullptr_t) const noexcept { return str != nullptr; } - - C4_ALWAYS_INLINE C4_PURE bool operator== (C const c) const noexcept { return this->compare(c) == 0; } - C4_ALWAYS_INLINE C4_PURE bool operator!= (C const c) const noexcept { return this->compare(c) != 0; } - C4_ALWAYS_INLINE C4_PURE bool operator< (C const c) const noexcept { return this->compare(c) < 0; } - C4_ALWAYS_INLINE C4_PURE bool operator> (C const c) const noexcept { return this->compare(c) > 0; } - C4_ALWAYS_INLINE C4_PURE bool operator<= (C const c) const noexcept { return this->compare(c) <= 0; } - C4_ALWAYS_INLINE C4_PURE bool operator>= (C const c) const noexcept { return this->compare(c) >= 0; } - - template C4_ALWAYS_INLINE C4_PURE bool operator== (basic_substring const that) const noexcept { return this->compare(that) == 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator!= (basic_substring const that) const noexcept { return this->compare(that) != 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator< (basic_substring const that) const noexcept { return this->compare(that) < 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator> (basic_substring const that) const noexcept { return this->compare(that) > 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator<= (basic_substring const that) const noexcept { return this->compare(that) <= 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator>= (basic_substring const that) const noexcept { return this->compare(that) >= 0; } - - template C4_ALWAYS_INLINE C4_PURE bool operator== (const char (&that)[N]) const noexcept { return this->compare(that, N-1) == 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator!= (const char (&that)[N]) const noexcept { return this->compare(that, N-1) != 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator< (const char (&that)[N]) const noexcept { return this->compare(that, N-1) < 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator> (const char (&that)[N]) const noexcept { return this->compare(that, N-1) > 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator<= (const char (&that)[N]) const noexcept { return this->compare(that, N-1) <= 0; } - template C4_ALWAYS_INLINE C4_PURE bool operator>= (const char (&that)[N]) const noexcept { return this->compare(that, N-1) >= 0; } - - /** @} */ - -public: - - /** @name Sub-selection methods */ - /** @{ */ - - /** true if *this is a substring of that (ie, from the same buffer) */ - C4_ALWAYS_INLINE C4_PURE bool is_sub(ro_substr const that) const noexcept - { - return that.is_super(*this); - } - - /** true if that is a substring of *this (ie, from the same buffer) */ - C4_ALWAYS_INLINE C4_PURE bool is_super(ro_substr const that) const noexcept - { - if(C4_LIKELY(len > 0)) - return that.str >= str && that.str+that.len <= str+len; - else - return that.len == 0 && that.str == str && str != nullptr; - } - - /** true if there is overlap of at least one element between that and *this */ - C4_ALWAYS_INLINE C4_PURE bool overlaps(ro_substr const that) const noexcept - { - // thanks @timwynants - return that.str+that.len > str && that.str < str+len; - } - -public: - - /** return [first,len[ */ - C4_ALWAYS_INLINE C4_PURE basic_substring sub(size_t first) const noexcept - { - C4_ASSERT(first >= 0 && first <= len); - return basic_substring(str + first, len - first); - } - - /** return [first,first+num[. If num==npos, return [first,len[ */ - C4_ALWAYS_INLINE C4_PURE basic_substring sub(size_t first, size_t num) const noexcept - { - C4_ASSERT(first >= 0 && first <= len); - C4_ASSERT((num >= 0 && num <= len) || (num == npos)); - size_t rnum = num != npos ? num : len - first; - C4_ASSERT((first >= 0 && first + rnum <= len) || (num == 0)); - return basic_substring(str + first, rnum); - } - - /** return [first,last[. If last==npos, return [first,len[ */ - C4_ALWAYS_INLINE C4_PURE basic_substring range(size_t first, size_t last=npos) const noexcept - { - C4_ASSERT(first >= 0 && first <= len); - last = last != npos ? last : len; - C4_ASSERT(first <= last); - C4_ASSERT(last >= 0 && last <= len); - return basic_substring(str + first, last - first); - } - - /** return the first @p num elements: [0,num[*/ - C4_ALWAYS_INLINE C4_PURE basic_substring first(size_t num) const noexcept - { - C4_ASSERT(num <= len || num == npos); - return basic_substring(str, num != npos ? num : len); - } - - /** return the last @num elements: [len-num,len[*/ - C4_ALWAYS_INLINE C4_PURE basic_substring last(size_t num) const noexcept - { - C4_ASSERT(num <= len || num == npos); - return num != npos ? - basic_substring(str + len - num, num) : - *this; - } - - /** offset from the ends: return [left,len-right[ ; ie, trim a - number of characters from the left and right. This is - equivalent to python's negative list indices. */ - C4_ALWAYS_INLINE C4_PURE basic_substring offs(size_t left, size_t right) const noexcept - { - C4_ASSERT(left >= 0 && left <= len); - C4_ASSERT(right >= 0 && right <= len); - C4_ASSERT(left <= len - right + 1); - return basic_substring(str + left, len - right - left); - } - - /** return [0, pos[ . Same as .first(pos), but provided for compatibility with .right_of() */ - C4_ALWAYS_INLINE C4_PURE basic_substring left_of(size_t pos) const noexcept - { - C4_ASSERT(pos <= len || pos == npos); - return (pos != npos) ? - basic_substring(str, pos) : - *this; - } - - /** return [0, pos+include_pos[ . Same as .first(pos+1), but provided for compatibility with .right_of() */ - C4_ALWAYS_INLINE C4_PURE basic_substring left_of(size_t pos, bool include_pos) const noexcept - { - C4_ASSERT(pos <= len || pos == npos); - return (pos != npos) ? - basic_substring(str, pos+include_pos) : - *this; - } - - /** return [pos+1, len[ */ - C4_ALWAYS_INLINE C4_PURE basic_substring right_of(size_t pos) const noexcept - { - C4_ASSERT(pos <= len || pos == npos); - return (pos != npos) ? - basic_substring(str + (pos + 1), len - (pos + 1)) : - basic_substring(str + len, size_t(0)); - } - - /** return [pos+!include_pos, len[ */ - C4_ALWAYS_INLINE C4_PURE basic_substring right_of(size_t pos, bool include_pos) const noexcept - { - C4_ASSERT(pos <= len || pos == npos); - return (pos != npos) ? - basic_substring(str + (pos + !include_pos), len - (pos + !include_pos)) : - basic_substring(str + len, size_t(0)); - } - -public: - - /** given @p subs a substring of the current string, get the - * portion of the current string to the left of it */ - C4_ALWAYS_INLINE C4_PURE basic_substring left_of(ro_substr const subs) const noexcept - { - C4_ASSERT(is_super(subs) || subs.empty()); - auto ssb = subs.begin(); - auto b = begin(); - auto e = end(); - if(ssb >= b && ssb <= e) - return sub(0, static_cast(ssb - b)); - else - return sub(0, 0); - } - - /** given @p subs a substring of the current string, get the - * portion of the current string to the right of it */ - C4_ALWAYS_INLINE C4_PURE basic_substring right_of(ro_substr const subs) const noexcept - { - C4_ASSERT(is_super(subs) || subs.empty()); - auto sse = subs.end(); - auto b = begin(); - auto e = end(); - if(sse >= b && sse <= e) - return sub(static_cast(sse - b), static_cast(e - sse)); - else - return sub(0, 0); - } - - /** @} */ - -public: - - /** @name Removing characters (trim()) / patterns (strip()) from the tips of the string */ - /** @{ */ - - /** trim left */ - basic_substring triml(const C c) const - { - if( ! empty()) - { - size_t pos = first_not_of(c); - if(pos != npos) - return sub(pos); - } - return sub(0, 0); - } - /** trim left ANY of the characters. - * @see stripl() to remove a pattern from the left */ - basic_substring triml(ro_substr chars) const - { - if( ! empty()) - { - size_t pos = first_not_of(chars); - if(pos != npos) - return sub(pos); - } - return sub(0, 0); - } - - /** trim the character c from the right */ - basic_substring trimr(const C c) const - { - if( ! empty()) - { - size_t pos = last_not_of(c, npos); - if(pos != npos) - return sub(0, pos+1); - } - return sub(0, 0); - } - /** trim right ANY of the characters - * @see stripr() to remove a pattern from the right */ - basic_substring trimr(ro_substr chars) const - { - if( ! empty()) - { - size_t pos = last_not_of(chars, npos); - if(pos != npos) - return sub(0, pos+1); - } - return sub(0, 0); - } - - /** trim the character c left and right */ - basic_substring trim(const C c) const - { - return triml(c).trimr(c); - } - /** trim left and right ANY of the characters - * @see strip() to remove a pattern from the left and right */ - basic_substring trim(ro_substr const chars) const - { - return triml(chars).trimr(chars); - } - - /** remove a pattern from the left - * @see triml() to remove characters*/ - basic_substring stripl(ro_substr pattern) const - { - if( ! begins_with(pattern)) - return *this; - return sub(pattern.len < len ? pattern.len : len); - } - - /** remove a pattern from the right - * @see trimr() to remove characters*/ - basic_substring stripr(ro_substr pattern) const - { - if( ! ends_with(pattern)) - return *this; - return left_of(len - (pattern.len < len ? pattern.len : len)); - } - - /** @} */ - -public: - - /** @name Lookup methods */ - /** @{ */ - - inline size_t find(const C c, size_t start_pos=0) const - { - return first_of(c, start_pos); - } - inline size_t find(ro_substr pattern, size_t start_pos=0) const - { - C4_ASSERT(start_pos == npos || (start_pos >= 0 && start_pos <= len)); - if(len < pattern.len) return npos; - for(size_t i = start_pos, e = len - pattern.len + 1; i < e; ++i) - { - bool gotit = true; - for(size_t j = 0; j < pattern.len; ++j) - { - C4_ASSERT(i + j < len); - if(str[i + j] != pattern.str[j]) - { - gotit = false; - break; - } - } - if(gotit) - { - return i; - } - } - return npos; - } - -public: - - /** count the number of occurrences of c */ - inline size_t count(const C c, size_t pos=0) const - { - C4_ASSERT(pos >= 0 && pos <= len); - size_t num = 0; - pos = find(c, pos); - while(pos != npos) - { - ++num; - pos = find(c, pos + 1); - } - return num; - } - - /** count the number of occurrences of s */ - inline size_t count(ro_substr c, size_t pos=0) const - { - C4_ASSERT(pos >= 0 && pos <= len); - size_t num = 0; - pos = find(c, pos); - while(pos != npos) - { - ++num; - pos = find(c, pos + c.len); - } - return num; - } - - /** get the substr consisting of the first occurrence of @p c after @p pos, or an empty substr if none occurs */ - inline basic_substring select(const C c, size_t pos=0) const - { - pos = find(c, pos); - return pos != npos ? sub(pos, 1) : basic_substring(); - } - - /** get the substr consisting of the first occurrence of @p pattern after @p pos, or an empty substr if none occurs */ - inline basic_substring select(ro_substr pattern, size_t pos=0) const - { - pos = find(pattern, pos); - return pos != npos ? sub(pos, pattern.len) : basic_substring(); - } - -public: - - struct first_of_any_result - { - size_t which; - size_t pos; - inline operator bool() const { return which != NONE && pos != npos; } - }; - - first_of_any_result first_of_any(ro_substr s0, ro_substr s1) const - { - ro_substr s[2] = {s0, s1}; - return first_of_any_iter(&s[0], &s[0] + 2); - } - - first_of_any_result first_of_any(ro_substr s0, ro_substr s1, ro_substr s2) const - { - ro_substr s[3] = {s0, s1, s2}; - return first_of_any_iter(&s[0], &s[0] + 3); - } - - first_of_any_result first_of_any(ro_substr s0, ro_substr s1, ro_substr s2, ro_substr s3) const - { - ro_substr s[4] = {s0, s1, s2, s3}; - return first_of_any_iter(&s[0], &s[0] + 4); - } - - first_of_any_result first_of_any(ro_substr s0, ro_substr s1, ro_substr s2, ro_substr s3, ro_substr s4) const - { - ro_substr s[5] = {s0, s1, s2, s3, s4}; - return first_of_any_iter(&s[0], &s[0] + 5); - } - - template - first_of_any_result first_of_any_iter(It first_span, It last_span) const - { - for(size_t i = 0; i < len; ++i) - { - size_t curr = 0; - for(It it = first_span; it != last_span; ++curr, ++it) - { - auto const& chars = *it; - if((i + chars.len) > len) continue; - bool gotit = true; - for(size_t j = 0; j < chars.len; ++j) - { - C4_ASSERT(i + j < len); - if(str[i + j] != chars[j]) - { - gotit = false; - break; - } - } - if(gotit) - { - return {curr, i}; - } - } - } - return {NONE, npos}; - } - -public: - - /** true if the first character of the string is @p c */ - bool begins_with(const C c) const - { - return len > 0 ? str[0] == c : false; - } - - /** true if the first @p num characters of the string are @p c */ - bool begins_with(const C c, size_t num) const - { - if(len < num) - { - return false; - } - for(size_t i = 0; i < num; ++i) - { - if(str[i] != c) - { - return false; - } - } - return true; - } - - /** true if the string begins with the given @p pattern */ - bool begins_with(ro_substr pattern) const - { - if(len < pattern.len) - { - return false; - } - for(size_t i = 0; i < pattern.len; ++i) - { - if(str[i] != pattern[i]) - { - return false; - } - } - return true; - } - - /** true if the first character of the string is any of the given @p chars */ - bool begins_with_any(ro_substr chars) const - { - if(len == 0) - { - return false; - } - for(size_t i = 0; i < chars.len; ++i) - { - if(str[0] == chars.str[i]) - { - return true; - } - } - return false; - } - - /** true if the last character of the string is @p c */ - bool ends_with(const C c) const - { - return len > 0 ? str[len-1] == c : false; - } - - /** true if the last @p num characters of the string are @p c */ - bool ends_with(const C c, size_t num) const - { - if(len < num) - { - return false; - } - for(size_t i = len - num; i < len; ++i) - { - if(str[i] != c) - { - return false; - } - } - return true; - } - - /** true if the string ends with the given @p pattern */ - bool ends_with(ro_substr pattern) const - { - if(len < pattern.len) - { - return false; - } - for(size_t i = 0, s = len-pattern.len; i < pattern.len; ++i) - { - if(str[s+i] != pattern[i]) - { - return false; - } - } - return true; - } - - /** true if the last character of the string is any of the given @p chars */ - bool ends_with_any(ro_substr chars) const - { - if(len == 0) - { - return false; - } - for(size_t i = 0; i < chars.len; ++i) - { - if(str[len - 1] == chars[i]) - { - return true; - } - } - return false; - } - -public: - - /** @return the first position where c is found in the string, or npos if none is found */ - size_t first_of(const C c, size_t start=0) const - { - C4_ASSERT(start == npos || (start >= 0 && start <= len)); - for(size_t i = start; i < len; ++i) - { - if(str[i] == c) - return i; - } - return npos; - } - - /** @return the last position where c is found in the string, or npos if none is found */ - size_t last_of(const C c, size_t start=npos) const - { - C4_ASSERT(start == npos || (start >= 0 && start <= len)); - if(start == npos) - start = len; - for(size_t i = start-1; i != size_t(-1); --i) - { - if(str[i] == c) - return i; - } - return npos; - } - - /** @return the first position where ANY of the chars is found in the string, or npos if none is found */ - size_t first_of(ro_substr chars, size_t start=0) const - { - C4_ASSERT(start == npos || (start >= 0 && start <= len)); - for(size_t i = start; i < len; ++i) - { - for(size_t j = 0; j < chars.len; ++j) - { - if(str[i] == chars[j]) - return i; - } - } - return npos; - } - - /** @return the last position where ANY of the chars is found in the string, or npos if none is found */ - size_t last_of(ro_substr chars, size_t start=npos) const - { - C4_ASSERT(start == npos || (start >= 0 && start <= len)); - if(start == npos) - start = len; - for(size_t i = start-1; i != size_t(-1); --i) - { - for(size_t j = 0; j < chars.len; ++j) - { - if(str[i] == chars[j]) - return i; - } - } - return npos; - } - -public: - - size_t first_not_of(const C c, size_t start=0) const - { - C4_ASSERT((start >= 0 && start <= len) || (start == len && len == 0)); - for(size_t i = start; i < len; ++i) - { - if(str[i] != c) - return i; - } - return npos; - } - - size_t last_not_of(const C c, size_t start=npos) const - { - C4_ASSERT(start == npos || (start >= 0 && start <= len)); - if(start == npos) - start = len; - for(size_t i = start-1; i != size_t(-1); --i) - { - if(str[i] != c) - return i; - } - return npos; - } - - size_t first_not_of(ro_substr chars, size_t start=0) const - { - C4_ASSERT((start >= 0 && start <= len) || (start == len && len == 0)); - for(size_t i = start; i < len; ++i) - { - bool gotit = true; - for(size_t j = 0; j < chars.len; ++j) - { - if(str[i] == chars.str[j]) - { - gotit = false; - break; - } - } - if(gotit) - { - return i; - } - } - return npos; - } - - size_t last_not_of(ro_substr chars, size_t start=npos) const - { - C4_ASSERT(start == npos || (start >= 0 && start <= len)); - if(start == npos) - start = len; - for(size_t i = start-1; i != size_t(-1); --i) - { - bool gotit = true; - for(size_t j = 0; j < chars.len; ++j) - { - if(str[i] == chars.str[j]) - { - gotit = false; - break; - } - } - if(gotit) - { - return i; - } - } - return npos; - } - - /** @} */ - -public: - - /** @name Range lookup methods */ - /** @{ */ - - /** get the range delimited by an open-close pair of characters. - * @note There must be no nested pairs. - * @note No checks for escapes are performed. */ - basic_substring pair_range(CC open, CC close) const - { - size_t b = find(open); - if(b == npos) - return basic_substring(); - size_t e = find(close, b+1); - if(e == npos) - return basic_substring(); - basic_substring ret = range(b, e+1); - C4_ASSERT(ret.sub(1).find(open) == npos); - return ret; - } - - /** get the range delimited by a single open-close character (eg, quotes). - * @note The open-close character can be escaped. */ - basic_substring pair_range_esc(CC open_close, CC escape=CC('\\')) - { - size_t b = find(open_close); - if(b == npos) return basic_substring(); - for(size_t i = b+1; i < len; ++i) - { - CC c = str[i]; - if(c == open_close) - { - if(str[i-1] != escape) - { - return range(b, i+1); - } - } - } - return basic_substring(); - } - - /** get the range delimited by an open-close pair of characters, - * with possibly nested occurrences. No checks for escapes are - * performed. */ - basic_substring pair_range_nested(CC open, CC close) const - { - size_t b = find(open); - if(b == npos) return basic_substring(); - size_t e, curr = b+1, count = 0; - const char both[] = {open, close, '\0'}; - while((e = first_of(both, curr)) != npos) - { - if(str[e] == open) - { - ++count; - curr = e+1; - } - else if(str[e] == close) - { - if(count == 0) return range(b, e+1); - --count; - curr = e+1; - } - } - return basic_substring(); - } - - basic_substring unquoted() const - { - constexpr const C dq('"'), sq('\''); - if(len >= 2 && (str[len - 2] != C('\\')) && - ((begins_with(sq) && ends_with(sq)) - || - (begins_with(dq) && ends_with(dq)))) - { - return range(1, len -1); - } - return *this; - } - - /** @} */ - -public: - - /** @name Number-matching query methods */ - /** @{ */ - - /** @return true if the substring contents are a floating-point or integer number. - * @note any leading or trailing whitespace will return false. */ - bool is_number() const - { - if(empty() || (first_non_empty_span().empty())) - return false; - if(first_uint_span() == *this) - return true; - if(first_int_span() == *this) - return true; - if(first_real_span() == *this) - return true; - return false; - } - - /** @return true if the substring contents are a real number. - * @note any leading or trailing whitespace will return false. */ - bool is_real() const - { - if(empty() || (first_non_empty_span().empty())) - return false; - if(first_real_span() == *this) - return true; - return false; - } - - /** @return true if the substring contents are an integer number. - * @note any leading or trailing whitespace will return false. */ - bool is_integer() const - { - if(empty() || (first_non_empty_span().empty())) - return false; - if(first_uint_span() == *this) - return true; - if(first_int_span() == *this) - return true; - return false; - } - - /** @return true if the substring contents are an unsigned integer number. - * @note any leading or trailing whitespace will return false. */ - bool is_unsigned_integer() const - { - if(empty() || (first_non_empty_span().empty())) - return false; - if(first_uint_span() == *this) - return true; - return false; - } - - /** get the first span consisting exclusively of non-empty characters */ - basic_substring first_non_empty_span() const - { - constexpr const ro_substr empty_chars(" \n\r\t"); - size_t pos = first_not_of(empty_chars); - if(pos == npos) - return first(0); - auto ret = sub(pos); - pos = ret.first_of(empty_chars); - return ret.first(pos); - } - - /** get the first span which can be interpreted as an unsigned integer */ - basic_substring first_uint_span() const - { - basic_substring ne = first_non_empty_span(); - if(ne.empty()) - return ne; - if(ne.str[0] == '-') - return first(0); - size_t skip_start = (ne.str[0] == '+') ? 1 : 0; - return ne._first_integral_span(skip_start); - } - - /** get the first span which can be interpreted as a signed integer */ - basic_substring first_int_span() const - { - basic_substring ne = first_non_empty_span(); - if(ne.empty()) - return ne; - size_t skip_start = (ne.str[0] == '+' || ne.str[0] == '-') ? 1 : 0; - return ne._first_integral_span(skip_start); - } - - basic_substring _first_integral_span(size_t skip_start) const - { - C4_ASSERT(!empty()); - if(skip_start == len) - return first(0); - C4_ASSERT(skip_start < len); - if(len >= skip_start + 3) - { - if(str[skip_start] != '0') - { - for(size_t i = skip_start; i < len; ++i) - { - char c = str[i]; - if(c < '0' || c > '9') - return i > skip_start && _is_delim_char(c) ? first(i) : first(0); - } - } - else - { - char next = str[skip_start + 1]; - if(next == 'x' || next == 'X') - { - skip_start += 2; - for(size_t i = skip_start; i < len; ++i) - { - const char c = str[i]; - if( ! _is_hex_char(c)) - return i > skip_start && _is_delim_char(c) ? first(i) : first(0); - } - return *this; - } - else if(next == 'b' || next == 'B') - { - skip_start += 2; - for(size_t i = skip_start; i < len; ++i) - { - const char c = str[i]; - if(c != '0' && c != '1') - return i > skip_start && _is_delim_char(c) ? first(i) : first(0); - } - return *this; - } - else if(next == 'o' || next == 'O') - { - skip_start += 2; - for(size_t i = skip_start; i < len; ++i) - { - const char c = str[i]; - if(c < '0' || c > '7') - return i > skip_start && _is_delim_char(c) ? first(i) : first(0); - } - return *this; - } - } - } - // must be a decimal, or it is not a an number - for(size_t i = skip_start; i < len; ++i) - { - const char c = str[i]; - if(c < '0' || c > '9') - return i > skip_start && _is_delim_char(c) ? first(i) : first(0); - } - return *this; - } - - /** get the first span which can be interpreted as a real (floating-point) number */ - basic_substring first_real_span() const - { - basic_substring ne = first_non_empty_span(); - if(ne.empty()) - return ne; - size_t skip_start = (ne.str[0] == '+' || ne.str[0] == '-'); - C4_ASSERT(skip_start == 0 || skip_start == 1); - // if we have at least three digits after the leading sign, it - // can be decimal, or hex, or bin or oct. Ex: - // non-decimal: 0x0, 0b0, 0o0 - // decimal: 1.0, 10., 1e1, 100, inf, nan, infinity - if(ne.len >= skip_start+3) - { - // if it does not have leading 0, it must be decimal, or it is not a real - if(ne.str[skip_start] != '0') - { - if(ne.str[skip_start] == 'i') // is it infinity or inf? - { - basic_substring word = ne._word_follows(skip_start + 1, "nfinity"); - if(word.len) - return word; - return ne._word_follows(skip_start + 1, "nf"); - } - else if(ne.str[skip_start] == 'n') // is it nan? - { - return ne._word_follows(skip_start + 1, "an"); - } - else // must be a decimal, or it is not a real - { - return ne._first_real_span_dec(skip_start); - } - } - else // starts with 0. is it 0x, 0b or 0o? - { - const char next = ne.str[skip_start + 1]; - // hexadecimal - if(next == 'x' || next == 'X') - return ne._first_real_span_hex(skip_start + 2); - // binary - else if(next == 'b' || next == 'B') - return ne._first_real_span_bin(skip_start + 2); - // octal - else if(next == 'o' || next == 'O') - return ne._first_real_span_oct(skip_start + 2); - // none of the above. may still be a decimal. - else - return ne._first_real_span_dec(skip_start); // do not skip the 0. - } - } - // less than 3 chars after the leading sign. It is either a - // decimal or it is not a real. (cannot be any of 0x0, etc). - return ne._first_real_span_dec(skip_start); - } - - /** true if the character is a delimiter character *at the end* */ - static constexpr C4_ALWAYS_INLINE C4_CONST bool _is_delim_char(char c) noexcept - { - return c == ' ' || c == '\n' - || c == ']' || c == ')' || c == '}' - || c == ',' || c == ';' || c == '\r' || c == '\t' || c == '\0'; - } - - /** true if the character is in [0-9a-fA-F] */ - static constexpr C4_ALWAYS_INLINE C4_CONST bool _is_hex_char(char c) noexcept - { - return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); - } - - C4_NO_INLINE C4_PURE basic_substring _word_follows(size_t pos, csubstr word) const noexcept - { - size_t posend = pos + word.len; - if(len >= posend && sub(pos, word.len) == word) - if(len == posend || _is_delim_char(str[posend])) - return first(posend); - return first(0); - } - - // this function is declared inside the class to avoid a VS error with __declspec(dllimport) - C4_NO_INLINE C4_PURE basic_substring _first_real_span_dec(size_t pos) const noexcept - { - bool intchars = false; - bool fracchars = false; - bool powchars; - // integral part - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c >= '0' && c <= '9') - { - intchars = true; - } - else if(c == '.') - { - ++pos; - goto fractional_part_dec; - } - else if(c == 'e' || c == 'E') - { - ++pos; - goto power_part_dec; - } - else if(_is_delim_char(c)) - { - return intchars ? first(pos) : first(0); - } - else - { - return first(0); - } - } - // no . or p were found; this is either an integral number - // or not a number at all - return intchars ? - *this : - first(0); - fractional_part_dec: - C4_ASSERT(pos > 0); - C4_ASSERT(str[pos - 1] == '.'); - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c >= '0' && c <= '9') - { - fracchars = true; - } - else if(c == 'e' || c == 'E') - { - ++pos; - goto power_part_dec; - } - else if(_is_delim_char(c)) - { - return intchars || fracchars ? first(pos) : first(0); - } - else - { - return first(0); - } - } - return intchars || fracchars ? - *this : - first(0); - power_part_dec: - C4_ASSERT(pos > 0); - C4_ASSERT(str[pos - 1] == 'e' || str[pos - 1] == 'E'); - // either a + or a - is expected here, followed by more chars. - // also, using (pos+1) in this check will cause an early - // return when no more chars follow the sign. - if(len <= (pos+1) || ((!intchars) && (!fracchars))) - return first(0); - ++pos; // this was the sign. - // ... so the (pos+1) ensures that we enter the loop and - // hence that there exist chars in the power part - powchars = false; - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c >= '0' && c <= '9') - powchars = true; - else if(powchars && _is_delim_char(c)) - return first(pos); - else - return first(0); - } - return *this; - } - - // this function is declared inside the class to avoid a VS error with __declspec(dllimport) - C4_NO_INLINE C4_PURE basic_substring _first_real_span_hex(size_t pos) const noexcept - { - bool intchars = false; - bool fracchars = false; - bool powchars; - // integral part - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(_is_hex_char(c)) - { - intchars = true; - } - else if(c == '.') - { - ++pos; - goto fractional_part_hex; - } - else if(c == 'p' || c == 'P') - { - ++pos; - goto power_part_hex; - } - else if(_is_delim_char(c)) - { - return intchars ? first(pos) : first(0); - } - else - { - return first(0); - } - } - // no . or p were found; this is either an integral number - // or not a number at all - return intchars ? - *this : - first(0); - fractional_part_hex: - C4_ASSERT(pos > 0); - C4_ASSERT(str[pos - 1] == '.'); - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(_is_hex_char(c)) - { - fracchars = true; - } - else if(c == 'p' || c == 'P') - { - ++pos; - goto power_part_hex; - } - else if(_is_delim_char(c)) - { - return intchars || fracchars ? first(pos) : first(0); - } - else - { - return first(0); - } - } - return intchars || fracchars ? - *this : - first(0); - power_part_hex: - C4_ASSERT(pos > 0); - C4_ASSERT(str[pos - 1] == 'p' || str[pos - 1] == 'P'); - // either a + or a - is expected here, followed by more chars. - // also, using (pos+1) in this check will cause an early - // return when no more chars follow the sign. - if(len <= (pos+1) || (str[pos] != '+' && str[pos] != '-') || ((!intchars) && (!fracchars))) - return first(0); - ++pos; // this was the sign. - // ... so the (pos+1) ensures that we enter the loop and - // hence that there exist chars in the power part - powchars = false; - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c >= '0' && c <= '9') - powchars = true; - else if(powchars && _is_delim_char(c)) - return first(pos); - else - return first(0); - } - return *this; - } - - // this function is declared inside the class to avoid a VS error with __declspec(dllimport) - C4_NO_INLINE C4_PURE basic_substring _first_real_span_bin(size_t pos) const noexcept - { - bool intchars = false; - bool fracchars = false; - bool powchars; - // integral part - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c == '0' || c == '1') - { - intchars = true; - } - else if(c == '.') - { - ++pos; - goto fractional_part_bin; - } - else if(c == 'p' || c == 'P') - { - ++pos; - goto power_part_bin; - } - else if(_is_delim_char(c)) - { - return intchars ? first(pos) : first(0); - } - else - { - return first(0); - } - } - // no . or p were found; this is either an integral number - // or not a number at all - return intchars ? - *this : - first(0); - fractional_part_bin: - C4_ASSERT(pos > 0); - C4_ASSERT(str[pos - 1] == '.'); - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c == '0' || c == '1') - { - fracchars = true; - } - else if(c == 'p' || c == 'P') - { - ++pos; - goto power_part_bin; - } - else if(_is_delim_char(c)) - { - return intchars || fracchars ? first(pos) : first(0); - } - else - { - return first(0); - } - } - return intchars || fracchars ? - *this : - first(0); - power_part_bin: - C4_ASSERT(pos > 0); - C4_ASSERT(str[pos - 1] == 'p' || str[pos - 1] == 'P'); - // either a + or a - is expected here, followed by more chars. - // also, using (pos+1) in this check will cause an early - // return when no more chars follow the sign. - if(len <= (pos+1) || (str[pos] != '+' && str[pos] != '-') || ((!intchars) && (!fracchars))) - return first(0); - ++pos; // this was the sign. - // ... so the (pos+1) ensures that we enter the loop and - // hence that there exist chars in the power part - powchars = false; - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c >= '0' && c <= '9') - powchars = true; - else if(powchars && _is_delim_char(c)) - return first(pos); - else - return first(0); - } - return *this; - } - - // this function is declared inside the class to avoid a VS error with __declspec(dllimport) - C4_NO_INLINE C4_PURE basic_substring _first_real_span_oct(size_t pos) const noexcept - { - bool intchars = false; - bool fracchars = false; - bool powchars; - // integral part - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c >= '0' && c <= '7') - { - intchars = true; - } - else if(c == '.') - { - ++pos; - goto fractional_part_oct; - } - else if(c == 'p' || c == 'P') - { - ++pos; - goto power_part_oct; - } - else if(_is_delim_char(c)) - { - return intchars ? first(pos) : first(0); - } - else - { - return first(0); - } - } - // no . or p were found; this is either an integral number - // or not a number at all - return intchars ? - *this : - first(0); - fractional_part_oct: - C4_ASSERT(pos > 0); - C4_ASSERT(str[pos - 1] == '.'); - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c >= '0' && c <= '7') - { - fracchars = true; - } - else if(c == 'p' || c == 'P') - { - ++pos; - goto power_part_oct; - } - else if(_is_delim_char(c)) - { - return intchars || fracchars ? first(pos) : first(0); - } - else - { - return first(0); - } - } - return intchars || fracchars ? - *this : - first(0); - power_part_oct: - C4_ASSERT(pos > 0); - C4_ASSERT(str[pos - 1] == 'p' || str[pos - 1] == 'P'); - // either a + or a - is expected here, followed by more chars. - // also, using (pos+1) in this check will cause an early - // return when no more chars follow the sign. - if(len <= (pos+1) || (str[pos] != '+' && str[pos] != '-') || ((!intchars) && (!fracchars))) - return first(0); - ++pos; // this was the sign. - // ... so the (pos+1) ensures that we enter the loop and - // hence that there exist chars in the power part - powchars = false; - for( ; pos < len; ++pos) - { - const char c = str[pos]; - if(c >= '0' && c <= '9') - powchars = true; - else if(powchars && _is_delim_char(c)) - return first(pos); - else - return first(0); - } - return *this; - } - - /** @} */ - -public: - - /** @name Splitting methods */ - /** @{ */ - - /** returns true if the string has not been exhausted yet, meaning - * it's ok to call next_split() again. When no instance of sep - * exists in the string, returns the full string. When the input - * is an empty string, the output string is the empty string. */ - bool next_split(C sep, size_t *C4_RESTRICT start_pos, basic_substring *C4_RESTRICT out) const - { - if(C4_LIKELY(*start_pos < len)) - { - for(size_t i = *start_pos, e = len; i < e; i++) - { - if(str[i] == sep) - { - out->assign(str + *start_pos, i - *start_pos); - *start_pos = i+1; - return true; - } - } - out->assign(str + *start_pos, len - *start_pos); - *start_pos = len + 1; - return true; - } - else - { - bool valid = len > 0 && (*start_pos == len); - if(valid && !empty() && str[len-1] == sep) - { - out->assign(str + len, (size_t)0); // the cast is needed to prevent overload ambiguity - } - else - { - out->assign(str + len + 1, (size_t)0); // the cast is needed to prevent overload ambiguity - } - *start_pos = len + 1; - return valid; - } - } - -private: - - struct split_proxy_impl - { - struct split_iterator_impl - { - split_proxy_impl const* m_proxy; - basic_substring m_str; - size_t m_pos; - NCC_ m_sep; - - split_iterator_impl(split_proxy_impl const* proxy, size_t pos, C sep) - : m_proxy(proxy), m_pos(pos), m_sep(sep) - { - _tick(); - } - - void _tick() - { - m_proxy->m_str.next_split(m_sep, &m_pos, &m_str); - } - - split_iterator_impl& operator++ () { _tick(); return *this; } - split_iterator_impl operator++ (int) { split_iterator_impl it = *this; _tick(); return it; } - - basic_substring& operator* () { return m_str; } - basic_substring* operator-> () { return &m_str; } - - bool operator!= (split_iterator_impl const& that) const - { - return !(this->operator==(that)); - } - bool operator== (split_iterator_impl const& that) const - { - C4_XASSERT((m_sep == that.m_sep) && "cannot compare split iterators with different separators"); - if(m_str.size() != that.m_str.size()) - return false; - if(m_str.data() != that.m_str.data()) - return false; - return m_pos == that.m_pos; - } - }; - - basic_substring m_str; - size_t m_start_pos; - C m_sep; - - split_proxy_impl(basic_substring str_, size_t start_pos, C sep) - : m_str(str_), m_start_pos(start_pos), m_sep(sep) - { - } - - split_iterator_impl begin() const - { - auto it = split_iterator_impl(this, m_start_pos, m_sep); - return it; - } - split_iterator_impl end() const - { - size_t pos = m_str.size() + 1; - auto it = split_iterator_impl(this, pos, m_sep); - return it; - } - }; - -public: - - using split_proxy = split_proxy_impl; - - /** a view into the splits */ - split_proxy split(C sep, size_t start_pos=0) const - { - C4_XASSERT((start_pos >= 0 && start_pos < len) || empty()); - auto ss = sub(0, len); - auto it = split_proxy(ss, start_pos, sep); - return it; - } - -public: - - /** pop right: return the first split from the right. Use - * gpop_left() to get the reciprocal part. - */ - basic_substring pop_right(C sep=C('/'), bool skip_empty=false) const - { - if(C4_LIKELY(len > 1)) - { - auto pos = last_of(sep); - if(pos != npos) - { - if(pos + 1 < len) // does not end with sep - { - return sub(pos + 1); // return from sep to end - } - else // the string ends with sep - { - if( ! skip_empty) - { - return sub(pos + 1, 0); - } - auto ppos = last_not_of(sep); // skip repeated seps - if(ppos == npos) // the string is all made of seps - { - return sub(0, 0); - } - // find the previous sep - auto pos0 = last_of(sep, ppos); - if(pos0 == npos) // only the last sep exists - { - return sub(0); // return the full string (because skip_empty is true) - } - ++pos0; - return sub(pos0); - } - } - else // no sep was found, return the full string - { - return *this; - } - } - else if(len == 1) - { - if(begins_with(sep)) - { - return sub(0, 0); - } - return *this; - } - else // an empty string - { - return basic_substring(); - } - } - - /** return the first split from the left. Use gpop_right() to get - * the reciprocal part. */ - basic_substring pop_left(C sep = C('/'), bool skip_empty=false) const - { - if(C4_LIKELY(len > 1)) - { - auto pos = first_of(sep); - if(pos != npos) - { - if(pos > 0) // does not start with sep - { - return sub(0, pos); // return everything up to it - } - else // the string starts with sep - { - if( ! skip_empty) - { - return sub(0, 0); - } - auto ppos = first_not_of(sep); // skip repeated seps - if(ppos == npos) // the string is all made of seps - { - return sub(0, 0); - } - // find the next sep - auto pos0 = first_of(sep, ppos); - if(pos0 == npos) // only the first sep exists - { - return sub(0); // return the full string (because skip_empty is true) - } - C4_XASSERT(pos0 > 0); - // return everything up to the second sep - return sub(0, pos0); - } - } - else // no sep was found, return the full string - { - return sub(0); - } - } - else if(len == 1) - { - if(begins_with(sep)) - { - return sub(0, 0); - } - return sub(0); - } - else // an empty string - { - return basic_substring(); - } - } - -public: - - /** greedy pop left. eg, csubstr("a/b/c").gpop_left('/')="c" */ - basic_substring gpop_left(C sep = C('/'), bool skip_empty=false) const - { - auto ss = pop_right(sep, skip_empty); - ss = left_of(ss); - if(ss.find(sep) != npos) - { - if(ss.ends_with(sep)) - { - if(skip_empty) - { - ss = ss.trimr(sep); - } - else - { - ss = ss.sub(0, ss.len-1); // safe to subtract because ends_with(sep) is true - } - } - } - return ss; - } - - /** greedy pop right. eg, csubstr("a/b/c").gpop_right('/')="a" */ - basic_substring gpop_right(C sep = C('/'), bool skip_empty=false) const - { - auto ss = pop_left(sep, skip_empty); - ss = right_of(ss); - if(ss.find(sep) != npos) - { - if(ss.begins_with(sep)) - { - if(skip_empty) - { - ss = ss.triml(sep); - } - else - { - ss = ss.sub(1); - } - } - } - return ss; - } - - /** @} */ - -public: - - /** @name Path-like manipulation methods */ - /** @{ */ - - basic_substring basename(C sep=C('/')) const - { - auto ss = pop_right(sep, /*skip_empty*/true); - ss = ss.trimr(sep); - return ss; - } - - basic_substring dirname(C sep=C('/')) const - { - auto ss = basename(sep); - ss = ss.empty() ? *this : left_of(ss); - return ss; - } - - C4_ALWAYS_INLINE basic_substring name_wo_extshort() const - { - return gpop_left('.'); - } - - C4_ALWAYS_INLINE basic_substring name_wo_extlong() const - { - return pop_left('.'); - } - - C4_ALWAYS_INLINE basic_substring extshort() const - { - return pop_right('.'); - } - - C4_ALWAYS_INLINE basic_substring extlong() const - { - return gpop_right('.'); - } - - /** @} */ - -public: - - /** @name Content-modification methods (only for non-const C) */ - /** @{ */ - - /** convert the string to upper-case - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(void) toupper() - { - for(size_t i = 0; i < len; ++i) - { - str[i] = static_cast(::toupper(str[i])); - } - } - - /** convert the string to lower-case - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(void) tolower() - { - for(size_t i = 0; i < len; ++i) - { - str[i] = static_cast(::tolower(str[i])); - } - } - -public: - - /** fill the entire contents with the given @p val - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(void) fill(C val) - { - for(size_t i = 0; i < len; ++i) - { - str[i] = val; - } - } - -public: - - /** set the current substring to a copy of the given csubstr - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(void) copy_from(ro_substr that, size_t ifirst=0, size_t num=npos) - { - C4_ASSERT(ifirst >= 0 && ifirst <= len); - num = num != npos ? num : len - ifirst; - num = num < that.len ? num : that.len; - C4_ASSERT(ifirst + num >= 0 && ifirst + num <= len); - // calling memcpy with null strings is undefined behavior - // and will wreak havoc in calling code's branches. - // see https://github.com/biojppm/rapidyaml/pull/264#issuecomment-1262133637 - if(num) - memcpy(str + sizeof(C) * ifirst, that.str, sizeof(C) * num); - } - -public: - - /** reverse in place - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(void) reverse() - { - if(len == 0) return; - detail::_do_reverse(str, str + len - 1); - } - - /** revert a subpart in place - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(void) reverse_sub(size_t ifirst, size_t num) - { - C4_ASSERT(ifirst >= 0 && ifirst <= len); - C4_ASSERT(ifirst + num >= 0 && ifirst + num <= len); - if(num == 0) return; - detail::_do_reverse(str + ifirst, str + ifirst + num - 1); - } - - /** revert a range in place - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(void) reverse_range(size_t ifirst, size_t ilast) - { - C4_ASSERT(ifirst >= 0 && ifirst <= len); - C4_ASSERT(ilast >= 0 && ilast <= len); - if(ifirst == ilast) return; - detail::_do_reverse(str + ifirst, str + ilast - 1); - } - -public: - - /** erase part of the string. eg, with char s[] = "0123456789", - * substr(s).erase(3, 2) = "01256789", and s is now "01245678989" - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(basic_substring) erase(size_t pos, size_t num) - { - C4_ASSERT(pos >= 0 && pos+num <= len); - size_t num_to_move = len - pos - num; - memmove(str + pos, str + pos + num, sizeof(C) * num_to_move); - return basic_substring{str, len - num}; - } - - /** @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(basic_substring) erase_range(size_t first, size_t last) - { - C4_ASSERT(first <= last); - return erase(first, static_cast(last-first)); - } - - /** erase a part of the string. - * @note @p sub must be a substring of this string - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(basic_substring) erase(ro_substr sub) - { - C4_ASSERT(is_super(sub)); - C4_ASSERT(sub.str >= str); - return erase(static_cast(sub.str - str), sub.len); - } - -public: - - /** replace every occurrence of character @p value with the character @p repl - * @return the number of characters that were replaced - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(size_t) replace(C value, C repl, size_t pos=0) - { - C4_ASSERT((pos >= 0 && pos <= len) || pos == npos); - size_t did_it = 0; - while((pos = find(value, pos)) != npos) - { - str[pos++] = repl; - ++did_it; - } - return did_it; - } - - /** replace every occurrence of each character in @p value with - * the character @p repl. - * @return the number of characters that were replaced - * @note this method requires that the string memory is writeable and is SFINAEd out for const C */ - C4_REQUIRE_RW(size_t) replace(ro_substr chars, C repl, size_t pos=0) - { - C4_ASSERT((pos >= 0 && pos <= len) || pos == npos); - size_t did_it = 0; - while((pos = first_of(chars, pos)) != npos) - { - str[pos++] = repl; - ++did_it; - } - return did_it; - } - - /** replace @p pattern with @p repl, and write the result into - * @dst. pattern and repl don't need equal sizes. - * - * @return the required size for dst. No overflow occurs if - * dst.len is smaller than the required size; this can be used to - * determine the required size for an existing container. */ - size_t replace_all(rw_substr dst, ro_substr pattern, ro_substr repl, size_t pos=0) const - { - C4_ASSERT( ! pattern.empty()); //!< @todo relax this precondition - C4_ASSERT( ! this ->overlaps(dst)); //!< @todo relax this precondition - C4_ASSERT( ! pattern.overlaps(dst)); - C4_ASSERT( ! repl .overlaps(dst)); - C4_ASSERT((pos >= 0 && pos <= len) || pos == npos); - C4_SUPPRESS_WARNING_GCC_PUSH - C4_SUPPRESS_WARNING_GCC("-Warray-bounds") // gcc11 has a false positive here - #if (!defined(__clang__)) && (defined(__GNUC__) && (__GNUC__ >= 7)) - C4_SUPPRESS_WARNING_GCC("-Wstringop-overflow") // gcc11 has a false positive here - #endif - #define _c4append(first, last) \ - { \ - C4_ASSERT((last) >= (first)); \ - size_t num = static_cast((last) - (first)); \ - if(num > 0 && sz + num <= dst.len) \ - { \ - memcpy(dst.str + sz, first, num * sizeof(C)); \ - } \ - sz += num; \ - } - size_t sz = 0; - size_t b = pos; - _c4append(str, str + pos); - do { - size_t e = find(pattern, b); - if(e == npos) - { - _c4append(str + b, str + len); - break; - } - _c4append(str + b, str + e); - _c4append(repl.begin(), repl.end()); - b = e + pattern.size(); - } while(b < len && b != npos); - return sz; - #undef _c4append - C4_SUPPRESS_WARNING_GCC_POP - } - - /** @} */ - -}; // template class basic_substring - - -#undef C4_REQUIRE_RW -#undef C4_REQUIRE_RO -#undef C4_NC2C - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** Because of a C++ limitation, substr cannot provide simultaneous - * overloads for constructing from a char[N] and a char*; the latter - * will always be chosen by the compiler. So this specialization is - * provided to simplify obtaining a substr from a char*. Being a - * function has the advantage of highlighting the strlen() cost. - * - * @see to_csubstr - * @see For a more detailed explanation on why the overloads cannot - * coexist, see http://cplusplus.bordoon.com/specializeForCharacterArrays.html */ -inline substr to_substr(char *s) -{ - return substr(s, s ? strlen(s) : 0); -} - -/** Because of a C++ limitation, substr cannot provide simultaneous - * overloads for constructing from a char[N] and a char*; the latter - * will always be chosen by the compiler. So this specialization is - * provided to simplify obtaining a substr from a char*. Being a - * function has the advantage of highlighting the strlen() cost. - * - * @see to_substr - * @see For a more detailed explanation on why the overloads cannot - * coexist, see http://cplusplus.bordoon.com/specializeForCharacterArrays.html */ -inline csubstr to_csubstr(char *s) -{ - return csubstr(s, s ? strlen(s) : 0); -} - -/** Because of a C++ limitation, substr cannot provide simultaneous - * overloads for constructing from a const char[N] and a const char*; - * the latter will always be chosen by the compiler. So this - * specialization is provided to simplify obtaining a substr from a - * char*. Being a function has the advantage of highlighting the - * strlen() cost. - * - * @overload to_csubstr - * @see to_substr - * @see For a more detailed explanation on why the overloads cannot - * coexist, see http://cplusplus.bordoon.com/specializeForCharacterArrays.html */ -inline csubstr to_csubstr(const char *s) -{ - return csubstr(s, s ? strlen(s) : 0); -} - - -/** neutral version for use in generic code */ -inline csubstr to_csubstr(csubstr s) -{ - return s; -} - -/** neutral version for use in generic code */ -inline csubstr to_csubstr(substr s) -{ - return s; -} - -/** neutral version for use in generic code */ -inline substr to_substr(substr s) -{ - return s; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template inline bool operator== (const C (&s)[N], basic_substring const that) { return that.compare(s) == 0; } -template inline bool operator!= (const C (&s)[N], basic_substring const that) { return that.compare(s) != 0; } -template inline bool operator< (const C (&s)[N], basic_substring const that) { return that.compare(s) > 0; } -template inline bool operator> (const C (&s)[N], basic_substring const that) { return that.compare(s) < 0; } -template inline bool operator<= (const C (&s)[N], basic_substring const that) { return that.compare(s) >= 0; } -template inline bool operator>= (const C (&s)[N], basic_substring const that) { return that.compare(s) <= 0; } - -template inline bool operator== (C const c, basic_substring const that) { return that.compare(c) == 0; } -template inline bool operator!= (C const c, basic_substring const that) { return that.compare(c) != 0; } -template inline bool operator< (C const c, basic_substring const that) { return that.compare(c) > 0; } -template inline bool operator> (C const c, basic_substring const that) { return that.compare(c) < 0; } -template inline bool operator<= (C const c, basic_substring const that) { return that.compare(c) >= 0; } -template inline bool operator>= (C const c, basic_substring const that) { return that.compare(c) <= 0; } - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** @define C4_SUBSTR_NO_OSTREAM_LSHIFT doctest does not deal well with - * template operator<< - * @see https://github.com/onqtam/doctest/pull/431 */ -#ifndef C4_SUBSTR_NO_OSTREAM_LSHIFT -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wsign-conversion" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wsign-conversion" -#endif - -/** output the string to a stream */ -template -inline OStream& operator<< (OStream& os, basic_substring s) -{ - os.write(s.str, s.len); - return os; -} - -// this causes ambiguity -///** this is used by google test */ -//template -//inline void PrintTo(basic_substring s, OStream* os) -//{ -// os->write(s.str, s.len); -//} - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif -#endif // !C4_SUBSTR_NO_OSTREAM_LSHIFT - -} // namespace c4 - - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -#endif /* _C4_SUBSTR_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/substr_fwd.hpp b/thirdparty/ryml/ext/c4core/src/c4/substr_fwd.hpp deleted file mode 100644 index 63d01b595..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/substr_fwd.hpp +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef _C4_SUBSTR_FWD_HPP_ -#define _C4_SUBSTR_FWD_HPP_ - -#include "c4/export.hpp" - -namespace c4 { - -#ifndef DOXYGEN -template struct basic_substring; -using csubstr = C4CORE_EXPORT basic_substring; -using substr = C4CORE_EXPORT basic_substring; -#endif // !DOXYGEN - -} // namespace c4 - -#endif /* _C4_SUBSTR_FWD_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/szconv.hpp b/thirdparty/ryml/ext/c4core/src/c4/szconv.hpp deleted file mode 100644 index 76cec1146..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/szconv.hpp +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef _C4_SZCONV_HPP_ -#define _C4_SZCONV_HPP_ - -/** @file szconv.hpp utilities to deal safely with narrowing conversions */ - -#include "c4/config.hpp" -#include "c4/error.hpp" - -#include - -namespace c4 { - -/** @todo this would be so much easier with calls to numeric_limits::max()... */ -template -struct is_narrower_size : std::conditional -< - (std::is_signed::value == std::is_signed::value) - ? - (sizeof(SizeOut) < sizeof(SizeIn)) - : - ( - (sizeof(SizeOut) < sizeof(SizeIn)) - || - ( - (sizeof(SizeOut) == sizeof(SizeIn)) - && - (std::is_signed::value && std::is_unsigned::value) - ) - ), - std::true_type, - std::false_type ->::type -{ - static_assert(std::is_integral::value, "must be integral type"); - static_assert(std::is_integral::value, "must be integral type"); -}; - - -/** when SizeOut is wider than SizeIn, assignment can occur without reservations */ -template -C4_ALWAYS_INLINE -typename std::enable_if< ! is_narrower_size::value, SizeOut>::type -szconv(SizeIn sz) noexcept -{ - return static_cast(sz); -} - -/** when SizeOut is narrower than SizeIn, narrowing will occur, so we check - * for overflow. Note that this check is done only if C4_XASSERT is enabled. - * @see C4_XASSERT */ -template -C4_ALWAYS_INLINE -typename std::enable_if::value, SizeOut>::type -szconv(SizeIn sz) C4_NOEXCEPT_X -{ - C4_XASSERT(sz >= 0); - C4_XASSERT_MSG((SizeIn)sz <= (SizeIn)std::numeric_limits::max(), "size conversion overflow: in=%zu", (size_t)sz); - SizeOut szo = static_cast(sz); - return szo; -} - -} // namespace c4 - -#endif /* _C4_SZCONV_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/type_name.hpp b/thirdparty/ryml/ext/c4core/src/c4/type_name.hpp deleted file mode 100644 index 41f13562b..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/type_name.hpp +++ /dev/null @@ -1,125 +0,0 @@ -#ifndef _C4_TYPENAME_HPP_ -#define _C4_TYPENAME_HPP_ - -/** @file type_name.hpp compile-time type name */ - -#include "c4/span.hpp" - -/// @cond dev -struct _c4t -{ - const char *str; - size_t sz; - template - constexpr _c4t(const char (&s)[N]) : str(s), sz(N-1) {} // take off the \0 -}; -// this is a more abbreviated way of getting the type name -// (if we used span in the return type, the name would involve -// templates and would create longer type name strings, -// as well as larger differences between compilers) -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE -_c4t _c4tn() -{ - auto p = _c4t(C4_PRETTY_FUNC); - return p; -} -/// @endcond - - -namespace c4 { - -/** compile-time type name - * @see http://stackoverflow.com/a/20170989/5875572 */ -template -C4_CONSTEXPR14 cspan type_name() -{ - const _c4t p = _c4tn(); - -#if (0) // _C4_THIS_IS_A_DEBUG_SCAFFOLD - for(size_t index = 0; index < p.sz; ++index) - { - printf(" %2c", p.str[index]); - } - printf("\n"); - for(size_t index = 0; index < p.sz; ++index) - { - printf(" %2d", (int)index); - } - printf("\n"); -#endif - -#if defined(_MSC_VER) -# if defined(__clang__) // Visual Studio has the clang toolset - // example: - // ..........................xxx. - // _c4t __cdecl _c4tn() [T = int] - enum : size_t { tstart = 26, tend = 1}; - -# elif defined(C4_MSVC_2015) || defined(C4_MSVC_2017) || defined(C4_MSVC_2019) || defined(C4_MSVC_2022) - // Note: subtract 7 at the end because the function terminates with ">(void)" in VS2015+ - cspan::size_type tstart = 26, tend = 7; - - const char *s = p.str + tstart; // look at the start - - // we're not using strcmp() or memcmp() to spare the #include - - // does it start with 'class '? - if(p.sz > 6 && s[0] == 'c' && s[1] == 'l' && s[2] == 'a' && s[3] == 's' && s[4] == 's' && s[5] == ' ') - { - tstart += 6; - } - // does it start with 'struct '? - else if(p.sz > 7 && s[0] == 's' && s[1] == 't' && s[2] == 'r' && s[3] == 'u' && s[4] == 'c' && s[5] == 't' && s[6] == ' ') - { - tstart += 7; - } - -# else - C4_NOT_IMPLEMENTED(); -# endif - -#elif defined(__ICC) - // example: - // ........................xxx. - // "_c4t _c4tn() [with T = int]" - enum : size_t { tstart = 23, tend = 1}; - -#elif defined(__clang__) - // example: - // ...................xxx. - // "_c4t _c4tn() [T = int]" - enum : size_t { tstart = 18, tend = 1}; - -#elif defined(__GNUC__) - #if __GNUC__ >= 7 && C4_CPP >= 14 - // example: - // ..................................xxx. - // "constexpr _c4t _c4tn() [with T = int]" - enum : size_t { tstart = 33, tend = 1 }; - #else - // example: - // ........................xxx. - // "_c4t _c4tn() [with T = int]" - enum : size_t { tstart = 23, tend = 1 }; - #endif -#else - C4_NOT_IMPLEMENTED(); -#endif - - cspan o(p.str + tstart, p.sz - tstart - tend); - - return o; -} - -/** compile-time type name - * @overload */ -template -C4_CONSTEXPR14 C4_ALWAYS_INLINE cspan type_name(T const&) -{ - return type_name(); -} - -} // namespace c4 - -#endif //_C4_TYPENAME_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/types.hpp b/thirdparty/ryml/ext/c4core/src/c4/types.hpp deleted file mode 100644 index 394a808fd..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/types.hpp +++ /dev/null @@ -1,492 +0,0 @@ -#ifndef _C4_TYPES_HPP_ -#define _C4_TYPES_HPP_ - -#include -#include -#include - -#if __cplusplus >= 201103L -#include // for integer_sequence and friends -#endif - -#include "c4/preprocessor.hpp" -#include "c4/language.hpp" - -/** @file types.hpp basic types, and utility macros and traits for types. - * @ingroup basic_headers */ - -/** @defgroup types Type utilities */ - -namespace c4 { - -/** @defgroup intrinsic_types Intrinsic types - * @ingroup types - * @{ */ - -using cbyte = const char; /**< a constant byte */ -using byte = char; /**< a mutable byte */ - -using i8 = int8_t; -using i16 = int16_t; -using i32 = int32_t; -using i64 = int64_t; -using u8 = uint8_t; -using u16 = uint16_t; -using u32 = uint32_t; -using u64 = uint64_t; - -using f32 = float; -using f64 = double; - -using ssize_t = typename std::make_signed::type; - -/** @} */ - -//-------------------------------------------------- - -/** @defgroup utility_types Utility types - * @ingroup types - * @{ */ - -// some tag types - -/** a tag type for initializing the containers with variadic arguments a la - * initializer_list, minus the initializer_list overload problems. - */ -struct aggregate_t {}; -/** @see aggregate_t */ -constexpr const aggregate_t aggregate{}; - -/** a tag type for specifying the initial capacity of allocatable contiguous storage */ -struct with_capacity_t {}; -/** @see with_capacity_t */ -constexpr const with_capacity_t with_capacity{}; - -/** a tag type for disambiguating template parameter packs in variadic template overloads */ -struct varargs_t {}; -/** @see with_capacity_t */ -constexpr const varargs_t varargs{}; - - -//-------------------------------------------------- - -/** whether a value should be used in place of a const-reference in argument passing. */ -template -struct cref_uses_val -{ - enum { value = ( - std::is_scalar::value - || - ( -#if C4_CPP >= 20 - (std::is_trivially_copyable::value && std::is_standard_layout::value) -#else - std::is_pod::value -#endif - && - sizeof(T) <= sizeof(size_t))) }; -}; -/** utility macro to override the default behaviour for c4::fastcref - @see fastcref */ -#define C4_CREF_USES_VAL(T) \ -template<> \ -struct cref_uses_val \ -{ \ - enum { value = true }; \ -}; - -/** Whether to use pass-by-value or pass-by-const-reference in a function argument - * or return type. */ -template -using fastcref = typename std::conditional::value, T, T const&>::type; - -//-------------------------------------------------- - -/** Just what its name says. Useful sometimes as a default empty policy class. */ -struct EmptyStruct -{ - template EmptyStruct(T && ...){} -}; - -/** Just what its name says. Useful sometimes as a default policy class to - * be inherited from. */ -struct EmptyStructVirtual -{ - virtual ~EmptyStructVirtual() = default; - template EmptyStructVirtual(T && ...){} -}; - - -/** */ -template -struct inheritfrom : public T {}; - -//-------------------------------------------------- -// Utilities to make a class obey size restrictions (eg, min size or size multiple of). -// DirectX usually makes this restriction with uniform buffers. -// This is also useful for padding to prevent false-sharing. - -/** how many bytes must be added to size such that the result is at least minsize? */ -C4_ALWAYS_INLINE constexpr size_t min_remainder(size_t size, size_t minsize) noexcept -{ - return size < minsize ? minsize-size : 0; -} - -/** how many bytes must be added to size such that the result is a multiple of multipleof? */ -C4_ALWAYS_INLINE constexpr size_t mult_remainder(size_t size, size_t multipleof) noexcept -{ - return (((size % multipleof) != 0) ? (multipleof-(size % multipleof)) : 0); -} - -/* force the following class to be tightly packed. */ -#pragma pack(push, 1) -/** pad a class with more bytes at the end. - * @see http://stackoverflow.com/questions/21092415/force-c-structure-to-pack-tightly */ -template -struct Padded : public T -{ - using T::T; - using T::operator=; - Padded(T const& val) : T(val) {} - Padded(T && val) : T(val) {} - char ___c4padspace___[BytesToPadAtEnd]; -}; -#pragma pack(pop) -/** When the padding argument is 0, we cannot declare the char[] array. */ -template -struct Padded : public T -{ - using T::T; - using T::operator=; - Padded(T const& val) : T(val) {} - Padded(T && val) : T(val) {} -}; - -/** make T have a size which is at least Min bytes */ -template -using MinSized = Padded; - -/** make T have a size which is a multiple of Mult bytes */ -template -using MultSized = Padded; - -/** make T have a size which is simultaneously: - * -bigger or equal than Min - * -a multiple of Mult */ -template -using MinMultSized = MultSized, Mult>; - -/** make T be suitable for use as a uniform buffer. (at least with DirectX). */ -template -using UbufSized = MinMultSized; - - -//----------------------------------------------------------------------------- - -#define C4_NO_COPY_CTOR(ty) ty(ty const&) = delete -#define C4_NO_MOVE_CTOR(ty) ty(ty &&) = delete -#define C4_NO_COPY_ASSIGN(ty) ty& operator=(ty const&) = delete -#define C4_NO_MOVE_ASSIGN(ty) ty& operator=(ty &&) = delete -#define C4_DEFAULT_COPY_CTOR(ty) ty(ty const&) noexcept = default -#define C4_DEFAULT_MOVE_CTOR(ty) ty(ty &&) noexcept = default -#define C4_DEFAULT_COPY_ASSIGN(ty) ty& operator=(ty const&) noexcept = default -#define C4_DEFAULT_MOVE_ASSIGN(ty) ty& operator=(ty &&) noexcept = default - -#define C4_NO_COPY_OR_MOVE_CTOR(ty) \ - C4_NO_COPY_CTOR(ty); \ - C4_NO_MOVE_CTOR(ty) - -#define C4_NO_COPY_OR_MOVE_ASSIGN(ty) \ - C4_NO_COPY_ASSIGN(ty); \ - C4_NO_MOVE_ASSIGN(ty) - -#define C4_NO_COPY_OR_MOVE(ty) \ - C4_NO_COPY_OR_MOVE_CTOR(ty); \ - C4_NO_COPY_OR_MOVE_ASSIGN(ty) - -#define C4_DEFAULT_COPY_AND_MOVE_CTOR(ty) \ - C4_DEFAULT_COPY_CTOR(ty); \ - C4_DEFAULT_MOVE_CTOR(ty) - -#define C4_DEFAULT_COPY_AND_MOVE_ASSIGN(ty) \ - C4_DEFAULT_COPY_ASSIGN(ty); \ - C4_DEFAULT_MOVE_ASSIGN(ty) - -#define C4_DEFAULT_COPY_AND_MOVE(ty) \ - C4_DEFAULT_COPY_AND_MOVE_CTOR(ty); \ - C4_DEFAULT_COPY_AND_MOVE_ASSIGN(ty) - -/** @see https://en.cppreference.com/w/cpp/named_req/TriviallyCopyable */ -#define C4_MUST_BE_TRIVIAL_COPY(ty) \ - static_assert(std::is_trivially_copyable::value, #ty " must be trivially copyable") - -/** @} */ - - -//----------------------------------------------------------------------------- - -/** @defgroup traits_types Type traits utilities - * @ingroup types - * @{ */ - -// http://stackoverflow.com/questions/10821380/is-t-an-instance-of-a-template-in-c -template class X, typename T> struct is_instance_of_tpl : std::false_type {}; -template class X, typename... Y> struct is_instance_of_tpl> : std::true_type {}; - -//----------------------------------------------------------------------------- - -/** SFINAE. use this macro to enable a template function overload -based on a compile-time condition. -@code -// define an overload for a non-pod type -template::value)> -void foo() { std::cout << "pod type\n"; } - -// define an overload for a non-pod type -template::value)> -void foo() { std::cout << "nonpod type\n"; } - -struct non_pod -{ - non_pod() : name("asdfkjhasdkjh") {} - const char *name; -}; - -int main() -{ - foo(); // prints "pod type" - foo(); // prints "nonpod type" -} -@endcode */ -#define C4_REQUIRE_T(cond) typename std::enable_if::type* = nullptr - -/** enable_if for a return type - * @see C4_REQUIRE_T */ -#define C4_REQUIRE_R(cond, type_) typename std::enable_if::type - -//----------------------------------------------------------------------------- -/** define a traits class reporting whether a type provides a member typedef */ -#define C4_DEFINE_HAS_TYPEDEF(member_typedef) \ -template \ -struct has_##stype \ -{ \ -private: \ - \ - typedef char yes; \ - typedef struct { char array[2]; } no; \ - \ - template \ - static yes _test(typename C::member_typedef*); \ - \ - template \ - static no _test(...); \ - \ -public: \ - \ - enum { value = (sizeof(_test(0)) == sizeof(yes)) }; \ - \ -} - - -/** @} */ - - -//----------------------------------------------------------------------------- - - -/** @defgroup type_declarations Type declaration utilities - * @ingroup types - * @{ */ - -#define _c4_DEFINE_ARRAY_TYPES_WITHOUT_ITERATOR(T, I) \ - \ - using size_type = I; \ - using ssize_type = typename std::make_signed::type; \ - using difference_type = typename std::make_signed::type; \ - \ - using value_type = T; \ - using pointer = T*; \ - using const_pointer = T const*; \ - using reference = T&; \ - using const_reference = T const& - -#define _c4_DEFINE_TUPLE_ARRAY_TYPES_WITHOUT_ITERATOR(interior_types, I) \ - \ - using size_type = I; \ - using ssize_type = typename std::make_signed::type; \ - using difference_type = typename std::make_signed::type; \ - \ - template using value_type = typename std::tuple_element< n, std::tuple>::type; \ - template using pointer = value_type*; \ - template using const_pointer = value_type const*; \ - template using reference = value_type&; \ - template using const_reference = value_type const& - - -#define _c4_DEFINE_ARRAY_TYPES(T, I) \ - \ - _c4_DEFINE_ARRAY_TYPES_WITHOUT_ITERATOR(T, I); \ - \ - using iterator = T*; \ - using const_iterator = T const*; \ - using reverse_iterator = std::reverse_iterator; \ - using const_reverse_iterator = std::reverse_iterator - - -#define _c4_DEFINE_TUPLE_ARRAY_TYPES(interior_types, I) \ - \ - _c4_DEFINE_TUPLE_ARRAY_TYPES_WITHOUT_ITERATOR(interior_types, I); \ - \ - template using iterator = value_type*; \ - template using const_iterator = value_type const*; \ - template using reverse_iterator = std::reverse_iterator< value_type*>; \ - template using const_reverse_iterator = std::reverse_iterator< value_type const*> - - - -/** @} */ - - -//----------------------------------------------------------------------------- - - -/** @defgroup compatility_utilities Backport implementation of some Modern C++ utilities - * @ingroup types - * @{ */ - -//----------------------------------------------------------------------------- -// index_sequence and friends are available only for C++14 and later. -// A C++11 implementation is provided here. -// This implementation was copied over from clang. -// see http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/utility?revision=211563&view=markup#l687 - -#if __cplusplus > 201103L - -using std::integer_sequence; -using std::index_sequence; -using std::make_integer_sequence; -using std::make_index_sequence; -using std::index_sequence_for; - -#else - -/** C++11 implementation of integer sequence - * @see https://en.cppreference.com/w/cpp/utility/integer_sequence - * @see taken from clang: http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/utility?revision=211563&view=markup#l687 */ -template -struct integer_sequence -{ - static_assert(std::is_integral<_Tp>::value, - "std::integer_sequence can only be instantiated with an integral type" ); - using value_type = _Tp; - static constexpr size_t size() noexcept { return sizeof...(_Ip); } -}; - -/** C++11 implementation of index sequence - * @see https://en.cppreference.com/w/cpp/utility/integer_sequence - * @see taken from clang: http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/utility?revision=211563&view=markup#l687 */ -template -using index_sequence = integer_sequence; - -/** @cond DONT_DOCUMENT_THIS */ -namespace __detail { - -template -struct __repeat; - -template -struct __repeat, _Extra...> -{ - using type = integer_sequence<_Tp, - _Np..., - sizeof...(_Np) + _Np..., - 2 * sizeof...(_Np) + _Np..., - 3 * sizeof...(_Np) + _Np..., - 4 * sizeof...(_Np) + _Np..., - 5 * sizeof...(_Np) + _Np..., - 6 * sizeof...(_Np) + _Np..., - 7 * sizeof...(_Np) + _Np..., - _Extra...>; -}; - -template struct __parity; -template struct __make : __parity<_Np % 8>::template __pmake<_Np> {}; - -template<> struct __make<0> { using type = integer_sequence; }; -template<> struct __make<1> { using type = integer_sequence; }; -template<> struct __make<2> { using type = integer_sequence; }; -template<> struct __make<3> { using type = integer_sequence; }; -template<> struct __make<4> { using type = integer_sequence; }; -template<> struct __make<5> { using type = integer_sequence; }; -template<> struct __make<6> { using type = integer_sequence; }; -template<> struct __make<7> { using type = integer_sequence; }; - -template<> struct __parity<0> { template struct __pmake : __repeat::type> {}; }; -template<> struct __parity<1> { template struct __pmake : __repeat::type, _Np - 1> {}; }; -template<> struct __parity<2> { template struct __pmake : __repeat::type, _Np - 2, _Np - 1> {}; }; -template<> struct __parity<3> { template struct __pmake : __repeat::type, _Np - 3, _Np - 2, _Np - 1> {}; }; -template<> struct __parity<4> { template struct __pmake : __repeat::type, _Np - 4, _Np - 3, _Np - 2, _Np - 1> {}; }; -template<> struct __parity<5> { template struct __pmake : __repeat::type, _Np - 5, _Np - 4, _Np - 3, _Np - 2, _Np - 1> {}; }; -template<> struct __parity<6> { template struct __pmake : __repeat::type, _Np - 6, _Np - 5, _Np - 4, _Np - 3, _Np - 2, _Np - 1> {}; }; -template<> struct __parity<7> { template struct __pmake : __repeat::type, _Np - 7, _Np - 6, _Np - 5, _Np - 4, _Np - 3, _Np - 2, _Np - 1> {}; }; - -template -struct __convert -{ - template struct __result; - template<_Tp ..._Np> struct __result> - { - using type = integer_sequence<_Up, _Np...>; - }; -}; - -template -struct __convert<_Tp, _Tp> -{ - template struct __result - { - using type = _Up; - }; -}; - -template -using __make_integer_sequence_unchecked = typename __detail::__convert::template __result::type>::type; - -template -struct __make_integer_sequence -{ - static_assert(std::is_integral<_Tp>::value, - "std::make_integer_sequence can only be instantiated with an integral type" ); - static_assert(0 <= _Ep, "std::make_integer_sequence input shall not be negative"); - typedef __make_integer_sequence_unchecked<_Tp, _Ep> type; -}; - -} // namespace __detail -/** @endcond */ - - -/** C++11 implementation of index sequence - * @see https://en.cppreference.com/w/cpp/utility/integer_sequence - * @see taken from clang: http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/utility?revision=211563&view=markup#l687 */ -template -using make_integer_sequence = typename __detail::__make_integer_sequence<_Tp, _Np>::type; - -/** C++11 implementation of index sequence - * @see https://en.cppreference.com/w/cpp/utility/integer_sequence - * @see taken from clang: http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/utility?revision=211563&view=markup#l687 */ -template -using make_index_sequence = make_integer_sequence; - -/** C++11 implementation of index sequence - * @see https://en.cppreference.com/w/cpp/utility/integer_sequence - * @see taken from clang: http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/utility?revision=211563&view=markup#l687 */ -template -using index_sequence_for = make_index_sequence; -#endif - -/** @} */ - - -} // namespace c4 - -#endif /* _C4_TYPES_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/unrestrict.hpp b/thirdparty/ryml/ext/c4core/src/c4/unrestrict.hpp deleted file mode 100644 index 552f9ccff..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/unrestrict.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#ifdef _C4_RESTRICT_HPP_ // must match the include guard from restrict.hpp - -/** @file unrestrict.hpp cleans up restrict macros */ - -#undef $ -#undef $$ -#undef c$ -#undef c$$ - -#undef _C4_RESTRICT_HPP_ - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -#endif - -#endif /* _C4_RESTRICT_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/utf.cpp b/thirdparty/ryml/ext/c4core/src/c4/utf.cpp deleted file mode 100644 index f1d509015..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/utf.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "c4/utf.hpp" -#include "c4/charconv.hpp" - -namespace c4 { - -size_t decode_code_point(uint8_t *C4_RESTRICT buf, size_t buflen, const uint32_t code) -{ - C4_UNUSED(buflen); - C4_ASSERT(buflen >= 4); - if (code <= UINT32_C(0x7f)) - { - buf[0] = (uint8_t)code; - return 1u; - } - else if(code <= UINT32_C(0x7ff)) - { - buf[0] = (uint8_t)(UINT32_C(0xc0) | (code >> 6)); /* 110xxxxx */ - buf[1] = (uint8_t)(UINT32_C(0x80) | (code & UINT32_C(0x3f))); /* 10xxxxxx */ - return 2u; - } - else if(code <= UINT32_C(0xffff)) - { - buf[0] = (uint8_t)(UINT32_C(0xe0) | ((code >> 12))); /* 1110xxxx */ - buf[1] = (uint8_t)(UINT32_C(0x80) | ((code >> 6) & UINT32_C(0x3f))); /* 10xxxxxx */ - buf[2] = (uint8_t)(UINT32_C(0x80) | ((code ) & UINT32_C(0x3f))); /* 10xxxxxx */ - return 3u; - } - else if(code <= UINT32_C(0x10ffff)) - { - buf[0] = (uint8_t)(UINT32_C(0xf0) | ((code >> 18))); /* 11110xxx */ - buf[1] = (uint8_t)(UINT32_C(0x80) | ((code >> 12) & UINT32_C(0x3f))); /* 10xxxxxx */ - buf[2] = (uint8_t)(UINT32_C(0x80) | ((code >> 6) & UINT32_C(0x3f))); /* 10xxxxxx */ - buf[3] = (uint8_t)(UINT32_C(0x80) | ((code ) & UINT32_C(0x3f))); /* 10xxxxxx */ - return 4u; - } - return 0; -} - -substr decode_code_point(substr out, csubstr code_point) -{ - C4_ASSERT(out.len >= 4); - C4_ASSERT(!code_point.begins_with("U+")); - C4_ASSERT(!code_point.begins_with("\\x")); - C4_ASSERT(!code_point.begins_with("\\u")); - C4_ASSERT(!code_point.begins_with("\\U")); - C4_ASSERT(!code_point.begins_with('0')); - C4_ASSERT(code_point.len <= 8); - C4_ASSERT(code_point.len > 0); - uint32_t code_point_val; - C4_CHECK(read_hex(code_point, &code_point_val)); - size_t ret = decode_code_point((uint8_t*)out.str, out.len, code_point_val); - C4_ASSERT(ret <= 4); - return out.first(ret); -} - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/src/c4/utf.hpp b/thirdparty/ryml/ext/c4core/src/c4/utf.hpp deleted file mode 100644 index 116b20593..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/utf.hpp +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef C4_UTF_HPP_ -#define C4_UTF_HPP_ - -#include "c4/language.hpp" -#include "c4/substr_fwd.hpp" -#include -#include - -namespace c4 { - -substr decode_code_point(substr out, csubstr code_point); -size_t decode_code_point(uint8_t *C4_RESTRICT buf, size_t buflen, const uint32_t code); - -} // namespace c4 - -#endif // C4_UTF_HPP_ diff --git a/thirdparty/ryml/ext/c4core/src/c4/windows.hpp b/thirdparty/ryml/ext/c4core/src/c4/windows.hpp deleted file mode 100644 index d94c66c55..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/windows.hpp +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef _C4_WINDOWS_HPP_ -#define _C4_WINDOWS_HPP_ - -#if defined(_WIN64) || defined(_WIN32) -#include "c4/windows_push.hpp" -#include -#include "c4/windows_pop.hpp" -#endif - -#endif /* _C4_WINDOWS_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/windows_pop.hpp b/thirdparty/ryml/ext/c4core/src/c4/windows_pop.hpp deleted file mode 100644 index e055af6fa..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/windows_pop.hpp +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef _C4_WINDOWS_POP_HPP_ -#define _C4_WINDOWS_POP_HPP_ - -#if defined(_WIN64) || defined(_WIN32) - -#ifdef _c4_AMD64_ -# undef _c4_AMD64_ -# undef _AMD64_ -#endif -#ifdef _c4_X86_ -# undef _c4_X86_ -# undef _X86_ -#endif -#ifdef _c4_ARM_ -# undef _c4_ARM_ -# undef _ARM_ -#endif - -#ifdef _c4_NOMINMAX -# undef _c4_NOMINMAX -# undef NOMINMAX -#endif - -#ifdef NOGDI -# undef _c4_NOGDI -# undef NOGDI -#endif - -#ifdef VC_EXTRALEAN -# undef _c4_VC_EXTRALEAN -# undef VC_EXTRALEAN -#endif - -#ifdef WIN32_LEAN_AND_MEAN -# undef _c4_WIN32_LEAN_AND_MEAN -# undef WIN32_LEAN_AND_MEAN -#endif - -#endif /* defined(_WIN64) || defined(_WIN32) */ - -#endif /* _C4_WINDOWS_POP_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/src/c4/windows_push.hpp b/thirdparty/ryml/ext/c4core/src/c4/windows_push.hpp deleted file mode 100644 index 156fe2fb4..000000000 --- a/thirdparty/ryml/ext/c4core/src/c4/windows_push.hpp +++ /dev/null @@ -1,102 +0,0 @@ -#ifndef _C4_WINDOWS_PUSH_HPP_ -#define _C4_WINDOWS_PUSH_HPP_ - -/** @file windows_push.hpp sets up macros to include windows header files - * without pulling in all of - * - * @see #include windows_pop.hpp to undefine these macros - * - * @see https://aras-p.info/blog/2018/01/12/Minimizing-windows.h/ */ - - -#if defined(_WIN64) || defined(_WIN32) - -#if defined(_M_AMD64) -# ifndef _AMD64_ -# define _c4_AMD64_ -# define _AMD64_ -# endif -#elif defined(_M_IX86) -# ifndef _X86_ -# define _c4_X86_ -# define _X86_ -# endif -#elif defined(_M_ARM64) -# ifndef _ARM64_ -# define _c4_ARM64_ -# define _ARM64_ -# endif -#elif defined(_M_ARM) -# ifndef _ARM_ -# define _c4_ARM_ -# define _ARM_ -# endif -#endif - -#ifndef NOMINMAX -# define _c4_NOMINMAX -# define NOMINMAX -#endif - -#ifndef NOGDI -# define _c4_NOGDI -# define NOGDI -#endif - -#ifndef VC_EXTRALEAN -# define _c4_VC_EXTRALEAN -# define VC_EXTRALEAN -#endif - -#ifndef WIN32_LEAN_AND_MEAN -# define _c4_WIN32_LEAN_AND_MEAN -# define WIN32_LEAN_AND_MEAN -#endif - -/* If defined, the following flags inhibit definition - * of the indicated items. - * - * NOGDICAPMASKS - CC_*, LC_*, PC_*, CP_*, TC_*, RC_ - * NOVIRTUALKEYCODES - VK_* - * NOWINMESSAGES - WM_*, EM_*, LB_*, CB_* - * NOWINSTYLES - WS_*, CS_*, ES_*, LBS_*, SBS_*, CBS_* - * NOSYSMETRICS - SM_* - * NOMENUS - MF_* - * NOICONS - IDI_* - * NOKEYSTATES - MK_* - * NOSYSCOMMANDS - SC_* - * NORASTEROPS - Binary and Tertiary raster ops - * NOSHOWWINDOW - SW_* - * OEMRESOURCE - OEM Resource values - * NOATOM - Atom Manager routines - * NOCLIPBOARD - Clipboard routines - * NOCOLOR - Screen colors - * NOCTLMGR - Control and Dialog routines - * NODRAWTEXT - DrawText() and DT_* - * NOGDI - All GDI defines and routines - * NOKERNEL - All KERNEL defines and routines - * NOUSER - All USER defines and routines - * NONLS - All NLS defines and routines - * NOMB - MB_* and MessageBox() - * NOMEMMGR - GMEM_*, LMEM_*, GHND, LHND, associated routines - * NOMETAFILE - typedef METAFILEPICT - * NOMINMAX - Macros min(a,b) and max(a,b) - * NOMSG - typedef MSG and associated routines - * NOOPENFILE - OpenFile(), OemToAnsi, AnsiToOem, and OF_* - * NOSCROLL - SB_* and scrolling routines - * NOSERVICE - All Service Controller routines, SERVICE_ equates, etc. - * NOSOUND - Sound driver routines - * NOTEXTMETRIC - typedef TEXTMETRIC and associated routines - * NOWH - SetWindowsHook and WH_* - * NOWINOFFSETS - GWL_*, GCL_*, associated routines - * NOCOMM - COMM driver routines - * NOKANJI - Kanji support stuff. - * NOHELP - Help engine interface. - * NOPROFILER - Profiler interface. - * NODEFERWINDOWPOS - DeferWindowPos routines - * NOMCX - Modem Configuration Extensions - */ - -#endif /* defined(_WIN64) || defined(_WIN32) */ - -#endif /* _C4_WINDOWS_PUSH_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/tbump.toml b/thirdparty/ryml/ext/c4core/tbump.toml deleted file mode 100644 index 310c606d6..000000000 --- a/thirdparty/ryml/ext/c4core/tbump.toml +++ /dev/null @@ -1,48 +0,0 @@ -# Uncomment this if your project is hosted on GitHub: -# github_url = "https://github.com///" - -[version] -current = "0.1.11" - -# Example of a semver regexp. -# Make sure this matches current_version before -# using tbump -regex = ''' - (?P\d+) - \. - (?P\d+) - \. - (?P\d+) - .* - ''' - -[git] -message_template = "Bump to {new_version}" -tag_template = "v{new_version}" - -# For each file to patch, add a [[file]] config -# section containing the path of the file, relative to the -# tbump.toml location. -[[file]] -src = "CMakeLists.txt" -search = "c4_project\\(VERSION {current_version}" -[[file]] -src = "test/test_install/CMakeLists.txt" -search = "c4_project\\(VERSION {current_version}" -[[file]] -src = "test/test_singleheader/CMakeLists.txt" -search = "c4_project\\(VERSION {current_version}" - -# You can specify a list of commands to -# run after the files have been patched -# and before the git commit is made - -# [[before_commit]] -# name = "check changelog" -# cmd = "grep -q {new_version} Changelog.rst" - -# Or run some commands after the git tag and the branch -# have been pushed: -# [[after_push]] -# name = "publish" -# cmd = "./publish.sh" diff --git a/thirdparty/ryml/ext/c4core/test/c4/libtest/archetypes.cpp b/thirdparty/ryml/ext/c4core/test/c4/libtest/archetypes.cpp deleted file mode 100644 index 9c4fb4310..000000000 --- a/thirdparty/ryml/ext/c4core/test/c4/libtest/archetypes.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "c4/libtest/archetypes.hpp" - -namespace c4 { -namespace archetypes { - -int IdOwner::s_current = 0; - -} // namespace archetypes -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/test/c4/libtest/archetypes.hpp b/thirdparty/ryml/ext/c4core/test/c4/libtest/archetypes.hpp deleted file mode 100644 index 87a4644e9..000000000 --- a/thirdparty/ryml/ext/c4core/test/c4/libtest/archetypes.hpp +++ /dev/null @@ -1,551 +0,0 @@ -#ifndef _C4_TEST_ARCHETYPES_HPP_ -#define _C4_TEST_ARCHETYPES_HPP_ - -#ifdef C4CORE_SINGLE_HEADER -#include "c4/c4core_all.hpp" -#else -#include "c4/memory_resource.hpp" -#include "c4/allocator.hpp" -#include "c4/char_traits.hpp" -#endif -#include "c4/test.hpp" - -#include -#include -#include - -namespace c4 { - -template< class String > class sstream; - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -namespace archetypes { - -template< class T > void check_archetype(T const& a) { a.check(); } -template< class T > void check_archetype(T const& a, T const& ref) { a.check(ref); } -inline void check_archetype(char ) {} -inline void check_archetype(wchar_t ) {} -inline void check_archetype(int8_t ) {} -inline void check_archetype(uint8_t ) {} -inline void check_archetype(int16_t ) {} -inline void check_archetype(uint16_t) {} -inline void check_archetype(int32_t ) {} -inline void check_archetype(uint32_t) {} -inline void check_archetype(int64_t ) {} -inline void check_archetype(uint64_t) {} -inline void check_archetype(float ) {} -inline void check_archetype(double ) {} -inline void check_archetype(char a, char ref) { CHECK_EQ(a, ref); } -inline void check_archetype(wchar_t a, wchar_t ref) { CHECK_EQ(a, ref); } -inline void check_archetype(int8_t a, int8_t ref) { CHECK_EQ(a, ref); } -inline void check_archetype(uint8_t a, uint8_t ref) { CHECK_EQ(a, ref); } -inline void check_archetype(int16_t a, int16_t ref) { CHECK_EQ(a, ref); } -inline void check_archetype(uint16_t a, uint16_t ref) { CHECK_EQ(a, ref); } -inline void check_archetype(int32_t a, int32_t ref) { CHECK_EQ(a, ref); } -inline void check_archetype(uint32_t a, uint32_t ref) { CHECK_EQ(a, ref); } -inline void check_archetype(int64_t a, int64_t ref) { CHECK_EQ(a, ref); } -inline void check_archetype(uint64_t a, uint64_t ref) { CHECK_EQ(a, ref); } -inline void check_archetype(float a, float ref) { CHECK_EQ((double)a, doctest::Approx((double)ref)); } -inline void check_archetype(double a, double ref) { CHECK_EQ(a, doctest::Approx(ref)); } - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template< class T, class Proto > -struct archetype_proto_base -{ - static T const& get(size_t which) - { - auto const& a = Proto::arr(); - C4_ASSERT(which < (int)a.size()); - return a[which]; - } - static std::array< T, 8 > dup() - { - std::array< T, 8 > d = Proto::arr; - return d; - } - static std::array< Counting, 8 > cdup() - { - std::array< Counting, 8 > d = Proto::carr; - return d; - } - static std::vector< T > dup(size_t n) - { - auto const& a = Proto::arr(); - std::vector< T > d; - d.reserve(n); - for(size_t i = 0, pos = 0; i < n; ++i, pos = ((pos+1)%a.size())) - { - d.push_back(a[pos]); - } - return d; - } - static std::vector< Counting > cdup(size_t n) - { - auto const& a = Proto::arr(); - std::vector< Counting > d; - d.reserve(n); - for(size_t i = 0, pos = 0; i < n; ++i, pos = ((pos+1)%a.size())) - { - d.push_back(a[pos]); - } - return d; - } -}; - -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wmissing-braces" // warning : suggest braces around initialization of subobject [-Wmissing-braces] -#endif - -// for scalar types: ints and floats -template< class T > -struct archetype_proto : public archetype_proto_base< T, archetype_proto > -{ - static_assert(std::is_fundamental< T >::value, "T must be a fundamental type"); - static std::array const& arr() - { - static const std::array arr_{0, 1, 2, 3, 4, 5, 6, 7}; - return arr_; - } - static std::array, 8> const& carr() - { - static const std::array, 8> arr_ = {0, 1, 2, 3, 4, 5, 6, 7}; - return arr_; - } - static std::initializer_list< T > il() - { - static const std::initializer_list< T > l{0, 1, 2, 3, 4, 5, 6, 7}; - return l; - } - static std::initializer_list< Counting > cil() - { - static const std::initializer_list< Counting > l = {0, 1, 2, 3, 4, 5, 6, 7}; - C4_ASSERT(l.size() == 8); - return l; - } -}; - -#define _C4_DECLARE_ARCHETYPE_PROTO(ty, ...) \ -template<> \ -struct archetype_proto : public archetype_proto_base< ty, archetype_proto > \ -{ \ - static std::array const& arr() \ - { \ - static const std::array arr_{__VA_ARGS__}; \ - return arr_; \ - } \ - static std::array, 8> const& carr() \ - { \ - static const std::array, 8> arr_{__VA_ARGS__}; \ - return arr_; \ - } \ - static std::initializer_list< ty > il() \ - { \ - static const std::initializer_list< ty > l{__VA_ARGS__}; \ - return l; \ - } \ - static std::initializer_list< Counting > cil() \ - { \ - static const std::initializer_list< Counting > l{__VA_ARGS__}; \ - return l; \ - } \ -} - -#define _C4_DECLARE_ARCHETYPE_PROTO_TPL1(tplparam1, ty, ...) \ -template< tplparam1 > \ -struct archetype_proto< ty > : public archetype_proto_base< ty, archetype_proto > \ -{ \ - static std::array const& arr() \ - { \ - static const std::array arr_{__VA_ARGS__}; \ - return arr_; \ - } \ - static std::array, 8> const& carr() \ - { \ - static const std::array, 8> arr_{__VA_ARGS__}; \ - return arr_; \ - } \ - static std::initializer_list< ty > il() \ - { \ - static const std::initializer_list< ty > l{__VA_ARGS__}; \ - return l; \ - } \ - static std::initializer_list< Counting > cil() \ - { \ - static const std::initializer_list< Counting > l{__VA_ARGS__}; \ - return l; \ - } \ -} - -_C4_DECLARE_ARCHETYPE_PROTO(std::string, - "str0", "str1", "str2", "str3", - "str4", "str5", "str6", "str7"); - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** Resource-owning archetype */ - -template< class T > -struct exvec3 -{ - T x, y, z; - bool operator== (exvec3 const& that) const - { - return x == that.x && y == that.y && z == that.z; - } -}; -template< class String, class T > -sstream< String >& operator<< (sstream< String >& ss, exvec3 const& v) -{ - using char_type = typename sstream< String >::char_type; - ss.printp(C4_TXTTY("({},{},{})", char_type), v.x, v.y, v.z); - return ss; -} -template< class String, class T > -sstream< String >& operator>> (sstream< String >& ss, exvec3 & v) -{ - using char_type = typename sstream< String >::char_type; - ss.scanp(C4_TXTTY("({},{},{})", char_type), v.x, v.y, v.z); - return ss; -} - -#define _ C4_COMMA -#define c4v(v0, v1, v2) exvec3{v0 _ v1 _ v2} -_C4_DECLARE_ARCHETYPE_PROTO_TPL1(class T, exvec3, - c4v(0, 1, 2), c4v(3, 4, 5), c4v(6, 7, 8), c4v(9, 10, 11), - c4v(100, 101, 102), c4v(103, 104, 105), c4v(106, 107, 108), c4v(109, 110, 111) - ); -#undef c4v -#undef _ - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** Resource-owning archetype */ -struct IdOwner -{ - static int s_current; - int id; - int val; - - void check() const - { - CHECK_UNARY(id > 0); - } - void check(IdOwner const& that) const - { - check(); - CHECK_NE(id, that.id); - } - - IdOwner(int v = 0) { id = ++s_current; val = v; } - ~IdOwner() { if(id > 0) --s_current; } - IdOwner(IdOwner const& that) { id = ++s_current; val = that.val; } - IdOwner(IdOwner && that) { id = that.id; val = that.val; that.id = 0; } - IdOwner& operator= (IdOwner const& that) { C4_CHECK(id > 0); --s_current; id = ++s_current; val = that.val; return *this; } - IdOwner& operator= (IdOwner && that) { C4_CHECK(id > 0); --s_current; id = that.id; val = that.val; that.id = 0; return *this; } - bool operator== (IdOwner const& that) const - { - return val == that.val; - } -}; - -_C4_DECLARE_ARCHETYPE_PROTO(IdOwner, 0, 1, 2, 3, 4, 5, 6, 7); - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** Memory-owning archetype, raw mem resource calls */ -template< class T > -struct MemOwner -{ - T *mem; - // prevent initialization order problems by using a memory resource here - MemoryResourceMalloc mr; - - void check() const - { - EXPECT_NE(mem, nullptr); - check_archetype(*mem); - } - void check(MemOwner const& that) const - { - EXPECT_NE(mem, that.mem); - } - - ~MemOwner() - { - if(!mem) return; - mem->~T(); - mr.deallocate(mem, sizeof(T), alignof(T)); - mem = nullptr; - } - - MemOwner() - { - mem = (T*)mr.allocate(sizeof(T), alignof(T)); - new (mem) T(); - } - template< class ...Args > - MemOwner(varargs_t, Args && ...args) - { - mem = (T*)mr.allocate(sizeof(T), alignof(T)); - new (mem) T(std::forward< Args >(args)...); - } - MemOwner(MemOwner const& that) - { - mem = (T*)mr.allocate(sizeof(T), alignof(T)); - new (mem) T(*that.mem); - } - MemOwner(MemOwner && that) - { - mem = that.mem; - that.mem = nullptr; - } - MemOwner& operator= (MemOwner const& that) - { - if(!mem) - { - mem = (T*)mr.allocate(sizeof(T), alignof(T)); - } - else - { - mem->~T(); - } - new (mem) T(*that.mem); - return *this; - } - MemOwner& operator= (MemOwner && that) - { - if(mem) - { - mem->~T(); - mr.deallocate(mem, sizeof(T), alignof(T)); - } - mem = that.mem; - that.mem = nullptr; - return *this; - } - bool operator== (MemOwner const& that) const - { - return *mem == *that.mem; - } -}; - -#define _ C4_COMMA -#define c4v(which) MemOwner{varargs _ archetype_proto::get(which)} -_C4_DECLARE_ARCHETYPE_PROTO_TPL1(class T, MemOwner, - c4v(0), c4v(1), c4v(2), c4v(3), - c4v(4), c4v(5), c4v(6), c4v(7) - ); -#undef c4v -#undef _ - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** Memory-owning archetype, with allocator */ -template< class T > -struct MemOwnerAlloc -{ - T *mem; - // prevent initialization order problems by using a memory resource here - MemoryResourceMalloc mr; - c4::Allocator< T > m_alloc; - using alloc_traits = std::allocator_traits< c4::Allocator< T > >; - - void check() const - { - EXPECT_NE(mem, nullptr); - check_archetype(*mem); - } - void check(MemOwnerAlloc const& that) const - { - check(); - EXPECT_NE(mem, that.mem); - } - - void free() - { - alloc_traits::destroy(m_alloc, mem); - alloc_traits::deallocate(m_alloc, mem, 1); - mem = nullptr; - } - ~MemOwnerAlloc() - { - if(!mem) return; - free(); - } - - MemOwnerAlloc() : m_alloc(&mr) - { - C4_ASSERT(m_alloc.resource() == &mr); - mem = alloc_traits::allocate(m_alloc, 1); - alloc_traits::construct(m_alloc, mem); - } - template< class ...Args > - MemOwnerAlloc(varargs_t, Args && ...args) : m_alloc(&mr) - { - mem = alloc_traits::allocate(m_alloc, 1); - alloc_traits::construct(m_alloc, mem, std::forward< Args >(args)...); - } - - MemOwnerAlloc(MemOwnerAlloc const& that) : m_alloc(&mr) - { - mem = alloc_traits::allocate(m_alloc, 1); - alloc_traits::construct(m_alloc, mem, *that.mem); - } - MemOwnerAlloc(MemOwnerAlloc && that) : m_alloc(&mr) - { - mem = that.mem; - that.mem = nullptr; - } - - MemOwnerAlloc& operator= (MemOwnerAlloc const& that) - { - if(!mem) - { - mem = alloc_traits::allocate(m_alloc, 1); - } - else - { - mem->~T(); - } - alloc_traits::construct(m_alloc, mem, *that.mem); - return *this; - } - MemOwnerAlloc& operator= (MemOwnerAlloc && that) - { - if(mem) - { - free(); - } - mem = that.mem; - that.mem = nullptr; - return *this; - } - - bool operator== (MemOwnerAlloc const& that) const - { - return *mem == *that.mem; - } -}; - -#define _ C4_COMMA -#define c4v(which) MemOwnerAlloc{varargs _ archetype_proto::get(which)} -_C4_DECLARE_ARCHETYPE_PROTO_TPL1(class T, MemOwnerAlloc, - c4v(0), c4v(1), c4v(2), c4v(3), - c4v(4), c4v(5), c4v(6), c4v(7) - ); -#undef c4v -#undef _ - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** base class archetype */ -struct Base -{ - virtual ~Base() = default; -protected: - Base() = default; -}; -/** derived class archetype */ -struct Derived : public Base -{ - -}; - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -template< class T > -struct InsidePtr -{ - T a; - T b; - T c; - T *ptr; - - InsidePtr(int which = 0) : a(), b(), c(), ptr(&a + (which % 3)) {} - InsidePtr(InsidePtr const& that) : a(that.a), b(that.b), c(that.c), ptr(&a + (that.ptr - &that.a)) {} - InsidePtr(InsidePtr && that) : a(std::move(that.a)), b(std::move(that.b)), c(std::move(that.c)), ptr(&a + (that.ptr - &that.a)) { that.ptr = nullptr; } - InsidePtr& operator= (InsidePtr const& that) { a = (that.a); b = (that.b); c = (that.c); ptr = (&a + (that.ptr - &that.a)); return *this; } - InsidePtr& operator= (InsidePtr && that) { a = std::move(that.a); b = std::move(that.b); c = std::move(that.c); ptr = (&a + (that.ptr - &that.a)); that.ptr = nullptr; return *this; } - ~InsidePtr() { EXPECT_TRUE(ptr == &a || ptr == &b || ptr == &c || ptr == nullptr); } - - void check() const - { - EXPECT_TRUE(ptr == &a || ptr == &b || ptr == &c); - } - void check(InsidePtr const& that) const - { - check(); - EXPECT_EQ(ptr - &a, that.ptr - &that.a); - } - bool operator== (InsidePtr const& that) const - { - return that.a == a && that.b == b && that.c == c && (ptr - &a) == (that.ptr - &that.a); - } - -}; - -#define _ C4_COMMA -_C4_DECLARE_ARCHETYPE_PROTO_TPL1(class T, InsidePtr, - 0, 1, 2, 3, 4, 5, 6, 7 - ); -#undef _ - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -# define CALL_FOR_SCALAR_ARCHETYPES(mcr) \ - mcr(int , int) \ - mcr(uint64_t , uint64_t) - -# define CALL_FOR_CONTAINEE_ARCHETYPES(mcr) \ - CALL_FOR_SCALAR_ARCHETYPES(mcr) \ - mcr(MemOwnerAlloc_std_string , archetypes::MemOwnerAlloc) - - - -using scalars_quick = std::tuple; -using scalars = std::tuple< - char, wchar_t, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t, float, double ->; -using containees_quick = std::tuple< - int, - uint64_t, - archetypes::MemOwnerAlloc ->; -using containees = std::tuple< - char, wchar_t, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t, float, double, - archetypes::exvec3, - archetypes::exvec3, - archetypes::IdOwner, - archetypes::MemOwner, - archetypes::MemOwner, - archetypes::MemOwnerAlloc, - archetypes::MemOwnerAlloc, - archetypes::InsidePtr, - archetypes::InsidePtr ->; - - -#ifdef __clang__ -# pragma clang diagnostic pop -#endif - -} // namespace archetypes -} // namespace c4 - -#endif // _C4_TEST_ARCHETYPES_HPP_ diff --git a/thirdparty/ryml/ext/c4core/test/c4/libtest/supprwarn_pop.hpp b/thirdparty/ryml/ext/c4core/test/c4/libtest/supprwarn_pop.hpp deleted file mode 100644 index 695035ef6..000000000 --- a/thirdparty/ryml/ext/c4core/test/c4/libtest/supprwarn_pop.hpp +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _C4_SUPPRWARN_POP_HPP_ -#define _C4_SUPPRWARN_POP_HPP_ - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#elif defined(_MSC_VER) -# pragma warning(pop) -#endif - -#endif /* SUPPRWARN_POP_H */ diff --git a/thirdparty/ryml/ext/c4core/test/c4/libtest/supprwarn_push.hpp b/thirdparty/ryml/ext/c4core/test/c4/libtest/supprwarn_push.hpp deleted file mode 100644 index 4651e227c..000000000 --- a/thirdparty/ryml/ext/c4core/test/c4/libtest/supprwarn_push.hpp +++ /dev/null @@ -1,48 +0,0 @@ -#ifndef _C4_LIBTEST_SUPPRWARN_PUSH_HPP_ -#define _C4_LIBTEST_SUPPRWARN_PUSH_HPP_ - -/** @file supprwarn_push.hpp this file contains directives to make the - * compiler ignore warnings in test code. It should NOT be used for c4stl - * itself, but only in test code. */ - -#ifdef __clang__ -# pragma clang diagnostic push - /* NOTE: using , ## __VA_ARGS__ to deal with zero-args calls to - * variadic macros is not portable, but works in clang, gcc, msvc, icc. - * clang requires switching off compiler warnings for pedantic mode. - * @see http://stackoverflow.com/questions/32047685/variadic-macro-without-arguments */ -# pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" // warning: token pasting of ',' and __VA_ARGS__ is a GNU extension -# pragma clang diagnostic ignored "-Wunused-local-typedef" -# pragma clang diagnostic ignored "-Wsign-compare" // warning: comparison of integers of different signs: 'const unsigned long' and 'const int' -# pragma clang diagnostic ignored "-Wfloat-equal" // warning: comparing floating point with == or != is unsafe -# pragma clang diagnostic ignored "-Wwritable-strings" // ISO C++11 does not allow conversion from string literal to char* -# pragma clang diagnostic ignored "-Wunused-variable" -# pragma clang diagnostic ignored "-Wunused-parameter" -#elif defined(__GNUC__) -# pragma GCC diagnostic push - /* GCC also issues a warning for zero-args calls to variadic macros. - * This warning is switched on with -pedantic and apparently there is no - * easy way to turn it off as with clang. But marking this as a system - * header works. - * @see https://gcc.gnu.org/onlinedocs/cpp/System-Headers.html - * @see http://stackoverflow.com/questions/35587137/ */ -# pragma GCC system_header -# pragma GCC diagnostic ignored "-Wvariadic-macros" -# pragma GCC diagnostic ignored "-Wwrite-strings" // ISO C++ forbids converting a string constant to ‘C* {aka char*}’ -# pragma GCC diagnostic ignored "-Wunused-local-typedefs" -# pragma GCC diagnostic ignored "-Wunused-variable" -# pragma GCC diagnostic ignored "-Wunused-parameter" -# pragma GCC diagnostic ignored "-Wsign-compare" // warning: comparison of integers of different signs: 'const unsigned long' and 'const int' -# pragma GCC diagnostic ignored "-Wfloat-equal" // warning: comparing floating point with == or != is unsafe -# pragma GCC diagnostic ignored "-Wpedantic" -# pragma GCC diagnostic ignored "-pedantic" -#elif defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable:4018) // '>=': signed/unsigned mismatch -# pragma warning(disable:4127) // conditional expression is constant -# pragma warning(disable:4189) // local variable is initialized but not referenced -# pragma warning(disable:4389) // '==': signed/unsigned mismatch -# pragma warning(disable:4702) // unreachable code -#endif - -#endif /* _C4_LIBTEST_SUPPRWARN_PUSH_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/test/c4/libtest/test.cpp b/thirdparty/ryml/ext/c4core/test/c4/libtest/test.cpp deleted file mode 100644 index f005d24ac..000000000 --- a/thirdparty/ryml/ext/c4core/test/c4/libtest/test.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "c4/test.hpp" - -namespace c4 { - -size_t TestErrorOccurs::num_errors = 0; - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/test/c4/main.cpp b/thirdparty/ryml/ext/c4core/test/c4/main.cpp deleted file mode 100644 index 36c8001e6..000000000 --- a/thirdparty/ryml/ext/c4core/test/c4/main.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#define DOCTEST_CONFIG_SUPER_FAST_ASSERTS -#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN -#include diff --git a/thirdparty/ryml/ext/c4core/test/c4/test.hpp b/thirdparty/ryml/ext/c4core/test/c4/test.hpp deleted file mode 100644 index 936283a2a..000000000 --- a/thirdparty/ryml/ext/c4core/test/c4/test.hpp +++ /dev/null @@ -1,324 +0,0 @@ -#ifndef _C4_TEST_HPP_ -#define _C4_TEST_HPP_ - -#ifdef C4CORE_SINGLE_HEADER -#include "c4/c4core_all.hpp" -#else -#include "c4/config.hpp" -#include "c4/memory_resource.hpp" -#include "c4/allocator.hpp" -#include "c4/substr.hpp" -#endif -#include -#include - -// FIXME - these are just dumb placeholders -#define C4_LOGF_ERR(...) fprintf(stderr, __VA_ARGS__) -#define C4_LOGF_WARN(...) fprintf(stderr, __VA_ARGS__) -#define C4_LOGP(msg, ...) printf(msg) - -#define DOCTEST_CONFIG_SUPER_FAST_ASSERTS -#include - -#define CHECK_STREQ(lhs, rhs) CHECK_EQ(c4::to_csubstr(lhs), c4::to_csubstr(rhs)) -#define CHECK_FLOAT_EQ(lhs, rhs) CHECK((double)(lhs) == doctest::Approx((double)(rhs))) - - -namespace c4 { - -template -inline std::ostream& operator<< (std::ostream& stream, c4::basic_substring s) -{ - stream.write(s.str, std::streamsize(s.len)); - return stream; -} - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** RAII class that tests whether an error occurs inside a scope. */ -struct TestErrorOccurs -{ - TestErrorOccurs(size_t num_expected_errors = 1) - : - expected_errors(num_expected_errors), - tmp_settings(c4::ON_ERROR_CALLBACK, &TestErrorOccurs::error_callback) - { - num_errors = 0; - } - ~TestErrorOccurs() - { - CHECK_EQ(num_errors, expected_errors); - num_errors = 0; - } - - size_t expected_errors; - static size_t num_errors; - ScopedErrorSettings tmp_settings; - static void error_callback(const char* /*msg*/, size_t /*msg_size*/) - { - ++num_errors; - } -}; - -#define C4_EXPECT_ERROR_OCCURS(...) \ - auto _testerroroccurs##__LINE__ = TestErrorOccurs(__VA_ARGS__) - -#if C4_USE_ASSERT -# define C4_EXPECT_ASSERT_TRIGGERS(...) C4_EXPECT_ERROR_OCCURS(__VA_ARGS__) -#else -# define C4_EXPECT_ASSERT_TRIGGERS(...) -#endif - -#if C4_USE_XASSERT -# define C4_EXPECT_XASSERT_TRIGGERS(...) C4_EXPECT_ERROR_OCCURS(__VA_ARGS__) -#else -# define C4_EXPECT_XASSERT_TRIGGERS(...) -#endif - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** count constructors, destructors and assigns */ -template< class T > -struct Counting -{ - using value_type = T; - - T obj; - - bool operator== (T const& that) const { return obj == that; } - - static bool log_ctors; - static size_t num_ctors; - template< class ...Args > - Counting(Args && ...args); - - static bool log_dtors; - static size_t num_dtors; - ~Counting(); - - static bool log_copy_ctors; - static size_t num_copy_ctors; - Counting(Counting const& that); - - static bool log_move_ctors; - static size_t num_move_ctors; - Counting(Counting && that); - - static bool log_copy_assigns; - static size_t num_copy_assigns; - Counting& operator= (Counting const& that); - - static bool log_move_assigns; - static size_t num_move_assigns; - Counting& operator= (Counting && that); - - - struct check_num - { - char const* name; - size_t const& what; - size_t const initial; - size_t const must_be_num; - check_num(char const* nm, size_t const& w, size_t n) : name(nm), what(w), initial(what), must_be_num(n) {} - ~check_num() - { - size_t del = what - initial; - INFO("# of " << name << " calls: expected " << must_be_num << ", but got " << del); - CHECK_EQ(del, must_be_num); - } - }; - - static check_num check_ctors(size_t n) { return check_num("ctor", num_ctors, n); } - static check_num check_dtors(size_t n) { return check_num("dtor", num_dtors, n); } - static check_num check_copy_ctors(size_t n) { return check_num("copy_ctor", num_copy_ctors, n); } - static check_num check_move_ctors(size_t n) { return check_num("move_ctor", num_move_ctors, n); } - static check_num check_copy_assigns(size_t n) { return check_num("copy_assign", num_copy_assigns, n); } - static check_num check_move_assigns(size_t n) { return check_num("move_assign", num_move_assigns, n); } - - struct check_num_ctors_dtors - { - check_num ctors, dtors; - check_num_ctors_dtors(size_t _ctors, size_t _dtors) - : - ctors(check_ctors(_ctors)), - dtors(check_dtors(_dtors)) - { - } - }; - static check_num_ctors_dtors check_ctors_dtors(size_t _ctors, size_t _dtors) - { - return check_num_ctors_dtors(_ctors, _dtors); - } - - struct check_num_all - { - check_num ctors, dtors, cp_ctors, mv_ctors, cp_assigns, mv_assigns; - check_num_all(size_t _ctors, size_t _dtors, size_t _cp_ctors, size_t _mv_ctors, size_t _cp_assigns, size_t _mv_assigns) - { - ctors = check_ctors(_ctors); - dtors = check_dtors(_dtors); - cp_ctors = check_copy_ctors(_cp_ctors); - mv_ctors = check_move_ctors(_mv_ctors); - cp_assigns = check_copy_assigns(_cp_assigns); - mv_assigns = check_move_assigns(_mv_assigns); - } - }; - static check_num_all check_all(size_t _ctors, size_t _dtors, size_t _cp_ctors, size_t _move_ctors, size_t _cp_assigns, size_t _mv_assigns) - { - return check_num_all(_ctors, _dtors, _cp_ctors, _move_ctors, _cp_assigns, _mv_assigns); - } - - static void reset() - { - num_ctors = 0; - num_dtors = 0; - num_copy_ctors = 0; - num_move_ctors = 0; - num_copy_assigns = 0; - num_move_assigns = 0; - } -}; - -template< class T > size_t Counting< T >::num_ctors = 0; -template< class T > bool Counting< T >::log_ctors = false; -template< class T > size_t Counting< T >::num_dtors = 0; -template< class T > bool Counting< T >::log_dtors = false; -template< class T > size_t Counting< T >::num_copy_ctors = 0; -template< class T > bool Counting< T >::log_copy_ctors = false; -template< class T > size_t Counting< T >::num_move_ctors = 0; -template< class T > bool Counting< T >::log_move_ctors = false; -template< class T > size_t Counting< T >::num_copy_assigns = 0; -template< class T > bool Counting< T >::log_copy_assigns = false; -template< class T > size_t Counting< T >::num_move_assigns = 0; -template< class T > bool Counting< T >::log_move_assigns = false; - -template< class T > -template< class ...Args > -Counting< T >::Counting(Args && ...args) : obj(std::forward< Args >(args)...) -{ - if(log_ctors) C4_LOGP("Counting[{}]::ctor #{}\n", (void*)this, num_ctors); - ++num_ctors; -} - -template< class T > -Counting< T >::~Counting() -{ - if(log_dtors) C4_LOGP("Counting[{}]::dtor #{}\n", (void*)this, num_dtors); - ++num_dtors; -} - -template< class T > -Counting< T >::Counting(Counting const& that) : obj(that.obj) -{ - if(log_copy_ctors) C4_LOGP("Counting[{}]::copy_ctor #{}\n", (void*)this, num_copy_ctors); - ++num_copy_ctors; -} - -template< class T > -Counting< T >::Counting(Counting && that) : obj(std::move(that.obj)) -{ - if(log_move_ctors) C4_LOGP("Counting[{}]::move_ctor #{}\n", (void*)this, num_move_ctors); - ++num_move_ctors; -} - -template< class T > -Counting< T >& Counting< T >::operator= (Counting const& that) -{ - obj = that.obj; - if(log_copy_assigns) C4_LOGP("Counting[{}]::copy_assign #{}\n", (void*)this, num_copy_assigns); - ++num_copy_assigns; - return *this; -} - -template< class T > -Counting< T >& Counting< T >::operator= (Counting && that) -{ - obj = std::move(that.obj); - if(log_move_assigns) C4_LOGP("Counting[{}]::move_assign #{}\n", (void*)this, num_move_assigns); - ++num_move_assigns; - return *this; -} - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** @todo refactor to use RAII @see Counting */ -struct AllocationCountsChecker : public ScopedMemoryResourceCounts -{ - AllocationCounts first; - -public: - - AllocationCountsChecker() - : - ScopedMemoryResourceCounts(), - first(mr.counts()) - { - } - - AllocationCountsChecker(MemoryResource *mr_) - : - ScopedMemoryResourceCounts(mr_), - first(mr.counts()) - { - } - - void reset() - { - first = mr.counts(); - } - - /** check value of curr allocations and size */ - void check_curr(ssize_t expected_allocs, ssize_t expected_size) const - { - CHECK_EQ(mr.counts().curr.allocs, expected_allocs); - CHECK_EQ(mr.counts().curr.size, expected_size); - } - /** check delta of curr allocations and size since construction or last reset() */ - void check_curr_delta(ssize_t expected_allocs, ssize_t expected_size) const - { - AllocationCounts delta = mr.counts() - first; - CHECK_EQ(delta.curr.allocs, expected_allocs); - CHECK_EQ(delta.curr.size, expected_size); - } - - /** check value of total allocations and size */ - void check_total(ssize_t expected_allocs, ssize_t expected_size) const - { - CHECK_EQ(mr.counts().total.allocs, expected_allocs); - CHECK_EQ(mr.counts().total.size, expected_size); - } - /** check delta of total allocations and size since construction or last reset() */ - void check_total_delta(ssize_t expected_allocs, ssize_t expected_size) const - { - AllocationCounts delta = mr.counts() - first; - CHECK_EQ(delta.total.allocs, expected_allocs); - CHECK_EQ(delta.total.size, expected_size); - } - - /** check value of max allocations and size */ - void check_max(ssize_t expected_max_allocs, ssize_t expected_max_size) const - { - CHECK_EQ(mr.counts().max.allocs, expected_max_allocs); - CHECK_EQ(mr.counts().max.size, expected_max_size); - } - - /** check that since construction or the last reset(): - * - num_allocs occcurred - * - totaling total_size - * - of which the largest is max_size */ - void check_all_delta(ssize_t num_allocs, ssize_t total_size, ssize_t max_size) const - { - check_curr_delta(num_allocs, total_size); - check_total_delta(num_allocs, total_size); - check_max(num_allocs > mr.counts().max.allocs ? num_allocs : mr.counts().max.allocs, - max_size > mr.counts().max.size ? max_size : mr.counts().max.size); - } -}; - -} // namespace c4 - -#endif // _C4_LIBTEST_TEST_HPP_ diff --git a/thirdparty/ryml/ext/c4core/test/printintegers.py b/thirdparty/ryml/ext/c4core/test/printintegers.py deleted file mode 100644 index de7532491..000000000 --- a/thirdparty/ryml/ext/c4core/test/printintegers.py +++ /dev/null @@ -1,107 +0,0 @@ - - -def nb(val): - return [val-1, val, val+1] - - -def ipowers2(min, max): - vals = [] - v = int(min / 2) - while v < -10: - vals += nb(v) - v = int(v / 2) - vals += upowers2(max) - return vals - - -def upowers2(max): - vals = [] - v = 16 - while v < max: - vals += nb(v) - v *= 2 - return vals - - -def ipowers10(min, max): - vals = [] - v = -100 - while v > min: - vals += nb(v) - v *= 10 - vals += upowers10(max) - return vals - - -def upowers10(max): - vals = [] - v = 10 - while v < max: - vals += nb(v) - v *= 10 - return vals - - -def idiv10(min, max): - vals = [] - v = int(min / 10) - while v < -10: - vals.append(v) - v = int(v / 10) - vals += udiv10(max) - return vals - - -def udiv10(max): - vals = [] - v = int(max / 10) - while v > 10: - vals += nb(v) - v = int(v / 10) - return vals - - -def ivals(bits): - min = -(1 << (bits-1)) - max = -min-1 - vals = [min, min+1, min+2, min+3, min+4, min+5] - vals += ipowers2(min, max) - vals += ipowers10(min, max) - vals += idiv10(min, max) - vals += [-11, -10, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] - vals += [max-5, max-4, max-3, max-2, max-1, max] - return sorted(list(set(vals))) - - -def uvals(bits): - max = (1 << bits) - 1 - vals = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] - vals += upowers2(max) - vals += upowers10(max) - vals += udiv10(max) - vals += [max-5, max-4, max-3, max-2, max-1, max] - return sorted(list(set(vals))) - - - -def p(v): - return f' nc({v}, "{v}", "{hex(v)}", "{oct(v)}", "{bin(v)}"),' - - -def pn(numbits, fn): - print() - for a in fn(numbits): - print(p(a)) - - -pn(8, ivals) -pn(8, uvals) - -pn(16, ivals) -pn(16, uvals) - -pn(32, ivals) -pn(32, uvals) - -pn(64, ivals) -pn(64, uvals) diff --git a/thirdparty/ryml/ext/c4core/test/test_allocator.cpp b/thirdparty/ryml/ext/c4core/test/test_allocator.cpp deleted file mode 100644 index 78d0daf88..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_allocator.cpp +++ /dev/null @@ -1,246 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/allocator.hpp" -#endif - -#include "c4/test.hpp" -#include "c4/libtest/supprwarn_push.hpp" - -#include -#include -#include - -namespace c4 { - -template< class T > using small_adapter = c4::small_allocator< T >; -template< class T > using small_adapter_mr = c4::small_allocator_mr< T >; - -#define _c4definealloctypes(Alloc) \ -using AllocInt = typename Alloc::template rebind::other;\ -using AllocChar = typename Alloc::template rebind::other;\ -using _string = std::basic_string< char, std::char_traits, AllocChar >;\ -using AllocString = typename Alloc::template rebind<_string>::other;\ -using AllocPair = typename Alloc::template rebind>::other;\ -using _vector_int = std::vector;\ -using _vector_string = std::vector<_string, AllocString >;\ -using _map_string_int = std::map<_string, int, std::less<_string>, AllocPair >; - -//----------------------------------------------------------------------------- -template< class Alloc > -void test_traits_compat_construct(typename Alloc::value_type const& val, Alloc &a) -{ - using atraits = std::allocator_traits< Alloc >; - using value_type = typename Alloc::value_type; - - value_type *mem = a.allocate(1); - REQUIRE_NE(mem, nullptr); - atraits::construct(a, mem, val); - CHECK_EQ(*mem, val); - - atraits::destroy(a, mem); - a.deallocate(mem, 1); -} - -TEST_CASE("allocator.traits_compat_construct") -{ - allocator a; - test_traits_compat_construct(1, a); -} - -TEST_CASE("small_allocator.traits_compat_construct") -{ - small_allocator a; - test_traits_compat_construct(1, a); -} - -TEST_CASE("allocator_mr_global.traits_compat_construct") -{ - allocator_mr a; - test_traits_compat_construct(1, a); -} - -TEST_CASE("allocator_mr_linear.traits_compat_construct") -{ - MemoryResourceLinear mr(1024); - allocator_mr a(&mr); - test_traits_compat_construct(1, a); -} - -TEST_CASE("allocator_mr_linear_arr.traits_compat_construct") -{ - MemoryResourceLinearArr<1024> mr; - allocator_mr a(&mr); - test_traits_compat_construct(1, a); -} - -TEST_CASE("small_allocator_mr_global.traits_compat_construct") -{ - small_allocator_mr a; - test_traits_compat_construct(1, a); -} - -TEST_CASE("small_allocator_mr_linear.traits_compat_construct") -{ - MemoryResourceLinear mr(1024); - small_allocator_mr a(&mr); - test_traits_compat_construct(1, a); -} - -TEST_CASE("small_allocator_mr_linear_arr.traits_compat_construct") -{ - MemoryResourceLinearArr<1024> mr; - small_allocator_mr a(&mr); - test_traits_compat_construct(1, a); -} - -//----------------------------------------------------------------------------- - -template< class Alloc > -void clear_mr(Alloc a) -{ - auto mrl = dynamic_cast(a.resource()); - if(mrl) - { - mrl->clear(); - } -} - -template< class Alloc > -void do_std_containers_test(Alloc alloc) -{ - _c4definealloctypes(Alloc); - - { - _string v(alloc); - v.reserve(256); - v = "adskjhsdfkjdflkjsdfkjhsdfkjh"; - CHECK_EQ(v, "adskjhsdfkjdflkjsdfkjhsdfkjh"); - } - - clear_mr(alloc); - - { - int arr[128]; - for(int &i : arr) - { - i = 42; - } - _vector_int vi(arr, arr+C4_COUNTOF(arr), alloc); - for(int i : vi) - { - CHECK_EQ(i, 42); - } - } - - clear_mr(alloc); - - { - AllocChar a = alloc; - _vector_string v({"foo", "bar", "baz", "bat", "bax"}, a); - CHECK_EQ(v.size(), 5); - CHECK_EQ(v[0], "foo"); - CHECK_EQ(v[1], "bar"); - CHECK_EQ(v[2], "baz"); - CHECK_EQ(v[3], "bat"); - CHECK_EQ(v[4], "bax"); - } - - clear_mr(alloc); - - { - AllocString a = alloc; - _vector_string v(a); - v.resize(4); - int count = 0; - for(auto &s : v) - { - _string ss(size_t(64), (char)('0' + count++)); - s = ss; - } - } - - clear_mr(alloc); - - { -#if !defined(__GNUC__) || (__GNUC__ >= 5) - /* constructor does not exist on gcc < 5) */ - AllocPair a = alloc; - _map_string_int v(a); -#else - _map_string_int v; -#endif - CHECK_EQ(v.size(), 0); - v["foo"] = 0; - v["bar"] = 1; - v["baz"] = 2; - v["bat"] = 3; - CHECK_EQ(v.size(), 4); - CHECK_EQ(v["foo"], 0); - CHECK_EQ(v["bar"], 1); - CHECK_EQ(v["baz"], 2); - CHECK_EQ(v["bat"], 3); - } -} - -TEST_CASE("allocator_global.std_containers") -{ - allocator a; - do_std_containers_test(a); -} - -TEST_CASE("small_allocator_global.std_containers") -{ - /* this is failing, investigate - small_allocator a; - do_std_containers_test(a); - */ -} - -TEST_CASE("allocator_mr_global.std_containers") -{ - allocator_mr a; - do_std_containers_test(a); -} - -TEST_CASE("allocator_mr_linear.std_containers") -{ - MemoryResourceLinear mr(1024); - allocator_mr a(&mr); - do_std_containers_test(a); -} - -TEST_CASE("allocator_mr_linear_arr.std_containers") -{ - MemoryResourceLinearArr<1024> mr; - allocator_mr a(&mr); - do_std_containers_test(a); -} - -TEST_CASE("small_allocator_mr_global.std_containers") -{ - /* this is failing, investigate - small_allocator_mr a; - do_std_containers_test(a); - */ -} - -TEST_CASE("small_allocator_mr_linear.std_containers") -{ - /* this is failing, investigate - MemoryResourceLinear mr(1024); - small_allocator_mr a(&mr); - do_std_containers_test(a); - */ -} - -TEST_CASE("small_allocator_mr_linear_arr.std_containers") -{ - /* this is failing, investigate - MemoryResourceLinearArr<1024> mr; - small_allocator_mr a(&mr); - do_std_containers_test(a); - */ -} - -} // namespace c4 - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_base64.cpp b/thirdparty/ryml/ext/c4core/test/test_base64.cpp deleted file mode 100644 index e638a0d81..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_base64.cpp +++ /dev/null @@ -1,265 +0,0 @@ -#include "c4/test.hpp" -#ifndef C4CORE_SINGLE_HEADER -#include "c4/std/string.hpp" -#include "c4/std/vector.hpp" -#include "c4/format.hpp" -#include "c4/base64.hpp" -#endif - -#include "c4/libtest/supprwarn_push.hpp" - -#include - -C4_SUPPRESS_WARNING_MSVC_WITH_PUSH(4310) // cast truncates constant value - -namespace c4 { - -namespace detail { -void base64_test_tables(); -TEST_CASE("base64.infrastructure") -{ - #ifndef NDEBUG - detail::base64_test_tables(); - #endif -} -// Since some the macros in c4/cpu.cpp cannot identify endanness at compile -// time, we use a simple runtime endianness-detection routine. -bool is_little_endian() -{ - unsigned long const v = 1UL; - unsigned char b[sizeof(v)]; - std::memcpy(&b[0], &v, sizeof(v)); - return !!b[0]; -} -} // namespace detail - -csubstr native(csubstr little_endian, csubstr big_endian) -{ - return detail::is_little_endian() ? little_endian : big_endian; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_base64_str_roundtrip(T const& val, csubstr expected, U *ws) -{ - char buf_[512]; - substr buf(buf_); - - csubstr encoded = to_chars_sub(buf, fmt::base64(val)); - CHECK(base64_valid(encoded)); - CHECK_EQ(encoded, expected); - CHECK_EQ(encoded.len % 4, 0); - - auto req = fmt::base64(*ws); - size_t written = from_chars(encoded, &req); - CHECK_EQ(ws->first(written), val); -} - -template -void test_base64_roundtrip(T const& val, csubstr expected) -{ - char buf_[512]; - substr buf(buf_); - - csubstr encoded = to_chars_sub(buf, fmt::base64(val)); - CHECK(base64_valid(encoded)); - CHECK_EQ(encoded, expected); - CHECK_EQ(encoded.len % 4, 0); - - T ws = {}; - auto req = fmt::base64(ws); - size_t written = from_chars(encoded, &req); - CHECK_EQ(written, sizeof(T)); - CHECK_EQ(ws, val); -} - -template -struct base64_test_pair -{ - T val; - csubstr encoded; -}; - -base64_test_pair base64_str_pairs[] = { -#define __(val, expected) {csubstr(val), csubstr(expected)} - __("" , "" ), - __("0" , "MA==" ), - __("1" , "MQ==" ), - __("2" , "Mg==" ), - __("3" , "Mw==" ), - __("4" , "NA==" ), - __("5" , "NQ==" ), - __("6" , "Ng==" ), - __("7" , "Nw==" ), - __("8" , "OA==" ), - __("9" , "OQ==" ), - __("10" , "MTA=" ), - __("123" , "MTIz" ), - __("1234" , "MTIzNA==" ), - __("1235" , "MTIzNQ==" ), - __("Man" , "TWFu" ), - __("Ma" , "TWE=" ), - __("M" , "TQ==" ), - __("deadbeef" , "ZGVhZGJlZWY=" ), - __("any carnal pleasure.", "YW55IGNhcm5hbCBwbGVhc3VyZS4="), - __("any carnal pleasure" , "YW55IGNhcm5hbCBwbGVhc3VyZQ=="), - __("any carnal pleasur" , "YW55IGNhcm5hbCBwbGVhc3Vy" ), - __("any carnal pleasu" , "YW55IGNhcm5hbCBwbGVhc3U=" ), - __("any carnal pleas" , "YW55IGNhcm5hbCBwbGVhcw==" ), - __( "pleasure.", "cGxlYXN1cmUu" ), - __( "leasure.", "bGVhc3VyZS4=" ), - __( "easure.", "ZWFzdXJlLg==" ), - __( "asure.", "YXN1cmUu" ), - __( "sure.", "c3VyZS4=" ), -#undef __ -}; - - -TEST_CASE("base64.str") -{ - char buf_[512]; - substr buf(buf_); - for(auto p : base64_str_pairs) - { - INFO(p.val); - test_base64_str_roundtrip(p.val, p.encoded, &buf); - } -} - -TEST_CASE_TEMPLATE("base64.8bit", T, int8_t, uint8_t) -{ - base64_test_pair pairs[] = { - {(T) 0, csubstr("AA==")}, - {(T) 1, csubstr("AQ==")}, - {(T) 2, csubstr("Ag==")}, - {(T) 3, csubstr("Aw==")}, - {(T) 4, csubstr("BA==")}, - {(T) 5, csubstr("BQ==")}, - {(T) 6, csubstr("Bg==")}, - {(T) 7, csubstr("Bw==")}, - {(T) 8, csubstr("CA==")}, - {(T) 9, csubstr("CQ==")}, - {(T) 10, csubstr("Cg==")}, - {(T) 11, csubstr("Cw==")}, - {(T) 12, csubstr("DA==")}, - {(T) 13, csubstr("DQ==")}, - {(T) 14, csubstr("Dg==")}, - {(T) 15, csubstr("Dw==")}, - {(T) 16, csubstr("EA==")}, - {(T) 17, csubstr("EQ==")}, - {(T) 18, csubstr("Eg==")}, - {(T) 19, csubstr("Ew==")}, - {(T) 20, csubstr("FA==")}, - {(T)127, csubstr("fw==")}, - {(T)128, csubstr("gA==")}, - {(T)254, csubstr("/g==")}, - {(T)255, csubstr("/w==")}, - }; - for(auto p : pairs) - { - INFO("val=" << (int)p.val << " expected=" << p.encoded); - test_base64_roundtrip(p.val, p.encoded); - } -} - -TEST_CASE_TEMPLATE("base64.16bit", T, int16_t, uint16_t) -{ - base64_test_pair pairs[] = { - { 0, native("AAA=", "AAA=")}, - { 1, native("AQA=", "AAE=")}, - { 2, native("AgA=", "AAI=")}, - { 3, native("AwA=", "AAM=")}, - { 4, native("BAA=", "AAQ=")}, - { 5, native("BQA=", "AAU=")}, - { 6, native("BgA=", "AAY=")}, - { 7, native("BwA=", "AAc=")}, - { 8, native("CAA=", "AAg=")}, - { 9, native("CQA=", "AAk=")}, - { 10, native("CgA=", "AAo=")}, - {1234, native("0gQ=", "BNI=")}, - }; - for(auto p : pairs) - { - INFO("val=" << p.val << " expected=" << p.encoded); - test_base64_roundtrip(p.val, p.encoded); - } -} - -TEST_CASE_TEMPLATE("base64.32bit", T, int32_t, uint32_t) -{ - base64_test_pair pairs[] = { - { 0, native("AAAAAA==", "AAAAAA==")}, - { 1, native("AQAAAA==", "AAAAAQ==")}, - { 2, native("AgAAAA==", "AAAAAg==")}, - { 3, native("AwAAAA==", "AAAAAw==")}, - { 4, native("BAAAAA==", "AAAABA==")}, - { 5, native("BQAAAA==", "AAAABQ==")}, - { 6, native("BgAAAA==", "AAAABg==")}, - { 7, native("BwAAAA==", "AAAABw==")}, - { 8, native("CAAAAA==", "AAAACA==")}, - { 9, native("CQAAAA==", "AAAACQ==")}, - { 10, native("CgAAAA==", "AAAACg==")}, - {1234, native("0gQAAA==", "AAAE0g==")}, - }; - for(auto p : pairs) - { - INFO("val=" << p.val << " expected=" << p.encoded); - test_base64_roundtrip(p.val, p.encoded); - } -} - -TEST_CASE_TEMPLATE("base64.64bit", T, int64_t, uint64_t) -{ - base64_test_pair pairs[] = { - { 0, native("AAAAAAAAAAA=", "AAAAAAAAAAA=")}, - { 1, native("AQAAAAAAAAA=", "AAAAAAAAAAE=")}, - { 2, native("AgAAAAAAAAA=", "AAAAAAAAAAI=")}, - { 3, native("AwAAAAAAAAA=", "AAAAAAAAAAM=")}, - { 4, native("BAAAAAAAAAA=", "AAAAAAAAAAQ=")}, - { 5, native("BQAAAAAAAAA=", "AAAAAAAAAAU=")}, - { 6, native("BgAAAAAAAAA=", "AAAAAAAAAAY=")}, - { 7, native("BwAAAAAAAAA=", "AAAAAAAAAAc=")}, - { 8, native("CAAAAAAAAAA=", "AAAAAAAAAAg=")}, - { 9, native("CQAAAAAAAAA=", "AAAAAAAAAAk=")}, - { 10, native("CgAAAAAAAAA=", "AAAAAAAAAAo=")}, - {1234, native("0gQAAAAAAAA=", "AAAAAAAABNI=")}, - {0xdeadbeef, native("776t3gAAAAA=", "AAAAAN6tvu8=")}, - }; - for(auto p : pairs) - { - INFO("val=" << p.val << " expected=" << p.encoded); - test_base64_roundtrip(p.val, p.encoded); - } -} - -TEST_CASE("base64.high_bits_u32") -{ - test_base64_roundtrip(UINT32_C(0xdeadbeef), native("776t3g==", "3q2+7w==")); - test_base64_roundtrip(UINT32_MAX, native("/////w==", "/////w==")); -} - -TEST_CASE("base64.high_bits_i32") -{ - test_base64_roundtrip(INT32_C(0x7fffffff), native("////fw==", "f////w==")); - test_base64_roundtrip(INT32_MAX, native("////fw==", "f////w==")); -} - -TEST_CASE("base64.high_bits_u64") -{ - test_base64_roundtrip(UINT64_MAX, native("//////////8=", "//////////8=")); -} - -TEST_CASE("base64.high_bits_i64") -{ - test_base64_roundtrip(INT64_MAX, native("/////////38=", "f/////////8=")); -} - -} // namespace c4 - -C4_SUPPRESS_WARNING_MSVC_POP - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_bitmask.cpp b/thirdparty/ryml/ext/c4core/test/test_bitmask.cpp deleted file mode 100644 index e7137d2a5..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_bitmask.cpp +++ /dev/null @@ -1,385 +0,0 @@ -#include -#include - -#ifndef C4CORE_SINGLE_HEADER -#include -#include -#endif - -#include - -#include "./test_enum_common.hpp" - -template -void cmp_enum(Enum lhs, Enum rhs) -{ - using I = typename std::underlying_type::type; - CHECK_EQ(static_cast(lhs), static_cast(rhs)); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -TEST_CASE("str2bm.simple_bitmask") -{ - using namespace c4; - std::vector str; - - CHECK_EQ(BM_NONE, str2bm("BM_NONE")); - CHECK_EQ(BM_NONE, str2bm("NONE")); - CHECK_EQ(BM_NONE, str2bm("0")); - CHECK_EQ(BM_NONE, str2bm("0x0")); - - CHECK_EQ(BM_FOO, str2bm("BM_FOO")); - CHECK_EQ(BM_FOO, str2bm("FOO")); - CHECK_EQ(BM_FOO, str2bm("1")); - CHECK_EQ(BM_FOO, str2bm("0x1")); - CHECK_EQ(BM_FOO, str2bm("BM_NONE|0x1")); - - CHECK_EQ(BM_BAR, str2bm("BM_BAR")); - CHECK_EQ(BM_BAR, str2bm("BAR")); - CHECK_EQ(BM_BAR, str2bm("2")); - CHECK_EQ(BM_BAR, str2bm("0x2")); - CHECK_EQ(BM_BAR, str2bm("BM_NONE|0x2")); - - CHECK_EQ(BM_BAZ, str2bm("BM_BAZ")); - CHECK_EQ(BM_BAZ, str2bm("BAZ")); - CHECK_EQ(BM_BAZ, str2bm("4")); - CHECK_EQ(BM_BAZ, str2bm("0x4")); - - CHECK_EQ(BM_FOO_BAR, str2bm("BM_FOO|BM_BAR")); - CHECK_EQ(BM_FOO_BAR, str2bm("BM_FOO|BAR")); - CHECK_EQ(BM_FOO_BAR, str2bm("FOO|BM_BAR")); - CHECK_EQ(BM_FOO_BAR, str2bm("BM_FOO_BAR")); - CHECK_EQ(BM_FOO_BAR, str2bm("FOO_BAR")); - CHECK_EQ(BM_FOO_BAR, str2bm("FOO|BAR")); - CHECK_EQ(BM_FOO_BAR, str2bm("0x1|0x2")); - CHECK_EQ(BM_FOO_BAR, str2bm("1|2")); - CHECK_EQ(BM_FOO_BAR, str2bm("0x3")); - CHECK_EQ(BM_FOO_BAR, str2bm("3")); - - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("BM_FOO|BM_BAR|BM_BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("BM_FOO|BM_BAR|BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("BM_FOO|BAR|BM_BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("FOO|BM_BAR|BM_BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("FOO|BM_BAR|BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("FOO|BAR|BM_BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("FOO|BAR|BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("FOO_BAR|BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("BM_FOO_BAR|BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("0x1|BAR|BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("FOO|0x2|BAZ")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("FOO|BAR|0x4")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("0x1|0x2|0x4")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("0x7")); - CHECK_EQ(BM_FOO_BAR_BAZ, str2bm("7")); -} - -TEST_CASE("str2bm.scoped_bitmask") -{ - using namespace c4; - std::vector str; - using bmt = MyBitmaskClass; - - cmp_enum(bmt::BM_NONE, (bmt)str2bm("MyBitmaskClass::BM_NONE")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("MyBitmaskClass::BM_FOO")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("MyBitmaskClass::BM_BAR")); - cmp_enum(bmt::BM_BAZ, (bmt)str2bm("MyBitmaskClass::BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("MyBitmaskClass::BM_FOO_BAR")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("MyBitmaskClass::BM_FOO_BAR_BAZ")); - cmp_enum(bmt::BM_NONE, (bmt)str2bm("BM_NONE")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("BM_FOO")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("BM_BAR")); - cmp_enum(bmt::BM_BAZ, (bmt)str2bm("BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("BM_FOO_BAR")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("BM_FOO_BAR_BAZ")); - cmp_enum(bmt::BM_NONE, (bmt)str2bm("NONE")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("FOO")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("BAR")); - cmp_enum(bmt::BM_BAZ, (bmt)str2bm("BAZ")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("FOO_BAR")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO_BAR_BAZ")); - - cmp_enum(bmt::BM_NONE, (bmt)str2bm("NONE")); - cmp_enum(bmt::BM_NONE, (bmt)str2bm("BM_NONE")); - cmp_enum(bmt::BM_NONE, (bmt)str2bm("MyBitmaskClass::BM_NONE")); - cmp_enum(bmt::BM_NONE, (bmt)str2bm("0")); - cmp_enum(bmt::BM_NONE, (bmt)str2bm("0x0")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("FOO")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("BM_FOO")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("MyBitmaskClass::BM_FOO")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("1")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("0x1")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("NONE|0x1")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("BM_NONE|0x1")); - cmp_enum(bmt::BM_FOO, (bmt)str2bm("MyBitmaskClass::BM_NONE|0x1")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("BAR")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("BM_BAR")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("MyBitmaskClass::BM_BAR")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("2")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("0x2")); - cmp_enum(bmt::BM_BAZ, (bmt)str2bm("BAZ")); - cmp_enum(bmt::BM_BAZ, (bmt)str2bm("BM_BAZ")); - cmp_enum(bmt::BM_BAZ, (bmt)str2bm("MyBitmaskClass::BM_BAZ")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("BM_NONE|0x2")); - cmp_enum(bmt::BM_BAR, (bmt)str2bm("MyBitmaskClass::BM_NONE|0x2")); - cmp_enum(bmt::BM_BAZ, (bmt)str2bm("4")); - cmp_enum(bmt::BM_BAZ, (bmt)str2bm("0x4")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("BM_FOO|BM_BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("MyBitmaskClass::BM_FOO|MyBitmaskClass::BM_BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("BM_FOO|BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("MyBitmaskClass::BM_FOO|BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("FOO|BM_BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("FOO|MyBitmaskClass::BM_BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("BM_FOO_BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("MyBitmaskClass::BM_FOO_BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("FOO_BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("FOO|BAR")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("0x1|0x2")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("1|2")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("0x3")); - cmp_enum(bmt::BM_FOO_BAR, (bmt)str2bm("3")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("BM_FOO|BM_BAR|BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("MyBitmaskClass::BM_FOO|MyBitmaskClass::BM_BAR|MyBitmaskClass::BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("BM_FOO|BM_BAR|BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("MyBitmaskClass::BM_FOO|MyBitmaskClass::BM_BAR|BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("BM_FOO|BAR|BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("BM_FOO|BAR|MyBitmaskClass::BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO|BM_BAR|BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO|MyBitmaskClass::BM_BAR|MyBitmaskClass::BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO|BM_BAR|BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO|MyBitmaskClass::BM_BAR|BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO|BAR|BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO|BAR|MyBitmaskClass::BM_BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO|BAR|BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO_BAR|BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("MyBitmaskClass::BM_FOO_BAR|BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("0x1|BAR|BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO|0x2|BAZ")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("FOO|BAR|0x4")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("0x1|0x2|0x4")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("0x7")); - cmp_enum(bmt::BM_FOO_BAR_BAZ, (bmt)str2bm("0x7")); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -template -const char* do_bm2str(Enum e, std::vector *s, c4::EnumOffsetType which) -{ - size_t len = c4::bm2str(e, nullptr, 0, which); - C4_CHECK(len > 0); - s->resize(len); - C4_CHECK(s->data() != nullptr); - c4::bm2str(e, s->data(), s->size(), which); - return s->data(); -} - -TEST_CASE("bm2str.simple_bitmask") -{ - using namespace c4; - std::vector str; - - CHECK_STREQ(do_bm2str(BM_NONE, &str, EOFFS_NONE), "BM_NONE"); - CHECK_STREQ(do_bm2str(BM_FOO, &str, EOFFS_NONE), "BM_FOO"); - CHECK_STREQ(do_bm2str(BM_BAR, &str, EOFFS_NONE), "BM_BAR"); - CHECK_STREQ(do_bm2str(BM_BAZ, &str, EOFFS_NONE), "BM_BAZ"); - CHECK_STREQ(do_bm2str(BM_FOO_BAR, &str, EOFFS_NONE), "BM_FOO_BAR"); - CHECK_STREQ(do_bm2str(BM_FOO_BAR_BAZ, &str, EOFFS_NONE), "BM_FOO_BAR_BAZ"); - CHECK_STREQ(do_bm2str(BM_NONE, &str, EOFFS_CLS ), "BM_NONE"); - CHECK_STREQ(do_bm2str(BM_FOO, &str, EOFFS_CLS ), "BM_FOO"); - CHECK_STREQ(do_bm2str(BM_BAR, &str, EOFFS_CLS ), "BM_BAR"); - CHECK_STREQ(do_bm2str(BM_BAZ, &str, EOFFS_CLS ), "BM_BAZ"); - CHECK_STREQ(do_bm2str(BM_FOO_BAR, &str, EOFFS_CLS ), "BM_FOO_BAR"); - CHECK_STREQ(do_bm2str(BM_FOO_BAR_BAZ, &str, EOFFS_CLS ), "BM_FOO_BAR_BAZ"); - CHECK_STREQ(do_bm2str(BM_NONE, &str, EOFFS_PFX ), "NONE"); - CHECK_STREQ(do_bm2str(BM_FOO, &str, EOFFS_PFX ), "FOO"); - CHECK_STREQ(do_bm2str(BM_BAR, &str, EOFFS_PFX ), "BAR"); - CHECK_STREQ(do_bm2str(BM_BAZ, &str, EOFFS_PFX ), "BAZ"); - CHECK_STREQ(do_bm2str(BM_FOO_BAR, &str, EOFFS_PFX ), "FOO_BAR"); - CHECK_STREQ(do_bm2str(BM_FOO_BAR_BAZ, &str, EOFFS_PFX ), "FOO_BAR_BAZ"); -} - -TEST_CASE("bm2str.scoped_bitmask") -{ - using namespace c4; - std::vector str; - - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_NONE, &str, EOFFS_NONE), "MyBitmaskClass::BM_NONE"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_FOO, &str, EOFFS_NONE), "MyBitmaskClass::BM_FOO"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_BAR, &str, EOFFS_NONE), "MyBitmaskClass::BM_BAR"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_BAZ, &str, EOFFS_NONE), "MyBitmaskClass::BM_BAZ"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_FOO_BAR, &str, EOFFS_NONE), "MyBitmaskClass::BM_FOO_BAR"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_FOO_BAR_BAZ, &str, EOFFS_NONE), "MyBitmaskClass::BM_FOO_BAR_BAZ"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_NONE, &str, EOFFS_CLS ), "BM_NONE"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_FOO, &str, EOFFS_CLS ), "BM_FOO"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_BAR, &str, EOFFS_CLS ), "BM_BAR"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_BAZ, &str, EOFFS_CLS ), "BM_BAZ"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_FOO_BAR, &str, EOFFS_CLS ), "BM_FOO_BAR"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_FOO_BAR_BAZ, &str, EOFFS_CLS ), "BM_FOO_BAR_BAZ"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_NONE, &str, EOFFS_PFX ), "NONE"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_FOO, &str, EOFFS_PFX ), "FOO"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_BAR, &str, EOFFS_PFX ), "BAR"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_BAZ, &str, EOFFS_PFX ), "BAZ"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_FOO_BAR, &str, EOFFS_PFX ), "FOO_BAR"); - CHECK_STREQ(do_bm2str(MyBitmaskClass::BM_FOO_BAR_BAZ, &str, EOFFS_PFX ), "FOO_BAR_BAZ"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -template -std::string do_bm2stream(Enum e, c4::EnumOffsetType which) -{ - std::stringstream ss; - c4::bm2stream(ss, e, which); - return ss.str(); -} - -TEST_CASE("bm2stream.simple_bitmask") -{ - using namespace c4; - - CHECK_EQ(do_bm2stream(BM_NONE, EOFFS_NONE), "BM_NONE"); - CHECK_EQ(do_bm2stream(BM_FOO, EOFFS_NONE), "BM_FOO"); - CHECK_EQ(do_bm2stream(BM_BAR, EOFFS_NONE), "BM_BAR"); - CHECK_EQ(do_bm2stream(BM_BAZ, EOFFS_NONE), "BM_BAZ"); - CHECK_EQ(do_bm2stream(BM_FOO_BAR, EOFFS_NONE), "BM_FOO_BAR"); - CHECK_EQ(do_bm2stream(BM_FOO_BAR_BAZ, EOFFS_NONE), "BM_FOO_BAR_BAZ"); - CHECK_EQ(do_bm2stream(BM_NONE, EOFFS_CLS ), "BM_NONE"); - CHECK_EQ(do_bm2stream(BM_FOO, EOFFS_CLS ), "BM_FOO"); - CHECK_EQ(do_bm2stream(BM_BAR, EOFFS_CLS ), "BM_BAR"); - CHECK_EQ(do_bm2stream(BM_BAZ, EOFFS_CLS ), "BM_BAZ"); - CHECK_EQ(do_bm2stream(BM_FOO_BAR, EOFFS_CLS ), "BM_FOO_BAR"); - CHECK_EQ(do_bm2stream(BM_FOO_BAR_BAZ, EOFFS_CLS ), "BM_FOO_BAR_BAZ"); - CHECK_EQ(do_bm2stream(BM_NONE, EOFFS_PFX ), "NONE"); - CHECK_EQ(do_bm2stream(BM_FOO, EOFFS_PFX ), "FOO"); - CHECK_EQ(do_bm2stream(BM_BAR, EOFFS_PFX ), "BAR"); - CHECK_EQ(do_bm2stream(BM_BAZ, EOFFS_PFX ), "BAZ"); - CHECK_EQ(do_bm2stream(BM_FOO_BAR, EOFFS_PFX ), "FOO_BAR"); - CHECK_EQ(do_bm2stream(BM_FOO_BAR_BAZ, EOFFS_PFX ), "FOO_BAR_BAZ"); -} - -TEST_CASE("bm2stream.scoped_bitmask") -{ - using namespace c4; - - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_NONE, EOFFS_NONE), "MyBitmaskClass::BM_NONE"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_FOO, EOFFS_NONE), "MyBitmaskClass::BM_FOO"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_BAR, EOFFS_NONE), "MyBitmaskClass::BM_BAR"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_BAZ, EOFFS_NONE), "MyBitmaskClass::BM_BAZ"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_FOO_BAR, EOFFS_NONE), "MyBitmaskClass::BM_FOO_BAR"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_FOO_BAR_BAZ, EOFFS_NONE), "MyBitmaskClass::BM_FOO_BAR_BAZ"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_NONE, EOFFS_CLS ), "BM_NONE"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_FOO, EOFFS_CLS ), "BM_FOO"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_BAR, EOFFS_CLS ), "BM_BAR"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_BAZ, EOFFS_CLS ), "BM_BAZ"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_FOO_BAR, EOFFS_CLS ), "BM_FOO_BAR"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_FOO_BAR_BAZ, EOFFS_CLS ), "BM_FOO_BAR_BAZ"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_NONE, EOFFS_PFX ), "NONE"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_FOO, EOFFS_PFX ), "FOO"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_BAR, EOFFS_PFX ), "BAR"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_BAZ, EOFFS_PFX ), "BAZ"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_FOO_BAR, EOFFS_PFX ), "FOO_BAR"); - CHECK_EQ(do_bm2stream(MyBitmaskClass::BM_FOO_BAR_BAZ, EOFFS_PFX ), "FOO_BAR_BAZ"); -} - -TEST_CASE("bm2stream.simple_bitmask_without_null_symbol") -{ - using namespace c4; - - CHECK_EQ(do_bm2stream(BM_KABOOM, EOFFS_NONE), "KABOOM"); - CHECK_EQ(do_bm2stream((BmWithoutNull)0, EOFFS_NONE), "0"); -} - -TEST_CASE("bm2stream.bitmask_class_without_null_symbol") -{ - using namespace c4; - - CHECK_EQ(do_bm2stream(BmClassWithoutNull::BM_KABOOM, EOFFS_PFX), "KABOOM"); - CHECK_EQ(do_bm2stream((BmClassWithoutNull)0, EOFFS_NONE), "0"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//---------------------------------------------------------------------------- - -// TODO -template -void test_bm2str() -{ - using namespace c4; - using I = typename std::underlying_type::type; - int combination_depth = 4; - auto syms = esyms(); - - std::vector indices; - std::string str; - std::vector ws; - I val = 0, res; - size_t len; - - for(int k = 1; k <= combination_depth; ++k) - { - indices.clear(); - indices.resize(static_cast(k)); - while(1) - { - str.clear(); - val = 0; - for(auto i : indices) - { - if(!str.empty()) str += '|'; - str += syms[i].name; - val |= static_cast(syms[i].value); - //printf("%d", i); - } - //len = bm2str(val); // needed length - //ws.resize(len); - //bm2str(val, &ws[0], len); - //printf(": %s (%zu) %s\n", str.c_str(), (uint64_t)val, ws.data()); - - res = str2bm(str.data()); - CHECK_EQ(res, val); - - len = bm2str(res); // needed length - ws.resize(len); - bm2str(val, &ws[0], len); - res = str2bm(ws.data()); - CHECK_EQ(res, val); - - // write a string with the bitmask as an int - c4::catrs(&ws, val); - res = str2bm(str.data()); - CHECK_EQ(res, val); - - bool carry = true; - for(size_t i = static_cast(k-1); i != size_t(-1); --i) - { - if(indices[i] + 1 < syms.size()) - { - ++indices[i]; - carry = false; - break; - } - else - { - indices[i] = 0; - } - } - if(carry) - { - break; - } - } // while(1) - } // for k -} diff --git a/thirdparty/ryml/ext/c4core/test/test_blob.cpp b/thirdparty/ryml/ext/c4core/test/test_blob.cpp deleted file mode 100644 index b803ba864..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_blob.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/blob.hpp" -#endif - -#include "c4/test.hpp" - -#ifdef __clang__ -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wcast-align" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wcast-align" -#endif - - -namespace c4 { - -template -void test_blob() -{ - T v; - blob b(v); - CHECK_EQ((T*)b.buf, &v); - CHECK_EQ(b.len, sizeof(T)); - - blob b2 = b; - CHECK_EQ((T*)b2.buf, &v); - CHECK_EQ(b2.len, sizeof(T)); -} - -TEST_CASE("blob.basic") -{ - test_blob(); -} - - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/test/test_char_traits.cpp b/thirdparty/ryml/ext/c4core/test/test_char_traits.cpp deleted file mode 100644 index 0f4e01518..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_char_traits.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/char_traits.hpp" -#endif - -#include "c4/test.hpp" - -namespace c4 { - -TEST_CASE("num_needed_chars.char") -{ - CHECK_EQ(num_needed_chars(0), 0); - CHECK_EQ(num_needed_chars(1), 1); - CHECK_EQ(num_needed_chars(2), 2); - CHECK_EQ(num_needed_chars(3), 3); - CHECK_EQ(num_needed_chars(4), 4); - for(int i = 0; i < 100; ++i) - { - CHECK_EQ(num_needed_chars(i), i); - } -} - -TEST_CASE("num_needed_chars.wchar_t") -{ -#if defined(_MSC_VER) -#pragma warning(push) -#pragma warning(disable : 4127) // C4127: conditional expression is constant -#endif - if(sizeof(wchar_t) == 2) - { - CHECK_EQ(num_needed_chars( 0), 0); - CHECK_EQ(num_needed_chars( 1), 1); - CHECK_EQ(num_needed_chars( 2), 1); - CHECK_EQ(num_needed_chars( 3), 2); - CHECK_EQ(num_needed_chars( 4), 2); - CHECK_EQ(num_needed_chars( 97), 49); - CHECK_EQ(num_needed_chars( 98), 49); - CHECK_EQ(num_needed_chars( 99), 50); - CHECK_EQ(num_needed_chars(100), 50); - CHECK_EQ(num_needed_chars(101), 51); - } - else if(sizeof(wchar_t) == 4) - { - CHECK_EQ(num_needed_chars( 0), 0); - CHECK_EQ(num_needed_chars( 1), 1); - CHECK_EQ(num_needed_chars( 2), 1); - CHECK_EQ(num_needed_chars( 3), 1); - CHECK_EQ(num_needed_chars( 4), 1); - CHECK_EQ(num_needed_chars( 5), 2); - CHECK_EQ(num_needed_chars( 6), 2); - CHECK_EQ(num_needed_chars( 7), 2); - CHECK_EQ(num_needed_chars( 8), 2); - CHECK_EQ(num_needed_chars( 93), 24); - CHECK_EQ(num_needed_chars( 94), 24); - CHECK_EQ(num_needed_chars( 95), 24); - CHECK_EQ(num_needed_chars( 96), 24); - CHECK_EQ(num_needed_chars( 97), 25); - CHECK_EQ(num_needed_chars( 98), 25); - CHECK_EQ(num_needed_chars( 99), 25); - CHECK_EQ(num_needed_chars(100), 25); - CHECK_EQ(num_needed_chars(101), 26); - } -#if defined(_MSC_VER) -#pragma warning(pop) -#endif -} - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/test/test_charconv.cpp b/thirdparty/ryml/ext/c4core/test/test_charconv.cpp deleted file mode 100644 index baa840d1e..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_charconv.cpp +++ /dev/null @@ -1,2743 +0,0 @@ -#ifdef C4CORE_SINGLE_HEADER -#include "c4/c4core_all.hpp" -#else -#include "c4/std/std.hpp" -#include "c4/charconv.hpp" -#include "c4/format.hpp" -#include "c4/type_name.hpp" -#endif - -#include "c4/libtest/supprwarn_push.hpp" - -C4_SUPPRESS_WARNING_GCC_PUSH -C4_SUPPRESS_WARNING_GCC("-Wfloat-equal") -C4_SUPPRESS_WARNING_GCC("-Wuseless-cast") -C4_SUPPRESS_WARNING_GCC("-Wconversion") -C4_SUPPRESS_WARNING_GCC("-Wtype-limits") -C4_SUPPRESS_WARNING_GCC("-Wfloat-equal") -#if defined (__GNUC__) && __GNUC_MAJOR__ >= 7 -C4_SUPPRESS_WARNING_GCC("-Wno-noexcept-type") -#endif -C4_SUPPRESS_WARNING_CLANG_PUSH -C4_SUPPRESS_WARNING_CLANG("-Wfloat-equal") - -#include - -#include "./test_numbers.hpp" - -namespace c4 { - -namespace { - -// skip the radix prefix: 0x, -0x, 0X, -0X, 0b, -0B, etc -csubstr nopfx(csubstr num) -{ - if(num.begins_with('-')) - num = num.sub(1); - if(num.len >= 2 && num[0] == '0') - { - switch(num[1]) - { - case 'x': case 'X': - case 'o': case 'O': - case 'b': case 'B': - num = num.sub(2); - } - } - return num; -} - -// filter out the radix prefix from anywhere: 0x, -0x, 0X, -0X, 0b, -0B, etc -csubstr nopfx(substr buf, csubstr num) -{ - REQUIRE_GE(buf.len, num.len); - if(num.begins_with('-')) - num = num.sub(1); - size_t pos = 0; - for(size_t i = 0; i < num.len; ++i) - { - const char c = num.str[i]; - if(c == '0') - { - const char n = i+1 < num.len ? num.str[i+1] : '\0'; - switch(n) - { - case 'x': case 'X': - case 'o': case 'O': - case 'b': case 'B': - ++i; - break; - default: - buf[pos++] = c; - break; - } - } - else - { - buf[pos++] = c; - } - } - return buf.first(pos); -} - -// capitalize the alphabetical characters -// eg 0xdeadbeef --> 0XDEADBEEF -substr capitalize(substr buf, csubstr str) -{ - C4_ASSERT(!buf.overlaps(str)); - memcpy(buf.str, str.str, str.len); - substr ret = buf.first(str.len); - ret.toupper(); - return ret; -} - -// prepend zeroes to the left of the number: -// eg 1234 --> 00001234 -// eg -1234 --> -00001234 -// eg 0x1234 --> 0x00001234 -// eg -0x1234 --> -0x00001234 -substr zpad(substr buf, csubstr str, size_t num_zeroes) -{ - C4_ASSERT(!buf.overlaps(str)); - size_t pos = 0; - if(str.len > 0 && str[0] == '-') - buf.str[pos++] = '-'; - if(str.len >= pos+2 && str[pos] == '0') - { - switch(str[pos+1]) - { - case 'x': case 'X': - case 'o': case 'O': - case 'b': case 'B': - memcpy(buf.str + pos, str.str + pos, 2); - pos += 2; - } - } - memset(buf.str + pos, '0', num_zeroes); - csubstr rem = str.sub(pos); - memcpy(buf.str + pos + num_zeroes, rem.str, rem.len); - return buf.first(str.len + num_zeroes); -} - -// get the front element of the type's test numbers -template -number_case const& front(size_t skip=0) -{ - return *(numbers::vals + skip); -} - -// get the back element of the type's test numbers -template -number_case const& back(size_t skip=0) -{ - return *(numbers::vals + C4_COUNTOF(numbers::vals) - 1 - skip); -} - -// given an element, get the n-th element previous to that -template -number_case const& prev(number_case const& curr, size_t less=1) -{ - C4_ASSERT(less >= 0); - size_t num = C4_COUNTOF(numbers::vals); - C4_ASSERT(&curr >= numbers::vals); - C4_ASSERT(&curr < numbers::vals + num); - size_t icurr = (size_t)(&curr - numbers::vals); - size_t prev = (icurr + num - less) % num; - return *(numbers::vals + prev); -} - -// given an element, get the n-th element after that -template -number_case const& next(number_case const& curr, size_t more=1) -{ - C4_ASSERT(more >= 0); - size_t num = C4_COUNTOF(numbers::vals); - C4_ASSERT(&curr >= numbers::vals); - C4_ASSERT(&curr < numbers::vals + num); - size_t icurr = (size_t)(&curr - numbers::vals); - size_t next = (icurr + more) % num; - return *(numbers::vals + next); -} - -// construct a string of a value such that it overflows an original value by a given amount -template -csubstr overflow_by(substr buf, T val, T how_much, T radix) -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_STATIC_ASSERT(sizeof(T) < sizeof(int64_t)); - using upcast_t = typename std::conditional::value, int64_t, uint64_t>::type; - upcast_t uval = (upcast_t) val; - uval += (upcast_t) how_much; - size_t len = xtoa(buf, uval, (upcast_t)radix); - REQUIRE_GE(buf.len, len); - return buf.first(len); -} - -// construct a string of a value such that it underflows an original value by a given amount -template -csubstr underflow_by(substr buf, T val, T how_much, T radix) -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_STATIC_ASSERT(sizeof(T) < sizeof(int64_t)); - using upcast_t = typename std::conditional::value, int64_t, uint64_t>::type; - upcast_t uval = (upcast_t) val; - uval -= (upcast_t) how_much; - size_t len = xtoa(buf, uval, (upcast_t)radix); - REQUIRE_GE(buf.len, len); - return buf.first(len); -} - -} // namespace - -TEST_CASE("charconv.to_chars_format") -{ -#if C4CORE_HAVE_STD_TO_CHARS - CHECK(FTOA_FLOAT == static_cast::type>(std::chars_format::fixed)); - CHECK(FTOA_SCIENT == static_cast::type>(std::chars_format::scientific)); - CHECK(FTOA_FLEX == static_cast::type>(std::chars_format::general)); - CHECK(FTOA_HEXA == static_cast::type>(std::chars_format::hex)); -#elif !C4CORE_HAVE_FAST_FLOAT - CHECK(FTOA_FLOAT == 'f'); - CHECK(FTOA_SCIENT == 'e'); - CHECK(FTOA_FLEX == 'g'); - CHECK(FTOA_HEXA == 'a'); -#endif -} - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("test_util.number_cases", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - CHECK_GT(number.dec.len, 0); - CHECK_GT(number.hex.len, 2); - CHECK_GT(number.oct.len, 2); - CHECK_GT(number.bin.len, 2); - CHECK_UNARY(number.hex.begins_with("0x")); - CHECK_UNARY(number.oct.begins_with("0o")); - CHECK_UNARY(number.bin.begins_with("0b")); - } - REQUIRE_GT(C4_COUNTOF(numbers::vals), 2); - // - CHECK_EQ(&front(), numbers::vals + 0); - CHECK_EQ(&front(0), numbers::vals + 0); - CHECK_EQ(&front(1), numbers::vals + 1); - // - CHECK_EQ(&back(), numbers::vals + C4_COUNTOF(numbers::vals) - 1); - CHECK_EQ(&back(0), numbers::vals + C4_COUNTOF(numbers::vals) - 1); - CHECK_EQ(&back(1), numbers::vals + C4_COUNTOF(numbers::vals) - 2); - // - CHECK_EQ(&next(front() ), numbers::vals + 1); - CHECK_EQ(&next(front(), T(1)), numbers::vals + 1); - CHECK_EQ(&next(front(), T(2)), numbers::vals + 2); - // - CHECK_EQ(&next(back() ), numbers::vals + 0); - CHECK_EQ(&next(back(), T(1)), numbers::vals + 0); - CHECK_EQ(&next(back(), T(2)), numbers::vals + 1); - CHECK_EQ(&next(back(), T(3)), numbers::vals + 2); - // - CHECK_EQ(&prev(front()), numbers::vals + C4_COUNTOF(numbers::vals) - 1); - CHECK_EQ(&prev(front(), T(1)), numbers::vals + C4_COUNTOF(numbers::vals) - 1); - CHECK_EQ(&prev(front(), T(2)), numbers::vals + C4_COUNTOF(numbers::vals) - 2); - // - CHECK_EQ(&prev(back()), numbers::vals + C4_COUNTOF(numbers::vals) - 2); - CHECK_EQ(&prev(back(), T(1)), numbers::vals + C4_COUNTOF(numbers::vals) - 2); - CHECK_EQ(&prev(back(), T(2)), numbers::vals + C4_COUNTOF(numbers::vals) - 3); -} - -TEST_CASE("test_util.overflow_by") -{ - char buf_[128]; - substr buf = buf_; - REQUIRE_EQ(overflow_by(buf, INT8_C(127), INT8_C(0), INT8_C(10)), "127"); - REQUIRE_EQ(overflow_by(buf, INT8_C(127), INT8_C(0), INT8_C(16)), "0x7f"); - REQUIRE_EQ(overflow_by(buf, INT8_C(127), INT8_C(1), INT8_C(10)), "128"); - REQUIRE_EQ(overflow_by(buf, INT8_C(127), INT8_C(1), INT8_C(16)), "0x80"); - REQUIRE_EQ(overflow_by(buf, INT8_C(127), INT8_C(2), INT8_C(10)), "129"); - REQUIRE_EQ(overflow_by(buf, INT8_C(127), INT8_C(2), INT8_C(16)), "0x81"); -} - -TEST_CASE("test_util.underflow_by") -{ - char buf_[128]; - substr buf = buf_; - REQUIRE_EQ(underflow_by(buf, INT8_C(-128), INT8_C(0), INT8_C(10)), "-128"); - REQUIRE_EQ(underflow_by(buf, INT8_C(-128), INT8_C(0), INT8_C(16)), "-0x80"); - REQUIRE_EQ(underflow_by(buf, INT8_C(-128), INT8_C(1), INT8_C(10)), "-129"); - REQUIRE_EQ(underflow_by(buf, INT8_C(-128), INT8_C(1), INT8_C(16)), "-0x81"); - REQUIRE_EQ(underflow_by(buf, INT8_C(-128), INT8_C(2), INT8_C(10)), "-130"); - REQUIRE_EQ(underflow_by(buf, INT8_C(-128), INT8_C(2), INT8_C(16)), "-0x82"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("digits_dec", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - CHECK_EQ(digits_dec(number.val), nopfx(number.dec).len); - } -} - -TEST_CASE_TEMPLATE("digits_hex", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - CHECK_EQ(digits_hex(number.val), nopfx(number.hex).len); - } -} - -TEST_CASE_TEMPLATE("digits_oct", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - CHECK_EQ(digits_oct(number.val), nopfx(number.oct).len); - } -} - - -TEST_CASE_TEMPLATE("digits_bin", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - CHECK_EQ(digits_bin(number.val), nopfx(number.bin).len); - } -} - - -//----------------------------------------------------------------------------- -TEST_CASE_TEMPLATE("write_dec_unchecked", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - unsigned digits = digits_dec(number.val); - REQUIRE_GE(buf.len, digits); - write_dec_unchecked(buf, number.val, digits); - CHECK_EQ(buf.first(digits), nopfx(number.dec)); - } -} - -TEST_CASE_TEMPLATE("write_hex_unchecked", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - unsigned digits = digits_hex(number.val); - REQUIRE_GE(buf.len, digits); - write_hex_unchecked(buf, number.val, digits); - CHECK_EQ(buf.first(digits), nopfx(number.hex)); - } -} - -TEST_CASE_TEMPLATE("write_oct_unchecked", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - unsigned digits = digits_hex(number.val); - REQUIRE_GE(buf.len, digits); - write_hex_unchecked(buf, number.val, digits); - CHECK_EQ(buf.first(digits), nopfx(number.hex)); - } -} - -TEST_CASE_TEMPLATE("write_bin_unchecked", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - unsigned digits = digits_bin(number.val); - REQUIRE_GE(buf.len, digits); - write_bin_unchecked(buf, number.val, digits); - CHECK_EQ(buf.first(digits), nopfx(number.bin)); - } -} - - -//----------------------------------------------------------------------------- -TEST_CASE_TEMPLATE("write_dec", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - REQUIRE_GE(buf.len, number.dec.len); - size_t retn = write_dec(substr{}, number.val); - CHECK_EQ(retn, number.dec.len); - size_t retb = write_dec(buf, number.val); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, number.dec.len); - CHECK_EQ(buf.first(retb), number.dec); - } -} - -TEST_CASE_TEMPLATE("write_hex", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - REQUIRE_GE(buf.len, number.hex.sub(2).len); - size_t retn = write_hex(substr{}, number.val); - CHECK_EQ(retn, number.hex.sub(2).len); - size_t retb = write_hex(buf, number.val); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, number.hex.sub(2).len); - CHECK_EQ(buf.first(retb), number.hex.sub(2)); - } -} - -TEST_CASE_TEMPLATE("write_oct", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - REQUIRE_GE(buf.len, number.oct.sub(2).len); - size_t retn = write_oct(substr{}, number.val); - CHECK_EQ(retn, number.oct.sub(2).len); - size_t retb = write_oct(buf, number.val); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, number.oct.sub(2).len); - CHECK_EQ(buf.first(retb), nopfx(number.oct)); - } -} - -TEST_CASE_TEMPLATE("write_bin", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - REQUIRE_GE(buf.len, number.bin.sub(2).len); - size_t retb = write_bin(substr{}, number.val); - CHECK_EQ(retb, number.bin.sub(2).len); - size_t retn = write_bin(buf, number.val); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, number.bin.sub(2).len); - CHECK_EQ(buf.first(retb), nopfx(number.bin)); - } -} - - -//----------------------------------------------------------------------------- -TEST_CASE_TEMPLATE("write_dec_digits", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("num digits smaller than length") - { - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - for(int less_ : {0, 1, 2, 4, 8, (int)number.dec.len}) - { - size_t less = (size_t) less_; - if(less > number.dec.len) - continue; - size_t num_digits = number.dec.len - less; - INFO("less=" << less << " num_digits=" << num_digits); - size_t retn = write_dec(substr{}, number.val, num_digits); - CHECK_EQ(retn, number.dec.len); // the number must always be written - size_t retb = write_dec(buf, number.val, num_digits); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, number.dec.len); - CHECK_EQ(buf.first(retb), number.dec); - } - } - } - SUBCASE("num digits larger than length") - { - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - for(int more_ : {1, 2, 4, 8}) - { - size_t more = (size_t) more_; - size_t num_digits = number.dec.len + more; - INFO("more=" << more << " num_digits=" << num_digits); - size_t retn = write_dec(substr{}, number.val, num_digits); - CHECK_EQ(retn, num_digits); - size_t retb = write_dec(buf, number.val, num_digits); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, num_digits); - csubstr result = buf.first(retb); - CHECK_EQ(result.last(number.dec.len), number.dec); - if(number.val) - { - CHECK_EQ(result.triml('0'), number.dec); - CHECK_EQ(result.first_not_of('0'), more); - } - else - { - CHECK(result.begins_with('0')); - CHECK_EQ(result.first_not_of('0'), csubstr::npos); - } - } - } - } -} - -TEST_CASE_TEMPLATE("write_hex_digits", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("num digits smaller than length") - { - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - for(int less_ : {0, 1, 2, 4, 8, (int)number.hex.len}) - { - const csubstr hex = nopfx(number.hex); - size_t less = (size_t) less_; - if(less > hex.len) - continue; - size_t num_digits = hex.len - less; - INFO("less=" << less << " num_digits=" << num_digits); - size_t retn = write_hex(substr{}, number.val, num_digits); - CHECK_EQ(retn, hex.len); // the number must always be written - size_t retb = write_hex(buf, number.val, num_digits); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, hex.len); - CHECK_EQ(buf.first(retb), hex); - } - } - } - SUBCASE("num digits larger than length") - { - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - for(int more_ : {1, 2, 4, 8}) - { - const csubstr hex = nopfx(number.hex); - size_t more = (size_t) more_; - size_t num_digits = hex.len + more; - INFO("more=" << more << " num_digits=" << num_digits); - size_t retn = write_hex(substr{}, number.val, num_digits); - CHECK_EQ(retn, num_digits); - size_t retb = write_hex(buf, number.val, num_digits); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retn, num_digits); - csubstr result = buf.first(retn); - CHECK_EQ(result.last(hex.len), hex); - if(number.val) - { - CHECK_EQ(result.triml('0'), hex); - CHECK_EQ(result.first_not_of('0'), more); - } - else - { - CHECK(result.begins_with('0')); - CHECK_EQ(result.first_not_of('0'), csubstr::npos); - } - } - } - } -} - -TEST_CASE_TEMPLATE("write_oct_digits", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("num digits smaller than length") - { - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - for(int less_ : {0, 1, 2, 4, 8, (int)number.oct.len}) - { - const csubstr oct = nopfx(number.oct); - size_t less = (size_t) less_; - if(less > oct.len) - continue; - size_t num_digits = oct.len - less; - INFO("less=" << less << " num_digits=" << num_digits); - size_t retn = write_oct(substr{}, number.val, num_digits); - CHECK_EQ(retn, oct.len); // the number must always be written - size_t retb = write_oct(buf, number.val, num_digits); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, oct.len); - CHECK_EQ(buf.first(retb), oct); - } - } - } - SUBCASE("num digits larger than length") - { - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - for(int more_ : {1, 2, 4, 8}) - { - const csubstr oct = nopfx(number.oct); - size_t more = (size_t) more_; - size_t num_digits = oct.len + more; - INFO("more=" << more << " num_digits=" << num_digits); - size_t retn = write_oct(substr{}, number.val, num_digits); - CHECK_EQ(retn, num_digits); - size_t retb = write_oct(buf, number.val, num_digits); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, num_digits); - csubstr result = buf.first(retb); - CHECK_EQ(result.last(oct.len), oct); - if(number.val) - { - CHECK_EQ(result.triml('0'), oct); - CHECK_EQ(result.first_not_of('0'), more); - } - else - { - CHECK(result.begins_with('0')); - CHECK_EQ(result.first_not_of('0'), csubstr::npos); - } - } - } - } -} - -TEST_CASE_TEMPLATE("write_bin_digits", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("num digits smaller than length") - { - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - for(int less_ : {0, 1, 2, 4, 8, (int)number.bin.len}) - { - const csubstr bin = nopfx(number.bin); - size_t less = (size_t) less_; - if(less > bin.len) - continue; - size_t num_digits = bin.len - less; - INFO("less=" << less << " num_digits=" << num_digits); - size_t retn = write_bin(substr{}, number.val, num_digits); - CHECK_EQ(retn, bin.len); // the number must always be written - size_t retb = write_bin(buf, number.val, num_digits); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retb, bin.len); - CHECK_EQ(buf.first(retb), bin); - } - } - } - SUBCASE("num digits larger than length") - { - ITER_NUMBERS(T, number) - { - if(number.val < 0) - continue; - INFO(number); - for(int more_ : {1, 2, 4, 8}) - { - const csubstr bin = nopfx(number.bin); - size_t more = (size_t) more_; - size_t num_digits = bin.len + more; - INFO("more=" << more << " num_digits=" << num_digits); - size_t retn = write_bin(substr{}, number.val, num_digits); - CHECK_EQ(retn, num_digits); - size_t retb = write_bin(buf, number.val, num_digits); - CHECK_EQ(retb, retn); - REQUIRE_EQ(retn, num_digits); - csubstr result = buf.first(retn); - CHECK_EQ(result.last(bin.len), bin); - if(number.val) - { - CHECK_EQ(result.triml('0'), bin); - CHECK_EQ(result.first_not_of('0'), more); - } - else - { - CHECK(result.begins_with('0')); - CHECK_EQ(result.first_not_of('0'), csubstr::npos); - } - } - } - } -} - - -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("xtoa", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - INFO(number); - { - buf.fill('?'); - size_t retn = xtoa(substr{}, number.val); - CHECK_EQ(retn, number.dec.len); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK_EQ(buf.first(retb), number.dec); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - } -} - - -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("xtoa_radix.dec", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - INFO(number); - { - buf.fill('?'); - size_t retn = xtoa(substr{}, number.val, T(10)); - CHECK_EQ(retn, number.dec.len); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(10)); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK_EQ(buf.first(retb), number.dec); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - const size_t adj = size_t(number.val < 0); - REQUIRE_LT(adj, number.dec.len); - const size_t dec_digits = number.dec.len - adj; - for(size_t more_digits = 0; more_digits < 6; ++more_digits) - { - buf.fill('?'); - size_t reqdigits = dec_digits + more_digits; - INFO("dec_digits=" << dec_digits << " more_digits=" << more_digits << " req_digits=" << reqdigits); - size_t retn = xtoa(substr{}, number.val, T(10), reqdigits); - CHECK_EQ(retn, reqdigits + size_t(number.val < 0)); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(10), reqdigits); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK(buf.first(retb).ends_with(number.dec.sub(number.val < 0))); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - for(size_t less_digits = 0; less_digits < dec_digits; ++less_digits) - { - buf.fill('?'); - size_t reqdigits = dec_digits - less_digits; - INFO("dec_digits=" << dec_digits << " less_digits=" << less_digits << " req_digits=" << reqdigits); - size_t retn = xtoa(substr{}, number.val, T(10), reqdigits); - CHECK_EQ(retn, number.dec.len); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(10), reqdigits); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK(buf.first(retb).ends_with(number.dec.sub(number.val < 0))); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - } -} - -TEST_CASE_TEMPLATE("xtoa_radix.hex", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - INFO(number); - { - buf.fill('?'); - size_t retn = xtoa(substr{}, number.val, T(16)); - CHECK_EQ(retn, number.hex.len); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(16)); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK_EQ(buf.first(retb), number.hex); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - const size_t adj = size_t(number.val < 0) + size_t(2); // 2 for 0x - REQUIRE_LT(adj, number.hex.len); - const size_t hex_digits = number.hex.len - adj; - for(size_t more_digits = 0; more_digits < 6; ++more_digits) - { - buf.fill('?'); - size_t reqdigits = hex_digits + more_digits; - INFO("more_digits=" << more_digits << " reqdigits=" << reqdigits); - size_t retn = xtoa(substr{}, number.val, T(16), reqdigits); - CHECK_EQ(retn, reqdigits + adj); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(16), reqdigits); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - csubstr result = buf.first(retb); - csubstr ref = number.hex.sub(adj); - INFO("result=" << result << " ref=" << ref); - if(number.val < 0) - CHECK(buf.first(retb).begins_with('-')); - CHECK(result.ends_with(ref)); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - for(size_t less_digits = 0; less_digits < hex_digits; ++less_digits) - { - buf.fill('?'); - size_t reqdigits = hex_digits - less_digits; - INFO("hex_digits=" << hex_digits << " less_digits=" << less_digits << " req_digits=" << reqdigits); - size_t retn = xtoa(substr{}, number.val, T(16), reqdigits); - CHECK_EQ(retn, number.hex.len); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(16), reqdigits); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK(buf.first(retb).ends_with(number.hex.sub(number.val < 0))); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - } -} - -TEST_CASE_TEMPLATE("xtoa_radix.oct", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - INFO(number); - { - buf.fill('?'); - size_t retn = xtoa(substr{}, number.val, T(8)); - CHECK_EQ(retn, number.oct.len); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(8)); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK_EQ(buf.first(retb), number.oct); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - const size_t adj = size_t(number.val < 0) + size_t(2); // 2 for 0o - REQUIRE_LT(adj, number.oct.len); - const size_t oct_digits = number.oct.len - adj; - for(size_t more_digits = 0; more_digits < 6; ++more_digits) - { - buf.fill('?'); - size_t reqdigits = oct_digits + more_digits; - INFO("more_digits=" << more_digits << " reqdigits=" << reqdigits); - size_t retn = xtoa(substr{}, number.val, T(8), reqdigits); - CHECK_EQ(retn, reqdigits + adj); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(8), reqdigits); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - csubstr result = buf.first(retb); - csubstr ref = number.oct.sub(adj); - INFO("result=" << result << " ref=" << ref); - if(number.val < 0) - CHECK(buf.first(retb).begins_with('-')); - CHECK(result.ends_with(ref)); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - for(size_t less_digits = 0; less_digits < oct_digits; ++less_digits) - { - buf.fill('?'); - size_t reqdigits = oct_digits - less_digits; - INFO("oct_digits=" << oct_digits << " less_digits=" << less_digits << " req_digits=" << reqdigits); - size_t retn = xtoa(substr{}, number.val, T(8), reqdigits); - CHECK_EQ(retn, number.oct.len); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(8), reqdigits); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK(buf.first(retb).ends_with(number.oct.sub(number.val < 0))); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - } -} - -TEST_CASE_TEMPLATE("xtoa_radix.bin", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - ITER_NUMBERS(T, number) - { - INFO(number); - { - buf.fill('?'); - size_t retn = xtoa(substr{}, number.val, T(2)); - CHECK_EQ(retn, number.bin.len); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(2)); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK_EQ(buf.first(retb), number.bin); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - const size_t adj = size_t(number.val < 0) + size_t(2); // 2 for 0b - REQUIRE_LT(adj, number.bin.len); - const size_t bin_digits = number.bin.len - adj; - for(size_t more_digits = 0; more_digits < 6; ++more_digits) - { - buf.fill('?'); - size_t reqdigits = bin_digits + more_digits; - INFO("more_digits=" << more_digits << " reqdigits=" << reqdigits); - size_t retn = xtoa(substr{}, number.val, T(2), reqdigits); - CHECK_EQ(retn, reqdigits + adj); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(2), reqdigits); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - csubstr result = buf.first(retb); - csubstr ref = number.bin.sub(adj); - INFO("result=" << result << " ref=" << ref); - if(number.val < 0) - CHECK(buf.first(retb).begins_with('-')); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - for(size_t less_digits = 0; less_digits < bin_digits; ++less_digits) - { - buf.fill('?'); - size_t reqdigits = bin_digits - less_digits; - INFO("bin_digits=" << bin_digits << " less_digits=" << less_digits << " req_digits=" << reqdigits); - size_t retn = xtoa(substr{}, number.val, T(2), reqdigits); - CHECK_EQ(retn, number.bin.len); - CHECK_UNARY(buf.begins_with('?') && buf.first_not_of('?') == csubstr::npos); - size_t retb = xtoa(buf, number.val, T(2), reqdigits); - CHECK_EQ(retn, retb); - REQUIRE_LE(retb, buf.len); - CHECK(buf.first(retb).ends_with(number.bin.sub(number.val < 0))); - T after_roundtrip = number.val + T(1); - CHECK(atox(buf.first(retb), &after_roundtrip)); - CHECK_EQ(after_roundtrip, number.val); - } - } -} - - -//----------------------------------------------------------------------------- - - -TEST_CASE_TEMPLATE("overflows.in_range_does_not_overflow", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("dec") - { - ITER_NUMBERS(T, number) - { - INFO(number); - CHECK_FALSE(overflows(number.dec)); - CHECK_FALSE(overflows(capitalize(buf, number.dec))); - for(size_t numz : {1u, 4u, 6u}) - { - substr buf2 = zpad(buf, number.dec, numz); - CHECK_FALSE(overflows(buf2)); - buf2.toupper(); - CHECK_FALSE(overflows(buf2)); - } - } - } - SUBCASE("hex") - { - ITER_NUMBERS(T, number) - { - INFO(number); - CHECK_FALSE(overflows(number.hex)); - CHECK_FALSE(overflows(capitalize(buf, number.hex))); - for(size_t numz : {1u, 4u, 6u}) - { - substr buf2 = zpad(buf, number.hex, numz); - CHECK_FALSE(overflows(buf2)); - buf2.toupper(); - CHECK_FALSE(overflows(buf2)); - } - } - } - SUBCASE("oct") - { - ITER_NUMBERS(T, number) - { - INFO(number); - CHECK_FALSE(overflows(number.oct)); - CHECK_FALSE(overflows(capitalize(buf, number.oct))); - for(size_t numz : {1u, 4u, 6u}) - { - substr buf2 = zpad(buf, number.oct, numz); - CHECK_FALSE(overflows(buf2)); - buf2.toupper(); - CHECK_FALSE(overflows(buf2)); - } - } - } - SUBCASE("bin") - { - ITER_NUMBERS(T, number) - { - INFO(number); - CHECK_FALSE(overflows(number.bin)); - CHECK_FALSE(overflows(capitalize(buf, number.bin))); - for(size_t numz : {1u, 4u, 6u}) - { - substr buf2 = zpad(buf, number.bin, numz); - CHECK_FALSE(overflows(buf2)); - buf2.toupper(); - CHECK_FALSE(overflows(buf2)); - } - } - } -} - - -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("read_dec", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("numbers") - { - ITER_NUMBERS(T, number) - { - if(number.val < T(0)) - continue; - INFO(number); - { - T val = number.val + T(1); - CHECK(read_dec(number.dec, &val)); - CHECK_EQ(val, number.val); - } - // capitalize - { - T val = number.val + T(1); - csubstr cbuf = capitalize(buf, number.dec); - CHECK(read_dec(cbuf, &val)); - CHECK_EQ(val, number.val); - } - // zero-prefix - for(size_t numz : {1u, 4u, 6u}) - { - T val = number.val + T(1); - substr buf2 = zpad(buf, number.dec, numz); - INFO("zprefix=" << buf2); - CHECK(read_dec(buf2, &val)); - CHECK_EQ(val, number.val); - buf2.toupper(); - CHECK(read_dec(buf2, &val)); - CHECK_EQ(val, number.val); - } - } - } - SUBCASE("fail") - { - T val = {}; - for(auto ic : invalid_cases) - { - if(ic.dec.empty()) - continue; - INFO(ic.dec); - CHECK(!read_dec(ic.dec, &val)); - } - } -} - -TEST_CASE_TEMPLATE("read_hex", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("numbers") - { - ITER_NUMBERS(T, number) - { - if(number.val < T(0)) - continue; - INFO(number); - // must not accept 0x prefix - { - T val = number.val + T(1); - CHECK(!read_hex(number.hex, &val)); - } - // must accept without prefix - csubstr hex = nopfx(number.hex); - INFO("nopfx(hex)=" << hex); - { - T val = number.val + T(1); - CHECK(read_hex(hex, &val)); - CHECK_EQ(val, number.val); - } - // capitalize - { - csubstr cbuf = capitalize(buf, hex); - INFO("capitalized=" << buf); - REQUIRE_EQ(cbuf.len, hex.len); - T val = number.val + T(1); - CHECK(read_hex(cbuf, &val)); - CHECK_EQ(val, number.val); - } - // zero-prefix - for(size_t numz : {1u, 4u, 6u}) - { - T val = number.val + T(1); - substr zprefix = zpad(buf, hex, numz); - INFO("zprefix='" << zprefix << "'"); - CHECK(read_hex(zprefix, &val)); - CHECK_EQ(val, number.val); - zprefix.toupper(); - CHECK(read_hex(zprefix, &val)); - CHECK_EQ(val, number.val); - } - } - } - SUBCASE("fail") - { - char buf2_[128] = {}; - substr buf2 = buf2_; - size_t icase = 0; - for(auto const& ic : invalid_cases) - { - csubstr cbuf = nopfx(buf, ic.hex); - csubstr cbuf2 = capitalize(buf2, cbuf); - INFO("case#=" << icase << " hex='" << ic.hex << "' nopfx(hex)='" << cbuf << "' capitalize(nopfx(hex))='" << cbuf2 << "'"); - REQUIRE_EQ(cbuf2.len, cbuf.len); - // must not accept 0x prefix - if(ic.hex.len) - { - T val = {}; - CHECK(!read_hex(ic.hex, &val)); - } - // it is invalid; must not accept even without 0x prefix - if(cbuf.len) - { - T val = {}; - CHECK(!read_hex(cbuf, &val)); - } - // capitalize - if(cbuf2.len) - { - T val = {}; - CHECK(!read_hex(cbuf2, &val)); - } - ++icase; - } - } -} - -TEST_CASE_TEMPLATE("read_oct", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("numbers") - { - ITER_NUMBERS(T, number) - { - if(number.val < T(0)) - continue; - INFO(number); - // must not accept 0x prefix - { - T val = number.val + T(1); - CHECK(!read_oct(number.oct, &val)); - } - // must accept without prefix - csubstr oct = nopfx(number.oct); - INFO("nopfx(oct)=" << oct); - { - T val = number.val + T(1); - CHECK(read_oct(oct, &val)); - CHECK_EQ(val, number.val); - } - // capitalize - { - csubstr cbuf = capitalize(buf, oct); - INFO("capitalized=" << buf); - REQUIRE_EQ(cbuf.len, oct.len); - T val = number.val + T(1); - CHECK(read_oct(cbuf, &val)); - CHECK_EQ(val, number.val); - } - // zero-prefix - for(size_t numz : {1u, 4u, 6u}) - { - T val = number.val + T(1); - substr zprefix = zpad(buf, oct, numz); - INFO("zprefix=" << zprefix); - CHECK(read_oct(zprefix, &val)); - CHECK_EQ(val, number.val); - zprefix.toupper(); - CHECK(read_oct(zprefix, &val)); - CHECK_EQ(val, number.val); - } - } - } - SUBCASE("fail") - { - char buf2_[128] = {}; - substr buf2 = buf2_; - size_t icase = 0; - for(auto const& ic : invalid_cases) - { - csubstr cbuf = nopfx(buf, ic.oct); - csubstr cbuf2 = capitalize(buf2, cbuf); - INFO("case#=" << icase << " oct='" << ic.oct << "' nopfx(oct)='" << cbuf << "' capitalize(nopfx(oct))='" << cbuf2 << "'"); - REQUIRE_EQ(cbuf2.len, cbuf.len); - // must not accept 0x prefix - if(ic.oct.len) - { - T val = {}; - CHECK(!read_oct(ic.oct, &val)); - } - // it is invalid; must not accept even without 0x prefix - if(cbuf.len) - { - T val = {}; - CHECK(!read_oct(cbuf, &val)); - } - // capitalize - if(cbuf2.len) - { - T val = {}; - CHECK(!read_oct(cbuf2, &val)); - } - ++icase; - } - } -} - -TEST_CASE_TEMPLATE("read_bin", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("numbers") - { - ITER_NUMBERS(T, number) - { - if(number.val < T(0)) - continue; - INFO(number); - // must not accept 0x prefix - { - T val = number.val + T(1); - CHECK(!read_bin(number.bin, &val)); - } - // must accept without prefix - csubstr bin = nopfx(number.bin); - INFO("nopfx(bin)=" << bin); - { - T val = number.val + T(1); - CHECK(read_bin(bin, &val)); - CHECK_EQ(val, number.val); - } - // capitalize - { - csubstr cbuf = capitalize(buf, bin); - INFO("capitalized=" << buf); - REQUIRE_EQ(cbuf.len, bin.len); - T val = number.val + T(1); - CHECK(read_bin(cbuf, &val)); - CHECK_EQ(val, number.val); - } - // zero-prefix - for(size_t numz : {1u, 4u, 6u}) - { - T val = number.val + T(1); - substr zprefix = zpad(buf, bin, numz); - INFO("zprefix=" << zprefix); - CHECK(read_bin(zprefix, &val)); - CHECK_EQ(val, number.val); - zprefix.toupper(); - CHECK(read_bin(zprefix, &val)); - CHECK_EQ(val, number.val); - } - } - } - SUBCASE("fail") - { - char buf2_[128] = {}; - substr buf2 = buf2_; - size_t icase = 0; - for(auto const& ic : invalid_cases) - { - csubstr cbuf = nopfx(buf, ic.bin); - csubstr cbuf2 = capitalize(buf2, cbuf); - INFO("case#=" << icase << " bin='" << ic.bin << "' nopfx(bin)='" << cbuf << "' capitalize(nopfx(bin))='" << cbuf2 << "'"); - REQUIRE_EQ(cbuf2.len, cbuf.len); - // must not accept 0x prefix - if(ic.bin.len) - { - T val = {}; - CHECK(!read_bin(ic.bin, &val)); - } - // it is invalid; must not accept even without 0x prefix - if(cbuf.len) - { - T val = {}; - CHECK(!read_bin(cbuf, &val)); - } - // capitalize - if(cbuf2.len) - { - T val = {}; - CHECK(!read_bin(cbuf2, &val)); - } - ++icase; - } - } -} - - -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("atox", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("dec") - { - ITER_NUMBERS(T, number) - { - INFO(number); - { - T val = number.val + T(1); - CHECK(atox(number.dec, &val)); - CHECK_EQ(val, number.val); - } - // zero-prefix - for(size_t numz : {1u, 4u, 6u}) - { - T val = number.val + T(1); - substr zprefix = zpad(buf, number.dec, numz); - INFO("zprefix=" << zprefix); - CHECK(atox(zprefix, &val)); - CHECK_EQ(val, number.val); - } - } - } - SUBCASE("hex") - { - ITER_NUMBERS(T, number) - { - INFO(number); - { - T val = number.val + T(1); - CHECK(atox(number.hex, &val)); - CHECK_EQ(val, number.val); - } - // capitalize - { - T val = number.val + T(1); - csubstr cbuf = capitalize(buf, number.hex); - CHECK(atox(cbuf, &val)); - CHECK_EQ(val, number.val); - } - // zero-prefix - for(size_t numz : {1u, 4u, 6u}) - { - T val = number.val + T(1); - substr zprefix = zpad(buf, number.hex, numz); - INFO("zprefix=" << zprefix); - CHECK(atox(zprefix, &val)); - CHECK_EQ(val, number.val); - zprefix.toupper(); - CHECK(atox(zprefix, &val)); - CHECK_EQ(val, number.val); - } - } - } - SUBCASE("oct") - { - ITER_NUMBERS(T, number) - { - INFO(number); - { - T val = number.val + T(1); - CHECK(atox(number.oct, &val)); - CHECK_EQ(val, number.val); - } - // capitalize - { - T val = number.val + T(1); - csubstr cbuf = capitalize(buf, number.oct); - CHECK(atox(cbuf, &val)); - CHECK_EQ(val, number.val); - } - // zero-prefix - for(size_t numz : {1u, 4u, 6u}) - { - T val = number.val + T(1); - substr zprefix = zpad(buf, number.oct, numz); - INFO("zprefix=" << zprefix); - CHECK(atox(zprefix, &val)); - CHECK_EQ(val, number.val); - zprefix.toupper(); - CHECK(atox(zprefix, &val)); - CHECK_EQ(val, number.val); - } - } - } - SUBCASE("bin") - { - ITER_NUMBERS(T, number) - { - INFO(number); - { - T val = number.val + T(1); - CHECK(atox(number.bin, &val)); - CHECK_EQ(val, number.val); - } - // capitalize - { - T val = number.val + T(1); - csubstr cbuf = capitalize(buf, number.bin); - CHECK(atox(cbuf, &val)); - CHECK_EQ(val, number.val); - } - // zero-prefix - for(size_t numz : {1u, 4u, 6u}) - { - T val = number.val + T(1); - substr zprefix = zpad(buf, number.oct, numz); - INFO("zprefix=" << zprefix); - CHECK(atox(zprefix, &val)); - CHECK_EQ(val, number.val); - zprefix.toupper(); - CHECK(atox(zprefix, &val)); - CHECK_EQ(val, number.val); - } - } - } -} - -TEST_CASE_TEMPLATE("atox.fail", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - SUBCASE("dec") - { - size_t icase = 0; - for(auto const& ic : invalid_cases) - { - csubstr cdec = capitalize(buf, ic.dec); - INFO("case#=" << icase << " dec='" << ic.dec << "' capitalize='" << cdec << "'"); - REQUIRE_EQ(cdec.len, ic.dec.len); - { - T val = {}; - CHECK(!atox(ic.dec, &val)); - } - { - T val = {}; - CHECK(!atox(cdec, &val)); - } - ++icase; - } - } - SUBCASE("hex") - { - size_t icase = 0; - for(auto const& ic : invalid_cases) - { - csubstr chex = capitalize(buf, ic.hex); - INFO("case#=" << icase << " hex='" << ic.hex << "' capitalize='" << chex << "'"); - REQUIRE_EQ(chex.len, ic.hex.len); - { - T val = {}; - CHECK(!atox(ic.hex, &val)); - } - { - T val = {}; - CHECK(!atox(chex, &val)); - } - ++icase; - } - } - SUBCASE("oct") - { - size_t icase = 0; - for(auto const& ic : invalid_cases) - { - csubstr coct = capitalize(buf, ic.oct); - INFO("case#=" << icase << " oct='" << ic.oct << "' capitalize='" << coct << "'"); - REQUIRE_EQ(coct.len, ic.oct.len); - { - T val = {}; - CHECK(!atox(ic.oct, &val)); - } - { - T val = {}; - CHECK(!atox(coct, &val)); - } - ++icase; - } - } - SUBCASE("bin") - { - size_t icase = 0; - for(auto const& ic : invalid_cases) - { - csubstr cbin = capitalize(buf, ic.bin); - INFO("case#=" << icase << " bin='" << ic.bin << "' capitalize='" << cbin << "'"); - REQUIRE_EQ(cbin.len, ic.bin.len); - { - T val = {}; - CHECK(!atox(ic.bin, &val)); - } - { - T val = {}; - CHECK(!atox(cbin, &val)); - } - ++icase; - } - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_overflows(std::initializer_list args) -{ - for(const char *s : args) - CHECK_MESSAGE(overflows(to_csubstr(s)), "num=" << s); -} - -template -void test_no_overflows(std::initializer_list args) -{ - for(const char *s : args) - CHECK_MESSAGE(!overflows(to_csubstr(s)), "num=" << s); -} - -template -auto test_no_overflow_zeroes() - -> typename std::enable_if::value, void>::type -{ - test_no_overflows({ "-", "-0", "-000", "-0b0", "-0B0", "-0x0", "-0X0", "-0o0", "-0O0" }); - test_no_overflows({ "", "0", "000", "0b0", "0B0", "0x0", "0X0", "0o0", "0O0" }); -} - -template -auto test_no_overflow_zeroes() - -> typename std::enable_if::value, void>::type -{ - test_no_overflows({ "", "0", "000", "0b0", "0B0", "0x0", "0X0", "0o0", "0O0" }); -} - - -// test overflow in sizes smaller than 64 bit by upcasting -TEST_CASE_TEMPLATE("atox.overflow", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t) -{ - char buf_[128]; - substr buf = buf_; - auto do_test = [](bool is_overflow, number_case const& num, csubstr exceeded, number_case const& wrapped){ - char buf2_[128] = {}; - substr buf2 = buf2_; - INFO("exceeded=" << exceeded << " is_overflow=" << is_overflow); - INFO("num=" << num); - INFO("wrapped=" << wrapped); - CHECK_EQ(is_overflow, overflows(exceeded)); - if(is_overflow) - CHECK_NE(&num, &wrapped); - else - CHECK_EQ(&num, &wrapped); - { - T val = num.val + T(1); - CHECK(atox(exceeded, &val)); - CHECK_EQ(val, wrapped.val); - } - // capitalize - buf2 = capitalize(buf2_, exceeded); - INFO(buf2); - CHECK_EQ(is_overflow, overflows(buf2)); - { - T val = num.val + T(1); - CHECK(atox(buf2, &val)); - CHECK_EQ(val, wrapped.val); - } - // zero-pad on the left - for(size_t numz : {1u, 4u, 6u}) - { - buf2 = zpad(buf2_, exceeded, numz); - CHECK_EQ(is_overflow, overflows(buf2)); - { - T val = num.val + T(1); - CHECK(atox(buf2, &val)); - CHECK_EQ(val, wrapped.val); - } - buf2.toupper(); - CHECK_EQ(is_overflow, overflows(buf2)); - { - T val = num.val + T(1); - CHECK(atox(buf2, &val)); - CHECK_EQ(val, wrapped.val); - } - } - }; - auto do_test_overflow = [&](T exceed_how_much, T radix){ - REQUIRE(exceed_how_much >= 0); - number_case const& backelm = back(); - number_case const& wrapelm = next(backelm, (size_t)exceed_how_much); - csubstr exceeded = overflow_by(buf, backelm.val, exceed_how_much, radix); - do_test(exceed_how_much > 0, backelm, exceeded, wrapelm); - }; - auto do_test_underflow = [&](T exceed_how_much, T radix){ - REQUIRE(exceed_how_much >= 0); - number_case const& frntelm = front(); - number_case const& wrapelm = prev(frntelm, (size_t)exceed_how_much); - csubstr exceeded = underflow_by(buf, frntelm.val, exceed_how_much, radix); - do_test(exceed_how_much > 0, frntelm, exceeded, wrapelm); - }; - SUBCASE("zeroes") - { - test_no_overflow_zeroes(); - } - SUBCASE("dec") - { - do_test_underflow(T(0), T(10)); - do_test_underflow(T(1), T(10)); - do_test_underflow(T(2), T(10)); - do_test_underflow(T(3), T(10)); - do_test_underflow(T(4), T(10)); - do_test_underflow(T(5), T(10)); - do_test_overflow(T(0), T(10)); - do_test_overflow(T(1), T(10)); - do_test_overflow(T(2), T(10)); - do_test_overflow(T(3), T(10)); - do_test_overflow(T(4), T(10)); - do_test_overflow(T(5), T(10)); - } - SUBCASE("hex") - { - do_test_underflow(T(0), T(16)); - do_test_underflow(T(1), T(16)); - do_test_underflow(T(2), T(16)); - do_test_underflow(T(3), T(16)); - do_test_underflow(T(4), T(16)); - do_test_underflow(T(5), T(16)); - do_test_overflow(T(0), T(16)); - do_test_overflow(T(1), T(16)); - do_test_overflow(T(2), T(16)); - do_test_overflow(T(3), T(16)); - do_test_overflow(T(4), T(16)); - do_test_overflow(T(5), T(16)); - } - SUBCASE("oct") - { - do_test_underflow(T(0), T(8)); - do_test_underflow(T(1), T(8)); - do_test_underflow(T(2), T(8)); - do_test_underflow(T(3), T(8)); - do_test_underflow(T(4), T(8)); - do_test_underflow(T(5), T(8)); - do_test_overflow(T(0), T(8)); - do_test_overflow(T(1), T(8)); - do_test_overflow(T(2), T(8)); - do_test_overflow(T(3), T(8)); - do_test_overflow(T(4), T(8)); - do_test_overflow(T(5), T(8)); - } - SUBCASE("bin") - { - do_test_underflow(T(0), T(2)); - do_test_underflow(T(1), T(2)); - do_test_underflow(T(2), T(2)); - do_test_underflow(T(3), T(2)); - do_test_underflow(T(4), T(2)); - do_test_underflow(T(5), T(2)); - do_test_overflow(T(0), T(2)); - do_test_overflow(T(1), T(2)); - do_test_overflow(T(2), T(2)); - do_test_overflow(T(3), T(2)); - do_test_overflow(T(4), T(2)); - do_test_overflow(T(5), T(2)); - } -} - -TEST_CASE_TEMPLATE("atox.overflow64", T, int64_t, uint64_t) -{ - char buf_[128] = {}; - substr buf = buf_; - auto test_atox = [](csubstr s, overflow64case const& c){ - INFO("s=" << s); - T val = c.wrapped + T(1); - if(std::is_signed::value || !s.begins_with('-')) - { - CHECK(atox(s, &val)); - CHECK_EQ(val, c.wrapped); - } - else - { - CHECK(!atox(s, &val)); - } - }; - SUBCASE("zeroes") - { - test_no_overflow_zeroes(); - } - SUBCASE("dec") - { - for(auto c : overflow64cases::values) - { - INFO(c.dec); - CHECK_EQ(c.is_overflow, overflows(c.dec)); - test_atox(c.dec, c); - csubstr capitalized = capitalize(buf, c.dec); - CHECK_EQ(c.is_overflow, overflows(capitalized)); - test_atox(capitalized, c); - for(size_t numz : {1u, 4u, 6u}) - { - substr buf2 = zpad(buf, c.dec, numz); - CHECK_EQ(c.is_overflow, overflows(buf2)); - test_atox(buf2, c); - buf2.toupper(); - CHECK_EQ(c.is_overflow, overflows(buf2)); - test_atox(buf2, c); - } - } - } - SUBCASE("hex") - { - for(auto c : overflow64cases::values) - { - INFO(c.hex); - CHECK_EQ(c.is_overflow, overflows(c.hex)); - test_atox(c.hex, c); - csubstr capitalized = capitalize(buf, c.hex); - CHECK_EQ(c.is_overflow, overflows(capitalized)); - test_atox(capitalized, c); - for(size_t numz : {1u, 4u, 6u}) - { - substr buf2 = zpad(buf, c.hex, numz); - CHECK_EQ(c.is_overflow, overflows(buf2)); - test_atox(buf2, c); - buf2.toupper(); - CHECK_EQ(c.is_overflow, overflows(buf2)); - test_atox(buf2, c); - } - } - } - SUBCASE("oct") - { - for(auto c : overflow64cases::values) - { - INFO(c.oct); - CHECK_EQ(c.is_overflow, overflows(c.oct)); - test_atox(c.oct, c); - csubstr capitalized = capitalize(buf, c.oct); - CHECK_EQ(c.is_overflow, overflows(capitalized)); - test_atox(capitalized, c); - for(size_t numz : {1u, 4u, 6u}) - { - substr buf2 = zpad(buf, c.oct, numz); - CHECK_EQ(c.is_overflow, overflows(buf2)); - test_atox(buf2, c); - buf2.toupper(); - CHECK_EQ(c.is_overflow, overflows(buf2)); - test_atox(buf2, c); - } - } - } - SUBCASE("bin") - { - for(auto c : overflow64cases::values) - { - INFO(c.bin); - CHECK_EQ(c.is_overflow, overflows(c.bin)); - test_atox(c.bin, c); - csubstr capitalized = capitalize(buf, c.bin); - CHECK_EQ(c.is_overflow, overflows(capitalized)); - test_atox(capitalized, c); - for(size_t numz : {1u, 4u, 6u}) - { - substr buf2 = zpad(buf, c.bin, numz); - CHECK_EQ(c.is_overflow, overflows(buf2)); - test_atox(buf2, c); - buf2.toupper(); - CHECK_EQ(c.is_overflow, overflows(buf2)); - test_atox(buf2, c); - } - } - } -} - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_overflows_hex() -{ - T x = {}; - std::string str; - if (std::is_unsigned::value) - { - /* with leading zeroes */ - str = "0x0" + std::string(sizeof (T) * 2, 'F'); - CHECK_MESSAGE(!overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::max(), x); - - str = "0x01" + std::string(sizeof (T) * 2, '0'); - CHECK_MESSAGE(overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::min(), x); - } - else - { - /* with leading zeroes */ - str = "0x07" + std::string(sizeof (T) * 2 - 1, 'F'); - CHECK_MESSAGE(!overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::max(), x); - - str = "0x0" + std::string(sizeof (T) * 2, 'F'); - CHECK_MESSAGE(overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(-1, x); - - str = "-0x08" + std::string(sizeof (T) * 2 - 1, '0'); - CHECK_MESSAGE(!overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::min(), x); - - str = "-0x08" + std::string(sizeof (T) * 2 - 2, '0') + "1"; - CHECK_MESSAGE(overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::max(), x); - } -} - -template -void test_overflows_bin() -{ - T x = {}; - std::string str; - if (std::is_unsigned::value) - { - /* with leading zeroes */ - str = "0b0" + std::string(sizeof (T) * 8, '1'); - CHECK_MESSAGE(!overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::max(), x); - - str = "0b01" + std::string(sizeof (T) * 8, '0'); - CHECK_MESSAGE(overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::min(), x); - } - else - { - /* with leading zeroes */ - str = "0b0" + std::string(sizeof (T) * 8 - 1, '1'); - CHECK_MESSAGE(!overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::max(), x); - - str = "0b0" + std::string(sizeof (T) * 8, '1'); - CHECK_MESSAGE(overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(-1, x); - - str = "-0b01" + std::string(sizeof (T) * 8 - 1, '0'); - CHECK_MESSAGE(!overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::min(), x); - - str = "-0b01" + std::string(sizeof (T) * 8 - 2, '0') + "1"; - CHECK_MESSAGE(overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::max(), x); - } -} - -// TODO: test_overflows_oct - -template -typename std::enable_if::value, void>::type -test_overflows() -{ - for(int radix : { 2, 8, 10, 16 }) - { - char bufc[100] = {0}; - substr s(bufc); - INFO("radix=" << radix << " num=" << s); - - uint64_t max = (uint64_t) std::numeric_limits::max(); - size_t sz = utoa(s, max, (uint64_t)radix); - REQUIRE_LE(sz, s.size()); - CHECK(!overflows(s.first(sz))); - memset(s.str, 0, s.len); - sz = utoa(s, max + 1, (uint64_t)radix); - REQUIRE_LE(sz, s.size()); - CHECK(overflows(s.first(sz))); - } - - test_overflows_hex(); - test_overflows_bin(); - // TODO: octal -} - -template -typename std::enable_if::value, void>::type -test_overflows() -{ - for(int radix : { 2, 8, 10, 16 }) - { - char bufc[100] = {0}; - substr s(bufc); - INFO("radix=" << radix << " num=" << s); - - int64_t max = (int64_t) std::numeric_limits::max(); - size_t sz = itoa(s, max, (int64_t)radix); - REQUIRE_LE(sz, s.size()); - CHECK(!overflows(s.first(sz))); - memset(s.str, 0, s.len); - sz = itoa(s, max + 1, (int64_t)radix); - REQUIRE_LE(sz, s.size()); - CHECK(overflows(s.first(sz))); - - int64_t min = (int64_t) std::numeric_limits::min(); - sz = itoa(s, min, (int64_t)radix); - REQUIRE_LE(sz, s.size()); - CHECK(!overflows(s.first(sz))); - memset(s.str, 0, s.len); - sz = itoa(s, min - 1, (int64_t)radix); - REQUIRE_LE(sz, s.size()); - CHECK(overflows(s.first(sz))); - } - - test_overflows_hex(); - test_overflows_bin(); - // TODO: octal -} - -TEST_CASE_TEMPLATE("overflows.8bit_32bit", T, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t) -{ - test_overflows(); -} - -TEST_CASE("overflows.u64") -{ - CHECK(!overflows("18446744073709551614")); - CHECK(!overflows("18446744073709551615")); - CHECK(overflows("18446744073709551616")); - - // more chars but leading zeroes - CHECK(!overflows("0018446744073709551615")); - - { /* with leading zeroes */ - std::string str; - uint64_t x = {}; - str = "0o01" + std::string(21, '7'); - CHECK_MESSAGE(!overflows(to_csubstr(str)), "num=" << str); - str = "0o02" + std::string(21, '0'); - CHECK_MESSAGE(overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(0, x); - } - - test_overflows_hex(); - test_overflows_bin(); -} - -TEST_CASE("overflows.i64") -{ - CHECK(!overflows("9223372036854775806")); - CHECK(!overflows("9223372036854775807")); - CHECK(overflows("9223372036854775808")); - CHECK(!overflows("-9223372036854775808")); - CHECK(overflows("-9223372036854775809")); - - // more chars, but leading zeroes - CHECK(!overflows("0009223372036854775807")); - CHECK(!overflows("-0009223372036854775807")); - - { /* with leading zeroes */ - std::string str; - int64_t x = {}; - str = "0o0" + std::string(21, '7'); - CHECK_MESSAGE(!overflows(to_csubstr(str)), "num=" << str); - str = "0o01" + std::string(21, '0'); - CHECK_MESSAGE(overflows(to_csubstr(str)), "num=" << str); - CHECK(atox(to_csubstr(str), &x)); - CHECK_EQ(std::numeric_limits::min(), x); - } - - test_overflows_hex(); - test_overflows_bin(); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** remove trailing digits after precision */ -template -T remprec10(T val, int precision) -{ - T fprec = T(1); - for(int i = 0; i < precision; ++i) - fprec *= T(10); - T rval = val * fprec; - return ((T)((int64_t)rval)) / fprec; -} - -template -T test_ator(csubstr s, T ref) -{ - INFO("str=" << s << " ref=" << ref); - T rval; - CHECK(atox(s, &rval)); - INFO("rval=" << rval); - CHECK_EQ(memcmp(&rval, &ref, sizeof(T)), 0); - return rval; -} - -template -void test_rtoa(substr buf, Real f, int precision, const char *scient, const char *flt, const char* flex, const char *hexa, const char *hexa_alternative=nullptr) -{ - size_t ret; - Real pf = remprec10(f, precision); - - { - INFO("num=" << f << " precision=" << precision << " scient=" << scient); - memset(buf.str, 0, buf.len); - ret = xtoa(buf, f, precision, FTOA_SCIENT); - REQUIRE_LE(ret, buf.len); - CHECK_EQ(buf.first(ret), to_csubstr(scient)); - test_ator(buf.first(ret), pf); - } - - { - INFO("num=" << f << " precision=" << precision << " flt=" << flt); - memset(buf.str, 0, ret); - ret = xtoa(buf, f, precision, FTOA_FLOAT); - REQUIRE_LE(ret, buf.len); - CHECK_EQ(buf.first(ret), to_csubstr(flt)); - test_ator(buf.first(ret), pf); - } - - { - INFO("num=" << f << " precision=" << precision << " flex=" << flex); - memset(buf.str, 0, ret); - ret = xtoa(buf, f, precision+1, FTOA_FLEX); - REQUIRE_LE(ret, buf.len); - CHECK_EQ(buf.first(ret), to_csubstr(flex)); - test_ator(buf.first(ret), pf); - } - - { - if(!hexa_alternative) - hexa_alternative = hexa; - INFO("num=" << f << " precision=" << precision << " hexa=" << hexa << " hexa_alternative=" << hexa_alternative); - memset(buf.str, 0, ret); - ret = xtoa(buf, f, precision, FTOA_HEXA); - REQUIRE_LE(ret, buf.len); - INFO("buf='" << buf.first(ret) << "'"); - - CHECK((buf.first(ret) == to_csubstr(hexa) || buf.first(ret) == to_csubstr(hexa_alternative))); - Real readback = {}; - CHECK(atox(buf.first(ret), &readback)); - INFO("readback=" << readback); - REQUIRE_EQ(xtoa(buf, readback, precision, FTOA_HEXA), ret); - Real readback2 = {}; - CHECK(atox(buf.first(ret), &readback2)); - INFO("readback2=" << readback2); - CHECK_EQ(memcmp(&readback2, &readback, sizeof(Real)), 0); - } -} - - -TEST_CASE("ftoa.basic") -{ - char bufc[128]; - substr buf(bufc); - C4_ASSERT(buf.len == sizeof(bufc)-1); - - // earlier versions of emscripten's sprintf() do not respect some - // precision values when printing in hexadecimal format. - // - // @see https://github.com/biojppm/c4core/pull/52 - #if defined(__EMSCRIPTEN__) && __EMSCRIPTEN_major__ < 3 - #define _c4emscripten_alt(alt) , alt - #define _c4emscripten_alt2(alt1, alt2) , alt2 - #else - #define _c4emscripten_alt(alt) - #define _c4emscripten_alt2(alt1, alt2) , alt1 - #endif - - float f = 1.1234123f; - double d = 1.1234123; - - test_rtoa(buf, f, 0, /*scient*/"1e+00", /*flt*/"1", /*flex*/"1", /*hexa*/"0x1p+0"); - test_rtoa(buf, d, 0, /*scient*/"1e+00", /*flt*/"1", /*flex*/"1", /*hexa*/"0x1p+0"); - - test_rtoa(buf, f, 1, /*scient*/"1.1e+00", /*flt*/"1.1", /*flex*/"1.1", /*hexa*/"0x1.2p+0"); - test_rtoa(buf, d, 1, /*scient*/"1.1e+00", /*flt*/"1.1", /*flex*/"1.1", /*hexa*/"0x1.2p+0"); - - test_rtoa(buf, f, 2, /*scient*/"1.12e+00", /*flt*/"1.12", /*flex*/"1.12", /*hexa*/"0x1.20p+0" _c4emscripten_alt("0x1.1f8p+0")); - test_rtoa(buf, d, 2, /*scient*/"1.12e+00", /*flt*/"1.12", /*flex*/"1.12", /*hexa*/"0x1.20p+0" _c4emscripten_alt("0x1.1f8p+0")); - - test_rtoa(buf, f, 3, /*scient*/"1.123e+00", /*flt*/"1.123", /*flex*/"1.123", /*hexa*/"0x1.1f9p+0" _c4emscripten_alt("0x1.1f98p+0")); - test_rtoa(buf, d, 3, /*scient*/"1.123e+00", /*flt*/"1.123", /*flex*/"1.123", /*hexa*/"0x1.1f9p+0" _c4emscripten_alt("0x1.1f98p+0")); - - test_rtoa(buf, f, 4, /*scient*/"1.1234e+00", /*flt*/"1.1234", /*flex*/"1.1234", /*hexa*/"0x1.1f98p+0"); - test_rtoa(buf, d, 4, /*scient*/"1.1234e+00", /*flt*/"1.1234", /*flex*/"1.1234", /*hexa*/"0x1.1f98p+0"); - - f = 1.01234123f; - d = 1.01234123; - - test_rtoa(buf, f, 0, /*scient*/"1e+00", /*flt*/"1", /*flex*/"1", /*hexa*/"0x1p+0"); - test_rtoa(buf, d, 0, /*scient*/"1e+00", /*flt*/"1", /*flex*/"1", /*hexa*/"0x1p+0"); - - test_rtoa(buf, f, 1, /*scient*/"1.0e+00", /*flt*/"1.0", /*flex*/"1", /*hexa*/"0x1.0p+0"); - test_rtoa(buf, d, 1, /*scient*/"1.0e+00", /*flt*/"1.0", /*flex*/"1", /*hexa*/"0x1.0p+0"); - - test_rtoa(buf, f, 2, /*scient*/"1.01e+00", /*flt*/"1.01", /*flex*/"1.01", /*hexa*/"0x1.03p+0"); - test_rtoa(buf, d, 2, /*scient*/"1.01e+00", /*flt*/"1.01", /*flex*/"1.01", /*hexa*/"0x1.03p+0"); - - test_rtoa(buf, f, 3, /*scient*/"1.012e+00", /*flt*/"1.012", /*flex*/"1.012", /*hexa*/"0x1.033p+0" _c4emscripten_alt2("0x1.032p+0", "0x1.0328p+0")); - test_rtoa(buf, d, 3, /*scient*/"1.012e+00", /*flt*/"1.012", /*flex*/"1.012", /*hexa*/"0x1.033p+0" _c4emscripten_alt2("0x1.032p+0", "0x1.0328p+0")); - - test_rtoa(buf, f, 4, /*scient*/"1.0123e+00", /*flt*/"1.0123", /*flex*/"1.0123", /*hexa*/"0x1.0329p+0"); - test_rtoa(buf, d, 4, /*scient*/"1.0123e+00", /*flt*/"1.0123", /*flex*/"1.0123", /*hexa*/"0x1.0329p+0"); - - f = 0.f; - d = 0.; - - test_rtoa(buf, f, 0, /*scient*/"0e+00", /*flt*/"0", /*flex*/"0", /*hexa*/"0x0p+0"); - test_rtoa(buf, d, 0, /*scient*/"0e+00", /*flt*/"0", /*flex*/"0", /*hexa*/"0x0p+0"); - - test_rtoa(buf, f, 1, /*scient*/"0.0e+00", /*flt*/"0.0", /*flex*/"0", /*hexa*/"0x0.0p+0"); - test_rtoa(buf, d, 1, /*scient*/"0.0e+00", /*flt*/"0.0", /*flex*/"0", /*hexa*/"0x0.0p+0"); - - test_rtoa(buf, f, 2, /*scient*/"0.00e+00", /*flt*/"0.00", /*flex*/"0", /*hexa*/"0x0.00p+0"); - test_rtoa(buf, d, 2, /*scient*/"0.00e+00", /*flt*/"0.00", /*flex*/"0", /*hexa*/"0x0.00p+0"); - - test_rtoa(buf, f, 3, /*scient*/"0.000e+00", /*flt*/"0.000", /*flex*/"0", /*hexa*/"0x0.000p+0" _c4emscripten_alt2("0x0.000p+0", "0x0.000p+0")); - test_rtoa(buf, d, 3, /*scient*/"0.000e+00", /*flt*/"0.000", /*flex*/"0", /*hexa*/"0x0.000p+0" _c4emscripten_alt2("0x0.000p+0", "0x0.000p+0")); - - test_rtoa(buf, f, 4, /*scient*/"0.0000e+00", /*flt*/"0.0000", /*flex*/"0", /*hexa*/"0x0.0000p+0"); - test_rtoa(buf, d, 4, /*scient*/"0.0000e+00", /*flt*/"0.0000", /*flex*/"0", /*hexa*/"0x0.0000p+0"); - - f = 1.f; - d = 1.; - - test_rtoa(buf, f, 0, /*scient*/"1e+00", /*flt*/"1", /*flex*/"1", /*hexa*/"0x1p+0"); - test_rtoa(buf, d, 0, /*scient*/"1e+00", /*flt*/"1", /*flex*/"1", /*hexa*/"0x1p+0"); - - test_rtoa(buf, f, 1, /*scient*/"1.0e+00", /*flt*/"1.0", /*flex*/"1", /*hexa*/"0x1.0p+0"); - test_rtoa(buf, d, 1, /*scient*/"1.0e+00", /*flt*/"1.0", /*flex*/"1", /*hexa*/"0x1.0p+0"); - - test_rtoa(buf, f, 2, /*scient*/"1.00e+00", /*flt*/"1.00", /*flex*/"1", /*hexa*/"0x1.00p+0"); - test_rtoa(buf, d, 2, /*scient*/"1.00e+00", /*flt*/"1.00", /*flex*/"1", /*hexa*/"0x1.00p+0"); - - test_rtoa(buf, f, 3, /*scient*/"1.000e+00", /*flt*/"1.000", /*flex*/"1", /*hexa*/"0x1.000p+0" _c4emscripten_alt2("0x1.000p+0", "0x1.000p+0")); - test_rtoa(buf, d, 3, /*scient*/"1.000e+00", /*flt*/"1.000", /*flex*/"1", /*hexa*/"0x1.000p+0" _c4emscripten_alt2("0x1.000p+0", "0x1.000p+0")); - - test_rtoa(buf, f, 4, /*scient*/"1.0000e+00", /*flt*/"1.0000", /*flex*/"1", /*hexa*/"0x1.0000p+0"); - test_rtoa(buf, d, 4, /*scient*/"1.0000e+00", /*flt*/"1.0000", /*flex*/"1", /*hexa*/"0x1.0000p+0"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -TEST_CASE_TEMPLATE("atof.integral", T, float, double) -{ - auto t_ = [](csubstr str, int val){ - T rval = (T)10 * (T)val; - INFO("str=" << str); - bool ret = atox(str, &rval); - CHECK_EQ(ret, true); - CHECK_EQ(static_cast(rval), val); - CHECK_EQ(rval, (T)val); - }; - - csubstr s = "12345678"; - t_(s, 12345678); - t_(s.first(8), 12345678); - t_(s.first(7), 1234567); - t_(s.first(6), 123456); - t_(s.first(5), 12345); - t_(s.first(4), 1234); - t_(s.first(3), 123); - t_(s.first(2), 12); - t_(s.first(1), 1); -} - -TEST_CASE_TEMPLATE("atof.hexa", T, float, double) -{ - auto t_ = [](csubstr str, bool isok){ - T rval = {}; - INFO("str=" << str); - CHECK_EQ(atox(str, &rval), isok); - }; - #if C4CORE_NO_FAST_FLOAT - #define _scanf_accepts(expected) !expected - #else - #define _scanf_accepts(expected) expected - #endif - t_("0x1.p+0", true); - t_("0x1.p", _scanf_accepts(false)); - t_("0x1.p+", _scanf_accepts(false)); - t_("0x12p+0", true); - t_("0x12p", _scanf_accepts(false)); - t_("0xabcdef.abcdefp+0", true); - t_("0xABCDEF.ABCDEFp+0", true); - t_("0x1g", _scanf_accepts(false)); - t_("0x1.2", true); - t_("0x1.", true); - t_("0x1.0329p+0", true); - t_("0x1.0329P+0", true); - t_("0x1.aAaAaAp+0", true); - t_("0x1.agA+0", _scanf_accepts(false)); -} - -TEST_CASE_TEMPLATE("atof.infnan", T, float, double) -{ - T pinf = std::numeric_limits::infinity(); - T ninf = -std::numeric_limits::infinity(); - T nan = std::numeric_limits::quiet_NaN(); - T rval = {}; - test_ator("infinity", pinf); - test_ator("inf", pinf); - test_ator("-infinity", ninf); - test_ator("-inf", ninf); - test_ator("nan", nan); -} - -TEST_CASE_TEMPLATE("atof.fail_parse", T, float, double) -{ - auto t_ = [](csubstr str){ - T rval; - INFO("str=" << str << " rval=" << rval); - CHECK_EQ(atox(str, &rval), false); - }; - t_(".inf"); - t_("-.inf"); - t_(".nan"); - t_("-.nan"); - t_("not a float!"); - #ifndef C4CORE_NO_FAST_FLOAT - t_("0xfonix!"); - #endif - //t_("123.45not a float!"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -TEST_CASE_TEMPLATE("to_chars.empty_buffer", T, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t, void*) -{ - char buf_[100]; - substr buf = buf_; - CHECK_EQ(to_chars({}, T(101)), to_chars(buf_, T(101))); - CHECK_EQ(to_chars({}, T(101)), to_chars(buf , T(101))); -} -// due to an implementation quirk with sprintf, for floats the empty is GE -TEST_CASE_TEMPLATE("to_chars.empty_buffer", T, float, double) -{ - char buf_[100]; - substr buf = buf_; - CHECK_GE(to_chars({}, T(101)), to_chars(buf_, T(101))); - CHECK_GE(to_chars({}, T(101)), to_chars(buf , T(101))); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE("to_chars.std_string") -{ - std::string foo("foo"); - char buf_[32]; - substr buf(buf_); - size_t result = to_chars(buf, foo); - CHECK_EQ(result, 3); - CHECK_EQ(buf.first(3), "foo"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE("to_chars.bool") -{ - char buf_[32]; - substr buf(buf_); - csubstr result = to_chars_sub(buf, true); - CHECK_EQ(result, "1"); - result = to_chars_sub(buf, false); - CHECK_EQ(result, "0"); -} - -TEST_CASE("from_chars.bool") -{ - bool result = false; - for(const char *s : {"1", "true", "True", "TRUE"}) - { - INFO("s='" << s << "'"); - bool ok = from_chars(to_csubstr(s), &result); - CHECK_UNARY(ok); - CHECK_UNARY(result); - } - for(const char *s : {"0", "false", "False", "FALSE"}) - { - INFO("s='" << s << "'"); - bool ok = from_chars(to_csubstr(s), &result); - CHECK_UNARY(ok); - CHECK_UNARY_FALSE(result); - } -} - -TEST_CASE("from_chars_first.bool") -{ - bool result = false; - for(const char *s : {"1", "10000", "2", "3", "10", "010", "001", "0001", "true", "True", "TRUE"}) - { - INFO("s='" << s << "'"); - bool ok = from_chars(to_csubstr(s), &result); - CHECK_UNARY(ok); - CHECK_UNARY(result); - } - for(const char *s : {"0", "00", "000", "0000", "false", "False", "FALSE"}) - { - INFO("s='" << s << "'"); - bool ok = from_chars(to_csubstr(s), &result); - CHECK_UNARY(ok); - CHECK_UNARY_FALSE(result); - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// test that no characters are trimmed at the end of -// the number due to printf-based implementations -// needing space for the \0 -template -void test_trimmed_fit(T v, csubstr expected) -{ - char buf_[128] = {}; - char buf2_[128] = {}; - substr buf(buf_); - substr buf2(buf_); - REQUIRE_GE(buf.len, expected.len); - REQUIRE_GE(buf2.len, expected.len); - csubstr result = to_chars_sub(buf, v); - CHECK_EQ(result, expected); - csubstr result2 = to_chars_sub(buf2.sub(result.len), v); - CHECK_EQ(result2, result); - std::string str; - catrs(&str, v); - CHECK_EQ(expected, to_csubstr(str)); - CHECK_EQ(result, to_csubstr(str)); -} - -TEST_CASE("to_chars.trimmed_fit_int") -{ - test_trimmed_fit(12345678, "12345678"); -} - -TEST_CASE("to_chars.trimmed_fit_float") -{ - test_trimmed_fit(0.374f, "0.374"); - test_trimmed_fit(12.374f, "12.374"); -} - -TEST_CASE("to_chars.trimmed_fit_double") -{ - test_trimmed_fit(0.374, "0.374"); - test_trimmed_fit(12.374, "12.374"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void to_chars_roundtrip(substr buf, T const& val, csubstr expected) -{ - T cp = {}; - INFO("val=" << val); - csubstr res = to_chars_sub(buf, val); - CHECK_EQ(res, expected); - bool ok = from_chars(res, &cp); - CHECK_UNARY(ok); - CHECK_EQ(cp, val); -} - -template -void to_chars_roundtrip(char (&buf)[N], csubstr val) -{ - char cp_[N] = {}; - substr cp = cp_; - INFO("val=" << val); - REQUIRE_LE(val.len, N); - csubstr res = to_chars_sub(buf, val); - CHECK_EQ(res.len, val.len); - CHECK_EQ(res, val); - bool ok = from_chars(res, &cp); - CHECK_UNARY(ok); - CHECK_EQ(cp, val); -} - - -TEST_CASE("to_chars.roundtrip_bool") -{ - char buf[128]; - to_chars_roundtrip(buf, false, "0"); - to_chars_roundtrip(buf, true, "1"); -} - - -TEST_CASE("to_chars.roundtrip_char") -{ - char buf[128]; - to_chars_roundtrip(buf, 'a', "a"); - to_chars_roundtrip(buf, 'b', "b"); - to_chars_roundtrip(buf, 'c', "c"); - to_chars_roundtrip(buf, 'd', "d"); -} - -#define C4_TEST_ROUNDTRIP_INT(ty) \ -TEST_CASE("to_chars.roundtrip_" #ty)\ -{\ - char buf[128];\ - to_chars_roundtrip(buf, 0, "0");\ - to_chars_roundtrip(buf, 1, "1");\ - to_chars_roundtrip(buf, 2, "2");\ - to_chars_roundtrip(buf, 3, "3");\ - to_chars_roundtrip(buf, 4, "4");\ -} -C4_TEST_ROUNDTRIP_INT(int8_t) -C4_TEST_ROUNDTRIP_INT(int16_t) -C4_TEST_ROUNDTRIP_INT(int32_t) -C4_TEST_ROUNDTRIP_INT(int64_t) -C4_TEST_ROUNDTRIP_INT(uint8_t) -C4_TEST_ROUNDTRIP_INT(uint16_t) -C4_TEST_ROUNDTRIP_INT(uint32_t) -C4_TEST_ROUNDTRIP_INT(uint64_t) -// some of the following types are not the same as above: -using ulong = unsigned long; -using uint = unsigned int; -C4_TEST_ROUNDTRIP_INT(int) -C4_TEST_ROUNDTRIP_INT(uint) -C4_TEST_ROUNDTRIP_INT(long) -C4_TEST_ROUNDTRIP_INT(ulong) -C4_TEST_ROUNDTRIP_INT(size_t) -C4_TEST_ROUNDTRIP_INT(intptr_t) -C4_TEST_ROUNDTRIP_INT(uintptr_t) - -#define C4_TEST_ROUNDTRIP_REAL(ty) \ -TEST_CASE("to_chars.roundtrip_" #ty)\ -{\ - char buf[128];\ - to_chars_roundtrip(buf, ty(0.0), "0");\ - to_chars_roundtrip(buf, ty(1.0), "1");\ - to_chars_roundtrip(buf, ty(2.0), "2");\ - to_chars_roundtrip(buf, ty(3.0), "3");\ - to_chars_roundtrip(buf, ty(4.0), "4");\ -} -C4_TEST_ROUNDTRIP_REAL(float) -C4_TEST_ROUNDTRIP_REAL(double) - -TEST_CASE("to_chars.roundtrip_substr") -{ - char buf[128]; - to_chars_roundtrip(buf, ""); - to_chars_roundtrip(buf, "0"); - to_chars_roundtrip(buf, "1"); - to_chars_roundtrip(buf, "2"); - to_chars_roundtrip(buf, "3"); - to_chars_roundtrip(buf, "4"); - to_chars_roundtrip(buf, "zhis iz a test"); -} - -TEST_CASE("to_chars.substr_enough_size") -{ - char orig_[] = "0123456789"; - substr orig = orig_; - char result_[20]; - substr result = result_; - size_t len = to_chars(result, orig); - CHECK_EQ(len, orig.len); - CHECK_NE(result.str, orig.str); - CHECK_EQ(result.first(10), orig); -} - -TEST_CASE("to_chars.substr_insufficient_size") -{ - char orig_[] = "0123456789"; - substr orig = orig_; - char result_[11] = {}; - substr result = result_; - result.len = 5; - size_t len = to_chars(result, orig); - CHECK_EQ(len, orig.len); - CHECK_NE(result.str, orig.str); - CHECK_EQ(result.first(5), "01234"); - CHECK_EQ(substr(result_).last(5), "\0\0\0\0\0"); -} - -TEST_CASE("from_chars.csubstr") -{ - csubstr orig = "0123456789"; - csubstr result; - CHECK_NE(result.str, orig.str); - CHECK_NE(result.len, orig.len); - bool ok = from_chars(orig, &result); - CHECK(ok); - CHECK_EQ(result.str, orig.str); - CHECK_EQ(result.len, orig.len); -} - -TEST_CASE("from_chars.substr_enough_size") -{ - char buf_[128] = {}; - substr result = buf_; - for(char r : result) - { - CHECK_EQ(r, '\0'); - } - bool ok = from_chars("0123456789", &result); - CHECK(ok); - CHECK_EQ(result.len, 10); - CHECK_EQ(result.str, buf_); - CHECK_EQ(result, "0123456789"); -} - -TEST_CASE("from_chars.substr_insufficient_size") -{ - char buf_[128] = {}; - substr buf = buf_; - buf.len = 0; - bool ok = from_chars("0123456789", &buf); - CHECK_FALSE(ok); - for(size_t i = 0; i < 10; ++i) - { - CHECK_EQ(buf_[i], '\0'); - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -struct ptr_expected { void *ptr; c4::csubstr str; }; -const ptr_expected ptr_cases[] = { - {(void*)0x0, c4::csubstr("0x0")}, - {(void*)0x1234, c4::csubstr("0x1234")}, - {(void*)-0x1234, c4::csubstr("-0x1234")}, -}; - -template -void test_xtoa_ptr(const char *type_name) -{ - INFO("type=" << type_name); - char buf_[128] = {}; - c4::substr buf(buf_); - for(auto &pe : ptr_cases) - { - INFO("val=" << pe.str); - size_t ret = xtoa(buf, (T const*)pe.ptr); - CHECK_EQ(ret, pe.str.len); - CHECK_EQ(buf.first(ret), pe.str); - } -} - -template -void test_to_chars_ptr(const char *type_name) -{ - INFO("type=" << type_name); - char buf_[128] = {}; - c4::substr buf(buf_); - for(auto &pe : ptr_cases) - { - INFO("val=" << pe.str); - size_t ret = to_chars(buf, (T const*)pe.ptr); - CHECK_EQ(ret, pe.str.len); - CHECK_EQ(buf.first(ret), pe.str); - } -} - -template -void test_atox_ptr(const char *type_name) -{ - INFO("type=" << type_name); - for(auto &pe : ptr_cases) - { - T *ptr = nullptr; - INFO("val=" << pe.str); - bool ret = atox(pe.str, &ptr); - CHECK(ret); - CHECK_EQ((void*)ptr, pe.ptr); - } -} - -template -void test_from_chars_ptr(const char *type_name) -{ - INFO("type=" << type_name); - for(auto &pe : ptr_cases) - { - T *ptr = nullptr; - INFO("val=" << pe.str); - bool ret = from_chars(pe.str, &ptr); - CHECK(ret); - CHECK_EQ((void*)ptr, pe.ptr); - } -} - -template -void test_from_chars_first_ptr(const char *type_name) -{ - INFO("type=" << type_name); - for(auto &pe : ptr_cases) - { - T *ptr = nullptr; - INFO("val=" << pe.str); - bool ret = from_chars(pe.str, &ptr); - CHECK(ret); - CHECK_EQ((void*)ptr, pe.ptr); - } -} - - -TEST_CASE("xtoa.ptr") -{ - test_xtoa_ptr("void"); - test_xtoa_ptr("int"); - test_xtoa_ptr>("std::vector"); -} - -TEST_CASE("atox.ptr") -{ - test_atox_ptr("void"); - test_atox_ptr("int"); - test_atox_ptr>("std::vector"); -} - -TEST_CASE("to_chars.ptr") -{ - test_to_chars_ptr("void"); - test_to_chars_ptr("int"); - test_to_chars_ptr>("std::vector"); -} - -TEST_CASE("from_chars.ptr") -{ - test_from_chars_ptr("void"); - test_from_chars_ptr("int"); - test_from_chars_ptr>("std::vector"); -} - -TEST_CASE("from_chars_first.ptr") -{ - test_from_chars_first_ptr("void"); - test_from_chars_first_ptr("int"); - test_from_chars_first_ptr>("std::vector"); -} - -} // namespace c4 - - -C4_SUPPRESS_WARNING_GCC_POP -C4_SUPPRESS_WARNING_CLANG_POP - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_ctor_dtor.cpp b/thirdparty/ryml/ext/c4core/test/test_ctor_dtor.cpp deleted file mode 100644 index f655fb17c..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_ctor_dtor.cpp +++ /dev/null @@ -1,306 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/ctor_dtor.hpp" -#endif - -#include "c4/libtest/supprwarn_push.hpp" - -#include -#include -#include - -namespace c4 { - -namespace { -struct subject -{ - static size_t ct_cp, ct_mv, cp, mv; - static void clear() { ct_cp = ct_mv = cp = mv = 0; } - subject(Counting const&) - { - ++ct_cp; - } - subject(Counting &&) - { - ++ct_mv; - } - subject(subject const&) - { - ++cp; - } - subject(subject &&) - { - ++mv; - } -}; -size_t subject::ct_cp = 0; -size_t subject::ct_mv = 0; -size_t subject::cp = 0; -size_t subject::mv = 0; -} // empty namespace - - -TEST_CASE("ctor_dtor.construct_n") -{ - using T = Counting; - C4_STATIC_ASSERT(sizeof(T) % alignof(T) == 0); - alignas(T) char buf1[100 * sizeof(T)]; - T* mem1 = reinterpret_cast(buf1); - - using cs = Counting; - - decltype(subject::ct_cp) num = 10; - - { - auto chc = T::check_num_ctors_dtors(num, 0); - auto ch = cs::check_num_ctors_dtors(1, 1); - cs s("bla"); - construct_n(mem1, num, s); - CHECK_EQ(subject::ct_cp, num); - subject::clear(); - } - - { - auto chc = T::check_num_ctors_dtors(num, 0); - auto ch = cs::check_num_ctors_dtors(1, 1); - construct_n(mem1, num, cs("bla")); // BAD!!! will call 10 moves - CHECK_EQ(subject::ct_cp, num); - subject::clear(); - } -} - - -//----------------------------------------------------------------------------- -template -void create_make_room_buffer(std::vector &orig) -{ - C4_STATIC_ASSERT(std::is_integral::value); - for(int i = 0, e = (int)orig.size(); i < e; ++i) - { - orig[static_cast(i)] = (T)(33 + i % (122 - 33)); // assign characters - } -} -template<> -void create_make_room_buffer(std::vector &orig) -{ - for(int i = 0, e = (int)orig.size(); i < e; ++i) - { - char c = (char)(33 + i % (122 - 33)); - orig[static_cast(i)].assign(10, c); - } -} - -template -void do_make_room_inplace(std::vector const& orig, std::vector & buf, - size_t bufsz, size_t room, size_t pos) -{ - buf = orig; - make_room(buf.data() + pos, bufsz, room); -} - -template -void do_make_room_srcdst(std::vector const& orig, std::vector & buf, - size_t bufsz, size_t room, size_t pos) -{ - buf.resize(orig.size()); - for(auto &t : buf) - { - t = T(); - } - make_room(buf.data(), orig.data(), bufsz, room, pos); -} - -template -void do_make_room_check(std::vector const& orig, std::vector & buf, - size_t bufsz, size_t room, size_t pos) -{ - for(size_t i = 0, e = orig.size(); i < e; ++i) - { - INFO("i=" << (int)i); - if(i < pos) - { - // memory before the move, must be untouched - CHECK_EQ(buf[i], orig[i]); - } - else - { - if(i >= pos && i < pos + room) - { - // this is the memory that was moved (at its origin) - //CHECK_EQ(buf[i], orig[i]) << "i=" << (int)i; - } - else if(i >= pos + room && i < pos + room + bufsz) - { - // this is the memory that was moved (at its destination) - CHECK_EQ(buf[i], orig[i - room]); - } - else - { - // this is memory at the end, must be untouched - CHECK_EQ(buf[i], orig[i]); - } - } - } -}; - -template -void do_make_room_inplace_test(std::vector const& orig, std::vector & buf, - size_t bufsz, size_t room, size_t pos) -{ - do_make_room_inplace(orig, buf, bufsz, room, pos); - do_make_room_check(orig, buf, bufsz, room, pos); -} - -template -void do_make_room_srcdst_test(std::vector const& orig, std::vector & buf, - size_t /*bufsz*/, size_t room, size_t pos) -{ - do_make_room_srcdst(orig, buf, buf.size() - room, room, pos); - do_make_room_check(orig, buf, buf.size() - room, room, pos); -} - -template -void test_make_room(Func test_func) -{ - std::vector orig(100), buf(100); - - create_make_room_buffer(orig); - - { - INFO("in the beginning without overlap"); - test_func(orig, buf, /*bufsz*/10, /*room*/10, /*pos*/0); - } - - { - INFO("in the beginning with overlap"); - test_func(orig, buf, /*bufsz*/10, /*room*/15, /*pos*/0); - } - - { - INFO("in the middle without overlap"); - test_func(orig, buf, /*bufsz*/10, /*room*/10, /*pos*/10); - } - - { - INFO("in the middle with overlap"); - test_func(orig, buf, /*bufsz*/10, /*room*/15, /*pos*/10); - } -} - -TEST_CASE_TEMPLATE("ctor_dtor.make_room_inplace", T, uint8_t, uint64_t, std::string) -{ - test_make_room(do_make_room_inplace_test); -} - -TEST_CASE_TEMPLATE("ctor_dtor.make_room_srcdst", T, uint8_t, uint64_t, std::string) -{ - test_make_room(&do_make_room_srcdst_test); -} - - -//----------------------------------------------------------------------------- - -template -void do_destroy_room_inplace(std::vector const& orig, std::vector & buf, - size_t bufsz, size_t room, size_t pos) -{ - buf = orig; - destroy_room(buf.data() + pos, bufsz - pos, room); -} - -template -void do_destroy_room_srcdst(std::vector const& orig, std::vector & buf, - size_t bufsz, size_t room, size_t pos) -{ - buf = orig; - destroy_room(buf.data(), orig.data(), bufsz, room, pos); -} - -template -void do_destroy_room_check(std::vector const& orig, std::vector & buf, - size_t bufsz, size_t room, size_t pos) -{ - for(size_t i = 0, e = orig.size(); i < e; ++i) - { - INFO("i=" << (int)i << " room=" << room << " pos=" << pos); - if(i < pos) - { - // memory before the destroy, should be untouched - CHECK_EQ(buf[i], orig[i]); - } - else - { - if(i >= pos && i < pos + room) - { - // this is the memory that was destroyed (at its origin) - } - else if(i >= pos + room && i < pos + room + bufsz) - { - // this is the memory that was moved (at its destination) - CHECK_EQ(buf[i - room], orig[i]); - } - else - { - // this is memory at the end, should be untouched - CHECK_EQ(buf[i], orig[i]); - } - } - } -}; - -template -void do_destroy_room_inplace_test(std::vector const& orig, std::vector & buf, - size_t room, size_t pos) -{ - do_destroy_room_inplace(orig, buf, buf.size(), room, pos); - do_destroy_room_check(orig, buf, buf.size(), room, pos); -} - -template -void do_destroy_room_srcdst_test(std::vector const& orig, std::vector & buf, - size_t room, size_t pos) -{ - do_destroy_room_srcdst(orig, buf, buf.size(), room, pos); - do_destroy_room_check(orig, buf, buf.size(), room, pos); -} - -template -void test_destroy_room(Func test_func) -{ - std::vector orig(100), buf(100); - - create_make_room_buffer(orig); - - { - INFO("in the beginning, room=10"); - test_func(orig, buf, /*room*/10, /*pos*/0); - } - - { - INFO("in the beginning, room=20"); - test_func(orig, buf, /*room*/20, /*pos*/0); - } - - { - INFO("in the middle, room=10"); - test_func(orig, buf, /*room*/10, /*pos*/10); - } - - { - INFO("in the middle, room=20"); - test_func(orig, buf, /*room*/20, /*pos*/10); - } -} - -TEST_CASE_TEMPLATE("ctor_dtor.destroy_room_inplace", T, uint8_t, uint64_t, std::string) -{ - test_destroy_room(do_destroy_room_inplace_test); -} - -TEST_CASE_TEMPLATE("ctor_dtor.destroy_room_srcdst", T, uint8_t, uint64_t, std::string) -{ - test_destroy_room(&do_destroy_room_srcdst_test); -} - -} // namespace c4 - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_dump.cpp b/thirdparty/ryml/ext/c4core/test/test_dump.cpp deleted file mode 100644 index cfa7d991c..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_dump.cpp +++ /dev/null @@ -1,1220 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/substr.hpp" -#include "c4/std/std.hpp" -#include "c4/dump.hpp" -#include "c4/format.hpp" -#endif - -#include -#include "c4/libtest/supprwarn_push.hpp" - -#ifdef __clang__ -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wuseless-cast" -#endif - -namespace c4 { - - -namespace example { - -std::string test_dumper_target = {}; -void test_dumper(csubstr str) -{ - test_dumper_target.append(str.str, str.len); -} - -template -void printf(csubstr fmt, Args&& ...args) -{ - static thread_local std::string writebuf(16, '\0'); - DumpResults results = format_dump_resume<&test_dumper>(c4::to_substr(writebuf), fmt, std::forward(args)...); - if(C4_UNLIKELY(results.bufsize > writebuf.size())) // bufsize will be that of the largest element serialized. Eg int(1), will require 1 byte. - { - size_t dup = 2 * writebuf.size(); - writebuf.resize(dup > results.bufsize ? dup : results.bufsize); - format_dump_resume<&test_dumper>(results, c4::to_substr(writebuf), fmt, std::forward(args)...); - } -} -} // namespace example - -TEST_CASE("printf_example") -{ - example::test_dumper_target.clear(); - SUBCASE("1") - { - example::printf("{} coffees per day.\n", 3); - CHECK_EQ(example::test_dumper_target, "3 coffees per day.\n"); - } - SUBCASE("2") - { - example::printf("{} would be {}.", "brecky", "nice"); - CHECK_EQ(example::test_dumper_target, "brecky would be nice."); - } - SUBCASE("resize writebuf") - { - // printed strings will not use the writebuf, so we write a zero-padded integer - size_t dim = 128; // pad with 128 zeroes - std::string s1(dim, '0'); - std::string s2(dim, '0'); - s1.back() = '1'; - s2.back() = '2'; - example::printf("{} cannot be {}", fmt::zpad(1, dim), fmt::zpad(2, dim)); - CHECK_EQ(example::test_dumper_target, s1 + " cannot be " + s2); - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE("DumpResults") -{ - DumpResults dr = {}; - CHECK_EQ(dr.bufsize, 0u); - CHECK_EQ(dr.lastok, DumpResults::noarg); - CHECK_UNARY(dr.write_arg(0)); - CHECK_FALSE(dr.success_until(0)); - CHECK_EQ(dr.argfail(), 0); -} - -struct DumpChecker -{ - static size_t s_num_calls; - static char s_workspace[100]; - static size_t s_accum_pos; - static char s_accum[100]; - static void s_reset() - { - s_num_calls = 0; - s_accum_pos = 0; - for(size_t i = 0; i < sizeof(s_workspace); ++i) - s_workspace[i] = '+'; - for(size_t i = 0; i < sizeof(s_accum); ++i) - s_accum[i] = '.'; - } - static void s_dump(csubstr buf) - { - REQUIRE_LT(buf.len, sizeof(s_workspace)); - REQUIRE_LT(s_accum_pos + buf.len, sizeof(s_accum)); - ++s_num_calls; - memcpy(s_accum + s_accum_pos, buf.str, buf.len); - s_accum_pos += buf.len; - } -}; -size_t DumpChecker::s_num_calls = 0; -char DumpChecker::s_workspace[100] = {}; -size_t DumpChecker::s_accum_pos = {}; -char DumpChecker::s_accum[100] = {}; - -struct CatDumpTplArg -{ - template - static size_t call_cat_dump(Args&& ...args) - { - return cat_dump<&DumpChecker::s_dump>(std::forward(args)...); - } - template - static DumpResults call_cat_dump_resume(Args&& ...args) - { - return cat_dump_resume<&DumpChecker::s_dump>(std::forward(args)...); - } - template - static size_t call_catsep_dump(Args&& ...args) - { - return catsep_dump<&DumpChecker::s_dump>(std::forward(args)...); - } - template - static DumpResults call_catsep_dump_resume(Args&& ...args) - { - return catsep_dump_resume<&DumpChecker::s_dump>(std::forward(args)...); - } - template - static size_t call_format_dump(Args&& ...args) - { - return format_dump<&DumpChecker::s_dump>(std::forward(args)...); - } - template - static DumpResults call_format_dump_resume(Args&& ...args) - { - return format_dump_resume<&DumpChecker::s_dump>(std::forward(args)...); - } -}; - -struct CatDumpFnArg -{ - template - static size_t call_cat_dump(Args&& ...args) - { - return cat_dump(&DumpChecker::s_dump, std::forward(args)...); - } - template - static DumpResults call_cat_dump_resume(Args&& ...args) - { - return cat_dump_resume(&DumpChecker::s_dump, std::forward(args)...); - } - template - static size_t call_catsep_dump(Args&& ...args) - { - return catsep_dump(&DumpChecker::s_dump, std::forward(args)...); - } - template - static DumpResults call_catsep_dump_resume(Args&& ...args) - { - return catsep_dump_resume(&DumpChecker::s_dump, std::forward(args)...); - } - template - static size_t call_format_dump(Args&& ...args) - { - return format_dump(&DumpChecker::s_dump, std::forward(args)...); - } - template - static DumpResults call_format_dump_resume(Args&& ...args) - { - return format_dump_resume(&DumpChecker::s_dump, std::forward(args)...); - } -}; - -namespace buffers { -int b1 = 1; -int b2 = 22; -int b3 = 333; -int b4 = 4444; -int sep = 90009; -size_t seplen = 5; -} - -TEST_CASE_TEMPLATE("cat_dump", T, CatDumpTplArg, CatDumpFnArg) -{ - using namespace buffers; - substr buf = DumpChecker::s_workspace; - auto accum = [&]{ return csubstr(DumpChecker::s_accum).first(DumpChecker::s_accum_pos); }; - SUBCASE("dont use the buffer with strings") - { - DumpChecker::s_reset(); - size_t needed_size = T::call_cat_dump(buf.first(0), b1); - CHECK_EQ(needed_size, 1); - CHECK_EQ(accum(), csubstr("")); - needed_size = T::call_cat_dump(buf.first(0), b1, b2); - CHECK_EQ(needed_size, 2); - CHECK_EQ(accum(), csubstr("")); - needed_size = T::call_cat_dump(buf.first(0), b1, b2, b3); - CHECK_EQ(needed_size, 3); - CHECK_EQ(accum(), csubstr("")); - needed_size = T::call_cat_dump(buf.first(0), b1, b2, b3, b4); - CHECK_EQ(needed_size, 4); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(0), ("1")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("")); - needed_size = T::call_cat_dump(buf.first(0), ("1"), ("22")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("")); - needed_size = T::call_cat_dump(buf.first(0), ("1"), ("22"), ("333")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("")); - needed_size = T::call_cat_dump(buf.first(0), ("1"), ("22"), ("333"), ("4444")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(0), csubstr("1")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("")); - needed_size = T::call_cat_dump(buf.first(0), csubstr("1"), csubstr("22")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("")); - needed_size = T::call_cat_dump(buf.first(0), csubstr("1"), csubstr("22"), csubstr("333")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("")); - needed_size = T::call_cat_dump(buf.first(0), csubstr("1"), csubstr("22"), csubstr("333"), csubstr("4444")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(1), b1); - CHECK_EQ(needed_size, 1); - CHECK_EQ(accum(), csubstr("1")); - needed_size = T::call_cat_dump(buf.first(1), b1, b2); - CHECK_EQ(needed_size, 2); - CHECK_EQ(accum(), csubstr("11")); - needed_size = T::call_cat_dump(buf.first(1), b1, b2, b3); - CHECK_EQ(needed_size, 3); - CHECK_EQ(accum(), csubstr("111")); - needed_size = T::call_cat_dump(buf.first(1), b1, b2, b3, b4); - CHECK_EQ(needed_size, 4); - CHECK_EQ(accum(), csubstr("1111")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(1), ("1")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("1")); - needed_size = T::call_cat_dump(buf.first(1), ("1"), ("22")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("1122")); - needed_size = T::call_cat_dump(buf.first(1), ("1"), ("22"), ("333")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("1122122333")); - needed_size = T::call_cat_dump(buf.first(1), ("1"), ("22"), ("333"), ("4444")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("11221223331223334444")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(1), csubstr("1")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("1")); - needed_size = T::call_cat_dump(buf.first(1), csubstr("1"), csubstr("22")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("1122")); - needed_size = T::call_cat_dump(buf.first(1), csubstr("1"), csubstr("22"), csubstr("333")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("1122122333")); - needed_size = T::call_cat_dump(buf.first(1), csubstr("1"), csubstr("22"), csubstr("333"), csubstr("4444")); - CHECK_EQ(needed_size, 0); - CHECK_EQ(accum(), csubstr("11221223331223334444")); - } - SUBCASE("1") - { - DumpChecker::s_reset(); - size_t needed_size = T::call_cat_dump(buf.first(0), b1); - CHECK_EQ(needed_size, 1u); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(1), csubstr("+")); // nothing was written - CHECK_EQ(accum(), csubstr("")); // nothing was written - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf, b1); - CHECK_EQ(needed_size, 1u); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(1), csubstr("1")); - CHECK_EQ(accum(), csubstr("1")); - } - SUBCASE("1 2") - { - DumpChecker::s_reset(); - size_t needed_size = T::call_cat_dump(buf.first(0), b1, b2); - CHECK_EQ(needed_size, 2); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(2), csubstr("++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(1), b1, b2); - CHECK_EQ(needed_size, 2); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(2), csubstr("1+")); // only the first character of b2 was written - CHECK_EQ(accum(), csubstr("1")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf, b1, b2); - CHECK_EQ(needed_size, 2); - CHECK_EQ(DumpChecker::s_num_calls, 2); - CHECK_EQ(buf.first(2), csubstr("22")); - CHECK_EQ(accum(), csubstr("122")); - } - SUBCASE("2 1") - { - DumpChecker::s_reset(); - size_t needed_size = T::call_cat_dump(buf, b2, b1); - CHECK_EQ(needed_size, 2); - CHECK_EQ(DumpChecker::s_num_calls, 2); - CHECK_EQ(buf.first(2), csubstr("12")); // wrote 2 then 1 - CHECK_EQ(accum(), csubstr("221")); - } - SUBCASE("1 2 3 4") - { - DumpChecker::s_reset(); - size_t needed_size = T::call_cat_dump(buf.first(0), b1, b2, b3, b4); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(1), b1, b2, b3, b4); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); - CHECK_EQ(accum(), csubstr("1")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(2), b1, b2, b3, b4); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 2); - CHECK_EQ(buf.first(4), csubstr("22++")); - CHECK_EQ(accum(), csubstr("122")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(3), b1, b2, b3, b4); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(4), csubstr("333+")); - CHECK_EQ(accum(), csubstr("122333")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf, b1, b2, b3, b4); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 4); - CHECK_EQ(buf.first(4), csubstr("4444")); - CHECK_EQ(accum(), csubstr("1223334444")); - } - SUBCASE("4 3 2 1") - { - DumpChecker::s_reset(); - size_t needed_size = T::call_cat_dump(buf.first(0), b4, b3, b2, b1); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(1), b4, b3, b2, b1); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(2), b4, b3, b2, b1); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf.first(3), b4, b3, b2, b1); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_cat_dump(buf, b4, b3, b2, b1); - CHECK_EQ(needed_size, 4); - CHECK_EQ(DumpChecker::s_num_calls, 4); - CHECK_EQ(buf.first(4), csubstr("1234")); - CHECK_EQ(accum(), csubstr("4444333221")); - } -} - -TEST_CASE_TEMPLATE("cat_dump_resume", T, CatDumpTplArg, CatDumpFnArg) -{ - using namespace buffers; - substr buf = DumpChecker::s_workspace; - auto accum = [&]{ return csubstr(DumpChecker::s_accum).first(DumpChecker::s_accum_pos); }; - SUBCASE("1") - { - DumpChecker::s_reset(); - DumpResults ret = T::call_cat_dump_resume(buf.first(0), b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_EQ(ret.bufsize, 1); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(1), csubstr("+")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - DumpResults retry = T::call_cat_dump_resume(ret, buf.first(1), b1); - CHECK_UNARY(retry.success_until(0)); - CHECK_UNARY(!retry.success_until(1)); - CHECK_UNARY(!retry.success_until(2)); - CHECK_EQ(retry.bufsize, 1); - CHECK_EQ(retry.lastok, 0); - CHECK_EQ(retry.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(1), csubstr("1")); - CHECK_EQ(accum(), csubstr("1")); - } - SUBCASE("1 2") - { - DumpChecker::s_reset(); - DumpResults ret = T::call_cat_dump_resume(buf.first(0), b1, b2); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_EQ(ret.bufsize, 2); // finds the buf size at once - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(2), csubstr("++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(1), b1, b2); - CHECK_UNARY(!ret.success_until(0)); // ret.bufsize signals buffer is at least 2 - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_EQ(ret.bufsize, 2); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(2), csubstr("++")); - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(2), b1, b2); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_EQ(ret.bufsize, 2); - CHECK_EQ(ret.lastok, 1); - CHECK_EQ(ret.argfail(), 2); - CHECK_EQ(DumpChecker::s_num_calls, 2); - CHECK_EQ(buf.first(2), csubstr("22")); - CHECK_EQ(accum(), csubstr("122")); - } - SUBCASE("1 2 3 4") - { - DumpChecker::s_reset(); - DumpResults ret = T::call_cat_dump_resume(buf.first(0), b1, b2, b3, b4); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_UNARY(!ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(4), csubstr("++++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(1), b1, b2, b3, b4); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_UNARY(!ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(2), b1, b2, b3, b4); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_UNARY(!ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(3), b1, b2, b3, b4); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_UNARY(!ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(4), b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_UNARY(ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, 3); - CHECK_EQ(ret.argfail(), 4); - CHECK_EQ(DumpChecker::s_num_calls, 4); - CHECK_EQ(buf.first(4), csubstr("4444")); - CHECK_EQ(accum(), csubstr("1223334444")); - } - SUBCASE("4 3 2 1") - { - DumpChecker::s_reset(); - DumpResults ret = T::call_cat_dump_resume(buf.first(0), b4, b3, b2, b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_UNARY(!ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(4), csubstr("++++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(1), b4, b3, b2, b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_UNARY(!ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(2), b4, b3, b2, b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_UNARY(!ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(3), b4, b3, b2, b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_UNARY(!ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - ret = T::call_cat_dump_resume(ret, buf.first(4), b4, b3, b2, b1); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_UNARY(ret.success_until(3)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, 3); - CHECK_EQ(ret.argfail(), 4); - CHECK_EQ(DumpChecker::s_num_calls, 4); - CHECK_EQ(buf.first(4), csubstr("1234")); - CHECK_EQ(accum(), csubstr("4444333221")); - } -} - - - -TEST_CASE_TEMPLATE("catsep_dump", T, CatDumpTplArg, CatDumpFnArg) -{ - using namespace buffers; - size_t needed_size; - substr buf = DumpChecker::s_workspace; - auto accum = [&]{ return csubstr(DumpChecker::s_accum).first(DumpChecker::s_accum_pos); }; - SUBCASE("1") - { - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(0), sep, b1); - CHECK_EQ(needed_size, 1); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(1), csubstr("+")); // nothing was written - CHECK_EQ(accum(), csubstr("")); // nothing was written - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(1), sep, b1); - CHECK_EQ(needed_size, 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(1), csubstr("1")); - CHECK_EQ(accum(), csubstr("1")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(1 + seplen), sep, b1); - CHECK_EQ(needed_size, 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); // sep was not written - CHECK_EQ(buf.first(1), csubstr("1")); - CHECK_EQ(accum(), csubstr("1")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf, sep, b1); - CHECK_EQ(needed_size, 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); // sep was not written - CHECK_EQ(buf.first(1), csubstr("1")); - CHECK_EQ(accum(), csubstr("1")); - } - SUBCASE("1 2") - { - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(0), sep, b1, b2); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(2), csubstr("++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(1), sep, b1, b2); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("1")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(seplen), sep, b1, b2); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("22")); - CHECK_EQ(accum(), csubstr("19000922")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf, sep, b1, b2); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("22")); - CHECK_EQ(accum(), csubstr("19000922")); - } - SUBCASE("2 1") - { - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(0), sep, b2, b1); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(2), csubstr("++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(1), sep, b2, b1); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(2), csubstr("++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(seplen), sep, b2, b1); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("10")); - CHECK_EQ(accum(), csubstr("22900091")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf, sep, b2, b1); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("10")); - CHECK_EQ(accum(), csubstr("22900091")); - } - SUBCASE("1 2 3 4") - { - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(0), sep, b1, b2, b3, b4); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(1), sep, b1, b2, b3, b4); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); - CHECK_EQ(accum(), csubstr("1")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(2), sep, b1, b2, b3, b4); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); - CHECK_EQ(accum(), csubstr("1")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(3), sep, b1, b2, b3, b4); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); - CHECK_EQ(accum(), csubstr("1")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf, sep, b1, b2, b3, b4); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 7); - CHECK_EQ(buf.first(4), csubstr("4444")); - CHECK_EQ(accum(), csubstr("1900092290009333900094444")); - } - SUBCASE("4 3 2 1") - { - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(0), sep, b4, b3, b2, b1); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(1), sep, b4, b3, b2, b1); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(2), sep, b4, b3, b2, b1); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf.first(3), sep, b4, b3, b2, b1); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - needed_size = T::call_catsep_dump(buf, sep, b4, b3, b2, b1); - CHECK_EQ(needed_size, seplen); - CHECK_EQ(DumpChecker::s_num_calls, 7); - CHECK_EQ(buf.first(4), csubstr("1000")); - CHECK_EQ(accum(), csubstr("4444900093339000922900091")); - } -} - - -TEST_CASE_TEMPLATE("catsep_dump_resume", T, CatDumpTplArg, CatDumpFnArg) -{ - using namespace buffers; - substr buf = DumpChecker::s_workspace; - auto accum = [&]{ return csubstr(DumpChecker::s_accum).first(DumpChecker::s_accum_pos); }; - SUBCASE("1") - { - DumpChecker::s_reset(); - DumpResults ret = T::call_catsep_dump_resume(buf.first(0), sep, b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_EQ(ret.bufsize, 1); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(1), csubstr("+")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_catsep_dump_resume(ret, buf.first(1), sep, b1); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_EQ(ret.bufsize, 1); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(1), csubstr("1")); - CHECK_EQ(accum(), csubstr("1")); - ret = T::call_catsep_dump_resume(ret, buf, sep, b1); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_EQ(ret.bufsize, 1); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(1), csubstr("1")); - CHECK_EQ(accum(), csubstr("1")); - } - SUBCASE("1 2") - { - DumpChecker::s_reset(); - DumpResults ret = T::call_catsep_dump_resume(buf.first(0), sep, b1, b2); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_EQ(ret.bufsize, seplen); // finds the buf size at once - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(2), csubstr("++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_catsep_dump_resume(ret, buf.first(1), sep, b1, b2); - CHECK_UNARY(ret.success_until(0)); // b1 - CHECK_UNARY(!ret.success_until(1)); // sep - CHECK_UNARY(!ret.success_until(2)); // b2 - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); // sep - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("1")); - ret = T::call_catsep_dump_resume(ret, buf.first(2), sep, b1, b2); - CHECK_UNARY(ret.success_until(0)); // b1 - CHECK_UNARY(!ret.success_until(1)); // sep - CHECK_UNARY(!ret.success_until(2)); // b2 - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); // sep - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("1")); - ret = T::call_catsep_dump_resume(ret, buf.first(seplen), sep, b1, b2); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, 2); - CHECK_EQ(ret.argfail(), 3); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("22")); - CHECK_EQ(accum(), csubstr("19000922")); - } - SUBCASE("1 2 3 4") - { - DumpChecker::s_reset(); - DumpResults ret = T::call_catsep_dump_resume(buf.first(0), sep, b1, b2, b3, b4); - CHECK_UNARY(!ret.success_until(0)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(4), csubstr("++++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_catsep_dump_resume(ret, buf.first(1), sep, b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); // failed while writing sep - CHECK_EQ(accum(), csubstr("1")); - ret = T::call_catsep_dump_resume(ret, buf.first(2), sep, b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); // failed while writing sep - CHECK_EQ(accum(), csubstr("1")); - ret = T::call_catsep_dump_resume(ret, buf.first(3), sep, b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); - CHECK_EQ(accum(), csubstr("1")); - ret = T::call_catsep_dump_resume(ret, buf.first(4), sep, b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); - CHECK_EQ(accum(), csubstr("1")); - ret = T::call_catsep_dump_resume(ret, buf.first(seplen), sep, b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_UNARY(ret.success_until(3)); - CHECK_UNARY(ret.success_until(4)); - CHECK_UNARY(ret.success_until(5)); - CHECK_UNARY(ret.success_until(6)); - CHECK_UNARY(!ret.success_until(7)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, 6); - CHECK_EQ(ret.argfail(), 7); - CHECK_EQ(DumpChecker::s_num_calls, 7); - CHECK_EQ(buf.first(seplen), csubstr("44449")); - CHECK_EQ(accum(), csubstr("1900092290009333900094444")); - } - SUBCASE("4 3 2 1") - { - DumpChecker::s_reset(); - DumpResults ret = T::call_catsep_dump_resume(buf.first(0), sep, b4, b3, b2, b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(4), csubstr("++++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_catsep_dump_resume(ret, buf.first(1), sep, b4, b3, b2, b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - ret = T::call_catsep_dump_resume(ret, buf.first(2), sep, b4, b3, b2, b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(4), csubstr("++++")); - CHECK_EQ(accum(), csubstr("")); - ret = T::call_catsep_dump_resume(ret, buf.first(seplen), sep, b4, b3, b2, b1); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_UNARY(ret.success_until(3)); - CHECK_UNARY(ret.success_until(4)); - CHECK_UNARY(ret.success_until(5)); - CHECK_UNARY(ret.success_until(6)); - CHECK_UNARY(!ret.success_until(7)); - CHECK_EQ(ret.bufsize, seplen); - CHECK_EQ(ret.lastok, 6); - CHECK_EQ(ret.argfail(), 7); - CHECK_EQ(DumpChecker::s_num_calls, 7); - CHECK_EQ(buf.first(seplen), csubstr("10009")); - CHECK_EQ(accum(), csubstr("4444900093339000922900091")); - } - SUBCASE("1 2 3 4 with seplen==3") - { - int s = 999; - DumpChecker::s_reset(); - DumpResults ret = T::call_catsep_dump_resume(buf.first(0), s, b1, b2, b3, b4); - CHECK_UNARY(!ret.success_until(0)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(4), csubstr("++++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_catsep_dump_resume(ret, buf.first(1), s, b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); // failed while writing sep - CHECK_EQ(accum(), csubstr("1")); - ret = T::call_catsep_dump_resume(ret, buf.first(2), s, b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); // b1 - CHECK_UNARY(!ret.success_until(1)); // s - CHECK_UNARY(!ret.success_until(2)); // b2 - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(4), csubstr("1+++")); // failed while writing sep - CHECK_EQ(accum(), csubstr("1")); - ret = T::call_catsep_dump_resume(ret, buf.first(3), s, b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_UNARY(ret.success_until(3)); - CHECK_UNARY(ret.success_until(4)); - CHECK_UNARY(ret.success_until(5)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, 5); - CHECK_EQ(ret.argfail(), 6); - CHECK_EQ(DumpChecker::s_num_calls, 6); - CHECK_EQ(buf.first(4), csubstr("999+")); // failed while writing b4 - CHECK_EQ(accum(), csubstr("199922999333999")); - ret = T::call_catsep_dump_resume(ret, buf.first(4), s, b1, b2, b3, b4); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_UNARY(ret.success_until(3)); - CHECK_UNARY(ret.success_until(4)); - CHECK_UNARY(ret.success_until(5)); - CHECK_UNARY(ret.success_until(6)); - CHECK_UNARY(!ret.success_until(7)); - CHECK_EQ(ret.bufsize, 4); - CHECK_EQ(ret.lastok, 6); - CHECK_EQ(ret.argfail(), 7); - CHECK_EQ(DumpChecker::s_num_calls, 7); - CHECK_EQ(buf.first(5), csubstr("4444+")); - CHECK_EQ(accum(), csubstr("1999229993339994444")); - } -} - - -TEST_CASE_TEMPLATE("format_dump", T, CatDumpTplArg, CatDumpFnArg) -{ - using namespace buffers; - size_t needed_size; - substr buf = DumpChecker::s_workspace; - auto accum = [&]{ return csubstr(DumpChecker::s_accum).first(DumpChecker::s_accum_pos); }; - SUBCASE("no buffer is needed for strings") - { - csubstr fmt = "{}-{}-{}-{}"; - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(0), fmt, "1", "22", "333", "4444"); // no buffer! - CHECK_EQ(needed_size, 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(1), csubstr("+")); - CHECK_EQ(accum(), csubstr("")); - DumpChecker::s_reset(); - CHECK_EQ(DumpChecker::s_num_calls, 0); - needed_size = T::call_format_dump(buf.first(1), fmt, "1", "22", "333", "4444"); // 1-len buffer, unused - CHECK_EQ(needed_size, 0); // no intermediate serialization is needed, since these are strings - CHECK_EQ(DumpChecker::s_num_calls, 7); // calls everything even when the buffer is empty - CHECK_EQ(accum(), csubstr("1-22-333-4444")); // dumped the full format string - } - SUBCASE("0") - { - csubstr fmt = "01234567890123456789"; - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(0), fmt, b1, b2, b3, b4); - CHECK_EQ(needed_size, 0); // the longest sized argument format argument - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls when the buffer is empty - CHECK_EQ(buf.first(needed_size), csubstr("")); // nothing was written - CHECK_EQ(accum(), csubstr("")); // dumped the full format string - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf, fmt, b1, b2, b3, b4); - CHECK_EQ(needed_size, 0); // the longest sized argument format argument - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(needed_size), csubstr("")); // nothing was written - CHECK_EQ(accum(), fmt); // dumped the full format string - } - SUBCASE("1") - { - // ____1____ 2 __3__ - csubstr fmt = "012345678_{}_34567"; - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(0), fmt, b1); - CHECK_EQ(needed_size, 1); // the longest sized argument format argument - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(2), csubstr("++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); // dumped first part of the format string - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(1), fmt, b1); - CHECK_EQ(needed_size, 1); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("012345678_1_34567")); - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(1), fmt, b1, b2, b3, b4); // check that extra arguments are ignored - CHECK_EQ(needed_size, 1); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("012345678_1_34567")); - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(1), fmt); // check that missing arguments are skipped - CHECK_EQ(needed_size, 0u); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(2), csubstr("++")); - CHECK_EQ(accum(), csubstr("012345678_{}_34567")); - } - SUBCASE("1 2") - { - // ____1____ 2 __3__ 4 _5_ - csubstr fmt = "012345678_{}_34567_{}_aaa"; - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(0), fmt, b1, b2); - CHECK_EQ(needed_size, 2); // the longest sized argument format argument - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(2), csubstr("++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); // dumped first part of the format string - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(1), fmt, b1, b2); - CHECK_EQ(needed_size, 2); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("012345678_1_34567_")); - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(2), fmt, b1, b2); - CHECK_EQ(needed_size, 2); - CHECK_EQ(DumpChecker::s_num_calls, 5); - CHECK_EQ(buf.first(2), csubstr("22")); - CHECK_EQ(accum(), csubstr("012345678_1_34567_22_aaa")); - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(1), fmt); // check that missing arguments are skipped - CHECK_EQ(needed_size, 0u); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(2), csubstr("++")); - CHECK_EQ(accum(), csubstr("012345678_{}_34567_{}_aaa")); - } - SUBCASE("1 2 3") - { - // 1 2 3 4 5 6 - csubstr fmt = "012345678_{}_34567_{}_aaa___{}"; - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(0), fmt, b1, b2, b3); - CHECK_EQ(needed_size, 3); // the longest sized argument format argument - CHECK_EQ(DumpChecker::s_num_calls, 0); - CHECK_EQ(buf.first(2), csubstr("++")); // nothing was written - CHECK_EQ(accum(), csubstr("")); // dumped first part of the format string - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(1), fmt, b1, b2, b3); - CHECK_EQ(needed_size, 3); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("012345678_1_34567_")); - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(2), fmt, b1, b2, b3); - CHECK_EQ(needed_size, 3); - CHECK_EQ(DumpChecker::s_num_calls, 5); - CHECK_EQ(buf.first(2), csubstr("22")); - CHECK_EQ(accum(), csubstr("012345678_1_34567_22_aaa___")); - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf.first(3), fmt, b1, b2, b3); - CHECK_EQ(needed_size, 3); - CHECK_EQ(DumpChecker::s_num_calls, 6); - CHECK_EQ(buf.first(2), csubstr("33")); - CHECK_EQ(accum(), csubstr("012345678_1_34567_22_aaa___333")); - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf, fmt); // check that missing arguments are skipped - CHECK_EQ(needed_size, 0u); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(2), csubstr("++")); - CHECK_EQ(accum(), csubstr("012345678_{}_34567_{}_aaa___{}")); - DumpChecker::s_reset(); - needed_size = T::call_format_dump(buf, fmt, b1); // check that missing arguments are skipped - CHECK_EQ(needed_size, 1); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("012345678_1_34567_{}_aaa___{}")); - } -} - - -TEST_CASE_TEMPLATE("format_dump_resume", T, CatDumpTplArg, CatDumpFnArg) -{ - using namespace buffers; - substr buf = DumpChecker::s_workspace; - auto accum = [&]{ return csubstr(DumpChecker::s_accum).first(DumpChecker::s_accum_pos); }; - SUBCASE("1") - { - csubstr fmt = "aaa_then_{}_then_bbb"; - DumpChecker::s_reset(); - DumpResults ret = T::call_format_dump_resume(buf.first(0), fmt, b1); - CHECK_UNARY(!ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_UNARY(!ret.success_until(2)); - CHECK_EQ(ret.bufsize, 1); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(1), csubstr("+")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_format_dump_resume(ret, buf.first(1), fmt, b1); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_EQ(ret.bufsize, 1); - CHECK_EQ(ret.lastok, 2); - CHECK_EQ(ret.argfail(), 3); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("aaa_then_1_then_bbb")); - } - SUBCASE("2") - { - csubstr fmt = "aaa_then_{}_then_bbb_then_{}__then_epilogue"; - DumpChecker::s_reset(); - DumpResults ret = T::call_format_dump_resume(buf.first(0), fmt, b1, b2); - CHECK_UNARY(!ret.success_until(0)); - CHECK_EQ(ret.bufsize, 2); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(1), csubstr("+")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_format_dump_resume(ret, buf.first(1), fmt, b1, b2); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_UNARY(!ret.success_until(3)); - CHECK_EQ(ret.bufsize, 2); - CHECK_EQ(ret.lastok, 2); - CHECK_EQ(ret.argfail(), 3); - CHECK_EQ(DumpChecker::s_num_calls, 3); - CHECK_EQ(buf.first(2), csubstr("1+")); - CHECK_EQ(accum(), csubstr("aaa_then_1_then_bbb_then_")); - ret = T::call_format_dump_resume(ret, buf.first(2), fmt, b1, b2); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(ret.success_until(1)); - CHECK_UNARY(ret.success_until(2)); - CHECK_UNARY(ret.success_until(3)); - CHECK_UNARY(ret.success_until(4)); - CHECK_EQ(ret.bufsize, 2); - CHECK_EQ(ret.lastok, 4); - CHECK_EQ(ret.argfail(), 5); - CHECK_EQ(DumpChecker::s_num_calls, 5); - CHECK_EQ(buf.first(2), csubstr("22")); - CHECK_EQ(accum(), csubstr("aaa_then_1_then_bbb_then_22__then_epilogue")); - } - SUBCASE("no args") - { - csubstr fmt = "no args { -- }"; - DumpChecker::s_reset(); - DumpResults ret = T::call_format_dump_resume(buf.first(0), fmt, b1, b2); - CHECK_UNARY(!ret.success_until(0)); - CHECK_EQ(ret.bufsize, 0); - CHECK_EQ(ret.lastok, DumpResults::noarg); - CHECK_EQ(ret.argfail(), 0); - CHECK_EQ(DumpChecker::s_num_calls, 0); // no calls to dump - CHECK_EQ(buf.first(1), csubstr("+")); // nothing was written - CHECK_EQ(accum(), csubstr("")); - ret = T::call_format_dump_resume(ret, buf.first(1), fmt, b1, b2); - CHECK_UNARY(ret.success_until(0)); - CHECK_UNARY(!ret.success_until(1)); - CHECK_EQ(ret.bufsize, 0); - CHECK_EQ(ret.lastok, 0); - CHECK_EQ(ret.argfail(), 1); - CHECK_EQ(DumpChecker::s_num_calls, 1); - CHECK_EQ(buf.first(2), "++"); - CHECK_EQ(accum(), fmt); - } -} - -} // namespace c4 - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_enum.cpp b/thirdparty/ryml/ext/c4core/test/test_enum.cpp deleted file mode 100644 index b2dad0bbd..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_enum.cpp +++ /dev/null @@ -1,158 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include -#include -#include -#endif - -#include - -#include "./test_enum_common.hpp" -#include "c4/libtest/supprwarn_push.hpp" -#include - - -TEST_CASE("eoffs.simple_enum") -{ - using namespace c4; - CHECK_EQ(eoffs_cls(), 0); - CHECK_EQ(eoffs_pfx(), 0); -} - -TEST_CASE("eoffs.scoped_enum") -{ - using namespace c4; - CHECK_EQ(eoffs_cls(), strlen("MyEnumClass::")); - CHECK_EQ(eoffs_pfx(), 0); -} - -TEST_CASE("eoffs.simple_bitmask") -{ - using namespace c4; - CHECK_EQ(eoffs_cls(), 0); - CHECK_EQ(eoffs_pfx(), strlen("BM_")); -} - -TEST_CASE("eoffs.scoped_bitmask") -{ - using namespace c4; - CHECK_EQ(eoffs_cls(), strlen("MyBitmaskClass::")); - CHECK_EQ(eoffs_pfx(), strlen("MyBitmaskClass::BM_")); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -#ifdef __clang__ -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# if __GNUC__ >= 6 -# pragma GCC diagnostic ignored "-Wnull-dereference" -# endif -#endif - -template -void cmp_enum(Enum lhs, Enum rhs) -{ - using I = typename std::underlying_type::type; - CHECK_EQ(static_cast(lhs), static_cast(rhs)); -} - -template -void test_esyms() -{ - auto ss = c4::esyms(); - CHECK_NE(ss.size(), 0); - CHECK_FALSE(ss.empty()); - for(auto s : ss) - { - REQUIRE_NE(ss.find(s.name), nullptr); - REQUIRE_NE(ss.find(s.value), nullptr); - CHECK_STREQ(ss.find(s.name)->name, s.name); - CHECK_STREQ(ss.find(s.value)->name, s.name); - cmp_enum(ss.find(s.name)->value, s.value); - cmp_enum(ss.find(s.value)->value, s.value); - } -} - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - - -TEST_CASE("esyms.simple_enum") -{ - test_esyms(); -} - -TEST_CASE("esyms.scoped_enum") -{ - test_esyms(); -} - -TEST_CASE("esyms.simple_bitmask") -{ - test_esyms(); -} - -TEST_CASE("esyms.scoped_bitmask") -{ - test_esyms(); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -template -void test_e2str() -{ - using namespace c4; - using I = typename std::underlying_type::type; - auto ss = esyms(); - CHECK_NE(ss.size(), 0); - CHECK_FALSE(ss.empty()); - for(auto const& p : ss) - { - // test round trips - cmp_enum(str2e(e2str(p.value)), p.value); - CHECK_STREQ(e2str(str2e(p.name)), p.name); - } -} - - -TEST_CASE("e2str.simple_enum") -{ - test_e2str(); -} - -TEST_CASE("e2str.scoped_enum") -{ - test_e2str(); - cmp_enum(c4::str2e("MyEnumClass::FOO"), MyEnumClass::FOO); - cmp_enum(c4::str2e("FOO"), MyEnumClass::FOO); -} - -TEST_CASE("e2str.simple_bitmask") -{ - test_e2str(); - cmp_enum(c4::str2e("BM_FOO"), BM_FOO); - cmp_enum(c4::str2e("FOO"), BM_FOO); -} - -TEST_CASE("e2str.scoped_bitmask") -{ - using I = typename std::underlying_type::type; - test_e2str(); - cmp_enum(c4::str2e("MyBitmaskClass::BM_FOO"), MyBitmaskClass::BM_FOO); - cmp_enum(c4::str2e("BM_FOO"), MyBitmaskClass::BM_FOO); - cmp_enum(c4::str2e("FOO"), MyBitmaskClass::BM_FOO); -} - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_enum_common.hpp b/thirdparty/ryml/ext/c4core/test/test_enum_common.hpp deleted file mode 100644 index 9acd621cf..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_enum_common.hpp +++ /dev/null @@ -1,213 +0,0 @@ -#ifndef _C4_ENUM_COMMON_HPP_ -#define _C4_ENUM_COMMON_HPP_ - -#ifndef C4CORE_SINGLE_HEADER -#include -#endif - -typedef enum { - FOO = 0, - BAR, - BAZ, -} MyEnum; - -namespace c4 { -template<> -inline const EnumSymbols esyms() -{ - static const EnumSymbols::Sym rs[] = - { - {FOO, "FOO"}, - {BAR, "BAR"}, - {BAZ, "BAZ"}, - }; - EnumSymbols r(rs); - return r; -} -} // namespace c4 - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -enum class MyEnumClass { - FOO = 0, - BAR, - BAZ, -}; - - -namespace c4 { -template<> -inline const EnumSymbols esyms() -{ - static const EnumSymbols::Sym rs[] = - { - {MyEnumClass::FOO, "MyEnumClass::FOO"}, - {MyEnumClass::BAR, "MyEnumClass::BAR"}, - {MyEnumClass::BAZ, "MyEnumClass::BAZ"}, - }; - EnumSymbols r(rs); - return r; -} - - -template<> -inline size_t eoffs_cls() -{ - return 13; // same as strlen("MyEnumClass::") -} -} // namespace c4 - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -typedef enum { - BM_NONE = 0, - BM_FOO = 1 << 0, - BM_BAR = 1 << 1, - BM_BAZ = 1 << 2, - BM_FOO_BAR = BM_FOO|BM_BAR, - BM_FOO_BAR_BAZ = BM_FOO|BM_BAR|BM_BAZ, -} MyBitmask; - -namespace c4 { -template<> -inline const EnumSymbols esyms() -{ - static const EnumSymbols::Sym rs[] = - { - {BM_NONE, "BM_NONE"}, - {BM_FOO, "BM_FOO"}, - {BM_BAR, "BM_BAR"}, - {BM_BAZ, "BM_BAZ"}, - {BM_FOO_BAR, "BM_FOO_BAR"}, - {BM_FOO_BAR_BAZ, "BM_FOO_BAR_BAZ"}, - }; - EnumSymbols r(rs); - return r; -} - -template<> -inline size_t eoffs_pfx() -{ - return 3; // same as strlen("BM_") -} -} // namespace c4 - - - -typedef enum { - // no null value - BM_KABOOM = 1, - BM_PAFF = 2, - BM_PEW = 4, - BM_POW = 7, -} BmWithoutNull; - - -namespace c4 { -template<> -inline const c4::EnumSymbols esyms() -{ - static const EnumSymbols::Sym rs[] = - { - {BM_KABOOM, "KABOOM"}, - {BM_PAFF , "PAFF"}, - {BM_PEW , "PEW"}, - {BM_POW , "POW"}, - }; - EnumSymbols r(rs); - return r; -} - -template<> -inline size_t eoffs_pfx() -{ - return 3; // same as strlen("BM_") -} -} // namespace c4 - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -enum class MyBitmaskClass { - BM_NONE = 0, - BM_FOO = 1 << 0, - BM_BAR = 1 << 1, - BM_BAZ = 1 << 2, - BM_FOO_BAR = BM_FOO|BM_BAR, - BM_FOO_BAR_BAZ = BM_FOO|BM_BAR|BM_BAZ, -}; - -namespace c4 { - -template<> -inline const EnumSymbols esyms() -{ - static const EnumSymbols::Sym rs[] = - { - {MyBitmaskClass::BM_NONE, "MyBitmaskClass::BM_NONE"}, - {MyBitmaskClass::BM_FOO, "MyBitmaskClass::BM_FOO"}, - {MyBitmaskClass::BM_BAR, "MyBitmaskClass::BM_BAR"}, - {MyBitmaskClass::BM_BAZ, "MyBitmaskClass::BM_BAZ"}, - {MyBitmaskClass::BM_FOO_BAR, "MyBitmaskClass::BM_FOO_BAR"}, - {MyBitmaskClass::BM_FOO_BAR_BAZ, "MyBitmaskClass::BM_FOO_BAR_BAZ"}, - }; - EnumSymbols r(rs); - return r; -} - -template<> inline size_t eoffs_cls< MyBitmaskClass >() -{ - return 16; // same as strlen("MyBitmaskClass::") -} -template<> inline size_t eoffs_pfx< MyBitmaskClass >() -{ - return 19; // same as strlen("MyBitmaskClass::BM_") -} - -} // namespace c4 - - -enum class BmClassWithoutNull { - // no null value - BM_KABOOM = 1, - BM_PAFF = 2, - BM_PEW = 4, - BM_POW = 7, -}; - - -namespace c4 { -template<> -inline const c4::EnumSymbols esyms() -{ - static const EnumSymbols::Sym rs[] = - { - {BmClassWithoutNull::BM_KABOOM, "BmClassWithoutNull::BM_KABOOM"}, - {BmClassWithoutNull::BM_PAFF , "BmClassWithoutNull::BM_PAFF"}, - {BmClassWithoutNull::BM_PEW , "BmClassWithoutNull::BM_PEW"}, - {BmClassWithoutNull::BM_POW , "BmClassWithoutNull::BM_POW"}, - }; - EnumSymbols r(rs); - return r; -} - -template<> inline size_t eoffs_cls() -{ - return strlen("BmClassWithoutNull::"); -} -template<> inline size_t eoffs_pfx() -{ - return strlen("BmClassWithoutNull::BM_"); -} -} // namespace c4 - - -#endif /* _C4_ENUM_COMMON_HPP_ */ diff --git a/thirdparty/ryml/ext/c4core/test/test_error.cpp b/thirdparty/ryml/ext/c4core/test/test_error.cpp deleted file mode 100644 index 96584ab92..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_error.cpp +++ /dev/null @@ -1,635 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/error.hpp" -#endif - -#include "c4/test.hpp" -#include "c4/libtest/supprwarn_push.hpp" - -C4_BEGIN_HIDDEN_NAMESPACE -bool got_an_error = false; -void error_callback(const char *msg, size_t msg_sz) -{ - CHECK_EQ(strncmp(msg, "bla bla", msg_sz), 0); - CHECK_EQ(msg_sz, 7); - got_an_error = true; -} -inline c4::ScopedErrorSettings tmp_err() -{ - got_an_error = false; - return c4::ScopedErrorSettings(c4::ON_ERROR_CALLBACK, error_callback); -} -C4_END_HIDDEN_NAMESPACE - -namespace c4 { - -TEST_CASE("Error.scoped_callback") -{ - auto orig = get_error_callback(); - { - auto tmp = tmp_err(); - CHECK_EQ(get_error_callback() == error_callback, true); - C4_ERROR("bla bla"); - CHECK_EQ(got_an_error, true); - } - CHECK_EQ(get_error_callback() == orig, true); -} - -} // namespace c4 - -TEST_CASE("Error.outside_of_c4_namespace") -{ - auto orig = c4::get_error_callback(); - { - auto tmp = tmp_err(); - CHECK_EQ(c4::get_error_callback() == error_callback, true); - C4_ERROR("bla bla"); - CHECK_EQ(got_an_error, true); - } - CHECK_EQ(c4::get_error_callback() == orig, true); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// WIP: new error handling code - - -#include // temporary; just for the exception example -#include // temporary; just for the exception example - -namespace c4 { - -#define C4_ERR_FMT_BUFFER_SIZE 256 - -using locref = c4::srcloc const& C4_RESTRICT; - -using pfn_err = void (*)(locref loc, void *data); -using pfn_warn = void (*)(locref loc, void *data); -using pfn_msg_begin = void (*)(locref loc, void *data); -using pfn_msg_part = void (*)(const char* msg, size_t size, void *data); -using pfn_msg_end = void (*)(void *data); - -struct ErrorCallbacks -{ - void *user_data; - - pfn_err err; - pfn_warn warn; - pfn_msg_begin msg_begin; - pfn_msg_part msg_part; - pfn_msg_end msg_end; - - bool msg_enabled() const { return msg_begin != nullptr; } - - template - void msg(const char (&s)[N]) - { - msg_part(s, N-1, user_data); - } - void msg(const char *msg, size_t sz) - { - msg_part(msg, sz, user_data); - } - void msg(char c) - { - msg_part(&c, 1, user_data); - } -}; - -#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmissing-field-initializers" -#endif -TEST_CASE("ErrorCallbacks.default_obj") -{ - ErrorCallbacks cb {}; - CHECK_EQ(cb.user_data, nullptr); - CHECK_EQ(cb.err, nullptr); - CHECK_EQ(cb.warn, nullptr); - CHECK_EQ(cb.msg_begin, nullptr); - CHECK_EQ(cb.msg_part, nullptr); - CHECK_EQ(cb.msg_end, nullptr); -} -#if __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 -#pragma GCC diagnostic pop -#endif - -template -struct ErrorCallbacksBridgeFull -{ - ErrorCallbacks callbacks() const - { - return { - (ErrBhv*)this, - ErrorCallbacksBridgeFull::on_err, - ErrorCallbacksBridgeFull::on_warn, - ErrorCallbacksBridgeFull::on_msg_begin, - ErrorCallbacksBridgeFull::on_msg_part, - ErrorCallbacksBridgeFull::on_msg_end, - }; - } - static void on_err(locref loc, void *data) - { - ((ErrBhv*)data)->err(loc); - } - static void on_warn(locref loc, void *data) - { - ((ErrBhv*)data)->warn(loc); - } - static void on_msg_begin(locref loc, void *data) - { - ((ErrBhv*)data)->msg_begin(loc); - } - static void on_msg_part(const char *part, size_t size, void *data) - { - ((ErrBhv*)data)->msg_part(part, size); - } - static void on_msg_end(void *data) - { - ((ErrBhv*)data)->msg_end(); - } -}; - -template -struct ErrorCallbacksBridge -{ - ErrorCallbacks callbacks() const - { - return { - (ErrBhv*)this, - ErrorCallbacksBridge::on_err, - ErrorCallbacksBridge::on_warn, - (pfn_msg_begin)nullptr, - (pfn_msg_part)nullptr, - (pfn_msg_end)nullptr - }; - } - static void on_err(locref loc, void *data) - { - ((ErrBhv*)data)->err(loc); - } - static void on_warn(locref loc, void *data) - { - ((ErrBhv*)data)->warn(loc); - } -}; - - -void fputi(int val, FILE *f); - -void _errmsg(locref loc) -{ - fputs(loc.file, stderr); - fputc(':', stderr); - fputi(loc.line, stderr); - fputs(": ", stderr); - fflush(stderr); -} - -void _errmsg(const char *part, size_t part_size) -{ - fwrite(part, 1u, part_size, stderr); - fflush(stderr); -} - -/** example implementation using old-style abort */ -struct ErrorBehaviorAbort : public ErrorCallbacksBridgeFull -{ - static void msg_begin(locref loc) - { - fputc('\n', stderr); - _errmsg(loc); - } - static void msg_part(const char *part, size_t part_size) - { - _errmsg(part, part_size); - } - static void msg_end() - { - fputc('\n', stderr); - fflush(stderr); - } - static void err(locref) - { - abort(); - } - static void warn(locref) - { - // nothing to do - } -}; - - -TEST_CASE("ErrorBehaviorAbort.default_obj") -{ - ErrorBehaviorAbort bhv; - auto cb = bhv.callbacks(); - CHECK_NE(cb.user_data, nullptr); - CHECK_NE(cb.err, nullptr); - CHECK_NE(cb.warn, nullptr); - CHECK_NE(cb.msg_begin, nullptr); - CHECK_NE(cb.msg_part, nullptr); - CHECK_NE(cb.msg_end, nullptr); -} - - -void fputi(int val, FILE *f); -void _append(std::string *s, int line); - -/** example implementation using vanilla c++ std::runtime_error */ -struct ErrorBehaviorRuntimeError : public ErrorCallbacksBridgeFull -{ - std::string exc_msg{}; - - void msg_begin(locref loc) - { - exc_msg.reserve(strlen(loc.file) + 16); - exc_msg = '\n'; - exc_msg += loc.file; - exc_msg += ':'; - _append(&exc_msg, loc.line); - exc_msg += ": "; - } - void msg_part(const char *part, size_t part_size) - { - exc_msg.append(part, part_size); - } - void msg_end() - { - std::cerr << exc_msg << "\n"; - } - void err(locref) - { - throw std::runtime_error(exc_msg); - } - void warn(locref) - { - } -}; - - - -TEST_CASE("ErrorBehaviorRuntimeError.default_obj") -{ - ErrorBehaviorRuntimeError bhv; - auto cb = bhv.callbacks(); - CHECK_NE(cb.user_data, nullptr); - CHECK_NE(cb.err, nullptr); - CHECK_NE(cb.warn, nullptr); - CHECK_NE(cb.msg_begin, nullptr); - CHECK_NE(cb.msg_part, nullptr); - CHECK_NE(cb.msg_end, nullptr); -} - - - - -ErrorBehaviorAbort s_err_abort = ErrorBehaviorAbort(); -ErrorCallbacks s_err_callbacks = s_err_abort.callbacks(); - - -void new_handle_error(locref loc, size_t msg_size, const char *msg) -{ - if(s_err_callbacks.msg_enabled()) - { - s_err_callbacks.msg_begin(loc, s_err_callbacks.user_data); - s_err_callbacks.msg("ERROR: "); - s_err_callbacks.msg(msg, msg_size); - s_err_callbacks.msg_end(s_err_callbacks.user_data); - } - s_err_callbacks.err(loc, s_err_callbacks.user_data); -} - -void new_handle_warning(locref loc, size_t msg_size, const char *msg) -{ - if(s_err_callbacks.msg_enabled()) - { - s_err_callbacks.msg_begin(loc, s_err_callbacks.user_data); - s_err_callbacks.msg("WARNING: "); - s_err_callbacks.msg(msg, msg_size); - s_err_callbacks.msg_end(s_err_callbacks.user_data); - } - s_err_callbacks.warn(loc, s_err_callbacks.user_data); -} - -template -C4_ALWAYS_INLINE void new_handle_error(locref loc, const char (&msg)[N]) -{ - new_handle_error(loc, N-1, msg); -} - -template -C4_ALWAYS_INLINE void new_handle_warning(locref loc, const char (&msg)[N]) -{ - new_handle_warning(loc, N-1, msg); -} - - -#define C4_ERROR_NEW(msg) c4::new_handle_error(C4_SRCLOC(), msg) -#define C4_WARNING_NEW(msg) c4::new_handle_warning(C4_SRCLOC(), msg) - -#define C4_ERROR_NEW_SZ(msg, msglen) c4::new_handle_error(C4_SRCLOC(), msglen, msg) -#define C4_WARNING_NEW_SZ(msg, msglen) c4::new_handle_warning(C4_SRCLOC(), msglen, msg) - -} // namespace c4 - -#ifndef C4CORE_SINGLE_HEADER -#include -#include -#endif - -namespace c4 { - -void fputi(int val, FILE *f) -{ - char buf[16]; - size_t ret = c4::itoa(buf, val); - ret = ret < sizeof(buf) ? ret : sizeof(buf); - fwrite(buf, 1u, ret, f); -} - -// to avoid using std::to_string() -void _append(std::string *s, int line) -{ - auto sz = s->size(); - s->resize(sz + 16); - auto ret = itoa(substr(&((*s)[0]) + sz, 16u), line); - s->resize(sz + ret); - if(ret >= sz) - { - itoa(substr(&((*s)[0]) + sz, 16u), line); - } -} - -} // namespace c4 -template -struct ScopedErrorBehavior -{ - c4::ErrorCallbacks m_prev; - ErrorBehavior m_tmp; - const char *m_name; - ScopedErrorBehavior(const char* name) : m_prev(c4::s_err_callbacks), m_tmp(), m_name(name) - { - c4::s_err_callbacks = m_tmp.callbacks(); - } - ~ScopedErrorBehavior() - { - c4::s_err_callbacks = m_prev; - } -}; -#define C4_TMP_ERR_BHV(bhv_ty) ScopedErrorBehavior(#bhv_ty) - -template -void test_error_exception(const char (&msg)[N]) -{ - INFO(msg); - { - auto tmp1 = C4_TMP_ERR_BHV(ErrorBehaviorAbort); - - { - auto tmp2 = C4_TMP_ERR_BHV(ErrorBehaviorRuntimeError); - - bool got_exc = false; - try { - C4_ERROR_NEW(msg); - } - catch(std::exception const& e) { - // check that the error terminates with the given message - auto what = c4::to_csubstr(e.what()).last(N-1); - CHECK_EQ(what, msg); - got_exc = (what == msg); - } - CHECK(got_exc); - - got_exc = false; - try { - C4_ERROR_NEW_SZ(msg, N-1); - } - catch(std::exception const& e) { - // check that the error terminates with the given message - auto what = c4::to_csubstr(e.what()).last(N-1); - CHECK_EQ(what, msg); - got_exc = (what == msg); - } - CHECK(got_exc); - } - } -} - -template -void test_warning_exception(const char (&msg)[N]) -{ - auto tmp = C4_TMP_ERR_BHV(ErrorBehaviorRuntimeError); - C4_WARNING_NEW(msg); - auto const& wmsg = tmp.m_tmp.exc_msg; - REQUIRE_FALSE(wmsg.empty()); - REQUIRE_GT(wmsg.size(), N); - auto what = c4::to_csubstr(wmsg.c_str()).last(N-1); - CHECK_EQ(what, msg); - - C4_WARNING_NEW_SZ(msg, N-1); - REQUIRE_FALSE(wmsg.empty()); - REQUIRE_GT(wmsg.size(), N); - what = c4::to_csubstr(wmsg.c_str()).last(N-1); - CHECK_EQ(what, msg); -} - -TEST_CASE("error.exception") -{ - test_error_exception("some error with some message"); - test_error_exception("some error with another message"); -} - -TEST_CASE("warning.exception") -{ - test_warning_exception("some warning"); - test_warning_exception("some other warning"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -#ifndef C4CORE_SINGLE_HEADER -#include -#include -#endif - -namespace c4 { - -template -void _err_send(T const& arg) -{ - char buf[C4_ERR_FMT_BUFFER_SIZE]; - size_t num = to_chars(buf, arg); - num = num < C4_ERR_FMT_BUFFER_SIZE ? num : C4_ERR_FMT_BUFFER_SIZE; - s_err_callbacks.msg_part(buf, num, s_err_callbacks.user_data); -} - - -size_t _find_fmt(const char *C4_RESTRICT fmt, size_t len) -{ - for(size_t i = 0; i < len; ++i) - { - if(fmt[i] != '{') - { - continue; - } - if(i + 1 == len) - { - break; - } - if(fmt[i+1] == '}') - { - return i; - } - } - return (size_t)-1; -} - -void _err_fmt(size_t fmt_size, const char *C4_RESTRICT fmt) -{ - s_err_callbacks.msg_part(fmt, fmt_size, s_err_callbacks.user_data); -} - -template -void _err_fmt(size_t fmt_size, const char *C4_RESTRICT fmt, Arg const& C4_RESTRICT arg, Args const& C4_RESTRICT ...args) -{ - size_t pos = _find_fmt(fmt, fmt_size); - if(pos == (size_t)-1) - { - s_err_callbacks.msg_part(fmt, fmt_size, s_err_callbacks.user_data); - return; - } - s_err_callbacks.msg_part(fmt, pos, s_err_callbacks.user_data); - _err_send(arg); - pos += 2; - _err_fmt(fmt_size - pos, fmt + pos, args...); -} - -template -void err_fmt(locref loc, size_t fmt_size, const char *fmt, Args const& C4_RESTRICT ...args) -{ - s_err_callbacks.msg_begin(loc, s_err_callbacks.user_data); - s_err_callbacks.msg("ERROR: "); - _err_fmt(fmt_size, fmt, args...); - s_err_callbacks.msg_end(s_err_callbacks.user_data); - s_err_callbacks.err(loc, s_err_callbacks.user_data); -} - -template -void warn_fmt(locref loc, size_t fmt_size, const char *fmt, Args const& C4_RESTRICT ...args) -{ - s_err_callbacks.msg_begin(loc, s_err_callbacks.user_data); - s_err_callbacks.msg("WARNING: "); - _err_fmt(fmt_size, fmt, args...); - s_err_callbacks.msg_end(s_err_callbacks.user_data); - s_err_callbacks.warn(loc, s_err_callbacks.user_data); -} - -template -void err_fmt(locref loc, const char (&fmt)[N], Args const& C4_RESTRICT ...args) -{ - err_fmt(loc, N-1, fmt, args...); -} - -template -void warn_fmt(locref loc, const char (&fmt)[N], Args const& C4_RESTRICT ...args) -{ - warn_fmt(loc, N-1, fmt, args...); -} - -#define C4_ERROR_FMT_NEW(fmt, ...) c4::err_fmt(C4_SRCLOC(), fmt, __VA_ARGS__) -#define C4_WARNING_FMT_NEW(msg, ...) c4::warn_fmt(C4_SRCLOC(), fmt, __VA_ARGS__) - -#define C4_ERROR_FMT_NEW_SZ(fmt, fmtlen, ...) c4::err_fmt(C4_SRCLOC(), fmtlen, fmt, __VA_ARGS__) -#define C4_WARNING_FMT_NEW_SZ(fmt, fmtlen, ...) c4::warn_fmt(C4_SRCLOC(), fmtlen, fmt, __VA_ARGS__) - -} // namespace c4 - -template -void test_error_fmt_exception(const char (&expected)[M], const char (&fmt)[N], Args const& ...args) -{ - INFO(expected); - { - auto tmp1 = C4_TMP_ERR_BHV(ErrorBehaviorAbort); - - { - auto tmp2 = C4_TMP_ERR_BHV(ErrorBehaviorRuntimeError); - - bool got_exc = false; - try { - C4_ERROR_FMT_NEW(fmt, args...); - } - catch(std::exception const& e) { - // check that the error terminates with the given message - auto what = c4::to_csubstr(e.what()).last(M-1); - CHECK_EQ(what, expected); - got_exc = (what == expected); - } - CHECK(got_exc); - - got_exc = false; - try { - C4_ERROR_FMT_NEW_SZ(fmt, N-1, args...); - } - catch(std::exception const& e) { - // check that the error terminates with the given message - auto what = c4::to_csubstr(e.what()).last(M-1); - CHECK_EQ(what, expected); - got_exc = (what == expected); - } - CHECK(got_exc); - } - } -} - -template -void test_warning_fmt_exception(const char (&expected)[M], const char (&fmt)[N], Args const& ...args) -{ - INFO(expected); - - auto tmp = C4_TMP_ERR_BHV(ErrorBehaviorRuntimeError); - auto const& wmsg = tmp.m_tmp.exc_msg; - C4_WARNING_FMT_NEW(fmt, args...); - REQUIRE_FALSE(wmsg.empty()); - REQUIRE_GT(wmsg.size(), M); - auto what = c4::to_csubstr(wmsg.c_str()).last(M-1); - CHECK_EQ(what, expected); - - C4_WARNING_FMT_NEW_SZ(fmt, N-1, args...); - REQUIRE_FALSE(wmsg.empty()); - REQUIRE_GT(wmsg.size(), M); - what = c4::to_csubstr(wmsg.c_str()).last(M-1); - CHECK_EQ(what, expected); -} - - -TEST_CASE("error.fmt") -{ - test_error_fmt_exception("aaa is 2 is it not?", - "{} is {} is it not?", "aaa", 2); - test_error_fmt_exception("aaa is bbb is it not?", - "{} is {} is it not?", "aaa", "bbb"); - test_error_fmt_exception("aaa is {} is it not?", - "{} is {} is it not?", "aaa"); - test_error_fmt_exception("aaa is {} is it not?", - "{} is {} is it not?", "aaa"); -} - -TEST_CASE("warning.fmt") -{ - test_warning_fmt_exception("aaa is 2 is it not?", - "{} is {} is it not?", "aaa", 2); - test_warning_fmt_exception("aaa is bbb is it not?", - "{} is {} is it not?", "aaa", "bbb"); - test_warning_fmt_exception("aaa is {} is it not?", - "{} is {} is it not?", "aaa"); - test_warning_fmt_exception("aaa is {} is it not?", - "{} is {} is it not?", "aaa"); -} - - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_error_exception.cpp b/thirdparty/ryml/ext/c4core/test/test_error_exception.cpp deleted file mode 100644 index fae0020b8..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_error_exception.cpp +++ /dev/null @@ -1,108 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/error.hpp" -#endif - -#include "c4/test.hpp" -#include -#include - - -C4_BEGIN_HIDDEN_NAMESPACE -bool got_an_error = false; -bool got_an_exception = false; -C4_END_HIDDEN_NAMESPACE - -void error_callback_throwing_exception(const char *msg_, size_t msg_sz) -{ - got_an_error = true; - c4::csubstr s(msg_, msg_sz); - if (s == "err1") throw 1; - else if(s == "err2") throw 2; - else if(s == "err3") throw 3; - else if(s == "err4") throw 4; - else throw std::runtime_error({msg_, msg_+msg_sz}); -} - -inline c4::ScopedErrorSettings tmp_err() -{ - got_an_error = false; - return c4::ScopedErrorSettings(c4::ON_ERROR_CALLBACK, error_callback_throwing_exception); -} - - -void test_exception(int which) -{ - if(which == 0) return; - CHECK_FALSE(got_an_exception); - CHECK_EQ(c4::get_error_callback() == error_callback_throwing_exception, false); - { - auto tmp = tmp_err(); - CHECK_EQ(got_an_error, false); - CHECK_EQ(c4::get_error_callback() == error_callback_throwing_exception, true); - try - { - if (which == 1) { C4_ERROR("err1"); } - else if(which == 2) { C4_ERROR("err2"); } - else if(which == 3) { C4_ERROR("err3"); } - else if(which == 4) { C4_ERROR("err4"); } - else { C4_ERROR("unknown error"); } - } - catch(int i) - { - got_an_exception = true; - CHECK_EQ(got_an_error, true); - CHECK_EQ(i, which); - throw; - } - catch(std::runtime_error const& e) - { - got_an_exception = true; - CHECK_EQ(got_an_error, true); - CHECK_STREQ(e.what(), "unknown error"); - throw; - } - // if we get here it means no exception was thrown - // so the test failed - FAIL("an exception was thrown"); - } - CHECK_EQ(c4::get_error_callback() == error_callback_throwing_exception, false); -} - - -// Although c4core does not use exceptions by default (*), you can have your -// error callback throw an exception which you can then catch on your code. -// This test covers that possibility. -// -// (*) Note that you can also configure c4core to throw an exception on error. - -TEST_CASE("error.exception_from_callback") -{ - // works! - got_an_exception = false; - CHECK_THROWS_AS(test_exception(-1), std::runtime_error); - CHECK(got_an_exception); - - got_an_exception = false; - CHECK_NOTHROW(test_exception(0)); - CHECK_FALSE(got_an_exception); - - got_an_exception = false; - CHECK_THROWS_AS(test_exception(1), int); - CHECK(got_an_exception); - - got_an_exception = false; - CHECK_THROWS_AS(test_exception(2), int); - CHECK(got_an_exception); - - got_an_exception = false; - CHECK_THROWS_AS(test_exception(3), int); - CHECK(got_an_exception); - - got_an_exception = false; - CHECK_THROWS_AS(test_exception(4), int); - CHECK(got_an_exception); - - got_an_exception = false; - CHECK_THROWS_AS(test_exception(6), std::runtime_error); - CHECK(got_an_exception); -} diff --git a/thirdparty/ryml/ext/c4core/test/test_format.cpp b/thirdparty/ryml/ext/c4core/test/test_format.cpp deleted file mode 100644 index 42a50dc75..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_format.cpp +++ /dev/null @@ -1,1054 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/substr.hpp" -#include "c4/std/std.hpp" -#include "c4/format.hpp" -#endif - -#include -#include "c4/libtest/supprwarn_push.hpp" - -#ifdef __clang__ -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wuseless-cast" -#endif - -namespace c4 { - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -TEST_CASE_TEMPLATE("to_chars.fmt.bin", T, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t) -{ - char bufc[128]; - substr buf(bufc); - - CHECK_EQ(to_chars_sub(buf, fmt::integral(T(21), T(2))), "0b10101"); - CHECK_EQ(to_chars_sub(buf, fmt::integral((T*)21, T(2))), "0b10101"); - CHECK_EQ(to_chars_sub(buf, fmt::integral((T const*)21, T(2))), "0b10101"); - CHECK_EQ(to_chars_sub(buf, fmt::integral(nullptr, T(2))), "0b0"); - CHECK_EQ(to_chars_sub(buf, fmt::bin(T(21))), "0b10101"); - CHECK_EQ(to_chars_sub(buf, fmt::bin((T*)21)), "0b10101"); - CHECK_EQ(to_chars_sub(buf, fmt::bin((T const*)21)), "0b10101"); - CHECK_EQ(to_chars_sub(buf, fmt::bin(nullptr)), "0b0"); -} - -TEST_CASE_TEMPLATE("to_chars.fmt.zpad.bin", T, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t) -{ - char bufc[128]; - substr buf(bufc); - using namespace fmt; - CHECK_EQ(to_chars_sub(buf, zpad(integral(T(21), T(2)), 8u)), "0b00010101"); - CHECK_EQ(to_chars_sub(buf, zpad(integral((T*)21, T(2)), 8u)), "0b00010101"); - CHECK_EQ(to_chars_sub(buf, zpad(integral((T const*)21, T(2)), 8u)), "0b00010101"); - CHECK_EQ(to_chars_sub(buf, zpad(bin(T(21)), 8u)), "0b00010101"); - CHECK_EQ(to_chars_sub(buf, zpad(bin((T*)21), 8u)), "0b00010101"); - CHECK_EQ(to_chars_sub(buf, zpad(bin((T const*)21), 8u)), "0b00010101"); -} - -TEST_CASE_TEMPLATE("to_chars.fmt.oct", T, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t) -{ - char bufc[128]; - substr buf(bufc); - - CHECK_EQ(to_chars_sub(buf, fmt::integral(T(65), T(8))), "0o101"); - CHECK_EQ(to_chars_sub(buf, fmt::integral((T*)65, T(8))), "0o101"); - CHECK_EQ(to_chars_sub(buf, fmt::integral((T const*)65, T(8))), "0o101"); - CHECK_EQ(to_chars_sub(buf, fmt::integral(nullptr, T(8))), "0o0"); - CHECK_EQ(to_chars_sub(buf, fmt::oct(T(65))), "0o101"); - CHECK_EQ(to_chars_sub(buf, fmt::oct((T*)65)), "0o101"); - CHECK_EQ(to_chars_sub(buf, fmt::oct((T const*)65)), "0o101"); - CHECK_EQ(to_chars_sub(buf, fmt::oct(nullptr)), "0o0"); -} - -TEST_CASE_TEMPLATE("to_chars.fmt.zpad.oct", T, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t) -{ - char bufc[128]; - substr buf(bufc); - using namespace fmt; - CHECK_EQ(to_chars_sub(buf, zpad(integral(T(65), T(8)), 5u)), "0o00101"); - CHECK_EQ(to_chars_sub(buf, zpad(integral((T*)65, T(8)), 5u)), "0o00101"); - CHECK_EQ(to_chars_sub(buf, zpad(integral((T const*)65, T(8)), 5u)), "0o00101"); - CHECK_EQ(to_chars_sub(buf, zpad(oct(T(65)), 5u)), "0o00101"); - CHECK_EQ(to_chars_sub(buf, zpad(oct((T*)65), 5u)), "0o00101"); - CHECK_EQ(to_chars_sub(buf, zpad(oct((T const*)65), 5u)), "0o00101"); -} - -TEST_CASE_TEMPLATE("to_chars.fmt.hex", T, uint8_t, int8_t, uint16_t, int16_t, uint32_t, int32_t, uint64_t, int64_t) -{ - char bufc[128]; - substr buf(bufc); - CHECK_EQ(to_chars_sub(buf, fmt::integral(T(0x7f), T(16))), "0x7f"); - CHECK_EQ(to_chars_sub(buf, fmt::integral((T*)0x7f, T(16))), "0x7f"); - CHECK_EQ(to_chars_sub(buf, fmt::integral((T const*)0x7f, T(16))), "0x7f"); - CHECK_EQ(to_chars_sub(buf, fmt::integral(nullptr, T(16))), "0x0"); - CHECK_EQ(to_chars_sub(buf, fmt::hex(T(0x7f))), "0x7f"); - CHECK_EQ(to_chars_sub(buf, fmt::hex((T*)0x7f)), "0x7f"); - CHECK_EQ(to_chars_sub(buf, fmt::hex((T const*)0x7f)), "0x7f"); - CHECK_EQ(to_chars_sub(buf, fmt::hex(nullptr)), "0x0"); -} - -TEST_CASE_TEMPLATE("to_chars.fmt.zpad.hex", T, uint8_t, int8_t) -{ - char bufc[128]; - substr buf(bufc); - using namespace fmt; - buf.fill('?'); - CHECK_EQ(to_chars_sub(buf, zpad(integral(T(0x7f), T(16)), 5u)), "0x0007f"); - CHECK_EQ(to_chars_sub(buf, zpad(integral((T*)0x7f, T(16)), 5u)), "0x0007f"); - CHECK_EQ(to_chars_sub(buf, zpad(integral((T const*)0x7f, T(16)), 5u)), "0x0007f"); - CHECK_EQ(to_chars_sub(buf, zpad(hex(T(0x7f)), 5u)), "0x0007f"); - CHECK_EQ(to_chars_sub(buf, zpad(hex((T*)0x7f), 5u)), "0x0007f"); - CHECK_EQ(to_chars_sub(buf, zpad(hex((T const*)0x7f), 5u)), "0x0007f"); -} - - -TEST_CASE_TEMPLATE("to_chars.fmt.zpad", T, uint8_t, int8_t) -{ - char bufc[128]; - substr buf(bufc); - using namespace fmt; - CHECK_EQ(to_chars_sub(buf, zpad(T(10), 0)), "10"); - CHECK_EQ(to_chars_sub(buf, zpad(T(10), 1)), "10"); - CHECK_EQ(to_chars_sub(buf, zpad(T(10), 2)), "10"); - CHECK_EQ(to_chars_sub(buf, zpad(T(10), 3)), "010"); - CHECK_EQ(to_chars_sub(buf, zpad(T(10), 4)), "0010"); - CHECK_EQ(to_chars_sub(buf, zpad((T const*)17, 0)), "0x11"); - CHECK_EQ(to_chars_sub(buf, zpad((T const*)17, 1)), "0x11"); - CHECK_EQ(to_chars_sub(buf, zpad((T const*)17, 2)), "0x11"); - CHECK_EQ(to_chars_sub(buf, zpad((T const*)17, 3)), "0x011"); - CHECK_EQ(to_chars_sub(buf, zpad((T const*)17, 4)), "0x0011"); - CHECK_EQ(to_chars_sub(buf, zpad((T *)17, 0)), "0x11"); - CHECK_EQ(to_chars_sub(buf, zpad((T *)17, 1)), "0x11"); - CHECK_EQ(to_chars_sub(buf, zpad((T *)17, 2)), "0x11"); - CHECK_EQ(to_chars_sub(buf, zpad((T *)17, 3)), "0x011"); - CHECK_EQ(to_chars_sub(buf, zpad((T *)17, 4)), "0x0011"); - CHECK_EQ(to_chars_sub(buf, zpad(nullptr, 0)), "0x0"); - CHECK_EQ(to_chars_sub(buf, zpad(nullptr, 1)), "0x0"); - CHECK_EQ(to_chars_sub(buf, zpad(nullptr, 2)), "0x00"); - CHECK_EQ(to_chars_sub(buf, zpad(nullptr, 3)), "0x000"); - CHECK_EQ(to_chars_sub(buf, zpad(nullptr, 4)), "0x0000"); - CHECK_EQ(to_chars_sub(buf, zpad(integral(nullptr, T(10)), 0u)), "0"); - CHECK_EQ(to_chars_sub(buf, zpad(integral(nullptr, T(16)), 0u)), "0x0"); - CHECK_EQ(to_chars_sub(buf, zpad(integral(nullptr, T(2)), 0u)), "0b0"); - CHECK_EQ(to_chars_sub(buf, zpad(integral(nullptr, T(8)), 0u)), "0o0"); - CHECK_EQ(to_chars_sub(buf, zpad(hex(nullptr), 0u)), "0x0"); - CHECK_EQ(to_chars_sub(buf, zpad(bin(nullptr), 0u)), "0b0"); - CHECK_EQ(to_chars_sub(buf, zpad(oct(nullptr), 0u)), "0o0"); - CHECK_EQ(to_chars_sub(buf, zpad(integral(nullptr, T(10)), 5u)), "00000"); - CHECK_EQ(to_chars_sub(buf, zpad(integral(nullptr, T(16)), 5u)), "0x00000"); - CHECK_EQ(to_chars_sub(buf, zpad(integral(nullptr, T(2)), 5u)), "0b00000"); - CHECK_EQ(to_chars_sub(buf, zpad(integral(nullptr, T(8)), 5u)), "0o00000"); - CHECK_EQ(to_chars_sub(buf, zpad(hex(nullptr), 5u)), "0x00000"); - CHECK_EQ(to_chars_sub(buf, zpad(bin(nullptr), 5u)), "0b00000"); - CHECK_EQ(to_chars_sub(buf, zpad(oct(nullptr), 5u)), "0o00000"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_to_chars_real(T f, int precision, const char* flt, T fltv, const char *scient, T scientv) -{ - char bufc[64]; - substr buf(bufc); - substr r; - T copy; - - INFO("num=" << f); - - r = to_chars_sub(buf, fmt::real(f, precision)); - CHECK_EQ(r, to_csubstr(flt)); - from_chars(r, ©); - if(sizeof(T) == sizeof(float)) - { - CHECK_FLOAT_EQ((float)fltv, (float)copy); - } - else - { - CHECK_FLOAT_EQ(fltv, copy); - } - - r = to_chars_sub(buf, fmt::real(f, precision, FTOA_SCIENT)); - CHECK_EQ(r, to_csubstr(scient)); - from_chars(r, ©); - if(sizeof(T) == sizeof(float)) - { - CHECK_FLOAT_EQ((float)scientv, (float)copy); - } - else - { - CHECK_FLOAT_EQ(scientv, copy); - } -} - -TEST_CASE_TEMPLATE("to_chars.fmt.real", T, float, double) -{ - char bufc[128]; - substr buf(bufc); - - T f = static_cast(256.064); - test_to_chars_real(f, 0, "256", T(256.), "3e+02", T(300.)); - test_to_chars_real(f, 1, "256.1", T(256.1), "2.6e+02", T(260.)); - test_to_chars_real(f, 2, "256.06", T(256.06), "2.56e+02", T(256.)); - test_to_chars_real(f, 3, "256.064", T(256.064), "2.561e+02", T(256.1)); - test_to_chars_real(f, 4, "256.0640", T(256.0640), "2.5606e+02", T(256.06)); - test_to_chars_real(f, 5, "256.06400", T(256.06400), "2.56064e+02", T(256.064)); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE("to_chars.fmt.boolalpha") -{ - char bufc[128]; - substr buf(bufc); - - CHECK_EQ(to_chars_sub(buf, true), "1"); - CHECK_EQ(to_chars_sub(buf, false), "0"); - CHECK_EQ(to_chars_sub(buf, fmt::boolalpha(true)), "true"); - CHECK_EQ(to_chars_sub(buf, 1), "1"); - CHECK_EQ(to_chars_sub(buf, fmt::boolalpha(1)), "true"); - CHECK_EQ(to_chars_sub(buf, fmt::boolalpha(10)), "true"); - CHECK_EQ(to_chars_sub(buf, fmt::boolalpha(false)), "false"); - CHECK_EQ(to_chars_sub(buf, fmt::boolalpha(0)), "false"); - CHECK_EQ(to_chars_sub(buf, fmt::boolalpha(0u)), "false"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE("align.left.overflow") -{ - CHECK_EQ(to_chars(substr(), fmt::left(' ', 91u)), 91u); - CHECK_EQ(to_chars(substr(), fmt::left("0123456789.123456789.123456789.123456789", 91u)), 91u); - CHECK_EQ(to_chars(substr(), fmt::left("0123456789.123456789.123456789.123456789", 30u)), 40u); -} - -TEST_CASE("align.right.overflow") -{ - CHECK_EQ(to_chars(substr(), fmt::right(' ', 91u)), 91u); - CHECK_EQ(to_chars(substr(), fmt::right("0123456789.123456789.123456789.123456789", 91u)), 91u); - CHECK_EQ(to_chars(substr(), fmt::right("0123456789.123456789.123456789.123456789", 30u)), 40u); -} - -TEST_CASE("align.left") -{ - char buf[128] = {}; - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 1)), "1"); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 2)), "1 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 3)), "1 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 4)), "1 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 5)), "1 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 6)), "1 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 7)), "1 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 8)), "1 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 9)), "1 "); - - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 1, '+')), "1"); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 2, '+')), "1+"); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 3, '+')), "1++"); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 4, '+')), "1+++"); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 5, '+')), "1++++"); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 6, '+')), "1+++++"); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 7, '+')), "1++++++"); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 8, '+')), "1+++++++"); - CHECK_EQ(to_chars_sub(buf, fmt::left("1", 9, '+')), "1++++++++"); - - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 0)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 1)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 2)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 3)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 4)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 5)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 6)), "01234 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 7)), "01234 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 8)), "01234 "); - CHECK_EQ(to_chars_sub(buf, fmt::left("01234", 9)), "01234 "); - - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 0)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 1)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 2)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 3)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 4)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 5)), "1234 "); - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 6)), "1234 "); - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 7)), "1234 "); - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 8)), "1234 "); - CHECK_EQ(to_chars_sub(buf, fmt::left(1234, 9)), "1234 "); -} - - -TEST_CASE("align.right") -{ - char buf[128] = {}; - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 1)), "1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 2)), " 1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 3)), " 1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 4)), " 1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 5)), " 1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 6)), " 1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 7)), " 1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 8)), " 1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 9)), " 1"); - - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 1, '+')), "1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 2, '+')), "+1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 3, '+')), "++1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 4, '+')), "+++1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 5, '+')), "++++1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 6, '+')), "+++++1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 7, '+')), "++++++1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 8, '+')), "+++++++1"); - CHECK_EQ(to_chars_sub(buf, fmt::right("1", 9, '+')), "++++++++1"); - - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 0)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 1)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 2)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 3)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 4)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 5)), "01234"); - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 6)), " 01234"); - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 7)), " 01234"); - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 8)), " 01234"); - CHECK_EQ(to_chars_sub(buf, fmt::right("01234", 9)), " 01234"); - - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 0)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 1)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 2)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 3)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 4)), "1234"); - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 5)), " 1234"); - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 6)), " 1234"); - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 7)), " 1234"); - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 8)), " 1234"); - CHECK_EQ(to_chars_sub(buf, fmt::right(1234, 9)), " 1234"); - - CHECK_EQ(to_chars_sub(buf, fmt::real(0.124, 1)), "0.1"); // we assume this in what follows - CHECK_EQ(to_chars_sub(buf, fmt::real(0.124, 2)), "0.12"); - CHECK_EQ(to_chars_sub(buf, fmt::real(0.124, 3)), "0.124"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 1), 0)), "0.1"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 1), 1)), "0.1"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 1), 2)), "0.1"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 1), 3)), "0.1"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 1), 4)), " 0.1"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 1), 5)), " 0.1"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 1), 6)), " 0.1"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 1), 7)), " 0.1"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 2), 0)), "0.12"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 2), 1)), "0.12"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 2), 2)), "0.12"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 2), 3)), "0.12"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 2), 4)), "0.12"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 2), 5)), " 0.12"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 2), 6)), " 0.12"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 2), 7)), " 0.12"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 3), 0)), "0.124"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 3), 1)), "0.124"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 3), 2)), "0.124"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 3), 3)), "0.124"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 3), 4)), "0.124"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 3), 5)), "0.124"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 3), 6)), " 0.124"); - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(0.124, 3), 7)), " 0.124"); - - CHECK_EQ(to_chars_sub(buf, fmt::right(fmt::real(1234.5222, 1), 7)), " 1234.5"); - auto r = [](double val, size_t width) { return fmt::right(fmt::real(val, 1), width); }; - CHECK_EQ(to_chars_sub(buf, r(1234.5, 7)), " 1234.5"); - c4::format(buf, "freq={}Hz\0", r(1234.5, 7)); - CHECK_EQ(to_csubstr(buf).len, to_csubstr("freq= 1234.5Hz").len); - CHECK_EQ(to_csubstr(buf), "freq= 1234.5Hz"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE("cat.vars") -{ - char buf[256]; - substr sp(buf); - csubstr result; - size_t sz; - - sz = cat(buf, 1, ' ', 2, ' ', 3, ' ', 4); - result = sp.left_of(sz); - CHECK_EQ(result, "1 2 3 4"); -} - -#ifdef C4_TUPLE_TO_STR -TEST_CASE("cat.tuple") -{ - char buf[256]; - substr sp(buf); - csubstr result; - size_t sz; - - sz = cat(buf, std::forward_as_tuple(1, ' ', 2, ' ', 3, ' ', 4)); - result = sp.left_of(sz); - CHECK_EQ(result, "1 2 3 4"); -} -#endif // C4_TUPLE_TO_STR - -TEST_CASE("uncat.vars") -{ - size_t sz; - size_t npos = csubstr::npos; - int v1 = 0, v2 = 0, v3 = 0, v4 = 0; - - sz = uncat("1 2 3 4", v1, v2, v3, v4); - CHECK_NE(sz, npos); - CHECK_EQ(sz, 7); - CHECK_EQ(v1, 1); - CHECK_EQ(v2, 2); - CHECK_EQ(v3, 3); - CHECK_EQ(v4, 4); -} - -#ifdef C4_TUPLE_TO_STR -TEST_CASE("uncat.tuple") -{ - size_t sz; - int v1 = 0, v2 = 0, v3 = 0, v4 = 0; - - auto tp = std::forward_as_tuple(v1, v2, v3, v4); - sz = uncat("1 2 3 4", tp); - CHECK_NE(sz, csubstr::npos); - CHECK_EQ(sz, 7); - CHECK_EQ(v1, 1); - CHECK_EQ(v2, 2); - CHECK_EQ(v3, 3); - CHECK_EQ(v4, 4); -} -#endif // C4_TUPLE_TO_STR - - -TEST_CASE("catsep.vars") -{ - char buf[256]; - substr sp(buf); - csubstr result; - size_t sz; - - sz = catsep(buf, ' ', 1, 2); - CHECK_EQ(sz, 3); - result = sp.left_of(sz); - CHECK_EQ(result, "1 2"); - - sz = catsep(buf, '/', 1, 2); - CHECK_EQ(sz, 3); - result = sp.left_of(sz); - CHECK_EQ(result, "1/2"); - - sz = catsep(buf, ' ', 1, 2, 3, 4); - CHECK_EQ(sz, 7); - result = sp.left_of(sz); - CHECK_EQ(result, "1 2 3 4"); - - sz = catsep(buf, '/', 1, 2, 3, 4); - CHECK_EQ(sz, 7); - result = sp.left_of(sz); - CHECK_EQ(result, "1/2/3/4"); -} - -#ifdef C4_TUPLE_TO_STR -TEST_CASE("catsep.tuple") -{ - char buf[256]; - substr sp(buf); - csubstr result; - size_t sz; - - sz = catsep(buf, ' ', std::forward_as_tuple(1, 2)); - CHECK_EQ(sz, 3); - result = sp.left_of(sz); - CHECK_EQ(result, "1 2"); - - sz = catsep(buf, '/', std::forward_as_tuple(1, 2)); - CHECK_EQ(sz, 3); - result = sp.left_of(sz); - CHECK_EQ(result, "1/2"); - - sz = catsep(buf, ' ', std::forward_as_tuple(1, 2, 3, 4)); - CHECK_EQ(sz, 7); - result = sp.left_of(sz); - CHECK_EQ(result, "1 2 3 4"); - - sz = catsep(buf, '/', std::forward_as_tuple(1, 2, 3, 4)); - CHECK_EQ(sz, 7); - result = sp.left_of(sz); - CHECK_EQ(result, "1/2/3/4"); -} -#endif // C4_TUPLE_TO_STR - -TEST_CASE("uncatsep.vars") -{ - size_t sz; - int v1 = 0, v2 = 0, v3 = 0, v4 = 0; - char sep; - - sz = uncatsep("1 2 3 4", sep, v1, v2, v3, v4); - CHECK_EQ(sz, 7); - CHECK_EQ(v1, 1); - CHECK_EQ(v2, 2); - CHECK_EQ(v3, 3); - CHECK_EQ(v4, 4); -} - -#ifdef C4_TUPLE_TO_STR -TEST_CASE("uncatsep.tuple") -{ - size_t sz; - int v1 = 0, v2 = 0, v3 = 0, v4 = 0; - char sep; - - auto tp = std::forward_as_tuple(v1, v2, v3, v4); - sz = uncatsep("1 2 3 4", sep, tp); - CHECK_EQ(sz, 7); - CHECK_EQ(v1, 1); - CHECK_EQ(v2, 2); - CHECK_EQ(v3, 3); - CHECK_EQ(v4, 4); -} -#endif // C4_TUPLE_TO_STR - -TEST_CASE("format.vars") -{ - char buf[256]; - substr sp(buf); - csubstr result; - size_t sz; - - sz = format(buf, "{} and {} and {} and {}", 1, 2, 3, 4); - CHECK_EQ(sz, strlen("1 and 2 and 3 and 4")); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and 2 and 3 and 4"); - - sz = format(buf, "{} and {} and {} and {}", 1, 2, 3, 4, 5, 6, 7); - CHECK_EQ(sz, 19); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and 2 and 3 and 4"); - - sz = format(buf, "{} and {} and {} and {}", 1, 2, 3); - CHECK_EQ(sz, 20); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and 2 and 3 and {}"); - - sz = format(buf, "{} and {} and {} and {}", 1, 2); - CHECK_EQ(sz, 21); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and 2 and {} and {}"); - - sz = format(buf, "{} and {} and {} and {}", 1); - CHECK_EQ(sz, 22); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and {} and {} and {}"); - - sz = format(buf, "{} and {} and {} and {}"); - CHECK_EQ(sz, 23); - result = sp.left_of(sz); - CHECK_EQ(result, "{} and {} and {} and {}"); - - sz = format(buf, "{} args only at the begin", 1); - CHECK_EQ(sz, csubstr("1 args only at the begin").len); - result = sp.left_of(sz); - CHECK_EQ(result, csubstr("1 args only at the begin")); -} - -TEST_CASE("format.empty_buffer") -{ - size_t sz = format({}, "{} and {} and {} and {}", 1, 2, 3, 4); - CHECK_EQ(sz, strlen("1 and 2 and 3 and 4")); - char buf_[128]; - substr buf = buf_; - sz = format(buf, "{} and {} and {} and {}", 1, 2, 3, 4); - CHECK_EQ(sz, strlen("1 and 2 and 3 and 4")); - CHECK_EQ(format(buf, "{} and {} and {} and {}", 1, 2, 3, 4), - format({} , "{} and {} and {} and {}", 1, 2, 3, 4)); - CHECK_EQ(to_chars({}, 101), to_chars(buf, 101)); // eq for all integers - CHECK_GE(to_chars({}, 0.1f), to_chars(buf, 0.1f)); // ge for all floats, due to a sprintf quirk - CHECK_EQ(format(buf, "a={} foo {} {} bar {}", 101, 10, 11, 12), - format({} , "a={} foo {} {} bar {}", 101, 10, 11, 12)); -} - -#ifdef C4_TUPLE_TO_STR -TEST_CASE("format.tuple") -{ - char buf[256]; - substr sp(buf); - csubstr result; - size_t sz; - - sz = format(buf, "{} and {} and {} and {}", std::forward_as_tuple(1, 2, 3, 4)); - CHECK_EQ(sz, 19); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and 2 and 3 and 4"); - - sz = format(buf, "{} and {} and {} and {}", std::forward_as_tuple(1, 2, 3, 4, 5, 6, 7)); - CHECK_EQ(sz, 19); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and 2 and 3 and 4"); - - sz = format(buf, "{} and {} and {} and {}", std::forward_as_tuple(1, 2, 3)); - CHECK_EQ(sz, 20); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and 2 and 3 and {}"); - - sz = format(buf, "{} and {} and {} and {}", std::forward_as_tuple(1, 2)); - CHECK_EQ(sz, 21); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and 2 and {} and {}"); - - sz = format(buf, "{} and {} and {} and {}", std::forward_as_tuple(1)); - CHECK_EQ(sz, 22); - result = sp.left_of(sz); - CHECK_EQ(result, "1 and {} and {} and {}"); - - sz = format(buf, "{} and {} and {} and {}"); - CHECK_EQ(sz, 23); - result = sp.left_of(sz); - CHECK_EQ(result, "{} and {} and {} and {}"); -} -#endif // C4_TUPLE_TO_STR - -TEST_CASE("unformat.vars") -{ - size_t sz; - int v1 = 0, v2 = 0, v3 = 0, v4 = 0; - - sz = unformat("1 and 2 and 3 and 4", "{} and {} and {} and {}", v1, v2, v3, v4); - CHECK_EQ(sz, 19); - CHECK_EQ(v1, 1); - CHECK_EQ(v2, 2); - CHECK_EQ(v3, 3); - CHECK_EQ(v4, 4); - - v1 = 0; - sz = unformat("1 and 2 and 3 and 4" , "3", v1); - CHECK_EQ(sz, 1); - CHECK_EQ(v1, 0); - - v1 = 0; - sz = unformat("1,2,3,,,", "{},{},{}", v1, v2, v3); - CHECK_EQ(sz, 5); - CHECK_EQ(v1, 1); - CHECK_EQ(v2, 2); - CHECK_EQ(v3, 3); - - v1 = v2 = v3 = 0; - sz = unformat("1,2,3,,,", "{},{},{},,,", v1, v2, v3); - CHECK_EQ(sz, 8); // make sure we count the trailing characters in the format - CHECK_EQ(v1, 1); - CHECK_EQ(v2, 2); - CHECK_EQ(v3, 3); -} - -#ifdef C4_TUPLE_TO_STR -TEST_CASE("unformat.tuple") -{ - size_t sz; - int v1 = 0, v2 = 0, v3 = 0, v4 = 0; - - auto tp = std::forward_as_tuple(v1, v2, v3, v4); - sz = unformat("1 and 2 and 3 and 4", "{} and {} and {} and {}", tp); - CHECK_EQ(sz, 19); - CHECK_EQ(v1, 1); - CHECK_EQ(v2, 2); - CHECK_EQ(v3, 3); - CHECK_EQ(v4, 4); -} -#endif // C4_TUPLE_TO_STR - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -TEST_CASE("catrs.basic") -{ - std::vector buf; - - catrs(&buf); - CHECK_EQ(to_csubstr(buf), ""); - - catrs(&buf, 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1234"); - catrs(&buf, 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5678"); -} - -TEST_CASE("catrs.basic_return") -{ - auto bufv = catrs>(9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - CHECK_EQ(to_csubstr(bufv), "9876543210"); - bufv = catrs>(); - CHECK_EQ(to_csubstr(bufv), ""); - CHECK(bufv.empty()); - - auto bufs = catrs(9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - CHECK_EQ(to_csubstr(bufs), "9876543210"); -} - -TEST_CASE("catrs.basic_append") -{ - std::vector buf; - - catrs(append, &buf); - CHECK_EQ(to_csubstr(buf), ""); - - catrs(append, &buf, 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1234"); - catrs(append, &buf, 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "12345678"); - catrs(append, &buf, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "123456789012345678"); -} - -template -void catrs_perfect_fwd(Args && ...args) -{ - catrs(std::forward(args)...); -} - -TEST_CASE("catrs.perfect_fwd") -{ - std::vector buf; - catrs_perfect_fwd(&buf, 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1234"); - catrs_perfect_fwd(&buf, 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5678"); -} - -template -void catrs_const_fwd(Args const& ...args) -{ - catrs(args...); -} - -TEST_CASE("catrs.const_fwd") -{ - std::vector buf; - catrs_const_fwd(&buf, 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1234"); - catrs_const_fwd(&buf, 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5678"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE("catseprs.basic") -{ - std::vector buf; - - catseprs(&buf, ' '); - CHECK_EQ(to_csubstr(buf), ""); - - catseprs(&buf, ' ', 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1 2 3 4"); - catseprs(&buf, ' ', 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5 6 7 8"); - - catseprs(&buf, ',', 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1,2,3,4"); - catseprs(&buf, ',', 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5,6,7,8"); - - catseprs(&buf, '/', 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1/2/3/4"); - catseprs(&buf, '/', 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5/6/7/8"); - - catseprs(&buf, "///", 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1///2///3///4"); - catseprs(&buf, "///", 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5///6///7///8"); - - catseprs(&buf, 5678, 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1567825678356784"); - catseprs(&buf, 1234, 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5123461234712348"); -} - -TEST_CASE("catseprs.basic_return") -{ - auto bufv = catseprs>('a', 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - CHECK_EQ(to_csubstr(bufv), "9a8a7a6a5a4a3a2a1a0"); - - auto bufs = catseprs('a', 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); - CHECK_EQ(to_csubstr(bufs), "9a8a7a6a5a4a3a2a1a0"); -} - -TEST_CASE("catseprs.basic_append") -{ - std::vector buf; - - auto ret = catseprs(append, &buf, ' '); - CHECK_EQ(to_csubstr(buf), ""); - CHECK_EQ(ret, ""); - - ret = catseprs(append, &buf, ' ', 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1 2 3 4"); - CHECK_EQ(ret, "1 2 3 4"); - ret = catseprs(append, &buf, ' ', 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "1 2 3 45 6 7 8"); - CHECK_EQ(ret, "5 6 7 8"); - ret = catseprs(append, &buf, ' ', 9, 0, 1, 2, 3, 4, 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "1 2 3 45 6 7 89 0 1 2 3 4 5 6 7 8"); - CHECK_EQ(ret, "9 0 1 2 3 4 5 6 7 8"); - - ret = catseprs(append, &buf, ' '); - CHECK_EQ(to_csubstr(buf), "1 2 3 45 6 7 89 0 1 2 3 4 5 6 7 8"); - CHECK_EQ(ret, ""); -} - -template -void catseprs_perfect_fwd(Args && ...args) -{ - catseprs(std::forward(args)...); -} - -template -void catseprs_const_fwd(Args const& ...args) -{ - catseprs(args...); -} - -TEST_CASE("catseprs.perfect_fwd") -{ - std::vector buf; - catseprs_perfect_fwd(&buf, '.', 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1.2.3.4"); - catseprs_perfect_fwd(&buf, 0, 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5060708"); -} - -TEST_CASE("catseprs.const_fwd") -{ - std::vector buf; - catseprs_const_fwd(&buf, '.', 1, 2, 3, 4); - CHECK_EQ(to_csubstr(buf), "1.2.3.4"); - catseprs_const_fwd(&buf, 0, 5, 6, 7, 8); - CHECK_EQ(to_csubstr(buf), "5060708"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST_CASE("formatrs.basic") -{ - std::vector buf; - - formatrs(&buf, ""); - CHECK(buf.empty()); - - formatrs(&buf, "{} goes with food, {} goes with heat, {} anytime", "wine", "beer", "coffee"); - CHECK_EQ(to_csubstr(buf), "wine goes with food, beer goes with heat, coffee anytime"); - - formatrs(&buf, ""); - CHECK(buf.empty()); -} - -TEST_CASE("formatrs.basic_return") -{ - auto bufv = formatrs>("{} goes with food, {} goes with heat, {} anytime", "wine", "beer", "coffee"); - CHECK_EQ(to_csubstr(bufv), "wine goes with food, beer goes with heat, coffee anytime"); - - auto bufs = formatrs("{} goes with food, {} goes with heat, {} anytime", "wine", "beer", "coffee"); - CHECK_EQ(to_csubstr(bufs), "wine goes with food, beer goes with heat, coffee anytime"); -} - -TEST_CASE("formatrs.basic_append") -{ - std::vector buf; - - formatrs(append, &buf, "{} goes with food", "wine"); - CHECK_EQ(to_csubstr(buf), "wine goes with food"); - formatrs(append, &buf, ", {} goes with heat", "beer"); - CHECK_EQ(to_csubstr(buf), "wine goes with food, beer goes with heat"); - formatrs(append, &buf, ", {} anytime", "coffee"); - CHECK_EQ(to_csubstr(buf), "wine goes with food, beer goes with heat, coffee anytime"); - - formatrs(append, &buf, ". And water. {} glass of {}cl in the morning clears you up for the day", 1, 40); - CHECK_EQ(to_csubstr(buf), "wine goes with food, beer goes with heat, coffee anytime. And water. 1 glass of 40cl in the morning clears you up for the day"); -} - -template -void formatrs_perfect_fwd(Args && ...args) -{ - formatrs(std::forward(args)...); -} - -template -void formatrs_const_fwd(Args const& ...args) -{ - formatrs(args...); -} - -TEST_CASE("formatrs.perfect_fwd") -{ - std::vector buf; - formatrs_perfect_fwd(&buf, "Too much of anything is bad, but too much {} is {}.", "Champagne", "just right"); - CHECK_EQ(to_csubstr(buf), "Too much of anything is bad, but too much Champagne is just right."); - formatrs_perfect_fwd(&buf, "{}, I am tasting the {}", "Come quickly", "stars!"); - CHECK_EQ(to_csubstr(buf), "Come quickly, I am tasting the stars!"); - formatrs_perfect_fwd(&buf, "{} the only wine that leaves a {} {} after {}.", "Champagne is", "woman", "beautiful", "drinking it"); - CHECK_EQ(to_csubstr(buf), "Champagne is the only wine that leaves a woman beautiful after drinking it."); - formatrs_perfect_fwd(&buf, "Remember {}, it's not just {} we are fighting for, it's {}", "gentlemen", "France", "Champagne!"); - CHECK_EQ(to_csubstr(buf), "Remember gentlemen, it's not just France we are fighting for, it's Champagne!"); - // https://www.townandcountrymag.com/leisure/drinks/how-to/g828/the-10-best-quotes-about-champagne/ -} - -TEST_CASE("formatrs.const_fwd") -{ - std::vector buf; - formatrs_const_fwd(&buf, "Too much of anything is bad, but too much {} is {}.", "Champagne", "just right"); - CHECK_EQ(to_csubstr(buf), "Too much of anything is bad, but too much Champagne is just right."); - formatrs_const_fwd(&buf, "{}, I am tasting the {}", "Come quickly", "stars!"); - CHECK_EQ(to_csubstr(buf), "Come quickly, I am tasting the stars!"); - formatrs_const_fwd(&buf, "{} the only wine that leaves a {} {} after {}.", "Champagne is", "woman", "beautiful", "drinking it"); - CHECK_EQ(to_csubstr(buf), "Champagne is the only wine that leaves a woman beautiful after drinking it."); - formatrs_const_fwd(&buf, "Remember {}, it's not just {} we are fighting for, it's {}", "gentlemen", "France", "Champagne!"); - CHECK_EQ(to_csubstr(buf), "Remember gentlemen, it's not just France we are fighting for, it's Champagne!"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_hex(T in, csubstr expected) -{ - INFO("expected=" << expected); - SUBCASE("charbuf") - { - char buf_[128] = {}; - substr buf = buf_; - buf.fill('?'); // this will be executed before each case - SUBCASE("sz=0") - { - CHECK_EQ(cat(buf.first(0), fmt::hex(in)), expected.len); - CHECK_EQ(buf[0], '?'); - CHECK_EQ(buf.trimr('?'), ""); - } - SUBCASE("sz=1") - { - CHECK_EQ(cat(buf.first(1), fmt::hex(in)), expected.len); - CHECK_EQ(buf[0], '?'); - CHECK_EQ(buf.trimr('?'), ""); - } - SUBCASE("sz=2") - { - CHECK_EQ(cat(buf.first(2), fmt::hex(in)), expected.len); - CHECK_EQ(buf[0], '?'); - CHECK_EQ(buf.trimr('?'), ""); - } - SUBCASE("full") - { - REQUIRE_EQ(cat(buf, fmt::hex(in)), expected.len); - CHECK_EQ(buf.first(expected.len), expected); - CHECK_EQ(buf.trimr('?'), expected); - } - } - SUBCASE("vector") - { - std::vector buf; - catrs(&buf, fmt::hex(in)); - CHECK_EQ(buf.size(), expected.len); - CHECK_EQ(to_csubstr(buf), expected); - } -} - -TEST_CASE("fmt.hex") -{ - test_hex(0, "0x0"); - test_hex(nullptr, "0x0"); - test_hex(254, "0xfe"); - test_hex(255, "0xff"); - test_hex(256, "0x100"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template void test_raw_roundtrip(const char *valstr, T const& orig) -{ - INFO("valstr=" << valstr); - alignas(alignof(T)) char buf_[2 * (sizeof(T) + alignof(T))] = {}; - substr buf = buf_; - - fmt::const_raw_wrapper rawwrap = fmt::raw(orig); - REQUIRE_EQ((void*)rawwrap.buf, (void*)&orig); - REQUIRE_EQ(rawwrap.len, sizeof(orig)); - - for(size_t i = 0; i < alignof(T); ++i) - { - INFO(" i=" << i); - // make sure to cover unaligned buffers - substr sbuf = buf.sub(i); - size_t szwrite = c4::to_chars(sbuf, fmt::raw(orig)); - REQUIRE_LE(szwrite, sbuf.len); - if(i == 0) - { - REQUIRE_EQ(szwrite, sizeof(T)); - } - else - { - REQUIRE_GT(szwrite, sizeof(T)); - } - T copy = {}; - REQUIRE_NE(copy, orig); - bool ok = c4::from_chars_first(sbuf, fmt::raw(copy)); - REQUIRE_EQ(ok, true); - CHECK_EQ(copy, orig); - - // cover also insufficient buffers - sbuf = sbuf.first(sizeof(T)-1); - memset(buf.str, 0, buf.len); - szwrite = c4::to_chars(sbuf, fmt::raw(orig)); - REQUIRE_GT(szwrite, sbuf.len); - for(char c : buf) - { - CHECK_EQ(c, 0); - } - } -} - -TEST_CASE("fmt.raw_int") -{ - #define _(v) test_raw_roundtrip(#v, v) - - _(int(1)); - _(int(2)); - _(int(-1)); - _(int(-2)); - - #undef _ -} - -} // namespace c4 - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_log.cpp b/thirdparty/ryml/ext/c4core/test/test_log.cpp deleted file mode 100644 index 2f5b56734..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_log.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "c4/log.hpp" - -#include "c4/libtest/supprwarn_push.hpp" -#include "c4/test.hpp" - -namespace c4 { - -TEST(LogBuffer, basic) -{ -#define _CHECK(s, str) \ - EXPECT_EQ(strncmp(s.rd(), str, s.pos), 0) << " string was '" << s.rd() << "'"; \ - s.clear(); \ - EXPECT_EQ(s.pos, 0);\ - EXPECT_EQ(s.buf[0], '\0') - - LogBuffer b; - const char *foo = "Foo"; - const char *bars_str = "123"; - int bars = 123; - - // raw writing - b.write("hello world I am "); - b.write(foo); - b.write(" and I frobnicate "); - b.write(bars_str); // only accepts const char* - b.write(" Bars"); - _CHECK(b, "hello world I am Foo and I frobnicate 123 Bars"); - - // chevron-style AKA iostream-style - b << "hello world I am " << foo << " and I frobnicate " << bars << " Bars"; - _CHECK(b, "hello world I am Foo and I frobnicate 123 Bars"); - - // c-style, not type safe - b.printf("hello world I am %s and I frobnicate %d Bars", foo, bars); - _CHECK(b, "hello world I am Foo and I frobnicate 123 Bars"); - - // python-style, type safe - b.print("hello world I am {} and I frobnicate {} Bars", foo, bars); - _CHECK(b, "hello world I am Foo and I frobnicate 123 Bars"); - - // r-style, type safe - b.cat("hello world I am ", foo, " and I frobnicate ", bars, " Bars"); - _CHECK(b, "hello world I am Foo and I frobnicate 123 Bars"); - - // using separators: this is unpractical... - const char *s[] = {"now", "we", "have", "11", "strings", "to", "cat", "one", "after", "the", "other"}; - b.cat(s[0], ' ', s[1], ' ', s[2], ' ', s[3], ' ', s[4], ' ', - s[5], ' ', s[6], ' ', s[7], ' ', s[8], ' ', s[9], ' ', s[10]); - _CHECK(b, "now we have 11 strings to cat one after the other"); - - // ... and this resolves it - b.catsep(' ', s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7], s[8], s[9], s[10]); - _CHECK(b, "now we have 11 strings to cat one after the other"); - - b.catsep('_', s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7], s[8], s[9], s[10]); - _CHECK(b, "now_we_have_11_strings_to_cat_one_after_the_other"); - - // can be a full string - b.catsep("____", s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7], s[8], s[9], s[10]); - _CHECK(b, "now____we____have____11____strings____to____cat____one____after____the____other"); - - // or just a general object - b.catsep(22, s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7], s[8], s[9], s[10]); - _CHECK(b, "now22we22have221122strings22to22cat22one22after22the22other"); - -} - -} // namespace c4 - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_memory_resource.cpp b/thirdparty/ryml/ext/c4core/test/test_memory_resource.cpp deleted file mode 100644 index 7ae7a71ac..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_memory_resource.cpp +++ /dev/null @@ -1,255 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/memory_resource.hpp" -#endif - -#include "c4/libtest/supprwarn_push.hpp" - -#include - -#include "c4/test.hpp" - -namespace c4 { - -TEST_CASE("set_aalloc.basic") -{ - auto a = get_aalloc(); - set_aalloc(nullptr); - CHECK_EQ(get_aalloc(), nullptr); - set_aalloc(a); - CHECK_EQ(get_aalloc(), a); -} - -TEST_CASE("set_afree.basic") -{ - auto a = get_afree(); - set_afree(nullptr); - CHECK_EQ(get_afree(), nullptr); - set_afree(a); - CHECK_EQ(get_afree(), a); -} - -TEST_CASE("set_arealloc.basic") -{ - auto a = get_arealloc(); - set_arealloc(nullptr); - CHECK_EQ(get_arealloc(), nullptr); - set_arealloc(a); - CHECK_EQ(get_arealloc(), a); -} - -//----------------------------------------------------------------------------- -namespace detail { -void* aalloc_impl(size_t size, size_t alignment=alignof(max_align_t)); -void* arealloc_impl(void *ptr, size_t oldsz, size_t newsz, size_t alignment=alignof(max_align_t)); -void afree_impl(void *ptr); -} // namespace detail - -TEST_CASE("aalloc_impl.error_bad_align") -{ -#if defined(C4_POSIX) - C4_EXPECT_ERROR_OCCURS(1); - auto *mem = detail::aalloc_impl(64, 9); // allocating with a non-power of two value is invalid - CHECK_EQ(mem, nullptr); -#endif -} - -TEST_CASE("aalloc_impl.error_out_of_mem") -{ -#if defined(C4_POSIX) - if(sizeof(size_t) != 8) return; // valgrind complains that size is -1 - C4_EXPECT_ERROR_OCCURS(1); - size_t sz = std::numeric_limits::max(); - sz /= 2; - auto *mem = detail::aalloc_impl(sz); - CHECK_EQ(mem, nullptr); -#endif -} - -//----------------------------------------------------------------------------- - -void do_test_realloc(arealloc_pfn fn) -{ -#define _set(dim) for(uint8_t i = 0; i < dim; ++i) { mem[i] = static_cast(i); } -#define _check(dim) for(uint8_t i = 0; i < dim; ++i) { CHECK_EQ(mem[i], i); } - - char *mem = (char*) aalloc(16, alignof(max_align_t)); - _set(16u); - _check(16u); - mem = (char*) fn(mem, 16, 20, alignof(max_align_t)); - _check(16u); - mem = (char*) fn(mem, 8, 20, alignof(max_align_t)); - _check(8u); - afree(mem); - -#undef _set -#undef _check -} - -TEST_CASE("realloc_impl.basic") -{ - do_test_realloc(&detail::arealloc_impl); -} - -TEST_CASE("realloc.basic") -{ - do_test_realloc(&arealloc); -} - - -//----------------------------------------------------------------------------- - -void do_memreslinear_realloc_test(MemoryResourceLinear &mr) -{ - C4_ASSERT(mr.capacity() >= 128); // this is needed for the tests below - - char * mem = (char*) mr.allocate(32); - CHECK_EQ(mem-(char*)mr.mem(), 0); - CHECK_EQ(mr.size(), 32); - CHECK_EQ(mr.slack(), mr.capacity() - 32); - - mem = (char*) mr.reallocate(mem, 32, 16); - CHECK_EQ(mem-(char*)mr.mem(), 0); - CHECK_EQ(mr.size(), 16); - CHECK_EQ(mr.slack(), mr.capacity() - 16); - - mem = (char*) mr.reallocate(mem, 16, 64); - CHECK_EQ(mem-(char*)mr.mem(), 0); - CHECK_EQ(mr.size(), 64); - CHECK_EQ(mr.slack(), mr.capacity() - 64); - - mem = (char*) mr.reallocate(mem, 64, 32); - CHECK_EQ(mem-(char*)mr.mem(), 0); - CHECK_EQ(mr.size(), 32); - CHECK_EQ(mr.slack(), mr.capacity() - 32); - - - char *mem2 = (char*) mr.allocate(32); - CHECK_EQ(mem-(char*)mr.mem(), 0); - CHECK_EQ(mem2-(char*)mr.mem(), 32); - CHECK_EQ(mr.size(), 64); - CHECK_EQ(mr.slack(), mr.capacity() - 64); - - mem = (char*) mr.reallocate(mem, 32, 16); - CHECK_EQ(mem-(char*)mr.mem(), 0); - CHECK_EQ(mr.size(), 64); - CHECK_EQ(mr.slack(), mr.capacity() - 64); -} - -TEST_CASE("MemoryResourceLinear.reallocate") -{ - MemoryResourceLinear mr(128); - do_memreslinear_realloc_test(mr); -} - -TEST_CASE("MemoryResourceLinearArr.reallocate") -{ - MemoryResourceLinearArr<128> mr; - do_memreslinear_realloc_test(mr); -} - - -//----------------------------------------------------------------------------- - -TEST_CASE("MemoryResourceLinear.error_out_of_mem") -{ - { - C4_EXPECT_ERROR_OCCURS(0); - MemoryResourceLinear mr(8); - mr.allocate(2); - } - - { - C4_EXPECT_ERROR_OCCURS(2); - MemoryResourceLinear mr(8); - mr.allocate(9); - } -} - -TEST_CASE("MemoryResourceLinearArr.error_out_of_mem") -{ - { - C4_EXPECT_ERROR_OCCURS(0); - MemoryResourceLinearArr<8> mr; - mr.allocate(2); - } - - { - C4_EXPECT_ERROR_OCCURS(2); - MemoryResourceLinearArr<8> mr; - mr.allocate(9); - } -} - - -//----------------------------------------------------------------------------- - -TEST_CASE("ScopedMemoryResource.basic") -{ - auto *before = get_memory_resource(); - { - MemoryResourceCounts mrc; - ScopedMemoryResource smr(&mrc); - CHECK_EQ(get_memory_resource(), &mrc); - } - CHECK_EQ(get_memory_resource(), before); -} - -TEST_CASE("ScopedMemoryResourceCounts.basic") -{ - auto *before = get_memory_resource(); - { - auto sac = ScopedMemoryResourceCounts{}; - CHECK_EQ(get_memory_resource(), &sac.mr); - } - CHECK_EQ(get_memory_resource(), before); -} - -TEST_CASE("ScopedMemoryResourceCounts.counts") -{ - auto *before = get_memory_resource(); - C4_UNUSED(before); - - { - auto checker = AllocationCountsChecker(); - auto *mr = &checker.mr; - - for(size_t sz : {16u, 32u, 64u, 128u}) - { - void *mem = mr->allocate(sz); - checker.check_all_delta(1, static_cast(sz), static_cast(sz)); - mr->deallocate(mem, sz); - checker.reset(); - } - checker.check_curr(0, 0); - checker.check_total(4, 240); - checker.check_max(1, 128); - } - - { - auto checker = AllocationCountsChecker(); - auto *mr = &checker.mr; - - std::pair mem[4] = {{0,16}, {0,32}, {0,64}, {0,128}}; - for(auto& m : mem) - { - m.first = mr->allocate(m.second); - checker.check_curr_delta(1, static_cast(m.second)); - checker.reset(); - } - checker.check_curr(4, 240); - checker.check_total(4, 240); - checker.check_max(4, 240); - for(auto& m : mem) - { - mr->deallocate(m.first, m.second); - } - checker.check_curr(0, 0); - checker.check_total(4, 240); - checker.check_max(4, 240); - } - -} - -} // namespace c4 - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_memory_util.cpp b/thirdparty/ryml/ext/c4core/test/test_memory_util.cpp deleted file mode 100644 index 36509070c..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_memory_util.cpp +++ /dev/null @@ -1,415 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/memory_util.hpp" -#endif - -#include "c4/libtest/supprwarn_push.hpp" - -#include -#include - -namespace c4 { - -TEST_CASE("mem_overlaps") -{ - csubstr buf = "0123456789012345678901234567890123456789"; - CHECK_EQ(buf.len, 40); - auto overlaps = [](csubstr lhs, csubstr rhs){ - bool res = mem_overlaps(lhs.str, rhs.str, lhs.len, rhs.len); - CHECK(res == lhs.overlaps(rhs)); - return res; - }; - CHECK(!overlaps(buf.first(0), buf.last(0))); - CHECK(!overlaps(buf.first(5), buf.last(5))); - CHECK(!overlaps(buf.first(10), buf.last(10))); - CHECK(!overlaps(buf.first(20), buf.last(20))); - CHECK(overlaps(buf.first(21), buf.last(20))); - CHECK(overlaps(buf.first(20), buf.last(21))); - CHECK(!overlaps(buf.first(0), buf)); - CHECK(overlaps(buf.first(1), buf)); - CHECK(!overlaps(buf, buf.last(0))); - CHECK(overlaps(buf, buf.last(1))); - CHECK(!overlaps(buf.first(20), buf.last(20))); - CHECK(overlaps(buf.first(21), buf.last(20))); - CHECK(overlaps(buf.first(20), buf.first(21))); -} - -TEST_CASE("mem_repeatT.one_repetition") -{ - char buf[120] = {0}; - - mem_repeat(buf, "123", 2, 1); - CHECK_EQ(strcmp(buf, "12"), 0); - - mem_repeat(buf, "123", 3, 1); - CHECK_EQ(strcmp(buf, "123"), 0); -} - -TEST_CASE("mem_repeatT.basic") -{ - char buf[120] = {0}; - - mem_zero(buf); - - mem_repeat(buf, "123", 2, 2); - CHECK_EQ(strcmp(buf, "1212"), 0); - CHECK_EQ(buf[4], '\0'); - - mem_zero(buf); - - mem_repeat(buf, "123", 3, 2); - CHECK_EQ(strcmp(buf, "123123"), 0); - CHECK_EQ(buf[6], '\0'); - - mem_zero(buf); - - mem_repeat(buf, "123", 2, 3); - CHECK_EQ(strcmp(buf, "121212"), 0); - CHECK_EQ(buf[6], '\0'); - - mem_zero(buf, sizeof(buf)); - - mem_repeat(buf, "123", 3, 3); - CHECK_EQ(strcmp(buf, "123123123"), 0); - CHECK_EQ(buf[9], '\0'); - - mem_zero(buf, sizeof(buf)); - - mem_repeat(buf, "123", 2, 4); - CHECK_EQ(strcmp(buf, "12121212"), 0); - CHECK_EQ(buf[8], '\0'); - - mem_zero(buf, sizeof(buf)); - - mem_repeat(buf, "123", 3, 4); - CHECK_EQ(strcmp(buf, "123123123123"), 0); - CHECK_EQ(buf[12], '\0'); - - mem_zero(buf, sizeof(buf)); - - mem_repeat(buf, "123", 2, 5); - CHECK_EQ(strcmp(buf, "1212121212"), 0); - CHECK_EQ(buf[10], '\0'); - - mem_zero(buf, sizeof(buf)); - - mem_repeat(buf, "123", 3, 5); - CHECK_EQ(strcmp(buf, "123123123123123"), 0); - CHECK_EQ(buf[15], '\0'); - - mem_zero(buf, sizeof(buf)); - - mem_repeat(buf, "123", 2, 6); - CHECK_EQ(strcmp(buf, "121212121212"), 0); - CHECK_EQ(buf[12], '\0'); - - mem_zero(buf, sizeof(buf)); - - mem_repeat(buf, "123", 3, 6); - CHECK_EQ(strcmp(buf, "123123123123123123"), 0); - CHECK_EQ(buf[18], '\0'); -} - - -//----------------------------------------------------------------------------- - -TEST_CASE("is_aligned.basic") -{ - CHECK(is_aligned((int*)0x0)); - CHECK_FALSE(is_aligned((int*)0x1)); - CHECK_FALSE(is_aligned((int*)0x2)); - CHECK_FALSE(is_aligned((int*)0x3)); - CHECK_FALSE(is_aligned((int*)0x3)); - CHECK(is_aligned((int*)0x4)); -} - - -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("lsb.basic", T, uint8_t, uint16_t, uint32_t, uint64_t) -{ - //CHECK_EQ(lsb( 0), T(0)); - CHECK_EQ(lsb( 1), T(0)); - CHECK_EQ(lsb( 2), T(1)); - CHECK_EQ(lsb( 3), T(0)); - CHECK_EQ(lsb( 4), T(2)); - CHECK_EQ(lsb( 5), T(0)); - CHECK_EQ(lsb( 6), T(1)); - CHECK_EQ(lsb( 7), T(0)); - CHECK_EQ(lsb( 8), T(3)); - CHECK_EQ(lsb( 9), T(0)); - CHECK_EQ(lsb(10), T(1)); - CHECK_EQ(lsb(11), T(0)); - CHECK_EQ(lsb(12), T(2)); - CHECK_EQ(lsb(13), T(0)); - CHECK_EQ(lsb(14), T(1)); - CHECK_EQ(lsb(15), T(0)); - CHECK_EQ(lsb(16), T(4)); -} - -TEST_CASE_TEMPLATE("lsb11.basic", T, uint8_t, uint16_t, uint32_t, uint64_t) -{ - //CHECK_EQ((lsb11::value), T(0)); - CHECK_EQ((lsb11::value), T(0)); - CHECK_EQ((lsb11::value), T(1)); - CHECK_EQ((lsb11::value), T(0)); - CHECK_EQ((lsb11::value), T(2)); - CHECK_EQ((lsb11::value), T(0)); - CHECK_EQ((lsb11::value), T(1)); - CHECK_EQ((lsb11::value), T(0)); - CHECK_EQ((lsb11::value), T(3)); - CHECK_EQ((lsb11::value), T(0)); - CHECK_EQ((lsb11::value), T(1)); - CHECK_EQ((lsb11::value), T(0)); - CHECK_EQ((lsb11::value), T(2)); - CHECK_EQ((lsb11::value), T(0)); - CHECK_EQ((lsb11::value), T(1)); - CHECK_EQ((lsb11::value), T(0)); - CHECK_EQ((lsb11::value), T(4)); -} - - -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("ipow.float", T, float, double) -{ - SUBCASE("base 1, signed exponent") - { - CHECK_FLOAT_EQ(ipow(T(1), int(0)), T(1)); - CHECK_FLOAT_EQ(ipow(T(1), int(1)), T(1)); - CHECK_FLOAT_EQ(ipow(T(1), int(2)), T(1)); - CHECK_FLOAT_EQ(ipow(T(1), -int(1)), T(1)); - CHECK_FLOAT_EQ(ipow(T(1), -int(2)), T(1)); - CHECK_FLOAT_EQ((ipow(int(0))), T(1)); - CHECK_FLOAT_EQ((ipow(int(1))), T(1)); - CHECK_FLOAT_EQ((ipow(int(2))), T(1)); - CHECK_FLOAT_EQ((ipow(-int(1))), T(1)); - CHECK_FLOAT_EQ((ipow(-int(2))), T(1)); - } - SUBCASE("base 1, unsigned exponent") - { - CHECK_FLOAT_EQ(ipow(T(1), unsigned(0)), T(1)); - CHECK_FLOAT_EQ(ipow(T(1), unsigned(1)), T(1)); - CHECK_FLOAT_EQ(ipow(T(1), unsigned(2)), T(1)); - CHECK_FLOAT_EQ((ipow(unsigned(0))), T(1)); - CHECK_FLOAT_EQ((ipow(unsigned(1))), T(1)); - CHECK_FLOAT_EQ((ipow(unsigned(2))), T(1)); - } - SUBCASE("base 2, signed exponent") - { - CHECK_FLOAT_EQ(ipow(T(2), int(0)), T(1)); - CHECK_FLOAT_EQ(ipow(T(2), int(1)), T(2)); - CHECK_FLOAT_EQ(ipow(T(2), int(2)), T(4)); - CHECK_FLOAT_EQ(ipow(T(2), int(7)), T(128)); - CHECK_FLOAT_EQ(ipow(T(2), -int(1)), T(0.5)); - CHECK_FLOAT_EQ(ipow(T(2), -int(2)), T(0.25)); - CHECK_FLOAT_EQ(ipow(T(2), -int(3)), T(0.125)); - CHECK_FLOAT_EQ(ipow(T(2), -int(4)), T(0.0625)); - CHECK_FLOAT_EQ((ipow(int(0))), T(1)); - CHECK_FLOAT_EQ((ipow(int(1))), T(2)); - CHECK_FLOAT_EQ((ipow(int(2))), T(4)); - CHECK_FLOAT_EQ((ipow(int(7))), T(128)); - CHECK_FLOAT_EQ((ipow(-int(1))), T(0.5)); - CHECK_FLOAT_EQ((ipow(-int(2))), T(0.25)); - CHECK_FLOAT_EQ((ipow(-int(3))), T(0.125)); - CHECK_FLOAT_EQ((ipow(-int(4))), T(0.0625)); - } - SUBCASE("base 2, unsigned exponent") - { - CHECK_FLOAT_EQ(ipow(T(2), unsigned(0)), T(1)); - CHECK_FLOAT_EQ(ipow(T(2), unsigned(1)), T(2)); - CHECK_FLOAT_EQ(ipow(T(2), unsigned(2)), T(4)); - CHECK_FLOAT_EQ(ipow(T(2), unsigned(6)), T(64)); - CHECK_FLOAT_EQ((ipow(unsigned(0))), T(1)); - CHECK_FLOAT_EQ((ipow(unsigned(1))), T(2)); - CHECK_FLOAT_EQ((ipow(unsigned(2))), T(4)); - CHECK_FLOAT_EQ((ipow(unsigned(6))), T(64)); - } -} - -TEST_CASE_TEMPLATE("ipow", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - SUBCASE("base 1, signed exponent") - { - CHECK_EQ(ipow(T(1), int(0)), T(1)); - CHECK_EQ(ipow(T(1), int(1)), T(1)); - CHECK_EQ(ipow(T(1), int(2)), T(1)); - CHECK_EQ(ipow(T(1), -int(1)), T(1)); - CHECK_EQ(ipow(T(1), -int(2)), T(1)); - CHECK_EQ(ipow(int(0)), T(1)); - CHECK_EQ(ipow(int(1)), T(1)); - CHECK_EQ(ipow(int(2)), T(1)); - CHECK_EQ(ipow(-int(1)), T(1)); - CHECK_EQ(ipow(-int(2)), T(1)); - } - SUBCASE("base 1, unsigned exponent") - { - CHECK_EQ(ipow(T(1), unsigned(0)), T(1)); - CHECK_EQ(ipow(T(1), unsigned(1)), T(1)); - CHECK_EQ(ipow(T(1), unsigned(2)), T(1)); - CHECK_EQ(ipow(unsigned(0)), T(1)); - CHECK_EQ(ipow(unsigned(1)), T(1)); - CHECK_EQ(ipow(unsigned(2)), T(1)); - } - SUBCASE("base 2, signed exponent") - { - CHECK_EQ(ipow(T(2), int(0)), T(1)); - CHECK_EQ(ipow(T(2), int(1)), T(2)); - CHECK_EQ(ipow(T(2), int(2)), T(4)); - CHECK_EQ(ipow(T(2), int(6)), T(64)); - CHECK_EQ(ipow(T(2), -int(1)), T(0)); - CHECK_EQ(ipow(T(2), -int(2)), T(0)); - CHECK_EQ(ipow(T(2), -int(6)), T(0)); - CHECK_EQ(ipow(int(0)), T(1)); - CHECK_EQ(ipow(int(1)), T(2)); - CHECK_EQ(ipow(int(2)), T(4)); - CHECK_EQ(ipow(int(6)), T(64)); - CHECK_EQ(ipow(-int(1)), T(0)); - CHECK_EQ(ipow(-int(2)), T(0)); - CHECK_EQ(ipow(-int(7)), T(0)); - } - SUBCASE("base 2, unsigned exponent") - { - CHECK_EQ(ipow(T(2), unsigned(0)), T(1)); - CHECK_EQ(ipow(T(2), unsigned(1)), T(2)); - CHECK_EQ(ipow(T(2), unsigned(2)), T(4)); - CHECK_EQ(ipow(T(2), unsigned(6)), T(64)); - CHECK_EQ(ipow(unsigned(0)), T(1)); - CHECK_EQ(ipow(unsigned(1)), T(2)); - CHECK_EQ(ipow(unsigned(2)), T(4)); - CHECK_EQ(ipow(unsigned(6)), T(64)); - } -} - - -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("msb.basic", T, uint8_t, uint16_t, uint32_t, uint64_t) -{ - CHECK_EQ(msb(T( 1)), 0u); - CHECK_EQ(msb(T( 2)), 1u); - CHECK_EQ(msb(T( 3)), 1u); - CHECK_EQ(msb(T( 4)), 2u); - CHECK_EQ(msb(T( 5)), 2u); - CHECK_EQ(msb(T( 6)), 2u); - CHECK_EQ(msb(T( 7)), 2u); - CHECK_EQ(msb(T( 8)), 3u); - CHECK_EQ(msb(T( 9)), 3u); - CHECK_EQ(msb(T(10)), 3u); - CHECK_EQ(msb(T(11)), 3u); - CHECK_EQ(msb(T(12)), 3u); - CHECK_EQ(msb(T(13)), 3u); - CHECK_EQ(msb(T(14)), 3u); - CHECK_EQ(msb(T(15)), 3u); - CHECK_EQ(msb(T(16)), 4u); - CHECK_EQ(msb(std::numeric_limits::max()), 8u * sizeof(T) - 1u); -} - -TEST_CASE_TEMPLATE("msb11.basic", T, uint8_t, uint16_t, uint32_t, uint64_t) -{ - CHECK_EQ((msb11::value), T(0)); - CHECK_EQ((msb11::value), T(1)); - CHECK_EQ((msb11::value), T(1)); - CHECK_EQ((msb11::value), T(2)); - CHECK_EQ((msb11::value), T(2)); - CHECK_EQ((msb11::value), T(2)); - CHECK_EQ((msb11::value), T(2)); - CHECK_EQ((msb11::value), T(3)); - CHECK_EQ((msb11::value), T(3)); - CHECK_EQ((msb11::value), T(3)); - CHECK_EQ((msb11::value), T(3)); - CHECK_EQ((msb11::value), T(3)); - CHECK_EQ((msb11::value), T(3)); - CHECK_EQ((msb11::value), T(3)); - CHECK_EQ((msb11::value), T(3)); - CHECK_EQ((msb11::value), T(4)); - CHECK_EQ((msb11::max()>::value), 8u * sizeof(T) - 1u); -} - - -//----------------------------------------------------------------------------- -// contiguous mask - -template T _mask() { return T(0); } -template T _mask(int bit, Bits ...bits) { return (T)(T(1) << bit | _mask(bits...)); } - -TEST_CASE_TEMPLATE("contiguous_mask.basic", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - CHECK_EQ(contiguous_mask(0, 0), _mask()); - CHECK_EQ(contiguous_mask(0, 1), _mask(0)); - CHECK_EQ(contiguous_mask(0, 2), _mask(0, 1)); - CHECK_EQ(contiguous_mask(0, 3), _mask(0, 1, 2)); - CHECK_EQ(contiguous_mask(0, 4), _mask(0, 1, 2, 3)); - CHECK_EQ(contiguous_mask(1, 4), _mask( 1, 2, 3)); - CHECK_EQ(contiguous_mask(2, 4), _mask( 2, 3)); - CHECK_EQ(contiguous_mask(3, 4), _mask( 3)); - CHECK_EQ(contiguous_mask(4, 4), _mask()); -} - -TEST_CASE_TEMPLATE("contiguous_mask11.basic", T, int8_t, uint8_t, int16_t, uint16_t, int32_t, uint32_t, int64_t, uint64_t) -{ - CHECK_EQ((contiguous_mask11::value), _mask()); - CHECK_EQ((contiguous_mask11::value), _mask(0)); - CHECK_EQ((contiguous_mask11::value), _mask(0, 1)); - CHECK_EQ((contiguous_mask11::value), _mask(0, 1, 2)); - CHECK_EQ((contiguous_mask11::value), _mask(0, 1, 2, 3)); - CHECK_EQ((contiguous_mask11::value), _mask( 1, 2, 3)); - CHECK_EQ((contiguous_mask11::value), _mask( 2, 3)); - CHECK_EQ((contiguous_mask11::value), _mask( 3)); - CHECK_EQ((contiguous_mask11::value), _mask()); -} - - -//----------------------------------------------------------------------------- - - -template struct sz { char buf[N]; }; -template< > struct sz<0> { }; -template void check_tp() -{ - #if defined(__clang__) - # pragma clang diagnostic push - #elif defined(__GNUC__) - # pragma GCC diagnostic push - # if __GNUC__ >= 7 - # pragma GCC diagnostic ignored "-Wduplicated-branches" - # endif - #endif - size_t expected; - if(F != 0 && S != 0) expected = F+S; - else if(F == 0 && S != 0) expected = S; - else if(F != 0 && S == 0) expected = F; // -Wduplicated-branches: false positive here - else /* F == 0 && S == 0)*/expected = 1; - #if defined(__clang__) - # pragma clang diagnostic pop - #elif defined(__GNUC__) - # pragma GCC diagnostic pop - #endif - INFO("F=" << F << " S=" << S); - CHECK_EQ(sizeof(tight_pair, sz>), expected); -} - - -TEST_CASE("tight_pair.basic") -{ - check_tp<0,0>(); - check_tp<0,1>(); - check_tp<0,2>(); - check_tp<0,3>(); - check_tp<0,4>(); - - check_tp<0,0>(); - check_tp<1,0>(); - check_tp<2,0>(); - check_tp<3,0>(); - check_tp<4,0>(); - - check_tp<0,0>(); - check_tp<1,1>(); - check_tp<2,2>(); - check_tp<3,3>(); - check_tp<4,4>(); -} - -} // namespace c4 - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_numbers.hpp b/thirdparty/ryml/ext/c4core/test/test_numbers.hpp deleted file mode 100644 index d310b36ba..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_numbers.hpp +++ /dev/null @@ -1,1863 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include -#include -#endif -#include - -namespace c4 { - -template -struct overflow64case -{ - bool is_overflow; - T wrapped; - csubstr dec; - csubstr hex; - csubstr oct; - csubstr bin; -}; - -template -struct overflow64cases; - -#define oc(T, is_overflow, wrapped, dec, hex, oct, bin) \ - overflow64case{is_overflow, wrapped, csubstr{dec}, csubstr{hex}, csubstr{oct}, csubstr{bin}} - -template<> -struct overflow64cases -{ - #define INT64_1(v) INT64_C(v) - INT64_C(1) // the min value is not representable! - static constexpr const overflow64case values[] = { - oc(int64_t, true , INT64_C( 9223372036854775803), "-9223372036854775813", "-0x8000000000000005", "-0o1000000000000000000005", "-0b1000000000000000000000000000000000000000000000000000000000000101"), - oc(int64_t, true , INT64_C( 9223372036854775804), "-9223372036854775812", "-0x8000000000000004", "-0o1000000000000000000004", "-0b1000000000000000000000000000000000000000000000000000000000000100"), - oc(int64_t, true , INT64_C( 9223372036854775805), "-9223372036854775811", "-0x8000000000000003", "-0o1000000000000000000003", "-0b1000000000000000000000000000000000000000000000000000000000000011"), - oc(int64_t, true , INT64_C( 9223372036854775806), "-9223372036854775810", "-0x8000000000000002", "-0o1000000000000000000002", "-0b1000000000000000000000000000000000000000000000000000000000000010"), - oc(int64_t, true , INT64_C( 9223372036854775807), "-9223372036854775809", "-0x8000000000000001", "-0o1000000000000000000001", "-0b1000000000000000000000000000000000000000000000000000000000000001"), - oc(int64_t, false, INT64_1(-9223372036854775807), "-9223372036854775808", "-0x8000000000000000", "-0o1000000000000000000000", "-0b1000000000000000000000000000000000000000000000000000000000000000"), - oc(int64_t, false, INT64_C(-9223372036854775807), "-9223372036854775807", "-0x7fffffffffffffff", "-0o777777777777777777777" , "-0b111111111111111111111111111111111111111111111111111111111111111"), - oc(int64_t, false, INT64_C(-9223372036854775806), "-9223372036854775806", "-0x7ffffffffffffffe", "-0o777777777777777777776" , "-0b111111111111111111111111111111111111111111111111111111111111110"), - oc(int64_t, false, INT64_C(-9223372036854775805), "-9223372036854775805", "-0x7ffffffffffffffd", "-0o777777777777777777775" , "-0b111111111111111111111111111111111111111111111111111111111111101"), - oc(int64_t, false, INT64_C( 9223372036854775804), "9223372036854775804", "0x7ffffffffffffffc", "0o777777777777777777774" , "0b111111111111111111111111111111111111111111111111111111111111100"), - oc(int64_t, false, INT64_C( 9223372036854775805), "9223372036854775805", "0x7ffffffffffffffd", "0o777777777777777777775" , "0b111111111111111111111111111111111111111111111111111111111111101"), - oc(int64_t, false, INT64_C( 9223372036854775806), "9223372036854775806", "0x7ffffffffffffffe", "0o777777777777777777776" , "0b111111111111111111111111111111111111111111111111111111111111110"), - oc(int64_t, false, INT64_C( 9223372036854775807), "9223372036854775807", "0x7fffffffffffffff", "0o777777777777777777777" , "0b111111111111111111111111111111111111111111111111111111111111111"), - oc(int64_t, true , INT64_1(-9223372036854775807), "9223372036854775808", "0x8000000000000000", "0o1000000000000000000000", "0b1000000000000000000000000000000000000000000000000000000000000000"), - oc(int64_t, true , INT64_C(-9223372036854775807), "9223372036854775809", "0x8000000000000001", "0o1000000000000000000001", "0b1000000000000000000000000000000000000000000000000000000000000001"), - oc(int64_t, true , INT64_C(-9223372036854775806), "9223372036854775810", "0x8000000000000002", "0o1000000000000000000002", "0b1000000000000000000000000000000000000000000000000000000000000010"), - oc(int64_t, true , INT64_C(-9223372036854775805), "9223372036854775811", "0x8000000000000003", "0o1000000000000000000003", "0b1000000000000000000000000000000000000000000000000000000000000011"), - oc(int64_t, true , INT64_C(-9223372036854775804), "9223372036854775812", "0x8000000000000004", "0o1000000000000000000004", "0b1000000000000000000000000000000000000000000000000000000000000100"), - }; -}; - -template<> -struct overflow64cases -{ - static constexpr const overflow64case values[] = { - oc(uint64_t, true , UINT64_C(18446744073709551611), "-5" , "-0x5" , "-0o5" , "-0b101"), - oc(uint64_t, true , UINT64_C(18446744073709551612), "-4" , "-0x4" , "-0o4" , "-0b100"), - oc(uint64_t, true , UINT64_C(18446744073709551613), "-3" , "-0x3" , "-0o3" , "-0b11"), - oc(uint64_t, true , UINT64_C(18446744073709551614), "-2" , "-0x2" , "-0o2" , "-0b10"), - oc(uint64_t, true , UINT64_C(18446744073709551615), "-1" , "-0x1" , "-0o1" , "-0b1"), - oc(uint64_t, false, UINT64_C( 0), "0" , "0x0" , "0o0" , "0b0"), - oc(uint64_t, false, UINT64_C( 1), "1" , "0x1" , "0o1" , "0b1"), - oc(uint64_t, false, UINT64_C( 2), "2" , "0x2" , "0o2" , "0b10"), - oc(uint64_t, false, UINT64_C( 3), "3" , "0x3" , "0o3" , "0b11"), - oc(uint64_t, false, UINT64_C(18446744073709551612), "18446744073709551612", "0xfffffffffffffffc" , "0o1777777777777777777774" , "0b1111111111111111111111111111111111111111111111111111111111111100"), - oc(uint64_t, false, UINT64_C(18446744073709551613), "18446744073709551613", "0xfffffffffffffffd" , "0o1777777777777777777775" , "0b1111111111111111111111111111111111111111111111111111111111111101"), - oc(uint64_t, false, UINT64_C(18446744073709551614), "18446744073709551614", "0xfffffffffffffffe" , "0o1777777777777777777776" , "0b1111111111111111111111111111111111111111111111111111111111111110"), - oc(uint64_t, false, UINT64_C(18446744073709551615), "18446744073709551615", "0xffffffffffffffff" , "0o1777777777777777777777" , "0b1111111111111111111111111111111111111111111111111111111111111111"), - oc(uint64_t, true , UINT64_C( 0), "18446744073709551616", "0x10000000000000000", "0o20000000000000000000000", "0b10000000000000000000000000000000000000000000000000000000000000000"), - oc(uint64_t, true , UINT64_C( 1), "18446744073709551617", "0x10000000000000001", "0o20000000000000000000001", "0b10000000000000000000000000000000000000000000000000000000000000001"), - oc(uint64_t, true , UINT64_C( 2), "18446744073709551618", "0x10000000000000002", "0o20000000000000000000002", "0b10000000000000000000000000000000000000000000000000000000000000010"), - oc(uint64_t, true , UINT64_C( 3), "18446744073709551619", "0x10000000000000003", "0o20000000000000000000003", "0b10000000000000000000000000000000000000000000000000000000000000011"), - oc(uint64_t, true , UINT64_C( 4), "18446744073709551620", "0x10000000000000004", "0o20000000000000000000004", "0b10000000000000000000000000000000000000000000000000000000000000100"), - oc(uint64_t, true , UINT64_C( 5), "18446744073709551621", "0x10000000000000005", "0o20000000000000000000005", "0b10000000000000000000000000000000000000000000000000000000000000101"), - }; -}; - -constexpr const overflow64case overflow64cases::values[]; -constexpr const overflow64case overflow64cases::values[]; - -#undef oc - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -struct invalid_case -{ - csubstr dec, hex, oct, bin; -}; - -C4_INLINE_CONSTEXPR const invalid_case invalid_cases[] = { -#define ic(dec, hex, oct, bin) \ - invalid_case{csubstr{dec}, csubstr{hex}, csubstr{oct}, csubstr{bin}} - ic("" , "" , "" , ""), - ic(" " , " " , " " , " "), - ic("." , "." , "." , "."), - ic("-" , "-" , "-" , "-"), - ic("\t" , "\t" , "\t" , "\t"), - ic("\n" , "\n" , "\n" , "\n"), - ic("...", "..." , "..." , "..."), - ic("===", "===" , "===" , "==="), - ic("=??", "???" , "???" , "???"), - ic("12a", "0x12g", "0o128", "0b102"), - ic("0.1", "0x1.2", "0o1.2", "0b1.1"), - ic("0,1", "0x1,2", "0o1,2", "0b1,1"), - ic("zz" , "0xzz" , "0ozz" , "0bzz"), - ic("" , "0x" , "0o" , "0b"), - ic("- " , "-0x" , "-0o" , "-0b"), - ic("2 0", "0x2 0", "0o2 0", "0b1 0"), - ic(" 2" , " 0x2" , " 0o2" , " 0b1"), - ic("\t2", "\t0x2", "\t0o2", "\t0b1"), - ic("\n2", "nt0x2", "\n0o2", "\n0b1"), - ic("2 " , "0x2 " , "0o2 " , "0b1 "), - ic("2\t", "0x2\t", "0o2\t", "0b1\t"), - ic("2\n", "0x2\n", "0o2\n", "0b1\n"), - ic("nan", "nan", "nan", "nan"), - ic("NaN", "NaN", "NaN", "NaN"), - ic("Inf", "Inf", "Inf", "Inf"), - ic("inf", "inf", "inf", "inf"), - ic("infinity", "infinity", "infinity", "infinity"), - ic("Infinity", "Infinity", "Infinity", "Infinity"), - ic("somevalue", "somevalue", "somevalue", "somevalue"), - ic("2345kjhiuy3245", "2345kjhiuy3245", "2345kjhiuy3245", "2345kjhiuy3245"), -#undef ic -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -struct number_case -{ - T val; - csubstr dec; - csubstr hex; - csubstr oct; - csubstr bin; -}; - -template -std::ostream& operator<< (std::ostream& s, number_case const& nc) -{ - using upcast_t = typename std::conditional::value, - int64_t, - uint64_t>::type>::type; - s << "val=" << (upcast_t)nc.val << " " - << "dec=" << nc.dec << " " - << "hex=" << nc.hex << " " - << "oct=" << nc.oct << " " - << "bin=" << nc.bin; - return s; -} - -template struct numbers; - -#define ITER_NUMBERS(ty, number_name) for(number_case const& C4_RESTRICT number_name : numbers::vals) - -C4_SUPPRESS_WARNING_MSVC_PUSH -C4_SUPPRESS_WARNING_MSVC(4146) -C4_SUPPRESS_WARNING_MSVC(4305) -C4_SUPPRESS_WARNING_MSVC(4310) - -// these numbers were generated with printintegers.py, in this dir. - -template<> -struct numbers -{ - using value_type = int8_t; - static C4_INLINE_CONSTEXPR const number_case vals[] = { -#define nc(val, dec, hex, bin, oct) \ - number_case{(value_type)INT8_C(val), csubstr{dec}, csubstr{hex}, csubstr{bin}, csubstr{oct}} - nc(-128, "-128", "-0x80", "-0o200", "-0b10000000"), - nc(-127, "-127", "-0x7f", "-0o177", "-0b1111111"), - nc(-126, "-126", "-0x7e", "-0o176", "-0b1111110"), - nc(-125, "-125", "-0x7d", "-0o175", "-0b1111101"), - nc(-124, "-124", "-0x7c", "-0o174", "-0b1111100"), - nc(-123, "-123", "-0x7b", "-0o173", "-0b1111011"), - nc(-101, "-101", "-0x65", "-0o145", "-0b1100101"), - nc(-100, "-100", "-0x64", "-0o144", "-0b1100100"), - nc(-99, "-99", "-0x63", "-0o143", "-0b1100011"), - nc(-65, "-65", "-0x41", "-0o101", "-0b1000001"), - nc(-64, "-64", "-0x40", "-0o100", "-0b1000000"), - nc(-63, "-63", "-0x3f", "-0o77", "-0b111111"), - nc(-33, "-33", "-0x21", "-0o41", "-0b100001"), - nc(-32, "-32", "-0x20", "-0o40", "-0b100000"), - nc(-31, "-31", "-0x1f", "-0o37", "-0b11111"), - nc(-17, "-17", "-0x11", "-0o21", "-0b10001"), - nc(-16, "-16", "-0x10", "-0o20", "-0b10000"), - nc(-15, "-15", "-0xf", "-0o17", "-0b1111"), - nc(-12, "-12", "-0xc", "-0o14", "-0b1100"), - nc(-11, "-11", "-0xb", "-0o13", "-0b1011"), - nc(-10, "-10", "-0xa", "-0o12", "-0b1010"), - nc(-9, "-9", "-0x9", "-0o11", "-0b1001"), - nc(-8, "-8", "-0x8", "-0o10", "-0b1000"), - nc(-7, "-7", "-0x7", "-0o7", "-0b111"), - nc(-6, "-6", "-0x6", "-0o6", "-0b110"), - nc(-5, "-5", "-0x5", "-0o5", "-0b101"), - nc(-4, "-4", "-0x4", "-0o4", "-0b100"), - nc(-3, "-3", "-0x3", "-0o3", "-0b11"), - nc(-2, "-2", "-0x2", "-0o2", "-0b10"), - nc(-1, "-1", "-0x1", "-0o1", "-0b1"), - nc(0, "0", "0x0", "0o0", "0b0"), - nc(1, "1", "0x1", "0o1", "0b1"), - nc(2, "2", "0x2", "0o2", "0b10"), - nc(3, "3", "0x3", "0o3", "0b11"), - nc(4, "4", "0x4", "0o4", "0b100"), - nc(5, "5", "0x5", "0o5", "0b101"), - nc(6, "6", "0x6", "0o6", "0b110"), - nc(7, "7", "0x7", "0o7", "0b111"), - nc(8, "8", "0x8", "0o10", "0b1000"), - nc(9, "9", "0x9", "0o11", "0b1001"), - nc(10, "10", "0xa", "0o12", "0b1010"), - nc(11, "11", "0xb", "0o13", "0b1011"), - nc(12, "12", "0xc", "0o14", "0b1100"), - nc(13, "13", "0xd", "0o15", "0b1101"), - nc(15, "15", "0xf", "0o17", "0b1111"), - nc(16, "16", "0x10", "0o20", "0b10000"), - nc(17, "17", "0x11", "0o21", "0b10001"), - nc(31, "31", "0x1f", "0o37", "0b11111"), - nc(32, "32", "0x20", "0o40", "0b100000"), - nc(33, "33", "0x21", "0o41", "0b100001"), - nc(63, "63", "0x3f", "0o77", "0b111111"), - nc(64, "64", "0x40", "0o100", "0b1000000"), - nc(65, "65", "0x41", "0o101", "0b1000001"), - nc(99, "99", "0x63", "0o143", "0b1100011"), - nc(100, "100", "0x64", "0o144", "0b1100100"), - nc(101, "101", "0x65", "0o145", "0b1100101"), - nc(122, "122", "0x7a", "0o172", "0b1111010"), - nc(123, "123", "0x7b", "0o173", "0b1111011"), - nc(124, "124", "0x7c", "0o174", "0b1111100"), - nc(125, "125", "0x7d", "0o175", "0b1111101"), - nc(126, "126", "0x7e", "0o176", "0b1111110"), - nc(127, "127", "0x7f", "0o177", "0b1111111"), -#undef nc - }; -}; - - -template<> -struct numbers -{ - using value_type = uint8_t; - static C4_INLINE_CONSTEXPR const number_case vals[] = { -#define nc(val, dec, hex, bin, oct) \ - number_case{(value_type)UINT8_C(val), csubstr{dec}, csubstr{hex}, csubstr{bin}, csubstr{oct}} - nc(0, "0", "0x0", "0o0", "0b0"), - nc(1, "1", "0x1", "0o1", "0b1"), - nc(2, "2", "0x2", "0o2", "0b10"), - nc(3, "3", "0x3", "0o3", "0b11"), - nc(4, "4", "0x4", "0o4", "0b100"), - nc(5, "5", "0x5", "0o5", "0b101"), - nc(6, "6", "0x6", "0o6", "0b110"), - nc(7, "7", "0x7", "0o7", "0b111"), - nc(8, "8", "0x8", "0o10", "0b1000"), - nc(9, "9", "0x9", "0o11", "0b1001"), - nc(10, "10", "0xa", "0o12", "0b1010"), - nc(11, "11", "0xb", "0o13", "0b1011"), - nc(15, "15", "0xf", "0o17", "0b1111"), - nc(16, "16", "0x10", "0o20", "0b10000"), - nc(17, "17", "0x11", "0o21", "0b10001"), - nc(24, "24", "0x18", "0o30", "0b11000"), - nc(25, "25", "0x19", "0o31", "0b11001"), - nc(26, "26", "0x1a", "0o32", "0b11010"), - nc(31, "31", "0x1f", "0o37", "0b11111"), - nc(32, "32", "0x20", "0o40", "0b100000"), - nc(33, "33", "0x21", "0o41", "0b100001"), - nc(63, "63", "0x3f", "0o77", "0b111111"), - nc(64, "64", "0x40", "0o100", "0b1000000"), - nc(65, "65", "0x41", "0o101", "0b1000001"), - nc(99, "99", "0x63", "0o143", "0b1100011"), - nc(100, "100", "0x64", "0o144", "0b1100100"), - nc(101, "101", "0x65", "0o145", "0b1100101"), - nc(127, "127", "0x7f", "0o177", "0b1111111"), - nc(128, "128", "0x80", "0o200", "0b10000000"), - nc(129, "129", "0x81", "0o201", "0b10000001"), - nc(251, "251", "0xfb", "0o373", "0b11111011"), - nc(252, "252", "0xfc", "0o374", "0b11111100"), - nc(253, "253", "0xfd", "0o375", "0b11111101"), - nc(254, "254", "0xfe", "0o376", "0b11111110"), - nc(255, "255", "0xff", "0o377", "0b11111111"), -#undef nc - }; -}; - - -template<> -struct numbers -{ - using value_type = int16_t; - static C4_INLINE_CONSTEXPR const number_case vals[] = { -#define nc(val, dec, hex, bin, oct) \ - number_case{(value_type)INT16_C(val), csubstr{dec}, csubstr{hex}, csubstr{bin}, csubstr{oct}} - nc(-32768, "-32768", "-0x8000", "-0o100000", "-0b1000000000000000"), - nc(-32767, "-32767", "-0x7fff", "-0o77777", "-0b111111111111111"), - nc(-32766, "-32766", "-0x7ffe", "-0o77776", "-0b111111111111110"), - nc(-32765, "-32765", "-0x7ffd", "-0o77775", "-0b111111111111101"), - nc(-32764, "-32764", "-0x7ffc", "-0o77774", "-0b111111111111100"), - nc(-32763, "-32763", "-0x7ffb", "-0o77773", "-0b111111111111011"), - nc(-16385, "-16385", "-0x4001", "-0o40001", "-0b100000000000001"), - nc(-16384, "-16384", "-0x4000", "-0o40000", "-0b100000000000000"), - nc(-16383, "-16383", "-0x3fff", "-0o37777", "-0b11111111111111"), - nc(-10001, "-10001", "-0x2711", "-0o23421", "-0b10011100010001"), - nc(-10000, "-10000", "-0x2710", "-0o23420", "-0b10011100010000"), - nc(-9999, "-9999", "-0x270f", "-0o23417", "-0b10011100001111"), - nc(-8193, "-8193", "-0x2001", "-0o20001", "-0b10000000000001"), - nc(-8192, "-8192", "-0x2000", "-0o20000", "-0b10000000000000"), - nc(-8191, "-8191", "-0x1fff", "-0o17777", "-0b1111111111111"), - nc(-4097, "-4097", "-0x1001", "-0o10001", "-0b1000000000001"), - nc(-4096, "-4096", "-0x1000", "-0o10000", "-0b1000000000000"), - nc(-4095, "-4095", "-0xfff", "-0o7777", "-0b111111111111"), - nc(-3276, "-3276", "-0xccc", "-0o6314", "-0b110011001100"), - nc(-2049, "-2049", "-0x801", "-0o4001", "-0b100000000001"), - nc(-2048, "-2048", "-0x800", "-0o4000", "-0b100000000000"), - nc(-2047, "-2047", "-0x7ff", "-0o3777", "-0b11111111111"), - nc(-1025, "-1025", "-0x401", "-0o2001", "-0b10000000001"), - nc(-1024, "-1024", "-0x400", "-0o2000", "-0b10000000000"), - nc(-1023, "-1023", "-0x3ff", "-0o1777", "-0b1111111111"), - nc(-1001, "-1001", "-0x3e9", "-0o1751", "-0b1111101001"), - nc(-1000, "-1000", "-0x3e8", "-0o1750", "-0b1111101000"), - nc(-999, "-999", "-0x3e7", "-0o1747", "-0b1111100111"), - nc(-513, "-513", "-0x201", "-0o1001", "-0b1000000001"), - nc(-512, "-512", "-0x200", "-0o1000", "-0b1000000000"), - nc(-511, "-511", "-0x1ff", "-0o777", "-0b111111111"), - nc(-327, "-327", "-0x147", "-0o507", "-0b101000111"), - nc(-257, "-257", "-0x101", "-0o401", "-0b100000001"), - nc(-256, "-256", "-0x100", "-0o400", "-0b100000000"), - nc(-255, "-255", "-0xff", "-0o377", "-0b11111111"), - nc(-129, "-129", "-0x81", "-0o201", "-0b10000001"), - nc(-128, "-128", "-0x80", "-0o200", "-0b10000000"), - nc(-127, "-127", "-0x7f", "-0o177", "-0b1111111"), - nc(-101, "-101", "-0x65", "-0o145", "-0b1100101"), - nc(-100, "-100", "-0x64", "-0o144", "-0b1100100"), - nc(-99, "-99", "-0x63", "-0o143", "-0b1100011"), - nc(-65, "-65", "-0x41", "-0o101", "-0b1000001"), - nc(-64, "-64", "-0x40", "-0o100", "-0b1000000"), - nc(-63, "-63", "-0x3f", "-0o77", "-0b111111"), - nc(-33, "-33", "-0x21", "-0o41", "-0b100001"), - nc(-32, "-32", "-0x20", "-0o40", "-0b100000"), - nc(-31, "-31", "-0x1f", "-0o37", "-0b11111"), - nc(-17, "-17", "-0x11", "-0o21", "-0b10001"), - nc(-16, "-16", "-0x10", "-0o20", "-0b10000"), - nc(-15, "-15", "-0xf", "-0o17", "-0b1111"), - nc(-11, "-11", "-0xb", "-0o13", "-0b1011"), - nc(-10, "-10", "-0xa", "-0o12", "-0b1010"), - nc(-9, "-9", "-0x9", "-0o11", "-0b1001"), - nc(-8, "-8", "-0x8", "-0o10", "-0b1000"), - nc(-7, "-7", "-0x7", "-0o7", "-0b111"), - nc(-6, "-6", "-0x6", "-0o6", "-0b110"), - nc(-5, "-5", "-0x5", "-0o5", "-0b101"), - nc(-4, "-4", "-0x4", "-0o4", "-0b100"), - nc(-3, "-3", "-0x3", "-0o3", "-0b11"), - nc(-2, "-2", "-0x2", "-0o2", "-0b10"), - nc(-1, "-1", "-0x1", "-0o1", "-0b1"), - nc(0, "0", "0x0", "0o0", "0b0"), - nc(1, "1", "0x1", "0o1", "0b1"), - nc(2, "2", "0x2", "0o2", "0b10"), - nc(3, "3", "0x3", "0o3", "0b11"), - nc(4, "4", "0x4", "0o4", "0b100"), - nc(5, "5", "0x5", "0o5", "0b101"), - nc(6, "6", "0x6", "0o6", "0b110"), - nc(7, "7", "0x7", "0o7", "0b111"), - nc(8, "8", "0x8", "0o10", "0b1000"), - nc(9, "9", "0x9", "0o11", "0b1001"), - nc(10, "10", "0xa", "0o12", "0b1010"), - nc(11, "11", "0xb", "0o13", "0b1011"), - nc(15, "15", "0xf", "0o17", "0b1111"), - nc(16, "16", "0x10", "0o20", "0b10000"), - nc(17, "17", "0x11", "0o21", "0b10001"), - nc(31, "31", "0x1f", "0o37", "0b11111"), - nc(32, "32", "0x20", "0o40", "0b100000"), - nc(33, "33", "0x21", "0o41", "0b100001"), - nc(63, "63", "0x3f", "0o77", "0b111111"), - nc(64, "64", "0x40", "0o100", "0b1000000"), - nc(65, "65", "0x41", "0o101", "0b1000001"), - nc(99, "99", "0x63", "0o143", "0b1100011"), - nc(100, "100", "0x64", "0o144", "0b1100100"), - nc(101, "101", "0x65", "0o145", "0b1100101"), - nc(127, "127", "0x7f", "0o177", "0b1111111"), - nc(128, "128", "0x80", "0o200", "0b10000000"), - nc(129, "129", "0x81", "0o201", "0b10000001"), - nc(255, "255", "0xff", "0o377", "0b11111111"), - nc(256, "256", "0x100", "0o400", "0b100000000"), - nc(257, "257", "0x101", "0o401", "0b100000001"), - nc(326, "326", "0x146", "0o506", "0b101000110"), - nc(327, "327", "0x147", "0o507", "0b101000111"), - nc(328, "328", "0x148", "0o510", "0b101001000"), - nc(511, "511", "0x1ff", "0o777", "0b111111111"), - nc(512, "512", "0x200", "0o1000", "0b1000000000"), - nc(513, "513", "0x201", "0o1001", "0b1000000001"), - nc(999, "999", "0x3e7", "0o1747", "0b1111100111"), - nc(1000, "1000", "0x3e8", "0o1750", "0b1111101000"), - nc(1001, "1001", "0x3e9", "0o1751", "0b1111101001"), - nc(1023, "1023", "0x3ff", "0o1777", "0b1111111111"), - nc(1024, "1024", "0x400", "0o2000", "0b10000000000"), - nc(1025, "1025", "0x401", "0o2001", "0b10000000001"), - nc(2047, "2047", "0x7ff", "0o3777", "0b11111111111"), - nc(2048, "2048", "0x800", "0o4000", "0b100000000000"), - nc(2049, "2049", "0x801", "0o4001", "0b100000000001"), - nc(3275, "3275", "0xccb", "0o6313", "0b110011001011"), - nc(3276, "3276", "0xccc", "0o6314", "0b110011001100"), - nc(3277, "3277", "0xccd", "0o6315", "0b110011001101"), - nc(4095, "4095", "0xfff", "0o7777", "0b111111111111"), - nc(4096, "4096", "0x1000", "0o10000", "0b1000000000000"), - nc(4097, "4097", "0x1001", "0o10001", "0b1000000000001"), - nc(8191, "8191", "0x1fff", "0o17777", "0b1111111111111"), - nc(8192, "8192", "0x2000", "0o20000", "0b10000000000000"), - nc(8193, "8193", "0x2001", "0o20001", "0b10000000000001"), - nc(9999, "9999", "0x270f", "0o23417", "0b10011100001111"), - nc(10000, "10000", "0x2710", "0o23420", "0b10011100010000"), - nc(10001, "10001", "0x2711", "0o23421", "0b10011100010001"), - nc(16383, "16383", "0x3fff", "0o37777", "0b11111111111111"), - nc(16384, "16384", "0x4000", "0o40000", "0b100000000000000"), - nc(16385, "16385", "0x4001", "0o40001", "0b100000000000001"), - nc(32762, "32762", "0x7ffa", "0o77772", "0b111111111111010"), - nc(32763, "32763", "0x7ffb", "0o77773", "0b111111111111011"), - nc(32764, "32764", "0x7ffc", "0o77774", "0b111111111111100"), - nc(32765, "32765", "0x7ffd", "0o77775", "0b111111111111101"), - nc(32766, "32766", "0x7ffe", "0o77776", "0b111111111111110"), - nc(32767, "32767", "0x7fff", "0o77777", "0b111111111111111"), -#undef nc - }; -}; - - -template<> -struct numbers -{ - using value_type = uint16_t; - static C4_INLINE_CONSTEXPR const number_case vals[] = { -#define nc(val, dec, hex, bin, oct) \ - number_case{(value_type)UINT16_C(val), csubstr{dec}, csubstr{hex}, csubstr{bin}, csubstr{oct}} - nc(0, "0", "0x0", "0o0", "0b0"), - nc(1, "1", "0x1", "0o1", "0b1"), - nc(2, "2", "0x2", "0o2", "0b10"), - nc(3, "3", "0x3", "0o3", "0b11"), - nc(4, "4", "0x4", "0o4", "0b100"), - nc(5, "5", "0x5", "0o5", "0b101"), - nc(6, "6", "0x6", "0o6", "0b110"), - nc(7, "7", "0x7", "0o7", "0b111"), - nc(8, "8", "0x8", "0o10", "0b1000"), - nc(9, "9", "0x9", "0o11", "0b1001"), - nc(10, "10", "0xa", "0o12", "0b1010"), - nc(11, "11", "0xb", "0o13", "0b1011"), - nc(15, "15", "0xf", "0o17", "0b1111"), - nc(16, "16", "0x10", "0o20", "0b10000"), - nc(17, "17", "0x11", "0o21", "0b10001"), - nc(31, "31", "0x1f", "0o37", "0b11111"), - nc(32, "32", "0x20", "0o40", "0b100000"), - nc(33, "33", "0x21", "0o41", "0b100001"), - nc(63, "63", "0x3f", "0o77", "0b111111"), - nc(64, "64", "0x40", "0o100", "0b1000000"), - nc(65, "65", "0x41", "0o101", "0b1000001"), - nc(66, "66", "0x42", "0o102", "0b1000010"), - nc(99, "99", "0x63", "0o143", "0b1100011"), - nc(100, "100", "0x64", "0o144", "0b1100100"), - nc(101, "101", "0x65", "0o145", "0b1100101"), - nc(127, "127", "0x7f", "0o177", "0b1111111"), - nc(128, "128", "0x80", "0o200", "0b10000000"), - nc(129, "129", "0x81", "0o201", "0b10000001"), - nc(255, "255", "0xff", "0o377", "0b11111111"), - nc(256, "256", "0x100", "0o400", "0b100000000"), - nc(257, "257", "0x101", "0o401", "0b100000001"), - nc(511, "511", "0x1ff", "0o777", "0b111111111"), - nc(512, "512", "0x200", "0o1000", "0b1000000000"), - nc(513, "513", "0x201", "0o1001", "0b1000000001"), - nc(654, "654", "0x28e", "0o1216", "0b1010001110"), - nc(655, "655", "0x28f", "0o1217", "0b1010001111"), - nc(656, "656", "0x290", "0o1220", "0b1010010000"), - nc(999, "999", "0x3e7", "0o1747", "0b1111100111"), - nc(1000, "1000", "0x3e8", "0o1750", "0b1111101000"), - nc(1001, "1001", "0x3e9", "0o1751", "0b1111101001"), - nc(1023, "1023", "0x3ff", "0o1777", "0b1111111111"), - nc(1024, "1024", "0x400", "0o2000", "0b10000000000"), - nc(1025, "1025", "0x401", "0o2001", "0b10000000001"), - nc(2047, "2047", "0x7ff", "0o3777", "0b11111111111"), - nc(2048, "2048", "0x800", "0o4000", "0b100000000000"), - nc(2049, "2049", "0x801", "0o4001", "0b100000000001"), - nc(4095, "4095", "0xfff", "0o7777", "0b111111111111"), - nc(4096, "4096", "0x1000", "0o10000", "0b1000000000000"), - nc(4097, "4097", "0x1001", "0o10001", "0b1000000000001"), - nc(6552, "6552", "0x1998", "0o14630", "0b1100110011000"), - nc(6553, "6553", "0x1999", "0o14631", "0b1100110011001"), - nc(6554, "6554", "0x199a", "0o14632", "0b1100110011010"), - nc(8191, "8191", "0x1fff", "0o17777", "0b1111111111111"), - nc(8192, "8192", "0x2000", "0o20000", "0b10000000000000"), - nc(8193, "8193", "0x2001", "0o20001", "0b10000000000001"), - nc(9999, "9999", "0x270f", "0o23417", "0b10011100001111"), - nc(10000, "10000", "0x2710", "0o23420", "0b10011100010000"), - nc(10001, "10001", "0x2711", "0o23421", "0b10011100010001"), - nc(16383, "16383", "0x3fff", "0o37777", "0b11111111111111"), - nc(16384, "16384", "0x4000", "0o40000", "0b100000000000000"), - nc(16385, "16385", "0x4001", "0o40001", "0b100000000000001"), - nc(32767, "32767", "0x7fff", "0o77777", "0b111111111111111"), - nc(32768, "32768", "0x8000", "0o100000", "0b1000000000000000"), - nc(32769, "32769", "0x8001", "0o100001", "0b1000000000000001"), - nc(65531, "65531", "0xfffb", "0o177773", "0b1111111111111011"), - nc(65532, "65532", "0xfffc", "0o177774", "0b1111111111111100"), - nc(65533, "65533", "0xfffd", "0o177775", "0b1111111111111101"), - nc(65534, "65534", "0xfffe", "0o177776", "0b1111111111111110"), - nc(65535, "65535", "0xffff", "0o177777", "0b1111111111111111"), -#undef nc - }; -}; - - -template<> -struct numbers -{ - using value_type = int32_t; - static C4_INLINE_CONSTEXPR const number_case vals[] = { -#define nc(val, dec, hex, bin, oct) \ - number_case{(value_type)INT32_C(val), csubstr{dec}, csubstr{hex}, csubstr{bin}, csubstr{oct}} - nc(-2147483648, "-2147483648", "-0x80000000", "-0o20000000000", "-0b10000000000000000000000000000000"), - nc(-2147483647, "-2147483647", "-0x7fffffff", "-0o17777777777", "-0b1111111111111111111111111111111"), - nc(-2147483646, "-2147483646", "-0x7ffffffe", "-0o17777777776", "-0b1111111111111111111111111111110"), - nc(-2147483645, "-2147483645", "-0x7ffffffd", "-0o17777777775", "-0b1111111111111111111111111111101"), - nc(-2147483644, "-2147483644", "-0x7ffffffc", "-0o17777777774", "-0b1111111111111111111111111111100"), - nc(-2147483643, "-2147483643", "-0x7ffffffb", "-0o17777777773", "-0b1111111111111111111111111111011"), - nc(-1073741825, "-1073741825", "-0x40000001", "-0o10000000001", "-0b1000000000000000000000000000001"), - nc(-1073741824, "-1073741824", "-0x40000000", "-0o10000000000", "-0b1000000000000000000000000000000"), - nc(-1073741823, "-1073741823", "-0x3fffffff", "-0o7777777777", "-0b111111111111111111111111111111"), - nc(-1000000001, "-1000000001", "-0x3b9aca01", "-0o7346545001", "-0b111011100110101100101000000001"), - nc(-1000000000, "-1000000000", "-0x3b9aca00", "-0o7346545000", "-0b111011100110101100101000000000"), - nc(-999999999, "-999999999", "-0x3b9ac9ff", "-0o7346544777", "-0b111011100110101100100111111111"), - nc(-536870913, "-536870913", "-0x20000001", "-0o4000000001", "-0b100000000000000000000000000001"), - nc(-536870912, "-536870912", "-0x20000000", "-0o4000000000", "-0b100000000000000000000000000000"), - nc(-536870911, "-536870911", "-0x1fffffff", "-0o3777777777", "-0b11111111111111111111111111111"), - nc(-268435457, "-268435457", "-0x10000001", "-0o2000000001", "-0b10000000000000000000000000001"), - nc(-268435456, "-268435456", "-0x10000000", "-0o2000000000", "-0b10000000000000000000000000000"), - nc(-268435455, "-268435455", "-0xfffffff", "-0o1777777777", "-0b1111111111111111111111111111"), - nc(-214748364, "-214748364", "-0xccccccc", "-0o1463146314", "-0b1100110011001100110011001100"), - nc(-134217729, "-134217729", "-0x8000001", "-0o1000000001", "-0b1000000000000000000000000001"), - nc(-134217728, "-134217728", "-0x8000000", "-0o1000000000", "-0b1000000000000000000000000000"), - nc(-134217727, "-134217727", "-0x7ffffff", "-0o777777777", "-0b111111111111111111111111111"), - nc(-100000001, "-100000001", "-0x5f5e101", "-0o575360401", "-0b101111101011110000100000001"), - nc(-100000000, "-100000000", "-0x5f5e100", "-0o575360400", "-0b101111101011110000100000000"), - nc(-99999999, "-99999999", "-0x5f5e0ff", "-0o575360377", "-0b101111101011110000011111111"), - nc(-67108865, "-67108865", "-0x4000001", "-0o400000001", "-0b100000000000000000000000001"), - nc(-67108864, "-67108864", "-0x4000000", "-0o400000000", "-0b100000000000000000000000000"), - nc(-67108863, "-67108863", "-0x3ffffff", "-0o377777777", "-0b11111111111111111111111111"), - nc(-33554433, "-33554433", "-0x2000001", "-0o200000001", "-0b10000000000000000000000001"), - nc(-33554432, "-33554432", "-0x2000000", "-0o200000000", "-0b10000000000000000000000000"), - nc(-33554431, "-33554431", "-0x1ffffff", "-0o177777777", "-0b1111111111111111111111111"), - nc(-21474836, "-21474836", "-0x147ae14", "-0o121727024", "-0b1010001111010111000010100"), - nc(-16777217, "-16777217", "-0x1000001", "-0o100000001", "-0b1000000000000000000000001"), - nc(-16777216, "-16777216", "-0x1000000", "-0o100000000", "-0b1000000000000000000000000"), - nc(-16777215, "-16777215", "-0xffffff", "-0o77777777", "-0b111111111111111111111111"), - nc(-10000001, "-10000001", "-0x989681", "-0o46113201", "-0b100110001001011010000001"), - nc(-10000000, "-10000000", "-0x989680", "-0o46113200", "-0b100110001001011010000000"), - nc(-9999999, "-9999999", "-0x98967f", "-0o46113177", "-0b100110001001011001111111"), - nc(-8388609, "-8388609", "-0x800001", "-0o40000001", "-0b100000000000000000000001"), - nc(-8388608, "-8388608", "-0x800000", "-0o40000000", "-0b100000000000000000000000"), - nc(-8388607, "-8388607", "-0x7fffff", "-0o37777777", "-0b11111111111111111111111"), - nc(-4194305, "-4194305", "-0x400001", "-0o20000001", "-0b10000000000000000000001"), - nc(-4194304, "-4194304", "-0x400000", "-0o20000000", "-0b10000000000000000000000"), - nc(-4194303, "-4194303", "-0x3fffff", "-0o17777777", "-0b1111111111111111111111"), - nc(-2147483, "-2147483", "-0x20c49b", "-0o10142233", "-0b1000001100010010011011"), - nc(-2097153, "-2097153", "-0x200001", "-0o10000001", "-0b1000000000000000000001"), - nc(-2097152, "-2097152", "-0x200000", "-0o10000000", "-0b1000000000000000000000"), - nc(-2097151, "-2097151", "-0x1fffff", "-0o7777777", "-0b111111111111111111111"), - nc(-1048577, "-1048577", "-0x100001", "-0o4000001", "-0b100000000000000000001"), - nc(-1048576, "-1048576", "-0x100000", "-0o4000000", "-0b100000000000000000000"), - nc(-1048575, "-1048575", "-0xfffff", "-0o3777777", "-0b11111111111111111111"), - nc(-1000001, "-1000001", "-0xf4241", "-0o3641101", "-0b11110100001001000001"), - nc(-1000000, "-1000000", "-0xf4240", "-0o3641100", "-0b11110100001001000000"), - nc(-999999, "-999999", "-0xf423f", "-0o3641077", "-0b11110100001000111111"), - nc(-524289, "-524289", "-0x80001", "-0o2000001", "-0b10000000000000000001"), - nc(-524288, "-524288", "-0x80000", "-0o2000000", "-0b10000000000000000000"), - nc(-524287, "-524287", "-0x7ffff", "-0o1777777", "-0b1111111111111111111"), - nc(-262145, "-262145", "-0x40001", "-0o1000001", "-0b1000000000000000001"), - nc(-262144, "-262144", "-0x40000", "-0o1000000", "-0b1000000000000000000"), - nc(-262143, "-262143", "-0x3ffff", "-0o777777", "-0b111111111111111111"), - nc(-214748, "-214748", "-0x346dc", "-0o643334", "-0b110100011011011100"), - nc(-131073, "-131073", "-0x20001", "-0o400001", "-0b100000000000000001"), - nc(-131072, "-131072", "-0x20000", "-0o400000", "-0b100000000000000000"), - nc(-131071, "-131071", "-0x1ffff", "-0o377777", "-0b11111111111111111"), - nc(-100001, "-100001", "-0x186a1", "-0o303241", "-0b11000011010100001"), - nc(-100000, "-100000", "-0x186a0", "-0o303240", "-0b11000011010100000"), - nc(-99999, "-99999", "-0x1869f", "-0o303237", "-0b11000011010011111"), - nc(-65537, "-65537", "-0x10001", "-0o200001", "-0b10000000000000001"), - nc(-65536, "-65536", "-0x10000", "-0o200000", "-0b10000000000000000"), - nc(-65535, "-65535", "-0xffff", "-0o177777", "-0b1111111111111111"), - nc(-32769, "-32769", "-0x8001", "-0o100001", "-0b1000000000000001"), - nc(-32768, "-32768", "-0x8000", "-0o100000", "-0b1000000000000000"), - nc(-32767, "-32767", "-0x7fff", "-0o77777", "-0b111111111111111"), - nc(-21474, "-21474", "-0x53e2", "-0o51742", "-0b101001111100010"), - nc(-16385, "-16385", "-0x4001", "-0o40001", "-0b100000000000001"), - nc(-16384, "-16384", "-0x4000", "-0o40000", "-0b100000000000000"), - nc(-16383, "-16383", "-0x3fff", "-0o37777", "-0b11111111111111"), - nc(-10001, "-10001", "-0x2711", "-0o23421", "-0b10011100010001"), - nc(-10000, "-10000", "-0x2710", "-0o23420", "-0b10011100010000"), - nc(-9999, "-9999", "-0x270f", "-0o23417", "-0b10011100001111"), - nc(-8193, "-8193", "-0x2001", "-0o20001", "-0b10000000000001"), - nc(-8192, "-8192", "-0x2000", "-0o20000", "-0b10000000000000"), - nc(-8191, "-8191", "-0x1fff", "-0o17777", "-0b1111111111111"), - nc(-4097, "-4097", "-0x1001", "-0o10001", "-0b1000000000001"), - nc(-4096, "-4096", "-0x1000", "-0o10000", "-0b1000000000000"), - nc(-4095, "-4095", "-0xfff", "-0o7777", "-0b111111111111"), - nc(-2147, "-2147", "-0x863", "-0o4143", "-0b100001100011"), - nc(-2049, "-2049", "-0x801", "-0o4001", "-0b100000000001"), - nc(-2048, "-2048", "-0x800", "-0o4000", "-0b100000000000"), - nc(-2047, "-2047", "-0x7ff", "-0o3777", "-0b11111111111"), - nc(-1025, "-1025", "-0x401", "-0o2001", "-0b10000000001"), - nc(-1024, "-1024", "-0x400", "-0o2000", "-0b10000000000"), - nc(-1023, "-1023", "-0x3ff", "-0o1777", "-0b1111111111"), - nc(-1001, "-1001", "-0x3e9", "-0o1751", "-0b1111101001"), - nc(-1000, "-1000", "-0x3e8", "-0o1750", "-0b1111101000"), - nc(-999, "-999", "-0x3e7", "-0o1747", "-0b1111100111"), - nc(-513, "-513", "-0x201", "-0o1001", "-0b1000000001"), - nc(-512, "-512", "-0x200", "-0o1000", "-0b1000000000"), - nc(-511, "-511", "-0x1ff", "-0o777", "-0b111111111"), - nc(-257, "-257", "-0x101", "-0o401", "-0b100000001"), - nc(-256, "-256", "-0x100", "-0o400", "-0b100000000"), - nc(-255, "-255", "-0xff", "-0o377", "-0b11111111"), - nc(-214, "-214", "-0xd6", "-0o326", "-0b11010110"), - nc(-129, "-129", "-0x81", "-0o201", "-0b10000001"), - nc(-128, "-128", "-0x80", "-0o200", "-0b10000000"), - nc(-127, "-127", "-0x7f", "-0o177", "-0b1111111"), - nc(-101, "-101", "-0x65", "-0o145", "-0b1100101"), - nc(-100, "-100", "-0x64", "-0o144", "-0b1100100"), - nc(-99, "-99", "-0x63", "-0o143", "-0b1100011"), - nc(-65, "-65", "-0x41", "-0o101", "-0b1000001"), - nc(-64, "-64", "-0x40", "-0o100", "-0b1000000"), - nc(-63, "-63", "-0x3f", "-0o77", "-0b111111"), - nc(-33, "-33", "-0x21", "-0o41", "-0b100001"), - nc(-32, "-32", "-0x20", "-0o40", "-0b100000"), - nc(-31, "-31", "-0x1f", "-0o37", "-0b11111"), - nc(-21, "-21", "-0x15", "-0o25", "-0b10101"), - nc(-17, "-17", "-0x11", "-0o21", "-0b10001"), - nc(-16, "-16", "-0x10", "-0o20", "-0b10000"), - nc(-15, "-15", "-0xf", "-0o17", "-0b1111"), - nc(-11, "-11", "-0xb", "-0o13", "-0b1011"), - nc(-10, "-10", "-0xa", "-0o12", "-0b1010"), - nc(-9, "-9", "-0x9", "-0o11", "-0b1001"), - nc(-8, "-8", "-0x8", "-0o10", "-0b1000"), - nc(-7, "-7", "-0x7", "-0o7", "-0b111"), - nc(-6, "-6", "-0x6", "-0o6", "-0b110"), - nc(-5, "-5", "-0x5", "-0o5", "-0b101"), - nc(-4, "-4", "-0x4", "-0o4", "-0b100"), - nc(-3, "-3", "-0x3", "-0o3", "-0b11"), - nc(-2, "-2", "-0x2", "-0o2", "-0b10"), - nc(-1, "-1", "-0x1", "-0o1", "-0b1"), - nc(0, "0", "0x0", "0o0", "0b0"), - nc(1, "1", "0x1", "0o1", "0b1"), - nc(2, "2", "0x2", "0o2", "0b10"), - nc(3, "3", "0x3", "0o3", "0b11"), - nc(4, "4", "0x4", "0o4", "0b100"), - nc(5, "5", "0x5", "0o5", "0b101"), - nc(6, "6", "0x6", "0o6", "0b110"), - nc(7, "7", "0x7", "0o7", "0b111"), - nc(8, "8", "0x8", "0o10", "0b1000"), - nc(9, "9", "0x9", "0o11", "0b1001"), - nc(10, "10", "0xa", "0o12", "0b1010"), - nc(11, "11", "0xb", "0o13", "0b1011"), - nc(15, "15", "0xf", "0o17", "0b1111"), - nc(16, "16", "0x10", "0o20", "0b10000"), - nc(17, "17", "0x11", "0o21", "0b10001"), - nc(20, "20", "0x14", "0o24", "0b10100"), - nc(21, "21", "0x15", "0o25", "0b10101"), - nc(22, "22", "0x16", "0o26", "0b10110"), - nc(31, "31", "0x1f", "0o37", "0b11111"), - nc(32, "32", "0x20", "0o40", "0b100000"), - nc(33, "33", "0x21", "0o41", "0b100001"), - nc(63, "63", "0x3f", "0o77", "0b111111"), - nc(64, "64", "0x40", "0o100", "0b1000000"), - nc(65, "65", "0x41", "0o101", "0b1000001"), - nc(99, "99", "0x63", "0o143", "0b1100011"), - nc(100, "100", "0x64", "0o144", "0b1100100"), - nc(101, "101", "0x65", "0o145", "0b1100101"), - nc(127, "127", "0x7f", "0o177", "0b1111111"), - nc(128, "128", "0x80", "0o200", "0b10000000"), - nc(129, "129", "0x81", "0o201", "0b10000001"), - nc(213, "213", "0xd5", "0o325", "0b11010101"), - nc(214, "214", "0xd6", "0o326", "0b11010110"), - nc(215, "215", "0xd7", "0o327", "0b11010111"), - nc(255, "255", "0xff", "0o377", "0b11111111"), - nc(256, "256", "0x100", "0o400", "0b100000000"), - nc(257, "257", "0x101", "0o401", "0b100000001"), - nc(511, "511", "0x1ff", "0o777", "0b111111111"), - nc(512, "512", "0x200", "0o1000", "0b1000000000"), - nc(513, "513", "0x201", "0o1001", "0b1000000001"), - nc(999, "999", "0x3e7", "0o1747", "0b1111100111"), - nc(1000, "1000", "0x3e8", "0o1750", "0b1111101000"), - nc(1001, "1001", "0x3e9", "0o1751", "0b1111101001"), - nc(1023, "1023", "0x3ff", "0o1777", "0b1111111111"), - nc(1024, "1024", "0x400", "0o2000", "0b10000000000"), - nc(1025, "1025", "0x401", "0o2001", "0b10000000001"), - nc(2047, "2047", "0x7ff", "0o3777", "0b11111111111"), - nc(2048, "2048", "0x800", "0o4000", "0b100000000000"), - nc(2049, "2049", "0x801", "0o4001", "0b100000000001"), - nc(2146, "2146", "0x862", "0o4142", "0b100001100010"), - nc(2147, "2147", "0x863", "0o4143", "0b100001100011"), - nc(2148, "2148", "0x864", "0o4144", "0b100001100100"), - nc(4095, "4095", "0xfff", "0o7777", "0b111111111111"), - nc(4096, "4096", "0x1000", "0o10000", "0b1000000000000"), - nc(4097, "4097", "0x1001", "0o10001", "0b1000000000001"), - nc(8191, "8191", "0x1fff", "0o17777", "0b1111111111111"), - nc(8192, "8192", "0x2000", "0o20000", "0b10000000000000"), - nc(8193, "8193", "0x2001", "0o20001", "0b10000000000001"), - nc(9999, "9999", "0x270f", "0o23417", "0b10011100001111"), - nc(10000, "10000", "0x2710", "0o23420", "0b10011100010000"), - nc(10001, "10001", "0x2711", "0o23421", "0b10011100010001"), - nc(16383, "16383", "0x3fff", "0o37777", "0b11111111111111"), - nc(16384, "16384", "0x4000", "0o40000", "0b100000000000000"), - nc(16385, "16385", "0x4001", "0o40001", "0b100000000000001"), - nc(21473, "21473", "0x53e1", "0o51741", "0b101001111100001"), - nc(21474, "21474", "0x53e2", "0o51742", "0b101001111100010"), - nc(21475, "21475", "0x53e3", "0o51743", "0b101001111100011"), - nc(32767, "32767", "0x7fff", "0o77777", "0b111111111111111"), - nc(32768, "32768", "0x8000", "0o100000", "0b1000000000000000"), - nc(32769, "32769", "0x8001", "0o100001", "0b1000000000000001"), - nc(65535, "65535", "0xffff", "0o177777", "0b1111111111111111"), - nc(65536, "65536", "0x10000", "0o200000", "0b10000000000000000"), - nc(65537, "65537", "0x10001", "0o200001", "0b10000000000000001"), - nc(99999, "99999", "0x1869f", "0o303237", "0b11000011010011111"), - nc(100000, "100000", "0x186a0", "0o303240", "0b11000011010100000"), - nc(100001, "100001", "0x186a1", "0o303241", "0b11000011010100001"), - nc(131071, "131071", "0x1ffff", "0o377777", "0b11111111111111111"), - nc(131072, "131072", "0x20000", "0o400000", "0b100000000000000000"), - nc(131073, "131073", "0x20001", "0o400001", "0b100000000000000001"), - nc(214747, "214747", "0x346db", "0o643333", "0b110100011011011011"), - nc(214748, "214748", "0x346dc", "0o643334", "0b110100011011011100"), - nc(214749, "214749", "0x346dd", "0o643335", "0b110100011011011101"), - nc(262143, "262143", "0x3ffff", "0o777777", "0b111111111111111111"), - nc(262144, "262144", "0x40000", "0o1000000", "0b1000000000000000000"), - nc(262145, "262145", "0x40001", "0o1000001", "0b1000000000000000001"), - nc(524287, "524287", "0x7ffff", "0o1777777", "0b1111111111111111111"), - nc(524288, "524288", "0x80000", "0o2000000", "0b10000000000000000000"), - nc(524289, "524289", "0x80001", "0o2000001", "0b10000000000000000001"), - nc(999999, "999999", "0xf423f", "0o3641077", "0b11110100001000111111"), - nc(1000000, "1000000", "0xf4240", "0o3641100", "0b11110100001001000000"), - nc(1000001, "1000001", "0xf4241", "0o3641101", "0b11110100001001000001"), - nc(1048575, "1048575", "0xfffff", "0o3777777", "0b11111111111111111111"), - nc(1048576, "1048576", "0x100000", "0o4000000", "0b100000000000000000000"), - nc(1048577, "1048577", "0x100001", "0o4000001", "0b100000000000000000001"), - nc(2097151, "2097151", "0x1fffff", "0o7777777", "0b111111111111111111111"), - nc(2097152, "2097152", "0x200000", "0o10000000", "0b1000000000000000000000"), - nc(2097153, "2097153", "0x200001", "0o10000001", "0b1000000000000000000001"), - nc(2147482, "2147482", "0x20c49a", "0o10142232", "0b1000001100010010011010"), - nc(2147483, "2147483", "0x20c49b", "0o10142233", "0b1000001100010010011011"), - nc(2147484, "2147484", "0x20c49c", "0o10142234", "0b1000001100010010011100"), - nc(4194303, "4194303", "0x3fffff", "0o17777777", "0b1111111111111111111111"), - nc(4194304, "4194304", "0x400000", "0o20000000", "0b10000000000000000000000"), - nc(4194305, "4194305", "0x400001", "0o20000001", "0b10000000000000000000001"), - nc(8388607, "8388607", "0x7fffff", "0o37777777", "0b11111111111111111111111"), - nc(8388608, "8388608", "0x800000", "0o40000000", "0b100000000000000000000000"), - nc(8388609, "8388609", "0x800001", "0o40000001", "0b100000000000000000000001"), - nc(9999999, "9999999", "0x98967f", "0o46113177", "0b100110001001011001111111"), - nc(10000000, "10000000", "0x989680", "0o46113200", "0b100110001001011010000000"), - nc(10000001, "10000001", "0x989681", "0o46113201", "0b100110001001011010000001"), - nc(16777215, "16777215", "0xffffff", "0o77777777", "0b111111111111111111111111"), - nc(16777216, "16777216", "0x1000000", "0o100000000", "0b1000000000000000000000000"), - nc(16777217, "16777217", "0x1000001", "0o100000001", "0b1000000000000000000000001"), - nc(21474835, "21474835", "0x147ae13", "0o121727023", "0b1010001111010111000010011"), - nc(21474836, "21474836", "0x147ae14", "0o121727024", "0b1010001111010111000010100"), - nc(21474837, "21474837", "0x147ae15", "0o121727025", "0b1010001111010111000010101"), - nc(33554431, "33554431", "0x1ffffff", "0o177777777", "0b1111111111111111111111111"), - nc(33554432, "33554432", "0x2000000", "0o200000000", "0b10000000000000000000000000"), - nc(33554433, "33554433", "0x2000001", "0o200000001", "0b10000000000000000000000001"), - nc(67108863, "67108863", "0x3ffffff", "0o377777777", "0b11111111111111111111111111"), - nc(67108864, "67108864", "0x4000000", "0o400000000", "0b100000000000000000000000000"), - nc(67108865, "67108865", "0x4000001", "0o400000001", "0b100000000000000000000000001"), - nc(99999999, "99999999", "0x5f5e0ff", "0o575360377", "0b101111101011110000011111111"), - nc(100000000, "100000000", "0x5f5e100", "0o575360400", "0b101111101011110000100000000"), - nc(100000001, "100000001", "0x5f5e101", "0o575360401", "0b101111101011110000100000001"), - nc(134217727, "134217727", "0x7ffffff", "0o777777777", "0b111111111111111111111111111"), - nc(134217728, "134217728", "0x8000000", "0o1000000000", "0b1000000000000000000000000000"), - nc(134217729, "134217729", "0x8000001", "0o1000000001", "0b1000000000000000000000000001"), - nc(214748363, "214748363", "0xccccccb", "0o1463146313", "0b1100110011001100110011001011"), - nc(214748364, "214748364", "0xccccccc", "0o1463146314", "0b1100110011001100110011001100"), - nc(214748365, "214748365", "0xccccccd", "0o1463146315", "0b1100110011001100110011001101"), - nc(268435455, "268435455", "0xfffffff", "0o1777777777", "0b1111111111111111111111111111"), - nc(268435456, "268435456", "0x10000000", "0o2000000000", "0b10000000000000000000000000000"), - nc(268435457, "268435457", "0x10000001", "0o2000000001", "0b10000000000000000000000000001"), - nc(536870911, "536870911", "0x1fffffff", "0o3777777777", "0b11111111111111111111111111111"), - nc(536870912, "536870912", "0x20000000", "0o4000000000", "0b100000000000000000000000000000"), - nc(536870913, "536870913", "0x20000001", "0o4000000001", "0b100000000000000000000000000001"), - nc(999999999, "999999999", "0x3b9ac9ff", "0o7346544777", "0b111011100110101100100111111111"), - nc(1000000000, "1000000000", "0x3b9aca00", "0o7346545000", "0b111011100110101100101000000000"), - nc(1000000001, "1000000001", "0x3b9aca01", "0o7346545001", "0b111011100110101100101000000001"), - nc(1073741823, "1073741823", "0x3fffffff", "0o7777777777", "0b111111111111111111111111111111"), - nc(1073741824, "1073741824", "0x40000000", "0o10000000000", "0b1000000000000000000000000000000"), - nc(1073741825, "1073741825", "0x40000001", "0o10000000001", "0b1000000000000000000000000000001"), - nc(2147483642, "2147483642", "0x7ffffffa", "0o17777777772", "0b1111111111111111111111111111010"), - nc(2147483643, "2147483643", "0x7ffffffb", "0o17777777773", "0b1111111111111111111111111111011"), - nc(2147483644, "2147483644", "0x7ffffffc", "0o17777777774", "0b1111111111111111111111111111100"), - nc(2147483645, "2147483645", "0x7ffffffd", "0o17777777775", "0b1111111111111111111111111111101"), - nc(2147483646, "2147483646", "0x7ffffffe", "0o17777777776", "0b1111111111111111111111111111110"), - nc(2147483647, "2147483647", "0x7fffffff", "0o17777777777", "0b1111111111111111111111111111111"), -#undef nc - }; -}; - - -template<> -struct numbers -{ - using value_type = uint32_t; - static C4_INLINE_CONSTEXPR const number_case vals[] = { -#define nc(val, dec, hex, bin, oct) \ - number_case{(value_type)UINT32_C(val), csubstr{dec}, csubstr{hex}, csubstr{bin}, csubstr{oct}} - nc(0, "0", "0x0", "0o0", "0b0"), - nc(1, "1", "0x1", "0o1", "0b1"), - nc(2, "2", "0x2", "0o2", "0b10"), - nc(3, "3", "0x3", "0o3", "0b11"), - nc(4, "4", "0x4", "0o4", "0b100"), - nc(5, "5", "0x5", "0o5", "0b101"), - nc(6, "6", "0x6", "0o6", "0b110"), - nc(7, "7", "0x7", "0o7", "0b111"), - nc(8, "8", "0x8", "0o10", "0b1000"), - nc(9, "9", "0x9", "0o11", "0b1001"), - nc(10, "10", "0xa", "0o12", "0b1010"), - nc(11, "11", "0xb", "0o13", "0b1011"), - nc(15, "15", "0xf", "0o17", "0b1111"), - nc(16, "16", "0x10", "0o20", "0b10000"), - nc(17, "17", "0x11", "0o21", "0b10001"), - nc(31, "31", "0x1f", "0o37", "0b11111"), - nc(32, "32", "0x20", "0o40", "0b100000"), - nc(33, "33", "0x21", "0o41", "0b100001"), - nc(41, "41", "0x29", "0o51", "0b101001"), - nc(42, "42", "0x2a", "0o52", "0b101010"), - nc(43, "43", "0x2b", "0o53", "0b101011"), - nc(63, "63", "0x3f", "0o77", "0b111111"), - nc(64, "64", "0x40", "0o100", "0b1000000"), - nc(65, "65", "0x41", "0o101", "0b1000001"), - nc(99, "99", "0x63", "0o143", "0b1100011"), - nc(100, "100", "0x64", "0o144", "0b1100100"), - nc(101, "101", "0x65", "0o145", "0b1100101"), - nc(127, "127", "0x7f", "0o177", "0b1111111"), - nc(128, "128", "0x80", "0o200", "0b10000000"), - nc(129, "129", "0x81", "0o201", "0b10000001"), - nc(255, "255", "0xff", "0o377", "0b11111111"), - nc(256, "256", "0x100", "0o400", "0b100000000"), - nc(257, "257", "0x101", "0o401", "0b100000001"), - nc(428, "428", "0x1ac", "0o654", "0b110101100"), - nc(429, "429", "0x1ad", "0o655", "0b110101101"), - nc(430, "430", "0x1ae", "0o656", "0b110101110"), - nc(511, "511", "0x1ff", "0o777", "0b111111111"), - nc(512, "512", "0x200", "0o1000", "0b1000000000"), - nc(513, "513", "0x201", "0o1001", "0b1000000001"), - nc(999, "999", "0x3e7", "0o1747", "0b1111100111"), - nc(1000, "1000", "0x3e8", "0o1750", "0b1111101000"), - nc(1001, "1001", "0x3e9", "0o1751", "0b1111101001"), - nc(1023, "1023", "0x3ff", "0o1777", "0b1111111111"), - nc(1024, "1024", "0x400", "0o2000", "0b10000000000"), - nc(1025, "1025", "0x401", "0o2001", "0b10000000001"), - nc(2047, "2047", "0x7ff", "0o3777", "0b11111111111"), - nc(2048, "2048", "0x800", "0o4000", "0b100000000000"), - nc(2049, "2049", "0x801", "0o4001", "0b100000000001"), - nc(4095, "4095", "0xfff", "0o7777", "0b111111111111"), - nc(4096, "4096", "0x1000", "0o10000", "0b1000000000000"), - nc(4097, "4097", "0x1001", "0o10001", "0b1000000000001"), - nc(4293, "4293", "0x10c5", "0o10305", "0b1000011000101"), - nc(4294, "4294", "0x10c6", "0o10306", "0b1000011000110"), - nc(4295, "4295", "0x10c7", "0o10307", "0b1000011000111"), - nc(8191, "8191", "0x1fff", "0o17777", "0b1111111111111"), - nc(8192, "8192", "0x2000", "0o20000", "0b10000000000000"), - nc(8193, "8193", "0x2001", "0o20001", "0b10000000000001"), - nc(9999, "9999", "0x270f", "0o23417", "0b10011100001111"), - nc(10000, "10000", "0x2710", "0o23420", "0b10011100010000"), - nc(10001, "10001", "0x2711", "0o23421", "0b10011100010001"), - nc(16383, "16383", "0x3fff", "0o37777", "0b11111111111111"), - nc(16384, "16384", "0x4000", "0o40000", "0b100000000000000"), - nc(16385, "16385", "0x4001", "0o40001", "0b100000000000001"), - nc(32767, "32767", "0x7fff", "0o77777", "0b111111111111111"), - nc(32768, "32768", "0x8000", "0o100000", "0b1000000000000000"), - nc(32769, "32769", "0x8001", "0o100001", "0b1000000000000001"), - nc(42948, "42948", "0xa7c4", "0o123704", "0b1010011111000100"), - nc(42949, "42949", "0xa7c5", "0o123705", "0b1010011111000101"), - nc(42950, "42950", "0xa7c6", "0o123706", "0b1010011111000110"), - nc(65535, "65535", "0xffff", "0o177777", "0b1111111111111111"), - nc(65536, "65536", "0x10000", "0o200000", "0b10000000000000000"), - nc(65537, "65537", "0x10001", "0o200001", "0b10000000000000001"), - nc(99999, "99999", "0x1869f", "0o303237", "0b11000011010011111"), - nc(100000, "100000", "0x186a0", "0o303240", "0b11000011010100000"), - nc(100001, "100001", "0x186a1", "0o303241", "0b11000011010100001"), - nc(131071, "131071", "0x1ffff", "0o377777", "0b11111111111111111"), - nc(131072, "131072", "0x20000", "0o400000", "0b100000000000000000"), - nc(131073, "131073", "0x20001", "0o400001", "0b100000000000000001"), - nc(262143, "262143", "0x3ffff", "0o777777", "0b111111111111111111"), - nc(262144, "262144", "0x40000", "0o1000000", "0b1000000000000000000"), - nc(262145, "262145", "0x40001", "0o1000001", "0b1000000000000000001"), - nc(429495, "429495", "0x68db7", "0o1506667", "0b1101000110110110111"), - nc(429496, "429496", "0x68db8", "0o1506670", "0b1101000110110111000"), - nc(429497, "429497", "0x68db9", "0o1506671", "0b1101000110110111001"), - nc(524287, "524287", "0x7ffff", "0o1777777", "0b1111111111111111111"), - nc(524288, "524288", "0x80000", "0o2000000", "0b10000000000000000000"), - nc(524289, "524289", "0x80001", "0o2000001", "0b10000000000000000001"), - nc(999999, "999999", "0xf423f", "0o3641077", "0b11110100001000111111"), - nc(1000000, "1000000", "0xf4240", "0o3641100", "0b11110100001001000000"), - nc(1000001, "1000001", "0xf4241", "0o3641101", "0b11110100001001000001"), - nc(1048575, "1048575", "0xfffff", "0o3777777", "0b11111111111111111111"), - nc(1048576, "1048576", "0x100000", "0o4000000", "0b100000000000000000000"), - nc(1048577, "1048577", "0x100001", "0o4000001", "0b100000000000000000001"), - nc(2097151, "2097151", "0x1fffff", "0o7777777", "0b111111111111111111111"), - nc(2097152, "2097152", "0x200000", "0o10000000", "0b1000000000000000000000"), - nc(2097153, "2097153", "0x200001", "0o10000001", "0b1000000000000000000001"), - nc(4194303, "4194303", "0x3fffff", "0o17777777", "0b1111111111111111111111"), - nc(4194304, "4194304", "0x400000", "0o20000000", "0b10000000000000000000000"), - nc(4194305, "4194305", "0x400001", "0o20000001", "0b10000000000000000000001"), - nc(4294966, "4294966", "0x418936", "0o20304466", "0b10000011000100100110110"), - nc(4294967, "4294967", "0x418937", "0o20304467", "0b10000011000100100110111"), - nc(4294968, "4294968", "0x418938", "0o20304470", "0b10000011000100100111000"), - nc(8388607, "8388607", "0x7fffff", "0o37777777", "0b11111111111111111111111"), - nc(8388608, "8388608", "0x800000", "0o40000000", "0b100000000000000000000000"), - nc(8388609, "8388609", "0x800001", "0o40000001", "0b100000000000000000000001"), - nc(9999999, "9999999", "0x98967f", "0o46113177", "0b100110001001011001111111"), - nc(10000000, "10000000", "0x989680", "0o46113200", "0b100110001001011010000000"), - nc(10000001, "10000001", "0x989681", "0o46113201", "0b100110001001011010000001"), - nc(16777215, "16777215", "0xffffff", "0o77777777", "0b111111111111111111111111"), - nc(16777216, "16777216", "0x1000000", "0o100000000", "0b1000000000000000000000000"), - nc(16777217, "16777217", "0x1000001", "0o100000001", "0b1000000000000000000000001"), - nc(33554431, "33554431", "0x1ffffff", "0o177777777", "0b1111111111111111111111111"), - nc(33554432, "33554432", "0x2000000", "0o200000000", "0b10000000000000000000000000"), - nc(33554433, "33554433", "0x2000001", "0o200000001", "0b10000000000000000000000001"), - nc(42949671, "42949671", "0x28f5c27", "0o243656047", "0b10100011110101110000100111"), - nc(42949672, "42949672", "0x28f5c28", "0o243656050", "0b10100011110101110000101000"), - nc(42949673, "42949673", "0x28f5c29", "0o243656051", "0b10100011110101110000101001"), - nc(67108863, "67108863", "0x3ffffff", "0o377777777", "0b11111111111111111111111111"), - nc(67108864, "67108864", "0x4000000", "0o400000000", "0b100000000000000000000000000"), - nc(67108865, "67108865", "0x4000001", "0o400000001", "0b100000000000000000000000001"), - nc(99999999, "99999999", "0x5f5e0ff", "0o575360377", "0b101111101011110000011111111"), - nc(100000000, "100000000", "0x5f5e100", "0o575360400", "0b101111101011110000100000000"), - nc(100000001, "100000001", "0x5f5e101", "0o575360401", "0b101111101011110000100000001"), - nc(134217727, "134217727", "0x7ffffff", "0o777777777", "0b111111111111111111111111111"), - nc(134217728, "134217728", "0x8000000", "0o1000000000", "0b1000000000000000000000000000"), - nc(134217729, "134217729", "0x8000001", "0o1000000001", "0b1000000000000000000000000001"), - nc(268435455, "268435455", "0xfffffff", "0o1777777777", "0b1111111111111111111111111111"), - nc(268435456, "268435456", "0x10000000", "0o2000000000", "0b10000000000000000000000000000"), - nc(268435457, "268435457", "0x10000001", "0o2000000001", "0b10000000000000000000000000001"), - nc(429496728, "429496728", "0x19999998", "0o3146314630", "0b11001100110011001100110011000"), - nc(429496729, "429496729", "0x19999999", "0o3146314631", "0b11001100110011001100110011001"), - nc(429496730, "429496730", "0x1999999a", "0o3146314632", "0b11001100110011001100110011010"), - nc(536870911, "536870911", "0x1fffffff", "0o3777777777", "0b11111111111111111111111111111"), - nc(536870912, "536870912", "0x20000000", "0o4000000000", "0b100000000000000000000000000000"), - nc(536870913, "536870913", "0x20000001", "0o4000000001", "0b100000000000000000000000000001"), - nc(999999999, "999999999", "0x3b9ac9ff", "0o7346544777", "0b111011100110101100100111111111"), - nc(1000000000, "1000000000", "0x3b9aca00", "0o7346545000", "0b111011100110101100101000000000"), - nc(1000000001, "1000000001", "0x3b9aca01", "0o7346545001", "0b111011100110101100101000000001"), - nc(1073741823, "1073741823", "0x3fffffff", "0o7777777777", "0b111111111111111111111111111111"), - nc(1073741824, "1073741824", "0x40000000", "0o10000000000", "0b1000000000000000000000000000000"), - nc(1073741825, "1073741825", "0x40000001", "0o10000000001", "0b1000000000000000000000000000001"), - nc(2147483647, "2147483647", "0x7fffffff", "0o17777777777", "0b1111111111111111111111111111111"), - nc(2147483648, "2147483648", "0x80000000", "0o20000000000", "0b10000000000000000000000000000000"), - nc(2147483649, "2147483649", "0x80000001", "0o20000000001", "0b10000000000000000000000000000001"), - nc(4294967291, "4294967291", "0xfffffffb", "0o37777777773", "0b11111111111111111111111111111011"), - nc(4294967292, "4294967292", "0xfffffffc", "0o37777777774", "0b11111111111111111111111111111100"), - nc(4294967293, "4294967293", "0xfffffffd", "0o37777777775", "0b11111111111111111111111111111101"), - nc(4294967294, "4294967294", "0xfffffffe", "0o37777777776", "0b11111111111111111111111111111110"), - nc(4294967295, "4294967295", "0xffffffff", "0o37777777777", "0b11111111111111111111111111111111"), -#undef nc - }; -}; - - -template<> -struct numbers -{ - using value_type = int64_t; - static C4_INLINE_CONSTEXPR const number_case vals[] = { -#define nc(val, dec, hex, bin, oct) \ - number_case{(value_type)INT64_C(val), csubstr{dec}, csubstr{hex}, csubstr{bin}, csubstr{oct}} -#define ncm1(val, dec, hex, bin, oct) \ - number_case{(value_type)(INT64_C(val)-INT64_C(1)), csubstr{dec}, csubstr{hex}, csubstr{bin}, csubstr{oct}} - ncm1(-9223372036854775807, "-9223372036854775808", "-0x8000000000000000", "-0o1000000000000000000000", "-0b1000000000000000000000000000000000000000000000000000000000000000"), - nc(-9223372036854775807, "-9223372036854775807", "-0x7fffffffffffffff", "-0o777777777777777777777", "-0b111111111111111111111111111111111111111111111111111111111111111"), - nc(-9223372036854775806, "-9223372036854775806", "-0x7ffffffffffffffe", "-0o777777777777777777776", "-0b111111111111111111111111111111111111111111111111111111111111110"), - nc(-9223372036854775805, "-9223372036854775805", "-0x7ffffffffffffffd", "-0o777777777777777777775", "-0b111111111111111111111111111111111111111111111111111111111111101"), - nc(-9223372036854775804, "-9223372036854775804", "-0x7ffffffffffffffc", "-0o777777777777777777774", "-0b111111111111111111111111111111111111111111111111111111111111100"), - nc(-9223372036854775803, "-9223372036854775803", "-0x7ffffffffffffffb", "-0o777777777777777777773", "-0b111111111111111111111111111111111111111111111111111111111111011"), - nc(-4611686018427387905, "-4611686018427387905", "-0x4000000000000001", "-0o400000000000000000001", "-0b100000000000000000000000000000000000000000000000000000000000001"), - nc(-4611686018427387904, "-4611686018427387904", "-0x4000000000000000", "-0o400000000000000000000", "-0b100000000000000000000000000000000000000000000000000000000000000"), - nc(-4611686018427387903, "-4611686018427387903", "-0x3fffffffffffffff", "-0o377777777777777777777", "-0b11111111111111111111111111111111111111111111111111111111111111"), - nc(-2305843009213693953, "-2305843009213693953", "-0x2000000000000001", "-0o200000000000000000001", "-0b10000000000000000000000000000000000000000000000000000000000001"), - nc(-2305843009213693952, "-2305843009213693952", "-0x2000000000000000", "-0o200000000000000000000", "-0b10000000000000000000000000000000000000000000000000000000000000"), - nc(-2305843009213693951, "-2305843009213693951", "-0x1fffffffffffffff", "-0o177777777777777777777", "-0b1111111111111111111111111111111111111111111111111111111111111"), - nc(-1152921504606846977, "-1152921504606846977", "-0x1000000000000001", "-0o100000000000000000001", "-0b1000000000000000000000000000000000000000000000000000000000001"), - nc(-1152921504606846976, "-1152921504606846976", "-0x1000000000000000", "-0o100000000000000000000", "-0b1000000000000000000000000000000000000000000000000000000000000"), - nc(-1152921504606846975, "-1152921504606846975", "-0xfffffffffffffff", "-0o77777777777777777777", "-0b111111111111111111111111111111111111111111111111111111111111"), - nc(-1000000000000000001, "-1000000000000000001", "-0xde0b6b3a7640001", "-0o67405553164731000001", "-0b110111100000101101101011001110100111011001000000000000000001"), - nc(-1000000000000000000, "-1000000000000000000", "-0xde0b6b3a7640000", "-0o67405553164731000000", "-0b110111100000101101101011001110100111011001000000000000000000"), - nc(-999999999999999999, "-999999999999999999", "-0xde0b6b3a763ffff", "-0o67405553164730777777", "-0b110111100000101101101011001110100111011000111111111111111111"), - nc(-922337203685477632, "-922337203685477632", "-0xccccccccccccd00", "-0o63146314631463146400", "-0b110011001100110011001100110011001100110011001100110100000000"), - nc(-576460752303423489, "-576460752303423489", "-0x800000000000001", "-0o40000000000000000001", "-0b100000000000000000000000000000000000000000000000000000000001"), - nc(-576460752303423488, "-576460752303423488", "-0x800000000000000", "-0o40000000000000000000", "-0b100000000000000000000000000000000000000000000000000000000000"), - nc(-576460752303423487, "-576460752303423487", "-0x7ffffffffffffff", "-0o37777777777777777777", "-0b11111111111111111111111111111111111111111111111111111111111"), - nc(-288230376151711745, "-288230376151711745", "-0x400000000000001", "-0o20000000000000000001", "-0b10000000000000000000000000000000000000000000000000000000001"), - nc(-288230376151711744, "-288230376151711744", "-0x400000000000000", "-0o20000000000000000000", "-0b10000000000000000000000000000000000000000000000000000000000"), - nc(-288230376151711743, "-288230376151711743", "-0x3ffffffffffffff", "-0o17777777777777777777", "-0b1111111111111111111111111111111111111111111111111111111111"), - nc(-144115188075855873, "-144115188075855873", "-0x200000000000001", "-0o10000000000000000001", "-0b1000000000000000000000000000000000000000000000000000000001"), - nc(-144115188075855872, "-144115188075855872", "-0x200000000000000", "-0o10000000000000000000", "-0b1000000000000000000000000000000000000000000000000000000000"), - nc(-144115188075855871, "-144115188075855871", "-0x1ffffffffffffff", "-0o7777777777777777777", "-0b111111111111111111111111111111111111111111111111111111111"), - nc(-100000000000000001, "-100000000000000001", "-0x16345785d8a0001", "-0o5432127413542400001", "-0b101100011010001010111100001011101100010100000000000000001"), - nc(-100000000000000000, "-100000000000000000", "-0x16345785d8a0000", "-0o5432127413542400000", "-0b101100011010001010111100001011101100010100000000000000000"), - nc(-99999999999999999, "-99999999999999999", "-0x16345785d89ffff", "-0o5432127413542377777", "-0b101100011010001010111100001011101100010011111111111111111"), - nc(-92233720368547760, "-92233720368547760", "-0x147ae147ae147b0", "-0o5075341217270243660", "-0b101000111101011100001010001111010111000010100011110110000"), - nc(-72057594037927937, "-72057594037927937", "-0x100000000000001", "-0o4000000000000000001", "-0b100000000000000000000000000000000000000000000000000000001"), - nc(-72057594037927936, "-72057594037927936", "-0x100000000000000", "-0o4000000000000000000", "-0b100000000000000000000000000000000000000000000000000000000"), - nc(-72057594037927935, "-72057594037927935", "-0xffffffffffffff", "-0o3777777777777777777", "-0b11111111111111111111111111111111111111111111111111111111"), - nc(-36028797018963969, "-36028797018963969", "-0x80000000000001", "-0o2000000000000000001", "-0b10000000000000000000000000000000000000000000000000000001"), - nc(-36028797018963968, "-36028797018963968", "-0x80000000000000", "-0o2000000000000000000", "-0b10000000000000000000000000000000000000000000000000000000"), - nc(-36028797018963967, "-36028797018963967", "-0x7fffffffffffff", "-0o1777777777777777777", "-0b1111111111111111111111111111111111111111111111111111111"), - nc(-18014398509481985, "-18014398509481985", "-0x40000000000001", "-0o1000000000000000001", "-0b1000000000000000000000000000000000000000000000000000001"), - nc(-18014398509481984, "-18014398509481984", "-0x40000000000000", "-0o1000000000000000000", "-0b1000000000000000000000000000000000000000000000000000000"), - nc(-18014398509481983, "-18014398509481983", "-0x3fffffffffffff", "-0o777777777777777777", "-0b111111111111111111111111111111111111111111111111111111"), - nc(-10000000000000001, "-10000000000000001", "-0x2386f26fc10001", "-0o434157115760200001", "-0b100011100001101111001001101111110000010000000000000001"), - nc(-10000000000000000, "-10000000000000000", "-0x2386f26fc10000", "-0o434157115760200000", "-0b100011100001101111001001101111110000010000000000000000"), - nc(-9999999999999999, "-9999999999999999", "-0x2386f26fc0ffff", "-0o434157115760177777", "-0b100011100001101111001001101111110000001111111111111111"), - nc(-9223372036854776, "-9223372036854776", "-0x20c49ba5e353f8", "-0o406111564570651770", "-0b100000110001001001101110100101111000110101001111111000"), - nc(-9007199254740993, "-9007199254740993", "-0x20000000000001", "-0o400000000000000001", "-0b100000000000000000000000000000000000000000000000000001"), - nc(-9007199254740992, "-9007199254740992", "-0x20000000000000", "-0o400000000000000000", "-0b100000000000000000000000000000000000000000000000000000"), - nc(-9007199254740991, "-9007199254740991", "-0x1fffffffffffff", "-0o377777777777777777", "-0b11111111111111111111111111111111111111111111111111111"), - nc(-4503599627370497, "-4503599627370497", "-0x10000000000001", "-0o200000000000000001", "-0b10000000000000000000000000000000000000000000000000001"), - nc(-4503599627370496, "-4503599627370496", "-0x10000000000000", "-0o200000000000000000", "-0b10000000000000000000000000000000000000000000000000000"), - nc(-4503599627370495, "-4503599627370495", "-0xfffffffffffff", "-0o177777777777777777", "-0b1111111111111111111111111111111111111111111111111111"), - nc(-2251799813685249, "-2251799813685249", "-0x8000000000001", "-0o100000000000000001", "-0b1000000000000000000000000000000000000000000000000001"), - nc(-2251799813685248, "-2251799813685248", "-0x8000000000000", "-0o100000000000000000", "-0b1000000000000000000000000000000000000000000000000000"), - nc(-2251799813685247, "-2251799813685247", "-0x7ffffffffffff", "-0o77777777777777777", "-0b111111111111111111111111111111111111111111111111111"), - nc(-1125899906842625, "-1125899906842625", "-0x4000000000001", "-0o40000000000000001", "-0b100000000000000000000000000000000000000000000000001"), - nc(-1125899906842624, "-1125899906842624", "-0x4000000000000", "-0o40000000000000000", "-0b100000000000000000000000000000000000000000000000000"), - nc(-1125899906842623, "-1125899906842623", "-0x3ffffffffffff", "-0o37777777777777777", "-0b11111111111111111111111111111111111111111111111111"), - nc(-1000000000000001, "-1000000000000001", "-0x38d7ea4c68001", "-0o34327724461500001", "-0b11100011010111111010100100110001101000000000000001"), - nc(-1000000000000000, "-1000000000000000", "-0x38d7ea4c68000", "-0o34327724461500000", "-0b11100011010111111010100100110001101000000000000000"), - nc(-999999999999999, "-999999999999999", "-0x38d7ea4c67fff", "-0o34327724461477777", "-0b11100011010111111010100100110001100111111111111111"), - nc(-922337203685477, "-922337203685477", "-0x346dc5d638865", "-0o32155613530704145", "-0b11010001101101110001011101011000111000100001100101"), - nc(-562949953421313, "-562949953421313", "-0x2000000000001", "-0o20000000000000001", "-0b10000000000000000000000000000000000000000000000001"), - nc(-562949953421312, "-562949953421312", "-0x2000000000000", "-0o20000000000000000", "-0b10000000000000000000000000000000000000000000000000"), - nc(-562949953421311, "-562949953421311", "-0x1ffffffffffff", "-0o17777777777777777", "-0b1111111111111111111111111111111111111111111111111"), - nc(-281474976710657, "-281474976710657", "-0x1000000000001", "-0o10000000000000001", "-0b1000000000000000000000000000000000000000000000001"), - nc(-281474976710656, "-281474976710656", "-0x1000000000000", "-0o10000000000000000", "-0b1000000000000000000000000000000000000000000000000"), - nc(-281474976710655, "-281474976710655", "-0xffffffffffff", "-0o7777777777777777", "-0b111111111111111111111111111111111111111111111111"), - nc(-140737488355329, "-140737488355329", "-0x800000000001", "-0o4000000000000001", "-0b100000000000000000000000000000000000000000000001"), - nc(-140737488355328, "-140737488355328", "-0x800000000000", "-0o4000000000000000", "-0b100000000000000000000000000000000000000000000000"), - nc(-140737488355327, "-140737488355327", "-0x7fffffffffff", "-0o3777777777777777", "-0b11111111111111111111111111111111111111111111111"), - nc(-100000000000001, "-100000000000001", "-0x5af3107a4001", "-0o2657142036440001", "-0b10110101111001100010000011110100100000000000001"), - nc(-100000000000000, "-100000000000000", "-0x5af3107a4000", "-0o2657142036440000", "-0b10110101111001100010000011110100100000000000000"), - nc(-99999999999999, "-99999999999999", "-0x5af3107a3fff", "-0o2657142036437777", "-0b10110101111001100010000011110100011111111111111"), - nc(-92233720368547, "-92233720368547", "-0x53e2d6238da3", "-0o2476132610706643", "-0b10100111110001011010110001000111000110110100011"), - nc(-70368744177665, "-70368744177665", "-0x400000000001", "-0o2000000000000001", "-0b10000000000000000000000000000000000000000000001"), - nc(-70368744177664, "-70368744177664", "-0x400000000000", "-0o2000000000000000", "-0b10000000000000000000000000000000000000000000000"), - nc(-70368744177663, "-70368744177663", "-0x3fffffffffff", "-0o1777777777777777", "-0b1111111111111111111111111111111111111111111111"), - nc(-35184372088833, "-35184372088833", "-0x200000000001", "-0o1000000000000001", "-0b1000000000000000000000000000000000000000000001"), - nc(-35184372088832, "-35184372088832", "-0x200000000000", "-0o1000000000000000", "-0b1000000000000000000000000000000000000000000000"), - nc(-35184372088831, "-35184372088831", "-0x1fffffffffff", "-0o777777777777777", "-0b111111111111111111111111111111111111111111111"), - nc(-17592186044417, "-17592186044417", "-0x100000000001", "-0o400000000000001", "-0b100000000000000000000000000000000000000000001"), - nc(-17592186044416, "-17592186044416", "-0x100000000000", "-0o400000000000000", "-0b100000000000000000000000000000000000000000000"), - nc(-17592186044415, "-17592186044415", "-0xfffffffffff", "-0o377777777777777", "-0b11111111111111111111111111111111111111111111"), - nc(-10000000000001, "-10000000000001", "-0x9184e72a001", "-0o221411634520001", "-0b10010001100001001110011100101010000000000001"), - nc(-10000000000000, "-10000000000000", "-0x9184e72a000", "-0o221411634520000", "-0b10010001100001001110011100101010000000000000"), - nc(-9999999999999, "-9999999999999", "-0x9184e729fff", "-0o221411634517777", "-0b10010001100001001110011100101001111111111111"), - nc(-9223372036854, "-9223372036854", "-0x8637bd05af6", "-0o206157364055366", "-0b10000110001101111011110100000101101011110110"), - nc(-8796093022209, "-8796093022209", "-0x80000000001", "-0o200000000000001", "-0b10000000000000000000000000000000000000000001"), - nc(-8796093022208, "-8796093022208", "-0x80000000000", "-0o200000000000000", "-0b10000000000000000000000000000000000000000000"), - nc(-8796093022207, "-8796093022207", "-0x7ffffffffff", "-0o177777777777777", "-0b1111111111111111111111111111111111111111111"), - nc(-4398046511105, "-4398046511105", "-0x40000000001", "-0o100000000000001", "-0b1000000000000000000000000000000000000000001"), - nc(-4398046511104, "-4398046511104", "-0x40000000000", "-0o100000000000000", "-0b1000000000000000000000000000000000000000000"), - nc(-4398046511103, "-4398046511103", "-0x3ffffffffff", "-0o77777777777777", "-0b111111111111111111111111111111111111111111"), - nc(-2199023255553, "-2199023255553", "-0x20000000001", "-0o40000000000001", "-0b100000000000000000000000000000000000000001"), - nc(-2199023255552, "-2199023255552", "-0x20000000000", "-0o40000000000000", "-0b100000000000000000000000000000000000000000"), - nc(-2199023255551, "-2199023255551", "-0x1ffffffffff", "-0o37777777777777", "-0b11111111111111111111111111111111111111111"), - nc(-1099511627777, "-1099511627777", "-0x10000000001", "-0o20000000000001", "-0b10000000000000000000000000000000000000001"), - nc(-1099511627776, "-1099511627776", "-0x10000000000", "-0o20000000000000", "-0b10000000000000000000000000000000000000000"), - nc(-1099511627775, "-1099511627775", "-0xffffffffff", "-0o17777777777777", "-0b1111111111111111111111111111111111111111"), - nc(-1000000000001, "-1000000000001", "-0xe8d4a51001", "-0o16432451210001", "-0b1110100011010100101001010001000000000001"), - nc(-1000000000000, "-1000000000000", "-0xe8d4a51000", "-0o16432451210000", "-0b1110100011010100101001010001000000000000"), - nc(-999999999999, "-999999999999", "-0xe8d4a50fff", "-0o16432451207777", "-0b1110100011010100101001010000111111111111"), - nc(-922337203685, "-922337203685", "-0xd6bf94d5e5", "-0o15327745152745", "-0b1101011010111111100101001101010111100101"), - nc(-549755813889, "-549755813889", "-0x8000000001", "-0o10000000000001", "-0b1000000000000000000000000000000000000001"), - nc(-549755813888, "-549755813888", "-0x8000000000", "-0o10000000000000", "-0b1000000000000000000000000000000000000000"), - nc(-549755813887, "-549755813887", "-0x7fffffffff", "-0o7777777777777", "-0b111111111111111111111111111111111111111"), - nc(-274877906945, "-274877906945", "-0x4000000001", "-0o4000000000001", "-0b100000000000000000000000000000000000001"), - nc(-274877906944, "-274877906944", "-0x4000000000", "-0o4000000000000", "-0b100000000000000000000000000000000000000"), - nc(-274877906943, "-274877906943", "-0x3fffffffff", "-0o3777777777777", "-0b11111111111111111111111111111111111111"), - nc(-137438953473, "-137438953473", "-0x2000000001", "-0o2000000000001", "-0b10000000000000000000000000000000000001"), - nc(-137438953472, "-137438953472", "-0x2000000000", "-0o2000000000000", "-0b10000000000000000000000000000000000000"), - nc(-137438953471, "-137438953471", "-0x1fffffffff", "-0o1777777777777", "-0b1111111111111111111111111111111111111"), - nc(-100000000001, "-100000000001", "-0x174876e801", "-0o1351035564001", "-0b1011101001000011101101110100000000001"), - nc(-100000000000, "-100000000000", "-0x174876e800", "-0o1351035564000", "-0b1011101001000011101101110100000000000"), - nc(-99999999999, "-99999999999", "-0x174876e7ff", "-0o1351035563777", "-0b1011101001000011101101110011111111111"), - nc(-92233720368, "-92233720368", "-0x15798ee230", "-0o1257143561060", "-0b1010101111001100011101110001000110000"), - nc(-68719476737, "-68719476737", "-0x1000000001", "-0o1000000000001", "-0b1000000000000000000000000000000000001"), - nc(-68719476736, "-68719476736", "-0x1000000000", "-0o1000000000000", "-0b1000000000000000000000000000000000000"), - nc(-68719476735, "-68719476735", "-0xfffffffff", "-0o777777777777", "-0b111111111111111111111111111111111111"), - nc(-34359738369, "-34359738369", "-0x800000001", "-0o400000000001", "-0b100000000000000000000000000000000001"), - nc(-34359738368, "-34359738368", "-0x800000000", "-0o400000000000", "-0b100000000000000000000000000000000000"), - nc(-34359738367, "-34359738367", "-0x7ffffffff", "-0o377777777777", "-0b11111111111111111111111111111111111"), - nc(-17179869185, "-17179869185", "-0x400000001", "-0o200000000001", "-0b10000000000000000000000000000000001"), - nc(-17179869184, "-17179869184", "-0x400000000", "-0o200000000000", "-0b10000000000000000000000000000000000"), - nc(-17179869183, "-17179869183", "-0x3ffffffff", "-0o177777777777", "-0b1111111111111111111111111111111111"), - nc(-10000000001, "-10000000001", "-0x2540be401", "-0o112402762001", "-0b1001010100000010111110010000000001"), - nc(-10000000000, "-10000000000", "-0x2540be400", "-0o112402762000", "-0b1001010100000010111110010000000000"), - nc(-9999999999, "-9999999999", "-0x2540be3ff", "-0o112402761777", "-0b1001010100000010111110001111111111"), - nc(-9223372036, "-9223372036", "-0x225c17d04", "-0o104560276404", "-0b1000100101110000010111110100000100"), - nc(-8589934593, "-8589934593", "-0x200000001", "-0o100000000001", "-0b1000000000000000000000000000000001"), - nc(-8589934592, "-8589934592", "-0x200000000", "-0o100000000000", "-0b1000000000000000000000000000000000"), - nc(-8589934591, "-8589934591", "-0x1ffffffff", "-0o77777777777", "-0b111111111111111111111111111111111"), - nc(-4294967297, "-4294967297", "-0x100000001", "-0o40000000001", "-0b100000000000000000000000000000001"), - nc(-4294967296, "-4294967296", "-0x100000000", "-0o40000000000", "-0b100000000000000000000000000000000"), - nc(-4294967295, "-4294967295", "-0xffffffff", "-0o37777777777", "-0b11111111111111111111111111111111"), - nc(-2147483649, "-2147483649", "-0x80000001", "-0o20000000001", "-0b10000000000000000000000000000001"), - nc(-2147483648, "-2147483648", "-0x80000000", "-0o20000000000", "-0b10000000000000000000000000000000"), - nc(-2147483647, "-2147483647", "-0x7fffffff", "-0o17777777777", "-0b1111111111111111111111111111111"), - nc(-1073741825, "-1073741825", "-0x40000001", "-0o10000000001", "-0b1000000000000000000000000000001"), - nc(-1073741824, "-1073741824", "-0x40000000", "-0o10000000000", "-0b1000000000000000000000000000000"), - nc(-1073741823, "-1073741823", "-0x3fffffff", "-0o7777777777", "-0b111111111111111111111111111111"), - nc(-1000000001, "-1000000001", "-0x3b9aca01", "-0o7346545001", "-0b111011100110101100101000000001"), - nc(-1000000000, "-1000000000", "-0x3b9aca00", "-0o7346545000", "-0b111011100110101100101000000000"), - nc(-999999999, "-999999999", "-0x3b9ac9ff", "-0o7346544777", "-0b111011100110101100100111111111"), - nc(-922337203, "-922337203", "-0x36f9bfb3", "-0o6676337663", "-0b110110111110011011111110110011"), - nc(-536870913, "-536870913", "-0x20000001", "-0o4000000001", "-0b100000000000000000000000000001"), - nc(-536870912, "-536870912", "-0x20000000", "-0o4000000000", "-0b100000000000000000000000000000"), - nc(-536870911, "-536870911", "-0x1fffffff", "-0o3777777777", "-0b11111111111111111111111111111"), - nc(-268435457, "-268435457", "-0x10000001", "-0o2000000001", "-0b10000000000000000000000000001"), - nc(-268435456, "-268435456", "-0x10000000", "-0o2000000000", "-0b10000000000000000000000000000"), - nc(-268435455, "-268435455", "-0xfffffff", "-0o1777777777", "-0b1111111111111111111111111111"), - nc(-134217729, "-134217729", "-0x8000001", "-0o1000000001", "-0b1000000000000000000000000001"), - nc(-134217728, "-134217728", "-0x8000000", "-0o1000000000", "-0b1000000000000000000000000000"), - nc(-134217727, "-134217727", "-0x7ffffff", "-0o777777777", "-0b111111111111111111111111111"), - nc(-100000001, "-100000001", "-0x5f5e101", "-0o575360401", "-0b101111101011110000100000001"), - nc(-100000000, "-100000000", "-0x5f5e100", "-0o575360400", "-0b101111101011110000100000000"), - nc(-99999999, "-99999999", "-0x5f5e0ff", "-0o575360377", "-0b101111101011110000011111111"), - nc(-92233720, "-92233720", "-0x57f5ff8", "-0o537657770", "-0b101011111110101111111111000"), - nc(-67108865, "-67108865", "-0x4000001", "-0o400000001", "-0b100000000000000000000000001"), - nc(-67108864, "-67108864", "-0x4000000", "-0o400000000", "-0b100000000000000000000000000"), - nc(-67108863, "-67108863", "-0x3ffffff", "-0o377777777", "-0b11111111111111111111111111"), - nc(-33554433, "-33554433", "-0x2000001", "-0o200000001", "-0b10000000000000000000000001"), - nc(-33554432, "-33554432", "-0x2000000", "-0o200000000", "-0b10000000000000000000000000"), - nc(-33554431, "-33554431", "-0x1ffffff", "-0o177777777", "-0b1111111111111111111111111"), - nc(-16777217, "-16777217", "-0x1000001", "-0o100000001", "-0b1000000000000000000000001"), - nc(-16777216, "-16777216", "-0x1000000", "-0o100000000", "-0b1000000000000000000000000"), - nc(-16777215, "-16777215", "-0xffffff", "-0o77777777", "-0b111111111111111111111111"), - nc(-10000001, "-10000001", "-0x989681", "-0o46113201", "-0b100110001001011010000001"), - nc(-10000000, "-10000000", "-0x989680", "-0o46113200", "-0b100110001001011010000000"), - nc(-9999999, "-9999999", "-0x98967f", "-0o46113177", "-0b100110001001011001111111"), - nc(-9223372, "-9223372", "-0x8cbccc", "-0o43136314", "-0b100011001011110011001100"), - nc(-8388609, "-8388609", "-0x800001", "-0o40000001", "-0b100000000000000000000001"), - nc(-8388608, "-8388608", "-0x800000", "-0o40000000", "-0b100000000000000000000000"), - nc(-8388607, "-8388607", "-0x7fffff", "-0o37777777", "-0b11111111111111111111111"), - nc(-4194305, "-4194305", "-0x400001", "-0o20000001", "-0b10000000000000000000001"), - nc(-4194304, "-4194304", "-0x400000", "-0o20000000", "-0b10000000000000000000000"), - nc(-4194303, "-4194303", "-0x3fffff", "-0o17777777", "-0b1111111111111111111111"), - nc(-2097153, "-2097153", "-0x200001", "-0o10000001", "-0b1000000000000000000001"), - nc(-2097152, "-2097152", "-0x200000", "-0o10000000", "-0b1000000000000000000000"), - nc(-2097151, "-2097151", "-0x1fffff", "-0o7777777", "-0b111111111111111111111"), - nc(-1048577, "-1048577", "-0x100001", "-0o4000001", "-0b100000000000000000001"), - nc(-1048576, "-1048576", "-0x100000", "-0o4000000", "-0b100000000000000000000"), - nc(-1048575, "-1048575", "-0xfffff", "-0o3777777", "-0b11111111111111111111"), - nc(-1000001, "-1000001", "-0xf4241", "-0o3641101", "-0b11110100001001000001"), - nc(-1000000, "-1000000", "-0xf4240", "-0o3641100", "-0b11110100001001000000"), - nc(-999999, "-999999", "-0xf423f", "-0o3641077", "-0b11110100001000111111"), - nc(-922337, "-922337", "-0xe12e1", "-0o3411341", "-0b11100001001011100001"), - nc(-524289, "-524289", "-0x80001", "-0o2000001", "-0b10000000000000000001"), - nc(-524288, "-524288", "-0x80000", "-0o2000000", "-0b10000000000000000000"), - nc(-524287, "-524287", "-0x7ffff", "-0o1777777", "-0b1111111111111111111"), - nc(-262145, "-262145", "-0x40001", "-0o1000001", "-0b1000000000000000001"), - nc(-262144, "-262144", "-0x40000", "-0o1000000", "-0b1000000000000000000"), - nc(-262143, "-262143", "-0x3ffff", "-0o777777", "-0b111111111111111111"), - nc(-131073, "-131073", "-0x20001", "-0o400001", "-0b100000000000000001"), - nc(-131072, "-131072", "-0x20000", "-0o400000", "-0b100000000000000000"), - nc(-131071, "-131071", "-0x1ffff", "-0o377777", "-0b11111111111111111"), - nc(-100001, "-100001", "-0x186a1", "-0o303241", "-0b11000011010100001"), - nc(-100000, "-100000", "-0x186a0", "-0o303240", "-0b11000011010100000"), - nc(-99999, "-99999", "-0x1869f", "-0o303237", "-0b11000011010011111"), - nc(-92233, "-92233", "-0x16849", "-0o264111", "-0b10110100001001001"), - nc(-65537, "-65537", "-0x10001", "-0o200001", "-0b10000000000000001"), - nc(-65536, "-65536", "-0x10000", "-0o200000", "-0b10000000000000000"), - nc(-65535, "-65535", "-0xffff", "-0o177777", "-0b1111111111111111"), - nc(-32769, "-32769", "-0x8001", "-0o100001", "-0b1000000000000001"), - nc(-32768, "-32768", "-0x8000", "-0o100000", "-0b1000000000000000"), - nc(-32767, "-32767", "-0x7fff", "-0o77777", "-0b111111111111111"), - nc(-16385, "-16385", "-0x4001", "-0o40001", "-0b100000000000001"), - nc(-16384, "-16384", "-0x4000", "-0o40000", "-0b100000000000000"), - nc(-16383, "-16383", "-0x3fff", "-0o37777", "-0b11111111111111"), - nc(-10001, "-10001", "-0x2711", "-0o23421", "-0b10011100010001"), - nc(-10000, "-10000", "-0x2710", "-0o23420", "-0b10011100010000"), - nc(-9999, "-9999", "-0x270f", "-0o23417", "-0b10011100001111"), - nc(-9223, "-9223", "-0x2407", "-0o22007", "-0b10010000000111"), - nc(-8193, "-8193", "-0x2001", "-0o20001", "-0b10000000000001"), - nc(-8192, "-8192", "-0x2000", "-0o20000", "-0b10000000000000"), - nc(-8191, "-8191", "-0x1fff", "-0o17777", "-0b1111111111111"), - nc(-4097, "-4097", "-0x1001", "-0o10001", "-0b1000000000001"), - nc(-4096, "-4096", "-0x1000", "-0o10000", "-0b1000000000000"), - nc(-4095, "-4095", "-0xfff", "-0o7777", "-0b111111111111"), - nc(-2049, "-2049", "-0x801", "-0o4001", "-0b100000000001"), - nc(-2048, "-2048", "-0x800", "-0o4000", "-0b100000000000"), - nc(-2047, "-2047", "-0x7ff", "-0o3777", "-0b11111111111"), - nc(-1025, "-1025", "-0x401", "-0o2001", "-0b10000000001"), - nc(-1024, "-1024", "-0x400", "-0o2000", "-0b10000000000"), - nc(-1023, "-1023", "-0x3ff", "-0o1777", "-0b1111111111"), - nc(-1001, "-1001", "-0x3e9", "-0o1751", "-0b1111101001"), - nc(-1000, "-1000", "-0x3e8", "-0o1750", "-0b1111101000"), - nc(-999, "-999", "-0x3e7", "-0o1747", "-0b1111100111"), - nc(-922, "-922", "-0x39a", "-0o1632", "-0b1110011010"), - nc(-513, "-513", "-0x201", "-0o1001", "-0b1000000001"), - nc(-512, "-512", "-0x200", "-0o1000", "-0b1000000000"), - nc(-511, "-511", "-0x1ff", "-0o777", "-0b111111111"), - nc(-257, "-257", "-0x101", "-0o401", "-0b100000001"), - nc(-256, "-256", "-0x100", "-0o400", "-0b100000000"), - nc(-255, "-255", "-0xff", "-0o377", "-0b11111111"), - nc(-129, "-129", "-0x81", "-0o201", "-0b10000001"), - nc(-128, "-128", "-0x80", "-0o200", "-0b10000000"), - nc(-127, "-127", "-0x7f", "-0o177", "-0b1111111"), - nc(-101, "-101", "-0x65", "-0o145", "-0b1100101"), - nc(-100, "-100", "-0x64", "-0o144", "-0b1100100"), - nc(-99, "-99", "-0x63", "-0o143", "-0b1100011"), - nc(-92, "-92", "-0x5c", "-0o134", "-0b1011100"), - nc(-65, "-65", "-0x41", "-0o101", "-0b1000001"), - nc(-64, "-64", "-0x40", "-0o100", "-0b1000000"), - nc(-63, "-63", "-0x3f", "-0o77", "-0b111111"), - nc(-33, "-33", "-0x21", "-0o41", "-0b100001"), - nc(-32, "-32", "-0x20", "-0o40", "-0b100000"), - nc(-31, "-31", "-0x1f", "-0o37", "-0b11111"), - nc(-17, "-17", "-0x11", "-0o21", "-0b10001"), - nc(-16, "-16", "-0x10", "-0o20", "-0b10000"), - nc(-15, "-15", "-0xf", "-0o17", "-0b1111"), - nc(-11, "-11", "-0xb", "-0o13", "-0b1011"), - nc(-10, "-10", "-0xa", "-0o12", "-0b1010"), - nc(-9, "-9", "-0x9", "-0o11", "-0b1001"), - nc(-8, "-8", "-0x8", "-0o10", "-0b1000"), - nc(-7, "-7", "-0x7", "-0o7", "-0b111"), - nc(-6, "-6", "-0x6", "-0o6", "-0b110"), - nc(-5, "-5", "-0x5", "-0o5", "-0b101"), - nc(-4, "-4", "-0x4", "-0o4", "-0b100"), - nc(-3, "-3", "-0x3", "-0o3", "-0b11"), - nc(-2, "-2", "-0x2", "-0o2", "-0b10"), - nc(-1, "-1", "-0x1", "-0o1", "-0b1"), - nc(0, "0", "0x0", "0o0", "0b0"), - nc(1, "1", "0x1", "0o1", "0b1"), - nc(2, "2", "0x2", "0o2", "0b10"), - nc(3, "3", "0x3", "0o3", "0b11"), - nc(4, "4", "0x4", "0o4", "0b100"), - nc(5, "5", "0x5", "0o5", "0b101"), - nc(6, "6", "0x6", "0o6", "0b110"), - nc(7, "7", "0x7", "0o7", "0b111"), - nc(8, "8", "0x8", "0o10", "0b1000"), - nc(9, "9", "0x9", "0o11", "0b1001"), - nc(10, "10", "0xa", "0o12", "0b1010"), - nc(11, "11", "0xb", "0o13", "0b1011"), - nc(15, "15", "0xf", "0o17", "0b1111"), - nc(16, "16", "0x10", "0o20", "0b10000"), - nc(17, "17", "0x11", "0o21", "0b10001"), - nc(31, "31", "0x1f", "0o37", "0b11111"), - nc(32, "32", "0x20", "0o40", "0b100000"), - nc(33, "33", "0x21", "0o41", "0b100001"), - nc(63, "63", "0x3f", "0o77", "0b111111"), - nc(64, "64", "0x40", "0o100", "0b1000000"), - nc(65, "65", "0x41", "0o101", "0b1000001"), - nc(91, "91", "0x5b", "0o133", "0b1011011"), - nc(92, "92", "0x5c", "0o134", "0b1011100"), - nc(93, "93", "0x5d", "0o135", "0b1011101"), - nc(99, "99", "0x63", "0o143", "0b1100011"), - nc(100, "100", "0x64", "0o144", "0b1100100"), - nc(101, "101", "0x65", "0o145", "0b1100101"), - nc(127, "127", "0x7f", "0o177", "0b1111111"), - nc(128, "128", "0x80", "0o200", "0b10000000"), - nc(129, "129", "0x81", "0o201", "0b10000001"), - nc(255, "255", "0xff", "0o377", "0b11111111"), - nc(256, "256", "0x100", "0o400", "0b100000000"), - nc(257, "257", "0x101", "0o401", "0b100000001"), - nc(511, "511", "0x1ff", "0o777", "0b111111111"), - nc(512, "512", "0x200", "0o1000", "0b1000000000"), - nc(513, "513", "0x201", "0o1001", "0b1000000001"), - nc(921, "921", "0x399", "0o1631", "0b1110011001"), - nc(922, "922", "0x39a", "0o1632", "0b1110011010"), - nc(923, "923", "0x39b", "0o1633", "0b1110011011"), - nc(999, "999", "0x3e7", "0o1747", "0b1111100111"), - nc(1000, "1000", "0x3e8", "0o1750", "0b1111101000"), - nc(1001, "1001", "0x3e9", "0o1751", "0b1111101001"), - nc(1023, "1023", "0x3ff", "0o1777", "0b1111111111"), - nc(1024, "1024", "0x400", "0o2000", "0b10000000000"), - nc(1025, "1025", "0x401", "0o2001", "0b10000000001"), - nc(2047, "2047", "0x7ff", "0o3777", "0b11111111111"), - nc(2048, "2048", "0x800", "0o4000", "0b100000000000"), - nc(2049, "2049", "0x801", "0o4001", "0b100000000001"), - nc(4095, "4095", "0xfff", "0o7777", "0b111111111111"), - nc(4096, "4096", "0x1000", "0o10000", "0b1000000000000"), - nc(4097, "4097", "0x1001", "0o10001", "0b1000000000001"), - nc(8191, "8191", "0x1fff", "0o17777", "0b1111111111111"), - nc(8192, "8192", "0x2000", "0o20000", "0b10000000000000"), - nc(8193, "8193", "0x2001", "0o20001", "0b10000000000001"), - nc(9222, "9222", "0x2406", "0o22006", "0b10010000000110"), - nc(9223, "9223", "0x2407", "0o22007", "0b10010000000111"), - nc(9224, "9224", "0x2408", "0o22010", "0b10010000001000"), - nc(9999, "9999", "0x270f", "0o23417", "0b10011100001111"), - nc(10000, "10000", "0x2710", "0o23420", "0b10011100010000"), - nc(10001, "10001", "0x2711", "0o23421", "0b10011100010001"), - nc(16383, "16383", "0x3fff", "0o37777", "0b11111111111111"), - nc(16384, "16384", "0x4000", "0o40000", "0b100000000000000"), - nc(16385, "16385", "0x4001", "0o40001", "0b100000000000001"), - nc(32767, "32767", "0x7fff", "0o77777", "0b111111111111111"), - nc(32768, "32768", "0x8000", "0o100000", "0b1000000000000000"), - nc(32769, "32769", "0x8001", "0o100001", "0b1000000000000001"), - nc(65535, "65535", "0xffff", "0o177777", "0b1111111111111111"), - nc(65536, "65536", "0x10000", "0o200000", "0b10000000000000000"), - nc(65537, "65537", "0x10001", "0o200001", "0b10000000000000001"), - nc(92232, "92232", "0x16848", "0o264110", "0b10110100001001000"), - nc(92233, "92233", "0x16849", "0o264111", "0b10110100001001001"), - nc(92234, "92234", "0x1684a", "0o264112", "0b10110100001001010"), - nc(99999, "99999", "0x1869f", "0o303237", "0b11000011010011111"), - nc(100000, "100000", "0x186a0", "0o303240", "0b11000011010100000"), - nc(100001, "100001", "0x186a1", "0o303241", "0b11000011010100001"), - nc(131071, "131071", "0x1ffff", "0o377777", "0b11111111111111111"), - nc(131072, "131072", "0x20000", "0o400000", "0b100000000000000000"), - nc(131073, "131073", "0x20001", "0o400001", "0b100000000000000001"), - nc(262143, "262143", "0x3ffff", "0o777777", "0b111111111111111111"), - nc(262144, "262144", "0x40000", "0o1000000", "0b1000000000000000000"), - nc(262145, "262145", "0x40001", "0o1000001", "0b1000000000000000001"), - nc(524287, "524287", "0x7ffff", "0o1777777", "0b1111111111111111111"), - nc(524288, "524288", "0x80000", "0o2000000", "0b10000000000000000000"), - nc(524289, "524289", "0x80001", "0o2000001", "0b10000000000000000001"), - nc(922336, "922336", "0xe12e0", "0o3411340", "0b11100001001011100000"), - nc(922337, "922337", "0xe12e1", "0o3411341", "0b11100001001011100001"), - nc(922338, "922338", "0xe12e2", "0o3411342", "0b11100001001011100010"), - nc(999999, "999999", "0xf423f", "0o3641077", "0b11110100001000111111"), - nc(1000000, "1000000", "0xf4240", "0o3641100", "0b11110100001001000000"), - nc(1000001, "1000001", "0xf4241", "0o3641101", "0b11110100001001000001"), - nc(1048575, "1048575", "0xfffff", "0o3777777", "0b11111111111111111111"), - nc(1048576, "1048576", "0x100000", "0o4000000", "0b100000000000000000000"), - nc(1048577, "1048577", "0x100001", "0o4000001", "0b100000000000000000001"), - nc(2097151, "2097151", "0x1fffff", "0o7777777", "0b111111111111111111111"), - nc(2097152, "2097152", "0x200000", "0o10000000", "0b1000000000000000000000"), - nc(2097153, "2097153", "0x200001", "0o10000001", "0b1000000000000000000001"), - nc(4194303, "4194303", "0x3fffff", "0o17777777", "0b1111111111111111111111"), - nc(4194304, "4194304", "0x400000", "0o20000000", "0b10000000000000000000000"), - nc(4194305, "4194305", "0x400001", "0o20000001", "0b10000000000000000000001"), - nc(8388607, "8388607", "0x7fffff", "0o37777777", "0b11111111111111111111111"), - nc(8388608, "8388608", "0x800000", "0o40000000", "0b100000000000000000000000"), - nc(8388609, "8388609", "0x800001", "0o40000001", "0b100000000000000000000001"), - nc(9223371, "9223371", "0x8cbccb", "0o43136313", "0b100011001011110011001011"), - nc(9223372, "9223372", "0x8cbccc", "0o43136314", "0b100011001011110011001100"), - nc(9223373, "9223373", "0x8cbccd", "0o43136315", "0b100011001011110011001101"), - nc(9999999, "9999999", "0x98967f", "0o46113177", "0b100110001001011001111111"), - nc(10000000, "10000000", "0x989680", "0o46113200", "0b100110001001011010000000"), - nc(10000001, "10000001", "0x989681", "0o46113201", "0b100110001001011010000001"), - nc(16777215, "16777215", "0xffffff", "0o77777777", "0b111111111111111111111111"), - nc(16777216, "16777216", "0x1000000", "0o100000000", "0b1000000000000000000000000"), - nc(16777217, "16777217", "0x1000001", "0o100000001", "0b1000000000000000000000001"), - nc(33554431, "33554431", "0x1ffffff", "0o177777777", "0b1111111111111111111111111"), - nc(33554432, "33554432", "0x2000000", "0o200000000", "0b10000000000000000000000000"), - nc(33554433, "33554433", "0x2000001", "0o200000001", "0b10000000000000000000000001"), - nc(67108863, "67108863", "0x3ffffff", "0o377777777", "0b11111111111111111111111111"), - nc(67108864, "67108864", "0x4000000", "0o400000000", "0b100000000000000000000000000"), - nc(67108865, "67108865", "0x4000001", "0o400000001", "0b100000000000000000000000001"), - nc(92233719, "92233719", "0x57f5ff7", "0o537657767", "0b101011111110101111111110111"), - nc(92233720, "92233720", "0x57f5ff8", "0o537657770", "0b101011111110101111111111000"), - nc(92233721, "92233721", "0x57f5ff9", "0o537657771", "0b101011111110101111111111001"), - nc(99999999, "99999999", "0x5f5e0ff", "0o575360377", "0b101111101011110000011111111"), - nc(100000000, "100000000", "0x5f5e100", "0o575360400", "0b101111101011110000100000000"), - nc(100000001, "100000001", "0x5f5e101", "0o575360401", "0b101111101011110000100000001"), - nc(134217727, "134217727", "0x7ffffff", "0o777777777", "0b111111111111111111111111111"), - nc(134217728, "134217728", "0x8000000", "0o1000000000", "0b1000000000000000000000000000"), - nc(134217729, "134217729", "0x8000001", "0o1000000001", "0b1000000000000000000000000001"), - nc(268435455, "268435455", "0xfffffff", "0o1777777777", "0b1111111111111111111111111111"), - nc(268435456, "268435456", "0x10000000", "0o2000000000", "0b10000000000000000000000000000"), - nc(268435457, "268435457", "0x10000001", "0o2000000001", "0b10000000000000000000000000001"), - nc(536870911, "536870911", "0x1fffffff", "0o3777777777", "0b11111111111111111111111111111"), - nc(536870912, "536870912", "0x20000000", "0o4000000000", "0b100000000000000000000000000000"), - nc(536870913, "536870913", "0x20000001", "0o4000000001", "0b100000000000000000000000000001"), - nc(922337202, "922337202", "0x36f9bfb2", "0o6676337662", "0b110110111110011011111110110010"), - nc(922337203, "922337203", "0x36f9bfb3", "0o6676337663", "0b110110111110011011111110110011"), - nc(922337204, "922337204", "0x36f9bfb4", "0o6676337664", "0b110110111110011011111110110100"), - nc(999999999, "999999999", "0x3b9ac9ff", "0o7346544777", "0b111011100110101100100111111111"), - nc(1000000000, "1000000000", "0x3b9aca00", "0o7346545000", "0b111011100110101100101000000000"), - nc(1000000001, "1000000001", "0x3b9aca01", "0o7346545001", "0b111011100110101100101000000001"), - nc(1073741823, "1073741823", "0x3fffffff", "0o7777777777", "0b111111111111111111111111111111"), - nc(1073741824, "1073741824", "0x40000000", "0o10000000000", "0b1000000000000000000000000000000"), - nc(1073741825, "1073741825", "0x40000001", "0o10000000001", "0b1000000000000000000000000000001"), - nc(2147483647, "2147483647", "0x7fffffff", "0o17777777777", "0b1111111111111111111111111111111"), - nc(2147483648, "2147483648", "0x80000000", "0o20000000000", "0b10000000000000000000000000000000"), - nc(2147483649, "2147483649", "0x80000001", "0o20000000001", "0b10000000000000000000000000000001"), - nc(4294967295, "4294967295", "0xffffffff", "0o37777777777", "0b11111111111111111111111111111111"), - nc(4294967296, "4294967296", "0x100000000", "0o40000000000", "0b100000000000000000000000000000000"), - nc(4294967297, "4294967297", "0x100000001", "0o40000000001", "0b100000000000000000000000000000001"), - nc(8589934591, "8589934591", "0x1ffffffff", "0o77777777777", "0b111111111111111111111111111111111"), - nc(8589934592, "8589934592", "0x200000000", "0o100000000000", "0b1000000000000000000000000000000000"), - nc(8589934593, "8589934593", "0x200000001", "0o100000000001", "0b1000000000000000000000000000000001"), - nc(9223372035, "9223372035", "0x225c17d03", "0o104560276403", "0b1000100101110000010111110100000011"), - nc(9223372036, "9223372036", "0x225c17d04", "0o104560276404", "0b1000100101110000010111110100000100"), - nc(9223372037, "9223372037", "0x225c17d05", "0o104560276405", "0b1000100101110000010111110100000101"), - nc(9999999999, "9999999999", "0x2540be3ff", "0o112402761777", "0b1001010100000010111110001111111111"), - nc(10000000000, "10000000000", "0x2540be400", "0o112402762000", "0b1001010100000010111110010000000000"), - nc(10000000001, "10000000001", "0x2540be401", "0o112402762001", "0b1001010100000010111110010000000001"), - nc(17179869183, "17179869183", "0x3ffffffff", "0o177777777777", "0b1111111111111111111111111111111111"), - nc(17179869184, "17179869184", "0x400000000", "0o200000000000", "0b10000000000000000000000000000000000"), - nc(17179869185, "17179869185", "0x400000001", "0o200000000001", "0b10000000000000000000000000000000001"), - nc(34359738367, "34359738367", "0x7ffffffff", "0o377777777777", "0b11111111111111111111111111111111111"), - nc(34359738368, "34359738368", "0x800000000", "0o400000000000", "0b100000000000000000000000000000000000"), - nc(34359738369, "34359738369", "0x800000001", "0o400000000001", "0b100000000000000000000000000000000001"), - nc(68719476735, "68719476735", "0xfffffffff", "0o777777777777", "0b111111111111111111111111111111111111"), - nc(68719476736, "68719476736", "0x1000000000", "0o1000000000000", "0b1000000000000000000000000000000000000"), - nc(68719476737, "68719476737", "0x1000000001", "0o1000000000001", "0b1000000000000000000000000000000000001"), - nc(92233720367, "92233720367", "0x15798ee22f", "0o1257143561057", "0b1010101111001100011101110001000101111"), - nc(92233720368, "92233720368", "0x15798ee230", "0o1257143561060", "0b1010101111001100011101110001000110000"), - nc(92233720369, "92233720369", "0x15798ee231", "0o1257143561061", "0b1010101111001100011101110001000110001"), - nc(99999999999, "99999999999", "0x174876e7ff", "0o1351035563777", "0b1011101001000011101101110011111111111"), - nc(100000000000, "100000000000", "0x174876e800", "0o1351035564000", "0b1011101001000011101101110100000000000"), - nc(100000000001, "100000000001", "0x174876e801", "0o1351035564001", "0b1011101001000011101101110100000000001"), - nc(137438953471, "137438953471", "0x1fffffffff", "0o1777777777777", "0b1111111111111111111111111111111111111"), - nc(137438953472, "137438953472", "0x2000000000", "0o2000000000000", "0b10000000000000000000000000000000000000"), - nc(137438953473, "137438953473", "0x2000000001", "0o2000000000001", "0b10000000000000000000000000000000000001"), - nc(274877906943, "274877906943", "0x3fffffffff", "0o3777777777777", "0b11111111111111111111111111111111111111"), - nc(274877906944, "274877906944", "0x4000000000", "0o4000000000000", "0b100000000000000000000000000000000000000"), - nc(274877906945, "274877906945", "0x4000000001", "0o4000000000001", "0b100000000000000000000000000000000000001"), - nc(549755813887, "549755813887", "0x7fffffffff", "0o7777777777777", "0b111111111111111111111111111111111111111"), - nc(549755813888, "549755813888", "0x8000000000", "0o10000000000000", "0b1000000000000000000000000000000000000000"), - nc(549755813889, "549755813889", "0x8000000001", "0o10000000000001", "0b1000000000000000000000000000000000000001"), - nc(922337203684, "922337203684", "0xd6bf94d5e4", "0o15327745152744", "0b1101011010111111100101001101010111100100"), - nc(922337203685, "922337203685", "0xd6bf94d5e5", "0o15327745152745", "0b1101011010111111100101001101010111100101"), - nc(922337203686, "922337203686", "0xd6bf94d5e6", "0o15327745152746", "0b1101011010111111100101001101010111100110"), - nc(999999999999, "999999999999", "0xe8d4a50fff", "0o16432451207777", "0b1110100011010100101001010000111111111111"), - nc(1000000000000, "1000000000000", "0xe8d4a51000", "0o16432451210000", "0b1110100011010100101001010001000000000000"), - nc(1000000000001, "1000000000001", "0xe8d4a51001", "0o16432451210001", "0b1110100011010100101001010001000000000001"), - nc(1099511627775, "1099511627775", "0xffffffffff", "0o17777777777777", "0b1111111111111111111111111111111111111111"), - nc(1099511627776, "1099511627776", "0x10000000000", "0o20000000000000", "0b10000000000000000000000000000000000000000"), - nc(1099511627777, "1099511627777", "0x10000000001", "0o20000000000001", "0b10000000000000000000000000000000000000001"), - nc(2199023255551, "2199023255551", "0x1ffffffffff", "0o37777777777777", "0b11111111111111111111111111111111111111111"), - nc(2199023255552, "2199023255552", "0x20000000000", "0o40000000000000", "0b100000000000000000000000000000000000000000"), - nc(2199023255553, "2199023255553", "0x20000000001", "0o40000000000001", "0b100000000000000000000000000000000000000001"), - nc(4398046511103, "4398046511103", "0x3ffffffffff", "0o77777777777777", "0b111111111111111111111111111111111111111111"), - nc(4398046511104, "4398046511104", "0x40000000000", "0o100000000000000", "0b1000000000000000000000000000000000000000000"), - nc(4398046511105, "4398046511105", "0x40000000001", "0o100000000000001", "0b1000000000000000000000000000000000000000001"), - nc(8796093022207, "8796093022207", "0x7ffffffffff", "0o177777777777777", "0b1111111111111111111111111111111111111111111"), - nc(8796093022208, "8796093022208", "0x80000000000", "0o200000000000000", "0b10000000000000000000000000000000000000000000"), - nc(8796093022209, "8796093022209", "0x80000000001", "0o200000000000001", "0b10000000000000000000000000000000000000000001"), - nc(9223372036853, "9223372036853", "0x8637bd05af5", "0o206157364055365", "0b10000110001101111011110100000101101011110101"), - nc(9223372036854, "9223372036854", "0x8637bd05af6", "0o206157364055366", "0b10000110001101111011110100000101101011110110"), - nc(9223372036855, "9223372036855", "0x8637bd05af7", "0o206157364055367", "0b10000110001101111011110100000101101011110111"), - nc(9999999999999, "9999999999999", "0x9184e729fff", "0o221411634517777", "0b10010001100001001110011100101001111111111111"), - nc(10000000000000, "10000000000000", "0x9184e72a000", "0o221411634520000", "0b10010001100001001110011100101010000000000000"), - nc(10000000000001, "10000000000001", "0x9184e72a001", "0o221411634520001", "0b10010001100001001110011100101010000000000001"), - nc(17592186044415, "17592186044415", "0xfffffffffff", "0o377777777777777", "0b11111111111111111111111111111111111111111111"), - nc(17592186044416, "17592186044416", "0x100000000000", "0o400000000000000", "0b100000000000000000000000000000000000000000000"), - nc(17592186044417, "17592186044417", "0x100000000001", "0o400000000000001", "0b100000000000000000000000000000000000000000001"), - nc(35184372088831, "35184372088831", "0x1fffffffffff", "0o777777777777777", "0b111111111111111111111111111111111111111111111"), - nc(35184372088832, "35184372088832", "0x200000000000", "0o1000000000000000", "0b1000000000000000000000000000000000000000000000"), - nc(35184372088833, "35184372088833", "0x200000000001", "0o1000000000000001", "0b1000000000000000000000000000000000000000000001"), - nc(70368744177663, "70368744177663", "0x3fffffffffff", "0o1777777777777777", "0b1111111111111111111111111111111111111111111111"), - nc(70368744177664, "70368744177664", "0x400000000000", "0o2000000000000000", "0b10000000000000000000000000000000000000000000000"), - nc(70368744177665, "70368744177665", "0x400000000001", "0o2000000000000001", "0b10000000000000000000000000000000000000000000001"), - nc(92233720368546, "92233720368546", "0x53e2d6238da2", "0o2476132610706642", "0b10100111110001011010110001000111000110110100010"), - nc(92233720368547, "92233720368547", "0x53e2d6238da3", "0o2476132610706643", "0b10100111110001011010110001000111000110110100011"), - nc(92233720368548, "92233720368548", "0x53e2d6238da4", "0o2476132610706644", "0b10100111110001011010110001000111000110110100100"), - nc(99999999999999, "99999999999999", "0x5af3107a3fff", "0o2657142036437777", "0b10110101111001100010000011110100011111111111111"), - nc(100000000000000, "100000000000000", "0x5af3107a4000", "0o2657142036440000", "0b10110101111001100010000011110100100000000000000"), - nc(100000000000001, "100000000000001", "0x5af3107a4001", "0o2657142036440001", "0b10110101111001100010000011110100100000000000001"), - nc(140737488355327, "140737488355327", "0x7fffffffffff", "0o3777777777777777", "0b11111111111111111111111111111111111111111111111"), - nc(140737488355328, "140737488355328", "0x800000000000", "0o4000000000000000", "0b100000000000000000000000000000000000000000000000"), - nc(140737488355329, "140737488355329", "0x800000000001", "0o4000000000000001", "0b100000000000000000000000000000000000000000000001"), - nc(281474976710655, "281474976710655", "0xffffffffffff", "0o7777777777777777", "0b111111111111111111111111111111111111111111111111"), - nc(281474976710656, "281474976710656", "0x1000000000000", "0o10000000000000000", "0b1000000000000000000000000000000000000000000000000"), - nc(281474976710657, "281474976710657", "0x1000000000001", "0o10000000000000001", "0b1000000000000000000000000000000000000000000000001"), - nc(562949953421311, "562949953421311", "0x1ffffffffffff", "0o17777777777777777", "0b1111111111111111111111111111111111111111111111111"), - nc(562949953421312, "562949953421312", "0x2000000000000", "0o20000000000000000", "0b10000000000000000000000000000000000000000000000000"), - nc(562949953421313, "562949953421313", "0x2000000000001", "0o20000000000000001", "0b10000000000000000000000000000000000000000000000001"), - nc(922337203685476, "922337203685476", "0x346dc5d638864", "0o32155613530704144", "0b11010001101101110001011101011000111000100001100100"), - nc(922337203685477, "922337203685477", "0x346dc5d638865", "0o32155613530704145", "0b11010001101101110001011101011000111000100001100101"), - nc(922337203685478, "922337203685478", "0x346dc5d638866", "0o32155613530704146", "0b11010001101101110001011101011000111000100001100110"), - nc(999999999999999, "999999999999999", "0x38d7ea4c67fff", "0o34327724461477777", "0b11100011010111111010100100110001100111111111111111"), - nc(1000000000000000, "1000000000000000", "0x38d7ea4c68000", "0o34327724461500000", "0b11100011010111111010100100110001101000000000000000"), - nc(1000000000000001, "1000000000000001", "0x38d7ea4c68001", "0o34327724461500001", "0b11100011010111111010100100110001101000000000000001"), - nc(1125899906842623, "1125899906842623", "0x3ffffffffffff", "0o37777777777777777", "0b11111111111111111111111111111111111111111111111111"), - nc(1125899906842624, "1125899906842624", "0x4000000000000", "0o40000000000000000", "0b100000000000000000000000000000000000000000000000000"), - nc(1125899906842625, "1125899906842625", "0x4000000000001", "0o40000000000000001", "0b100000000000000000000000000000000000000000000000001"), - nc(2251799813685247, "2251799813685247", "0x7ffffffffffff", "0o77777777777777777", "0b111111111111111111111111111111111111111111111111111"), - nc(2251799813685248, "2251799813685248", "0x8000000000000", "0o100000000000000000", "0b1000000000000000000000000000000000000000000000000000"), - nc(2251799813685249, "2251799813685249", "0x8000000000001", "0o100000000000000001", "0b1000000000000000000000000000000000000000000000000001"), - nc(4503599627370495, "4503599627370495", "0xfffffffffffff", "0o177777777777777777", "0b1111111111111111111111111111111111111111111111111111"), - nc(4503599627370496, "4503599627370496", "0x10000000000000", "0o200000000000000000", "0b10000000000000000000000000000000000000000000000000000"), - nc(4503599627370497, "4503599627370497", "0x10000000000001", "0o200000000000000001", "0b10000000000000000000000000000000000000000000000000001"), - nc(9007199254740991, "9007199254740991", "0x1fffffffffffff", "0o377777777777777777", "0b11111111111111111111111111111111111111111111111111111"), - nc(9007199254740992, "9007199254740992", "0x20000000000000", "0o400000000000000000", "0b100000000000000000000000000000000000000000000000000000"), - nc(9007199254740993, "9007199254740993", "0x20000000000001", "0o400000000000000001", "0b100000000000000000000000000000000000000000000000000001"), - nc(9223372036854775, "9223372036854775", "0x20c49ba5e353f7", "0o406111564570651767", "0b100000110001001001101110100101111000110101001111110111"), - nc(9223372036854776, "9223372036854776", "0x20c49ba5e353f8", "0o406111564570651770", "0b100000110001001001101110100101111000110101001111111000"), - nc(9223372036854777, "9223372036854777", "0x20c49ba5e353f9", "0o406111564570651771", "0b100000110001001001101110100101111000110101001111111001"), - nc(9999999999999999, "9999999999999999", "0x2386f26fc0ffff", "0o434157115760177777", "0b100011100001101111001001101111110000001111111111111111"), - nc(10000000000000000, "10000000000000000", "0x2386f26fc10000", "0o434157115760200000", "0b100011100001101111001001101111110000010000000000000000"), - nc(10000000000000001, "10000000000000001", "0x2386f26fc10001", "0o434157115760200001", "0b100011100001101111001001101111110000010000000000000001"), - nc(18014398509481983, "18014398509481983", "0x3fffffffffffff", "0o777777777777777777", "0b111111111111111111111111111111111111111111111111111111"), - nc(18014398509481984, "18014398509481984", "0x40000000000000", "0o1000000000000000000", "0b1000000000000000000000000000000000000000000000000000000"), - nc(18014398509481985, "18014398509481985", "0x40000000000001", "0o1000000000000000001", "0b1000000000000000000000000000000000000000000000000000001"), - nc(36028797018963967, "36028797018963967", "0x7fffffffffffff", "0o1777777777777777777", "0b1111111111111111111111111111111111111111111111111111111"), - nc(36028797018963968, "36028797018963968", "0x80000000000000", "0o2000000000000000000", "0b10000000000000000000000000000000000000000000000000000000"), - nc(36028797018963969, "36028797018963969", "0x80000000000001", "0o2000000000000000001", "0b10000000000000000000000000000000000000000000000000000001"), - nc(72057594037927935, "72057594037927935", "0xffffffffffffff", "0o3777777777777777777", "0b11111111111111111111111111111111111111111111111111111111"), - nc(72057594037927936, "72057594037927936", "0x100000000000000", "0o4000000000000000000", "0b100000000000000000000000000000000000000000000000000000000"), - nc(72057594037927937, "72057594037927937", "0x100000000000001", "0o4000000000000000001", "0b100000000000000000000000000000000000000000000000000000001"), - nc(92233720368547759, "92233720368547759", "0x147ae147ae147af", "0o5075341217270243657", "0b101000111101011100001010001111010111000010100011110101111"), - nc(92233720368547760, "92233720368547760", "0x147ae147ae147b0", "0o5075341217270243660", "0b101000111101011100001010001111010111000010100011110110000"), - nc(92233720368547761, "92233720368547761", "0x147ae147ae147b1", "0o5075341217270243661", "0b101000111101011100001010001111010111000010100011110110001"), - nc(99999999999999999, "99999999999999999", "0x16345785d89ffff", "0o5432127413542377777", "0b101100011010001010111100001011101100010011111111111111111"), - nc(100000000000000000, "100000000000000000", "0x16345785d8a0000", "0o5432127413542400000", "0b101100011010001010111100001011101100010100000000000000000"), - nc(100000000000000001, "100000000000000001", "0x16345785d8a0001", "0o5432127413542400001", "0b101100011010001010111100001011101100010100000000000000001"), - nc(144115188075855871, "144115188075855871", "0x1ffffffffffffff", "0o7777777777777777777", "0b111111111111111111111111111111111111111111111111111111111"), - nc(144115188075855872, "144115188075855872", "0x200000000000000", "0o10000000000000000000", "0b1000000000000000000000000000000000000000000000000000000000"), - nc(144115188075855873, "144115188075855873", "0x200000000000001", "0o10000000000000000001", "0b1000000000000000000000000000000000000000000000000000000001"), - nc(288230376151711743, "288230376151711743", "0x3ffffffffffffff", "0o17777777777777777777", "0b1111111111111111111111111111111111111111111111111111111111"), - nc(288230376151711744, "288230376151711744", "0x400000000000000", "0o20000000000000000000", "0b10000000000000000000000000000000000000000000000000000000000"), - nc(288230376151711745, "288230376151711745", "0x400000000000001", "0o20000000000000000001", "0b10000000000000000000000000000000000000000000000000000000001"), - nc(576460752303423487, "576460752303423487", "0x7ffffffffffffff", "0o37777777777777777777", "0b11111111111111111111111111111111111111111111111111111111111"), - nc(576460752303423488, "576460752303423488", "0x800000000000000", "0o40000000000000000000", "0b100000000000000000000000000000000000000000000000000000000000"), - nc(576460752303423489, "576460752303423489", "0x800000000000001", "0o40000000000000000001", "0b100000000000000000000000000000000000000000000000000000000001"), - nc(922337203685477631, "922337203685477631", "0xcccccccccccccff", "0o63146314631463146377", "0b110011001100110011001100110011001100110011001100110011111111"), - nc(922337203685477632, "922337203685477632", "0xccccccccccccd00", "0o63146314631463146400", "0b110011001100110011001100110011001100110011001100110100000000"), - nc(922337203685477633, "922337203685477633", "0xccccccccccccd01", "0o63146314631463146401", "0b110011001100110011001100110011001100110011001100110100000001"), - nc(999999999999999999, "999999999999999999", "0xde0b6b3a763ffff", "0o67405553164730777777", "0b110111100000101101101011001110100111011000111111111111111111"), - nc(1000000000000000000, "1000000000000000000", "0xde0b6b3a7640000", "0o67405553164731000000", "0b110111100000101101101011001110100111011001000000000000000000"), - nc(1000000000000000001, "1000000000000000001", "0xde0b6b3a7640001", "0o67405553164731000001", "0b110111100000101101101011001110100111011001000000000000000001"), - nc(1152921504606846975, "1152921504606846975", "0xfffffffffffffff", "0o77777777777777777777", "0b111111111111111111111111111111111111111111111111111111111111"), - nc(1152921504606846976, "1152921504606846976", "0x1000000000000000", "0o100000000000000000000", "0b1000000000000000000000000000000000000000000000000000000000000"), - nc(1152921504606846977, "1152921504606846977", "0x1000000000000001", "0o100000000000000000001", "0b1000000000000000000000000000000000000000000000000000000000001"), - nc(2305843009213693951, "2305843009213693951", "0x1fffffffffffffff", "0o177777777777777777777", "0b1111111111111111111111111111111111111111111111111111111111111"), - nc(2305843009213693952, "2305843009213693952", "0x2000000000000000", "0o200000000000000000000", "0b10000000000000000000000000000000000000000000000000000000000000"), - nc(2305843009213693953, "2305843009213693953", "0x2000000000000001", "0o200000000000000000001", "0b10000000000000000000000000000000000000000000000000000000000001"), - nc(4611686018427387903, "4611686018427387903", "0x3fffffffffffffff", "0o377777777777777777777", "0b11111111111111111111111111111111111111111111111111111111111111"), - nc(4611686018427387904, "4611686018427387904", "0x4000000000000000", "0o400000000000000000000", "0b100000000000000000000000000000000000000000000000000000000000000"), - nc(4611686018427387905, "4611686018427387905", "0x4000000000000001", "0o400000000000000000001", "0b100000000000000000000000000000000000000000000000000000000000001"), - nc(9223372036854775802, "9223372036854775802", "0x7ffffffffffffffa", "0o777777777777777777772", "0b111111111111111111111111111111111111111111111111111111111111010"), - nc(9223372036854775803, "9223372036854775803", "0x7ffffffffffffffb", "0o777777777777777777773", "0b111111111111111111111111111111111111111111111111111111111111011"), - nc(9223372036854775804, "9223372036854775804", "0x7ffffffffffffffc", "0o777777777777777777774", "0b111111111111111111111111111111111111111111111111111111111111100"), - nc(9223372036854775805, "9223372036854775805", "0x7ffffffffffffffd", "0o777777777777777777775", "0b111111111111111111111111111111111111111111111111111111111111101"), - nc(9223372036854775806, "9223372036854775806", "0x7ffffffffffffffe", "0o777777777777777777776", "0b111111111111111111111111111111111111111111111111111111111111110"), - nc(9223372036854775807, "9223372036854775807", "0x7fffffffffffffff", "0o777777777777777777777", "0b111111111111111111111111111111111111111111111111111111111111111"), -#undef nc - }; -}; - - -template<> -struct numbers -{ - using value_type = uint64_t; - static C4_INLINE_CONSTEXPR const number_case vals[] = { -#define nc(val, dec, hex, bin, oct) \ - number_case{UINT64_C(val), csubstr{dec}, csubstr{hex}, csubstr{bin}, csubstr{oct}} - nc(0, "0", "0x0", "0o0", "0b0"), - nc(1, "1", "0x1", "0o1", "0b1"), - nc(2, "2", "0x2", "0o2", "0b10"), - nc(3, "3", "0x3", "0o3", "0b11"), - nc(4, "4", "0x4", "0o4", "0b100"), - nc(5, "5", "0x5", "0o5", "0b101"), - nc(6, "6", "0x6", "0o6", "0b110"), - nc(7, "7", "0x7", "0o7", "0b111"), - nc(8, "8", "0x8", "0o10", "0b1000"), - nc(9, "9", "0x9", "0o11", "0b1001"), - nc(10, "10", "0xa", "0o12", "0b1010"), - nc(11, "11", "0xb", "0o13", "0b1011"), - nc(15, "15", "0xf", "0o17", "0b1111"), - nc(16, "16", "0x10", "0o20", "0b10000"), - nc(17, "17", "0x11", "0o21", "0b10001"), - nc(18, "18", "0x12", "0o22", "0b10010"), - nc(19, "19", "0x13", "0o23", "0b10011"), - nc(31, "31", "0x1f", "0o37", "0b11111"), - nc(32, "32", "0x20", "0o40", "0b100000"), - nc(33, "33", "0x21", "0o41", "0b100001"), - nc(63, "63", "0x3f", "0o77", "0b111111"), - nc(64, "64", "0x40", "0o100", "0b1000000"), - nc(65, "65", "0x41", "0o101", "0b1000001"), - nc(99, "99", "0x63", "0o143", "0b1100011"), - nc(100, "100", "0x64", "0o144", "0b1100100"), - nc(101, "101", "0x65", "0o145", "0b1100101"), - nc(127, "127", "0x7f", "0o177", "0b1111111"), - nc(128, "128", "0x80", "0o200", "0b10000000"), - nc(129, "129", "0x81", "0o201", "0b10000001"), - nc(183, "183", "0xb7", "0o267", "0b10110111"), - nc(184, "184", "0xb8", "0o270", "0b10111000"), - nc(185, "185", "0xb9", "0o271", "0b10111001"), - nc(255, "255", "0xff", "0o377", "0b11111111"), - nc(256, "256", "0x100", "0o400", "0b100000000"), - nc(257, "257", "0x101", "0o401", "0b100000001"), - nc(511, "511", "0x1ff", "0o777", "0b111111111"), - nc(512, "512", "0x200", "0o1000", "0b1000000000"), - nc(513, "513", "0x201", "0o1001", "0b1000000001"), - nc(999, "999", "0x3e7", "0o1747", "0b1111100111"), - nc(1000, "1000", "0x3e8", "0o1750", "0b1111101000"), - nc(1001, "1001", "0x3e9", "0o1751", "0b1111101001"), - nc(1023, "1023", "0x3ff", "0o1777", "0b1111111111"), - nc(1024, "1024", "0x400", "0o2000", "0b10000000000"), - nc(1025, "1025", "0x401", "0o2001", "0b10000000001"), - nc(1843, "1843", "0x733", "0o3463", "0b11100110011"), - nc(1844, "1844", "0x734", "0o3464", "0b11100110100"), - nc(1845, "1845", "0x735", "0o3465", "0b11100110101"), - nc(2047, "2047", "0x7ff", "0o3777", "0b11111111111"), - nc(2048, "2048", "0x800", "0o4000", "0b100000000000"), - nc(2049, "2049", "0x801", "0o4001", "0b100000000001"), - nc(4095, "4095", "0xfff", "0o7777", "0b111111111111"), - nc(4096, "4096", "0x1000", "0o10000", "0b1000000000000"), - nc(4097, "4097", "0x1001", "0o10001", "0b1000000000001"), - nc(8191, "8191", "0x1fff", "0o17777", "0b1111111111111"), - nc(8192, "8192", "0x2000", "0o20000", "0b10000000000000"), - nc(8193, "8193", "0x2001", "0o20001", "0b10000000000001"), - nc(9999, "9999", "0x270f", "0o23417", "0b10011100001111"), - nc(10000, "10000", "0x2710", "0o23420", "0b10011100010000"), - nc(10001, "10001", "0x2711", "0o23421", "0b10011100010001"), - nc(16383, "16383", "0x3fff", "0o37777", "0b11111111111111"), - nc(16384, "16384", "0x4000", "0o40000", "0b100000000000000"), - nc(16385, "16385", "0x4001", "0o40001", "0b100000000000001"), - nc(18445, "18445", "0x480d", "0o44015", "0b100100000001101"), - nc(18446, "18446", "0x480e", "0o44016", "0b100100000001110"), - nc(18447, "18447", "0x480f", "0o44017", "0b100100000001111"), - nc(32767, "32767", "0x7fff", "0o77777", "0b111111111111111"), - nc(32768, "32768", "0x8000", "0o100000", "0b1000000000000000"), - nc(32769, "32769", "0x8001", "0o100001", "0b1000000000000001"), - nc(65535, "65535", "0xffff", "0o177777", "0b1111111111111111"), - nc(65536, "65536", "0x10000", "0o200000", "0b10000000000000000"), - nc(65537, "65537", "0x10001", "0o200001", "0b10000000000000001"), - nc(99999, "99999", "0x1869f", "0o303237", "0b11000011010011111"), - nc(100000, "100000", "0x186a0", "0o303240", "0b11000011010100000"), - nc(100001, "100001", "0x186a1", "0o303241", "0b11000011010100001"), - nc(131071, "131071", "0x1ffff", "0o377777", "0b11111111111111111"), - nc(131072, "131072", "0x20000", "0o400000", "0b100000000000000000"), - nc(131073, "131073", "0x20001", "0o400001", "0b100000000000000001"), - nc(184466, "184466", "0x2d092", "0o550222", "0b101101000010010010"), - nc(184467, "184467", "0x2d093", "0o550223", "0b101101000010010011"), - nc(184468, "184468", "0x2d094", "0o550224", "0b101101000010010100"), - nc(262143, "262143", "0x3ffff", "0o777777", "0b111111111111111111"), - nc(262144, "262144", "0x40000", "0o1000000", "0b1000000000000000000"), - nc(262145, "262145", "0x40001", "0o1000001", "0b1000000000000000001"), - nc(524287, "524287", "0x7ffff", "0o1777777", "0b1111111111111111111"), - nc(524288, "524288", "0x80000", "0o2000000", "0b10000000000000000000"), - nc(524289, "524289", "0x80001", "0o2000001", "0b10000000000000000001"), - nc(999999, "999999", "0xf423f", "0o3641077", "0b11110100001000111111"), - nc(1000000, "1000000", "0xf4240", "0o3641100", "0b11110100001001000000"), - nc(1000001, "1000001", "0xf4241", "0o3641101", "0b11110100001001000001"), - nc(1048575, "1048575", "0xfffff", "0o3777777", "0b11111111111111111111"), - nc(1048576, "1048576", "0x100000", "0o4000000", "0b100000000000000000000"), - nc(1048577, "1048577", "0x100001", "0o4000001", "0b100000000000000000001"), - nc(1844673, "1844673", "0x1c25c1", "0o7022701", "0b111000010010111000001"), - nc(1844674, "1844674", "0x1c25c2", "0o7022702", "0b111000010010111000010"), - nc(1844675, "1844675", "0x1c25c3", "0o7022703", "0b111000010010111000011"), - nc(2097151, "2097151", "0x1fffff", "0o7777777", "0b111111111111111111111"), - nc(2097152, "2097152", "0x200000", "0o10000000", "0b1000000000000000000000"), - nc(2097153, "2097153", "0x200001", "0o10000001", "0b1000000000000000000001"), - nc(4194303, "4194303", "0x3fffff", "0o17777777", "0b1111111111111111111111"), - nc(4194304, "4194304", "0x400000", "0o20000000", "0b10000000000000000000000"), - nc(4194305, "4194305", "0x400001", "0o20000001", "0b10000000000000000000001"), - nc(8388607, "8388607", "0x7fffff", "0o37777777", "0b11111111111111111111111"), - nc(8388608, "8388608", "0x800000", "0o40000000", "0b100000000000000000000000"), - nc(8388609, "8388609", "0x800001", "0o40000001", "0b100000000000000000000001"), - nc(9999999, "9999999", "0x98967f", "0o46113177", "0b100110001001011001111111"), - nc(10000000, "10000000", "0x989680", "0o46113200", "0b100110001001011010000000"), - nc(10000001, "10000001", "0x989681", "0o46113201", "0b100110001001011010000001"), - nc(16777215, "16777215", "0xffffff", "0o77777777", "0b111111111111111111111111"), - nc(16777216, "16777216", "0x1000000", "0o100000000", "0b1000000000000000000000000"), - nc(16777217, "16777217", "0x1000001", "0o100000001", "0b1000000000000000000000001"), - nc(18446743, "18446743", "0x1197997", "0o106274627", "0b1000110010111100110010111"), - nc(18446744, "18446744", "0x1197998", "0o106274630", "0b1000110010111100110011000"), - nc(18446745, "18446745", "0x1197999", "0o106274631", "0b1000110010111100110011001"), - nc(33554431, "33554431", "0x1ffffff", "0o177777777", "0b1111111111111111111111111"), - nc(33554432, "33554432", "0x2000000", "0o200000000", "0b10000000000000000000000000"), - nc(33554433, "33554433", "0x2000001", "0o200000001", "0b10000000000000000000000001"), - nc(67108863, "67108863", "0x3ffffff", "0o377777777", "0b11111111111111111111111111"), - nc(67108864, "67108864", "0x4000000", "0o400000000", "0b100000000000000000000000000"), - nc(67108865, "67108865", "0x4000001", "0o400000001", "0b100000000000000000000000001"), - nc(99999999, "99999999", "0x5f5e0ff", "0o575360377", "0b101111101011110000011111111"), - nc(100000000, "100000000", "0x5f5e100", "0o575360400", "0b101111101011110000100000000"), - nc(100000001, "100000001", "0x5f5e101", "0o575360401", "0b101111101011110000100000001"), - nc(134217727, "134217727", "0x7ffffff", "0o777777777", "0b111111111111111111111111111"), - nc(134217728, "134217728", "0x8000000", "0o1000000000", "0b1000000000000000000000000000"), - nc(134217729, "134217729", "0x8000001", "0o1000000001", "0b1000000000000000000000000001"), - nc(184467439, "184467439", "0xafebfef", "0o1277537757", "0b1010111111101011111111101111"), - nc(184467440, "184467440", "0xafebff0", "0o1277537760", "0b1010111111101011111111110000"), - nc(184467441, "184467441", "0xafebff1", "0o1277537761", "0b1010111111101011111111110001"), - nc(268435455, "268435455", "0xfffffff", "0o1777777777", "0b1111111111111111111111111111"), - nc(268435456, "268435456", "0x10000000", "0o2000000000", "0b10000000000000000000000000000"), - nc(268435457, "268435457", "0x10000001", "0o2000000001", "0b10000000000000000000000000001"), - nc(536870911, "536870911", "0x1fffffff", "0o3777777777", "0b11111111111111111111111111111"), - nc(536870912, "536870912", "0x20000000", "0o4000000000", "0b100000000000000000000000000000"), - nc(536870913, "536870913", "0x20000001", "0o4000000001", "0b100000000000000000000000000001"), - nc(999999999, "999999999", "0x3b9ac9ff", "0o7346544777", "0b111011100110101100100111111111"), - nc(1000000000, "1000000000", "0x3b9aca00", "0o7346545000", "0b111011100110101100101000000000"), - nc(1000000001, "1000000001", "0x3b9aca01", "0o7346545001", "0b111011100110101100101000000001"), - nc(1073741823, "1073741823", "0x3fffffff", "0o7777777777", "0b111111111111111111111111111111"), - nc(1073741824, "1073741824", "0x40000000", "0o10000000000", "0b1000000000000000000000000000000"), - nc(1073741825, "1073741825", "0x40000001", "0o10000000001", "0b1000000000000000000000000000001"), - nc(1844674406, "1844674406", "0x6df37f66", "0o15574677546", "0b1101101111100110111111101100110"), - nc(1844674407, "1844674407", "0x6df37f67", "0o15574677547", "0b1101101111100110111111101100111"), - nc(1844674408, "1844674408", "0x6df37f68", "0o15574677550", "0b1101101111100110111111101101000"), - nc(2147483647, "2147483647", "0x7fffffff", "0o17777777777", "0b1111111111111111111111111111111"), - nc(2147483648, "2147483648", "0x80000000", "0o20000000000", "0b10000000000000000000000000000000"), - nc(2147483649, "2147483649", "0x80000001", "0o20000000001", "0b10000000000000000000000000000001"), - nc(4294967295, "4294967295", "0xffffffff", "0o37777777777", "0b11111111111111111111111111111111"), - nc(4294967296, "4294967296", "0x100000000", "0o40000000000", "0b100000000000000000000000000000000"), - nc(4294967297, "4294967297", "0x100000001", "0o40000000001", "0b100000000000000000000000000000001"), - nc(8589934591, "8589934591", "0x1ffffffff", "0o77777777777", "0b111111111111111111111111111111111"), - nc(8589934592, "8589934592", "0x200000000", "0o100000000000", "0b1000000000000000000000000000000000"), - nc(8589934593, "8589934593", "0x200000001", "0o100000000001", "0b1000000000000000000000000000000001"), - nc(9999999999, "9999999999", "0x2540be3ff", "0o112402761777", "0b1001010100000010111110001111111111"), - nc(10000000000, "10000000000", "0x2540be400", "0o112402762000", "0b1001010100000010111110010000000000"), - nc(10000000001, "10000000001", "0x2540be401", "0o112402762001", "0b1001010100000010111110010000000001"), - nc(17179869183, "17179869183", "0x3ffffffff", "0o177777777777", "0b1111111111111111111111111111111111"), - nc(17179869184, "17179869184", "0x400000000", "0o200000000000", "0b10000000000000000000000000000000000"), - nc(17179869185, "17179869185", "0x400000001", "0o200000000001", "0b10000000000000000000000000000000001"), - nc(18446744072, "18446744072", "0x44b82fa08", "0o211340575010", "0b10001001011100000101111101000001000"), - nc(18446744073, "18446744073", "0x44b82fa09", "0o211340575011", "0b10001001011100000101111101000001001"), - nc(18446744074, "18446744074", "0x44b82fa0a", "0o211340575012", "0b10001001011100000101111101000001010"), - nc(34359738367, "34359738367", "0x7ffffffff", "0o377777777777", "0b11111111111111111111111111111111111"), - nc(34359738368, "34359738368", "0x800000000", "0o400000000000", "0b100000000000000000000000000000000000"), - nc(34359738369, "34359738369", "0x800000001", "0o400000000001", "0b100000000000000000000000000000000001"), - nc(68719476735, "68719476735", "0xfffffffff", "0o777777777777", "0b111111111111111111111111111111111111"), - nc(68719476736, "68719476736", "0x1000000000", "0o1000000000000", "0b1000000000000000000000000000000000000"), - nc(68719476737, "68719476737", "0x1000000001", "0o1000000000001", "0b1000000000000000000000000000000000001"), - nc(99999999999, "99999999999", "0x174876e7ff", "0o1351035563777", "0b1011101001000011101101110011111111111"), - nc(100000000000, "100000000000", "0x174876e800", "0o1351035564000", "0b1011101001000011101101110100000000000"), - nc(100000000001, "100000000001", "0x174876e801", "0o1351035564001", "0b1011101001000011101101110100000000001"), - nc(137438953471, "137438953471", "0x1fffffffff", "0o1777777777777", "0b1111111111111111111111111111111111111"), - nc(137438953472, "137438953472", "0x2000000000", "0o2000000000000", "0b10000000000000000000000000000000000000"), - nc(137438953473, "137438953473", "0x2000000001", "0o2000000000001", "0b10000000000000000000000000000000000001"), - nc(184467440736, "184467440736", "0x2af31dc460", "0o2536307342140", "0b10101011110011000111011100010001100000"), - nc(184467440737, "184467440737", "0x2af31dc461", "0o2536307342141", "0b10101011110011000111011100010001100001"), - nc(184467440738, "184467440738", "0x2af31dc462", "0o2536307342142", "0b10101011110011000111011100010001100010"), - nc(274877906943, "274877906943", "0x3fffffffff", "0o3777777777777", "0b11111111111111111111111111111111111111"), - nc(274877906944, "274877906944", "0x4000000000", "0o4000000000000", "0b100000000000000000000000000000000000000"), - nc(274877906945, "274877906945", "0x4000000001", "0o4000000000001", "0b100000000000000000000000000000000000001"), - nc(549755813887, "549755813887", "0x7fffffffff", "0o7777777777777", "0b111111111111111111111111111111111111111"), - nc(549755813888, "549755813888", "0x8000000000", "0o10000000000000", "0b1000000000000000000000000000000000000000"), - nc(549755813889, "549755813889", "0x8000000001", "0o10000000000001", "0b1000000000000000000000000000000000000001"), - nc(999999999999, "999999999999", "0xe8d4a50fff", "0o16432451207777", "0b1110100011010100101001010000111111111111"), - nc(1000000000000, "1000000000000", "0xe8d4a51000", "0o16432451210000", "0b1110100011010100101001010001000000000000"), - nc(1000000000001, "1000000000001", "0xe8d4a51001", "0o16432451210001", "0b1110100011010100101001010001000000000001"), - nc(1099511627775, "1099511627775", "0xffffffffff", "0o17777777777777", "0b1111111111111111111111111111111111111111"), - nc(1099511627776, "1099511627776", "0x10000000000", "0o20000000000000", "0b10000000000000000000000000000000000000000"), - nc(1099511627777, "1099511627777", "0x10000000001", "0o20000000000001", "0b10000000000000000000000000000000000000001"), - nc(1844674407369, "1844674407369", "0x1ad7f29abc9", "0o32657712325711", "0b11010110101111111001010011010101111001001"), - nc(1844674407370, "1844674407370", "0x1ad7f29abca", "0o32657712325712", "0b11010110101111111001010011010101111001010"), - nc(1844674407371, "1844674407371", "0x1ad7f29abcb", "0o32657712325713", "0b11010110101111111001010011010101111001011"), - nc(2199023255551, "2199023255551", "0x1ffffffffff", "0o37777777777777", "0b11111111111111111111111111111111111111111"), - nc(2199023255552, "2199023255552", "0x20000000000", "0o40000000000000", "0b100000000000000000000000000000000000000000"), - nc(2199023255553, "2199023255553", "0x20000000001", "0o40000000000001", "0b100000000000000000000000000000000000000001"), - nc(4398046511103, "4398046511103", "0x3ffffffffff", "0o77777777777777", "0b111111111111111111111111111111111111111111"), - nc(4398046511104, "4398046511104", "0x40000000000", "0o100000000000000", "0b1000000000000000000000000000000000000000000"), - nc(4398046511105, "4398046511105", "0x40000000001", "0o100000000000001", "0b1000000000000000000000000000000000000000001"), - nc(8796093022207, "8796093022207", "0x7ffffffffff", "0o177777777777777", "0b1111111111111111111111111111111111111111111"), - nc(8796093022208, "8796093022208", "0x80000000000", "0o200000000000000", "0b10000000000000000000000000000000000000000000"), - nc(8796093022209, "8796093022209", "0x80000000001", "0o200000000000001", "0b10000000000000000000000000000000000000000001"), - nc(9999999999999, "9999999999999", "0x9184e729fff", "0o221411634517777", "0b10010001100001001110011100101001111111111111"), - nc(10000000000000, "10000000000000", "0x9184e72a000", "0o221411634520000", "0b10010001100001001110011100101010000000000000"), - nc(10000000000001, "10000000000001", "0x9184e72a001", "0o221411634520001", "0b10010001100001001110011100101010000000000001"), - nc(17592186044415, "17592186044415", "0xfffffffffff", "0o377777777777777", "0b11111111111111111111111111111111111111111111"), - nc(17592186044416, "17592186044416", "0x100000000000", "0o400000000000000", "0b100000000000000000000000000000000000000000000"), - nc(17592186044417, "17592186044417", "0x100000000001", "0o400000000000001", "0b100000000000000000000000000000000000000000001"), - nc(18446744073708, "18446744073708", "0x10c6f7a0b5ec", "0o414336750132754", "0b100001100011011110111101000001011010111101100"), - nc(18446744073709, "18446744073709", "0x10c6f7a0b5ed", "0o414336750132755", "0b100001100011011110111101000001011010111101101"), - nc(18446744073710, "18446744073710", "0x10c6f7a0b5ee", "0o414336750132756", "0b100001100011011110111101000001011010111101110"), - nc(35184372088831, "35184372088831", "0x1fffffffffff", "0o777777777777777", "0b111111111111111111111111111111111111111111111"), - nc(35184372088832, "35184372088832", "0x200000000000", "0o1000000000000000", "0b1000000000000000000000000000000000000000000000"), - nc(35184372088833, "35184372088833", "0x200000000001", "0o1000000000000001", "0b1000000000000000000000000000000000000000000001"), - nc(70368744177663, "70368744177663", "0x3fffffffffff", "0o1777777777777777", "0b1111111111111111111111111111111111111111111111"), - nc(70368744177664, "70368744177664", "0x400000000000", "0o2000000000000000", "0b10000000000000000000000000000000000000000000000"), - nc(70368744177665, "70368744177665", "0x400000000001", "0o2000000000000001", "0b10000000000000000000000000000000000000000000001"), - nc(99999999999999, "99999999999999", "0x5af3107a3fff", "0o2657142036437777", "0b10110101111001100010000011110100011111111111111"), - nc(100000000000000, "100000000000000", "0x5af3107a4000", "0o2657142036440000", "0b10110101111001100010000011110100100000000000000"), - nc(100000000000001, "100000000000001", "0x5af3107a4001", "0o2657142036440001", "0b10110101111001100010000011110100100000000000001"), - nc(140737488355327, "140737488355327", "0x7fffffffffff", "0o3777777777777777", "0b11111111111111111111111111111111111111111111111"), - nc(140737488355328, "140737488355328", "0x800000000000", "0o4000000000000000", "0b100000000000000000000000000000000000000000000000"), - nc(140737488355329, "140737488355329", "0x800000000001", "0o4000000000000001", "0b100000000000000000000000000000000000000000000001"), - nc(184467440737094, "184467440737094", "0xa7c5ac471b46", "0o5174265421615506", "0b101001111100010110101100010001110001101101000110"), - nc(184467440737095, "184467440737095", "0xa7c5ac471b47", "0o5174265421615507", "0b101001111100010110101100010001110001101101000111"), - nc(184467440737096, "184467440737096", "0xa7c5ac471b48", "0o5174265421615510", "0b101001111100010110101100010001110001101101001000"), - nc(281474976710655, "281474976710655", "0xffffffffffff", "0o7777777777777777", "0b111111111111111111111111111111111111111111111111"), - nc(281474976710656, "281474976710656", "0x1000000000000", "0o10000000000000000", "0b1000000000000000000000000000000000000000000000000"), - nc(281474976710657, "281474976710657", "0x1000000000001", "0o10000000000000001", "0b1000000000000000000000000000000000000000000000001"), - nc(562949953421311, "562949953421311", "0x1ffffffffffff", "0o17777777777777777", "0b1111111111111111111111111111111111111111111111111"), - nc(562949953421312, "562949953421312", "0x2000000000000", "0o20000000000000000", "0b10000000000000000000000000000000000000000000000000"), - nc(562949953421313, "562949953421313", "0x2000000000001", "0o20000000000000001", "0b10000000000000000000000000000000000000000000000001"), - nc(999999999999999, "999999999999999", "0x38d7ea4c67fff", "0o34327724461477777", "0b11100011010111111010100100110001100111111111111111"), - nc(1000000000000000, "1000000000000000", "0x38d7ea4c68000", "0o34327724461500000", "0b11100011010111111010100100110001101000000000000000"), - nc(1000000000000001, "1000000000000001", "0x38d7ea4c68001", "0o34327724461500001", "0b11100011010111111010100100110001101000000000000001"), - nc(1125899906842623, "1125899906842623", "0x3ffffffffffff", "0o37777777777777777", "0b11111111111111111111111111111111111111111111111111"), - nc(1125899906842624, "1125899906842624", "0x4000000000000", "0o40000000000000000", "0b100000000000000000000000000000000000000000000000000"), - nc(1125899906842625, "1125899906842625", "0x4000000000001", "0o40000000000000001", "0b100000000000000000000000000000000000000000000000001"), - nc(1844674407370954, "1844674407370954", "0x68db8bac710ca", "0o64333427261610312", "0b110100011011011100010111010110001110001000011001010"), - nc(1844674407370955, "1844674407370955", "0x68db8bac710cb", "0o64333427261610313", "0b110100011011011100010111010110001110001000011001011"), - nc(1844674407370956, "1844674407370956", "0x68db8bac710cc", "0o64333427261610314", "0b110100011011011100010111010110001110001000011001100"), - nc(2251799813685247, "2251799813685247", "0x7ffffffffffff", "0o77777777777777777", "0b111111111111111111111111111111111111111111111111111"), - nc(2251799813685248, "2251799813685248", "0x8000000000000", "0o100000000000000000", "0b1000000000000000000000000000000000000000000000000000"), - nc(2251799813685249, "2251799813685249", "0x8000000000001", "0o100000000000000001", "0b1000000000000000000000000000000000000000000000000001"), - nc(4503599627370495, "4503599627370495", "0xfffffffffffff", "0o177777777777777777", "0b1111111111111111111111111111111111111111111111111111"), - nc(4503599627370496, "4503599627370496", "0x10000000000000", "0o200000000000000000", "0b10000000000000000000000000000000000000000000000000000"), - nc(4503599627370497, "4503599627370497", "0x10000000000001", "0o200000000000000001", "0b10000000000000000000000000000000000000000000000000001"), - nc(9007199254740991, "9007199254740991", "0x1fffffffffffff", "0o377777777777777777", "0b11111111111111111111111111111111111111111111111111111"), - nc(9007199254740992, "9007199254740992", "0x20000000000000", "0o400000000000000000", "0b100000000000000000000000000000000000000000000000000000"), - nc(9007199254740993, "9007199254740993", "0x20000000000001", "0o400000000000000001", "0b100000000000000000000000000000000000000000000000000001"), - nc(9999999999999999, "9999999999999999", "0x2386f26fc0ffff", "0o434157115760177777", "0b100011100001101111001001101111110000001111111111111111"), - nc(10000000000000000, "10000000000000000", "0x2386f26fc10000", "0o434157115760200000", "0b100011100001101111001001101111110000010000000000000000"), - nc(10000000000000001, "10000000000000001", "0x2386f26fc10001", "0o434157115760200001", "0b100011100001101111001001101111110000010000000000000001"), - nc(18014398509481983, "18014398509481983", "0x3fffffffffffff", "0o777777777777777777", "0b111111111111111111111111111111111111111111111111111111"), - nc(18014398509481984, "18014398509481984", "0x40000000000000", "0o1000000000000000000", "0b1000000000000000000000000000000000000000000000000000000"), - nc(18014398509481985, "18014398509481985", "0x40000000000001", "0o1000000000000000001", "0b1000000000000000000000000000000000000000000000000000001"), - nc(18446744073709551, "18446744073709551", "0x4189374bc6a7ef", "0o1014223351361523757", "0b1000001100010010011011101001011110001101010011111101111"), - nc(18446744073709552, "18446744073709552", "0x4189374bc6a7f0", "0o1014223351361523760", "0b1000001100010010011011101001011110001101010011111110000"), - nc(18446744073709553, "18446744073709553", "0x4189374bc6a7f1", "0o1014223351361523761", "0b1000001100010010011011101001011110001101010011111110001"), - nc(36028797018963967, "36028797018963967", "0x7fffffffffffff", "0o1777777777777777777", "0b1111111111111111111111111111111111111111111111111111111"), - nc(36028797018963968, "36028797018963968", "0x80000000000000", "0o2000000000000000000", "0b10000000000000000000000000000000000000000000000000000000"), - nc(36028797018963969, "36028797018963969", "0x80000000000001", "0o2000000000000000001", "0b10000000000000000000000000000000000000000000000000000001"), - nc(72057594037927935, "72057594037927935", "0xffffffffffffff", "0o3777777777777777777", "0b11111111111111111111111111111111111111111111111111111111"), - nc(72057594037927936, "72057594037927936", "0x100000000000000", "0o4000000000000000000", "0b100000000000000000000000000000000000000000000000000000000"), - nc(72057594037927937, "72057594037927937", "0x100000000000001", "0o4000000000000000001", "0b100000000000000000000000000000000000000000000000000000001"), - nc(99999999999999999, "99999999999999999", "0x16345785d89ffff", "0o5432127413542377777", "0b101100011010001010111100001011101100010011111111111111111"), - nc(100000000000000000, "100000000000000000", "0x16345785d8a0000", "0o5432127413542400000", "0b101100011010001010111100001011101100010100000000000000000"), - nc(100000000000000001, "100000000000000001", "0x16345785d8a0001", "0o5432127413542400001", "0b101100011010001010111100001011101100010100000000000000001"), - nc(144115188075855871, "144115188075855871", "0x1ffffffffffffff", "0o7777777777777777777", "0b111111111111111111111111111111111111111111111111111111111"), - nc(144115188075855872, "144115188075855872", "0x200000000000000", "0o10000000000000000000", "0b1000000000000000000000000000000000000000000000000000000000"), - nc(144115188075855873, "144115188075855873", "0x200000000000001", "0o10000000000000000001", "0b1000000000000000000000000000000000000000000000000000000001"), - nc(184467440737095519, "184467440737095519", "0x28f5c28f5c28f5f", "0o12172702436560507537", "0b1010001111010111000010100011110101110000101000111101011111"), - nc(184467440737095520, "184467440737095520", "0x28f5c28f5c28f60", "0o12172702436560507540", "0b1010001111010111000010100011110101110000101000111101100000"), - nc(184467440737095521, "184467440737095521", "0x28f5c28f5c28f61", "0o12172702436560507541", "0b1010001111010111000010100011110101110000101000111101100001"), - nc(288230376151711743, "288230376151711743", "0x3ffffffffffffff", "0o17777777777777777777", "0b1111111111111111111111111111111111111111111111111111111111"), - nc(288230376151711744, "288230376151711744", "0x400000000000000", "0o20000000000000000000", "0b10000000000000000000000000000000000000000000000000000000000"), - nc(288230376151711745, "288230376151711745", "0x400000000000001", "0o20000000000000000001", "0b10000000000000000000000000000000000000000000000000000000001"), - nc(576460752303423487, "576460752303423487", "0x7ffffffffffffff", "0o37777777777777777777", "0b11111111111111111111111111111111111111111111111111111111111"), - nc(576460752303423488, "576460752303423488", "0x800000000000000", "0o40000000000000000000", "0b100000000000000000000000000000000000000000000000000000000000"), - nc(576460752303423489, "576460752303423489", "0x800000000000001", "0o40000000000000000001", "0b100000000000000000000000000000000000000000000000000000000001"), - nc(999999999999999999, "999999999999999999", "0xde0b6b3a763ffff", "0o67405553164730777777", "0b110111100000101101101011001110100111011000111111111111111111"), - nc(1000000000000000000, "1000000000000000000", "0xde0b6b3a7640000", "0o67405553164731000000", "0b110111100000101101101011001110100111011001000000000000000000"), - nc(1000000000000000001, "1000000000000000001", "0xde0b6b3a7640001", "0o67405553164731000001", "0b110111100000101101101011001110100111011001000000000000000001"), - nc(1152921504606846975, "1152921504606846975", "0xfffffffffffffff", "0o77777777777777777777", "0b111111111111111111111111111111111111111111111111111111111111"), - nc(1152921504606846976, "1152921504606846976", "0x1000000000000000", "0o100000000000000000000", "0b1000000000000000000000000000000000000000000000000000000000000"), - nc(1152921504606846977, "1152921504606846977", "0x1000000000000001", "0o100000000000000000001", "0b1000000000000000000000000000000000000000000000000000000000001"), - nc(1844674407370955263, "1844674407370955263", "0x19999999999999ff", "0o146314631463146314777", "0b1100110011001100110011001100110011001100110011001100111111111"), - nc(1844674407370955264, "1844674407370955264", "0x1999999999999a00", "0o146314631463146315000", "0b1100110011001100110011001100110011001100110011001101000000000"), - nc(1844674407370955265, "1844674407370955265", "0x1999999999999a01", "0o146314631463146315001", "0b1100110011001100110011001100110011001100110011001101000000001"), - nc(2305843009213693951, "2305843009213693951", "0x1fffffffffffffff", "0o177777777777777777777", "0b1111111111111111111111111111111111111111111111111111111111111"), - nc(2305843009213693952, "2305843009213693952", "0x2000000000000000", "0o200000000000000000000", "0b10000000000000000000000000000000000000000000000000000000000000"), - nc(2305843009213693953, "2305843009213693953", "0x2000000000000001", "0o200000000000000000001", "0b10000000000000000000000000000000000000000000000000000000000001"), - nc(4611686018427387903, "4611686018427387903", "0x3fffffffffffffff", "0o377777777777777777777", "0b11111111111111111111111111111111111111111111111111111111111111"), - nc(4611686018427387904, "4611686018427387904", "0x4000000000000000", "0o400000000000000000000", "0b100000000000000000000000000000000000000000000000000000000000000"), - nc(4611686018427387905, "4611686018427387905", "0x4000000000000001", "0o400000000000000000001", "0b100000000000000000000000000000000000000000000000000000000000001"), - nc(9223372036854775807, "9223372036854775807", "0x7fffffffffffffff", "0o777777777777777777777", "0b111111111111111111111111111111111111111111111111111111111111111"), - nc(9223372036854775808, "9223372036854775808", "0x8000000000000000", "0o1000000000000000000000", "0b1000000000000000000000000000000000000000000000000000000000000000"), - nc(9223372036854775809, "9223372036854775809", "0x8000000000000001", "0o1000000000000000000001", "0b1000000000000000000000000000000000000000000000000000000000000001"), - nc(9999999999999999999, "9999999999999999999", "0x8ac7230489e7ffff", "0o1053071060221171777777", "0b1000101011000111001000110000010010001001111001111111111111111111"), - nc(10000000000000000000, "10000000000000000000", "0x8ac7230489e80000", "0o1053071060221172000000", "0b1000101011000111001000110000010010001001111010000000000000000000"), - nc(10000000000000000001, "10000000000000000001", "0x8ac7230489e80001", "0o1053071060221172000001", "0b1000101011000111001000110000010010001001111010000000000000000001"), - nc(18446744073709551611, "18446744073709551611", "0xfffffffffffffffb", "0o1777777777777777777773", "0b1111111111111111111111111111111111111111111111111111111111111011"), - nc(18446744073709551612, "18446744073709551612", "0xfffffffffffffffc", "0o1777777777777777777774", "0b1111111111111111111111111111111111111111111111111111111111111100"), - nc(18446744073709551613, "18446744073709551613", "0xfffffffffffffffd", "0o1777777777777777777775", "0b1111111111111111111111111111111111111111111111111111111111111101"), - nc(18446744073709551614, "18446744073709551614", "0xfffffffffffffffe", "0o1777777777777777777776", "0b1111111111111111111111111111111111111111111111111111111111111110"), - nc(18446744073709551615, "18446744073709551615", "0xffffffffffffffff", "0o1777777777777777777777", "0b1111111111111111111111111111111111111111111111111111111111111111"), -#undef nc - }; -}; - -C4_INLINE_CONSTEXPR const number_case numbers::vals[]; -C4_INLINE_CONSTEXPR const number_case numbers::vals[]; - -C4_INLINE_CONSTEXPR const number_case numbers::vals[]; -C4_INLINE_CONSTEXPR const number_case numbers::vals[]; - -C4_INLINE_CONSTEXPR const number_case numbers::vals[]; -C4_INLINE_CONSTEXPR const number_case numbers::vals[]; - -C4_INLINE_CONSTEXPR const number_case numbers::vals[]; -C4_INLINE_CONSTEXPR const number_case numbers::vals[]; - -C4_SUPPRESS_WARNING_MSVC_POP - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/test/test_preprocessor.cpp b/thirdparty/ryml/ext/c4core/test/test_preprocessor.cpp deleted file mode 100644 index bd2619ae4..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_preprocessor.cpp +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/preprocessor.hpp" -#include "c4/language.hpp" -#endif - -#ifdef WE_LL_GET_THERE___MSVC_CANT_HANDLE_THE_FOREACH_MACRO___NEEDS_TO_BE_FIXED -#include -#include - -struct SomeStruct -{ - int32_t a; - int32_t b; - int32_t c; - int32_t d; -}; - -TEST(TestForEach, print_offsets) -{ -#define M_OFFS_(structure, field) m[#field] = offsetof(structure, field) -#define M_OFFS(field) M_OFFS_(SomeStruct, field) - - std::map< std::string, size_t > m; - - C4_FOR_EACH(M_OFFS, a, b, c); - C4_FOR_EACH(M_OFFS, d); - - EXPECT_EQ(m["a"], 0); - EXPECT_EQ(m["b"], 4); - EXPECT_EQ(m["c"], 8); - EXPECT_EQ(m["d"], 12); -} - -//----------------------------------------------------------------------------- -// C4_BEGIN_NAMESPACE()/C4_END_NAMESPACE() are implemented with C4_FOR_EACH(). -// Test these here too. - -namespace a, b, c { -int a_var = 0; -} // namespace c, b -int var = 1; // a::var -namespace b { -int var = 2; // a::b::var -namespace c { -int var = 3; // a::b::c::var -} // namespace c, b, a - -TEST(TestForEach, begin_end_namespace) -{ - EXPECT_EQ(a::b::c::a_var, 0); - EXPECT_EQ(a::var, 1); - EXPECT_EQ(a::b::var, 2); - EXPECT_EQ(a::b::c::var, 3); -} -#endif diff --git a/thirdparty/ryml/ext/c4core/test/test_singleheader/libc4core_singleheader.cpp b/thirdparty/ryml/ext/c4core/test/test_singleheader/libc4core_singleheader.cpp deleted file mode 100644 index 078c77d1e..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_singleheader/libc4core_singleheader.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#define C4CORE_SINGLE_HDR_DEFINE_NOW -#include diff --git a/thirdparty/ryml/ext/c4core/test/test_span.cpp b/thirdparty/ryml/ext/c4core/test/test_span.cpp deleted file mode 100644 index c492013d7..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_span.cpp +++ /dev/null @@ -1,944 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/span.hpp" -#endif - -#include "c4/libtest/supprwarn_push.hpp" - -#include - -namespace c4 { - -//----------------------------------------------------------------------------- -TEST_CASE_TEMPLATE("span.default_init", SpanClass, span, spanrs, spanrsl) -{ - SpanClass s; - CHECK_EQ(s.size(), 0); - CHECK_EQ(s.capacity(), 0); - CHECK_EQ(s.data(), nullptr); -} - - -template class Span, class T, class I> -Span cvt_to_const(Span const& s) -{ - Span ret = s; - return ret; -} - -TEST_CASE_TEMPLATE("span.convert_to_const", SpanClass, span, spanrs, spanrsl) -{ - SpanClass s; - auto cs = cvt_to_const(s); - CHECK_EQ(s.size(), cs.size()); - CHECK_EQ(s.data(), cs.data()); - CHECK_EQ(s.end(), cs.end()); -} - - -//----------------------------------------------------------------------------- -TEST_CASE("span.empty_init") -{ - int arr[10]; - span s(arr, 0); - CHECK_EQ(s.size(), 0); - CHECK_EQ(s.capacity(), 0); - CHECK_NE(s.data(), nullptr); -} - -TEST_CASE("spanrs.empty_init") -{ - int arr[10]; - - { - spanrs s(arr, 0); - CHECK_EQ(s.size(), 0); - CHECK_EQ(s.capacity(), 0); - CHECK_EQ(s.data(), arr); - } - - { - spanrs s(arr, 0, C4_COUNTOF(arr)); - CHECK_EQ(s.size(), 0); - CHECK_EQ(s.capacity(), 10); - CHECK_EQ(s.data(), arr); - } -} - -TEST_CASE("spanrsl.empty_init") -{ - int arr[10]; - - { - spanrsl s(arr, 0); - CHECK_EQ(s.size(), 0); - CHECK_EQ(s.capacity(), 0); - CHECK_EQ(s.data(), arr); - CHECK_EQ(s.offset(), 0); - } - - { - spanrsl s(arr, 0, C4_COUNTOF(arr)); - CHECK_EQ(s.size(), 0); - CHECK_EQ(s.capacity(), 10); - CHECK_EQ(s.data(), arr); - CHECK_EQ(s.offset(), 0); - } -} - -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("span.fromArray", SpanClass, - span, span, span, - spanrs, spanrs, spanrs, - spanrsl, spanrsl, spanrsl - ) -{ - using ConstSpanClass = typename SpanClass::const_type; - using T = typename SpanClass::value_type; - T arr1[10]; - T arr2[20]; - - T a = 0; - for(auto &v : arr1) { v = a; ++a; } - for(auto &v : arr2) { v = a; ++a; } - - { - SpanClass s(arr1); - CHECK_EQ(s.size(), C4_COUNTOF(arr1)); - CHECK_EQ(s.capacity(), C4_COUNTOF(arr1)); - CHECK_EQ(s.data(), arr1); - } - - { - ConstSpanClass s(arr1); - CHECK_EQ(s.size(), C4_COUNTOF(arr1)); - CHECK_EQ(s.capacity(), C4_COUNTOF(arr1)); - CHECK_EQ(s.data(), arr1); - } - - { - SpanClass s = arr1; - CHECK_EQ(s.size(), C4_COUNTOF(arr1)); - CHECK_EQ(s.capacity(), C4_COUNTOF(arr1)); - CHECK_EQ(s.data(), arr1); - } - - { - ConstSpanClass s = arr1; - CHECK_EQ(s.size(), C4_COUNTOF(arr1)); - CHECK_EQ(s.capacity(), C4_COUNTOF(arr1)); - CHECK_EQ(s.data(), arr1); - } - - { - SpanClass s = arr1; - CHECK_EQ(s.size(), C4_COUNTOF(arr1)); - CHECK_EQ(s.capacity(), C4_COUNTOF(arr1)); - CHECK_EQ(s.data(), arr1); - s = arr2; - CHECK_EQ(s.size(), C4_COUNTOF(arr2)); - CHECK_EQ(s.capacity(), C4_COUNTOF(arr2)); - CHECK_EQ(s.data(), arr2); - } - - { - ConstSpanClass s = arr1; - CHECK_EQ(s.size(), C4_COUNTOF(arr1)); - CHECK_EQ(s.capacity(), C4_COUNTOF(arr1)); - CHECK_EQ(s.data(), arr1); - s = arr2; - CHECK_EQ(s.size(), C4_COUNTOF(arr2)); - CHECK_EQ(s.capacity(), C4_COUNTOF(arr2)); - CHECK_EQ(s.data(), arr2); - } -} - - -//----------------------------------------------------------------------------- -TEST_CASE("span.subspan") -{ - int arr[10]; - span s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - { - auto ss = s.subspan(0, 5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), arr); - - ss = s.subspan(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), &arr[5]); - - // fine to obtain an empty span at the end - ss = s.subspan(10); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), std::end(arr)); - // fine to obtain an empty span at the end - ss = s.subspan(10, 0); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), std::end(arr)); - } - { - int buf10[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - int buf_5[] = {-1, 0, 1, 2, 3, 4}; - int *buf5 = buf_5 + 1; // to make sure that one does not immediately follow the other in memory - - span n(buf10); - span m(buf5, 5); - - auto ss = n.subspan(0); - CHECK_EQ(ss.data(), buf10); - CHECK_EQ(ss.size(), 10); - ss = m.subspan(0); - CHECK_EQ(ss.data(), buf5); - CHECK_EQ(ss.size(), 5); - ss = n.subspan(0, 0); - CHECK_NE(ss.data(), nullptr); - CHECK_EQ(ss.data(), &buf10[0]); - CHECK_EQ(ss.size(), 0); - ss = m.subspan(0, 0); - CHECK_NE(ss.data(), nullptr); - CHECK_EQ(ss.data(), &buf5[0]); - CHECK_EQ(ss.size(), 0); - } -} -TEST_CASE("spanrs.subspan") -{ - int arr[10]; - spanrs s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.subspan(0, 5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = s.subspan(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), &arr[5]); - - // fine to obtain an empty span at the end - ss = s.subspan(10); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), std::end(arr)); - // fine to obtain an empty span at the end - ss = s.subspan(10, 0); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), std::end(arr)); -} -TEST_CASE("spanrsl.subspan") -{ - int arr[10]; - spanrsl s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.subspan(0, 5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - CHECK_EQ(ss.offset(), 0); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - CHECK_EQ(ss.offset(), 0); - - ss = s.subspan(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), &arr[5]); - CHECK_EQ(ss.offset(), 5); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - CHECK_EQ(ss.offset(), 0); - - // fine to obtain an empty span at the end - ss = s.subspan(10); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), std::end(arr)); - // fine to obtain an empty span at the end - ss = s.subspan(10, 0); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), std::end(arr)); -} - -//----------------------------------------------------------------------------- -TEST_CASE("span.range") -{ - int arr[10]; - span s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.range(0, 5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), arr); - - ss = s.range(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), &arr[5]); - - ss = s.range(5, 10); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), &arr[5]); - - ss = s.range(10, 10); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), std::end(arr)); - - SUBCASE("empty_span") - { - s = {}; - ss = s.range(0, 0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), nullptr); - s = arr; - ss = s.range(0, 0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr); - ss = s.range(10, 10); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr + 10); - ss = s.range(10); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr + 10); - } -} -TEST_CASE("spanrs.range") -{ - int arr[10]; - spanrs s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.range(0, 5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = s.range(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), &arr[5]); - - ss = s.range(5, 10); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), &arr[5]); - - ss = s.range(10, 10); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), std::end(arr)); - - SUBCASE("empty_span") - { - s = {}; - ss = s.range(0, 0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), nullptr); - s = arr; - ss = s.range(0, 0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr); - ss = s.range(10, 10); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr + 10); - ss = s.range(10); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr + 10); - } -} -TEST_CASE("spanrsl.range") -{ - int arr[10]; - spanrsl s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.range(0, 5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = s.range(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), &arr[5]); - - ss = s.range(5, 10); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), &arr[5]); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = s.range(10, 10); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), std::end(arr)); - - SUBCASE("empty_span") - { - s = {}; - ss = s.range(0, 0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), nullptr); - s = arr; - ss = s.range(0, 0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr); - ss = s.range(10, 10); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr + 10); - ss = s.range(10); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr + 10); - } -} - -//----------------------------------------------------------------------------- -TEST_CASE("span.first") -{ - int arr[10]; - span s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.first(0); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), arr); - - ss = s.first(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), arr); - - SUBCASE("empty") - { - s = {}; - ss = s.first(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), nullptr); - s = arr; - ss = s.first(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr); - } -} -TEST_CASE("spanrs.first") -{ - int arr[10]; - spanrs s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.first(0); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = s.first(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - SUBCASE("empty") - { - s = {}; - ss = s.first(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), nullptr); - s = arr; - ss = s.first(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr); - } -} -TEST_CASE("spanrsl.first") -{ - int arr[10]; - spanrsl s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.first(0); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = s.first(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - SUBCASE("empty") - { - s = {}; - ss = s.first(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), nullptr); - s = arr; - ss = s.first(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr); - } -} - -//----------------------------------------------------------------------------- -TEST_CASE("span.last") -{ - int arr[10]; - span s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.last(0); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), arr + s.size()); - - ss = s.last(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), arr + 5); - - SUBCASE("empty") - { - s = {}; - ss = s.last(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), nullptr); - s = arr; - ss = s.last(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr + 10); - } -} -TEST_CASE("spanrs.last") -{ - int arr[10]; - spanrs s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.last(0); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), arr + s.size()); - - ss = s.last(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), arr + 5); - - SUBCASE("empty") - { - s = {}; - ss = s.last(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), nullptr); - s = arr; - ss = s.last(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr + 10); - } -} -TEST_CASE("spanrsl.last") -{ - int arr[10]; - spanrsl s(arr); - C4_STATIC_ASSERT((std::is_same::value)); - - auto ss = s.last(0); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.capacity(), 0); - CHECK_EQ(ss.data(), arr + s.size()); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss = s.last(5); - CHECK_EQ(ss.size(), 5); - CHECK_EQ(ss.capacity(), 5); - CHECK_EQ(ss.data(), arr + 5); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - SUBCASE("empty") - { - s = {}; - ss = s.last(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), nullptr); - s = arr; - ss = s.last(0); - CHECK(ss.empty()); - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.data(), arr + 10); - } -} - - -//----------------------------------------------------------------------------- -TEST_CASE_TEMPLATE("span.is_subspan", SpanClass, span, spanrs, spanrsl) -{ - int buf10[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - int buf_5[] = {-1, 0, 1, 2, 3, 4}; - int *buf5 = buf_5 + 1; // to make sure that one does not immediately follow the other in memory - - SpanClass n(buf10); - SpanClass m(buf5, 5); - - CHECK_EQ(n.data(), buf10); - CHECK_EQ(m.data(), buf5); - - CHECK_UNARY(n.is_subspan(n.subspan(0 ))); - CHECK_UNARY(n.is_subspan(n.subspan(0, 3))); - CHECK_UNARY(n.is_subspan(n.subspan(0, 0))); - - CHECK_FALSE(n.is_subspan(m.subspan(0 ))); - CHECK_FALSE(n.is_subspan(m.subspan(0, 3))); - CHECK_FALSE(n.is_subspan(m.subspan(0, 0))); -} - -//----------------------------------------------------------------------------- -TEST_CASE_TEMPLATE("span.compll", SpanClass, span, spanrs, spanrsl) -{ - int buf10[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - - SpanClass n(buf10); - - CHECK_EQ(n.compll(n.subspan(0)), n.subspan(0, 0)); - CHECK_EQ(n.is_subspan(n.compll(n.subspan(0))), true); - CHECK_EQ(n.compll(n.subspan(0, 0)), n.subspan(0, 0)); - CHECK_EQ(n.is_subspan(n.compll(n.subspan(0, 0))), true); - - CHECK_EQ(n.compll(n.subspan(0, 1)), n.subspan(0, 0)); - CHECK_EQ(n.compll(n.subspan(0, 3)), n.subspan(0, 0)); - - CHECK_EQ(n.compll(n.range(5, 10)), n.subspan(0, 5)); - CHECK_EQ(n.compll(n.range(5, 5)), n.subspan(0, 5)); - - CHECK_EQ(n.compll(n.subspan(n.size(), 0)), n); - CHECK_EQ(n.compll(n.range(n.size(), n.size())), n); -} - - -//----------------------------------------------------------------------------- - -TEST_CASE_TEMPLATE("span.complr", SpanClass, span, spanrs, spanrsl) -{ - int buf10[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - - SpanClass n(buf10); - - CHECK_EQ(n.complr(n.subspan(0)), n.subspan(0, 0)); - CHECK_EQ(n.is_subspan(n.complr(n.subspan(0))), true); - CHECK_EQ(n.complr(n.subspan(0, 0)), n.subspan(0)); - CHECK_EQ(n.is_subspan(n.complr(n.subspan(0, 0))), true); - - CHECK_EQ(n.complr(n.subspan(0, 1)), n.subspan(1)); - CHECK_EQ(n.complr(n.subspan(0, 3)), n.subspan(3)); - - CHECK_EQ(n.complr(n.subspan(5)), n.subspan(0, 0)); - CHECK_EQ(n.complr(n.range(5, 10)), n.subspan(0, 0)); - - CHECK_EQ(n.complr(n.subspan(5, 0)), n.subspan(5)); - CHECK_EQ(n.complr(n.range(5, 5)), n.subspan(5)); - - CHECK_EQ(n.complr(n.subspan(0, 0)), n); - CHECK_EQ(n.complr(n.range(0, 0)), n); -} - - -//----------------------------------------------------------------------------- -TEST_CASE("span.rtrim") -{ - int arr[10]; - span s(arr); - auto ss = s; - - ss.rtrim(0); - CHECK_EQ(ss.size(), s.size()); - CHECK_EQ(ss.capacity(), s.capacity()); - CHECK_EQ(ss.data(), arr); - - ss.rtrim(5); - CHECK_EQ(ss.size(), s.size() - 5); - CHECK_EQ(ss.capacity(), s.capacity() - 5); - CHECK_EQ(ss.data(), arr); -} -TEST_CASE("spanrs.rtrim") -{ - int arr[10]; - spanrs s(arr); - auto ss = s; - - ss.rtrim(0); - CHECK_EQ(ss.size(), s.size()); - CHECK_EQ(ss.capacity(), s.capacity()); - CHECK_EQ(ss.data(), arr); - - ss.rtrim(5); - CHECK_EQ(ss.size(), s.size() - 5); - CHECK_EQ(ss.capacity(), s.capacity()); - CHECK_EQ(ss.data(), arr); -} -TEST_CASE("spanrsl.rtrim") -{ - int arr[10]; - spanrsl s(arr); - auto ss = s; - - ss.rtrim(0); - CHECK_EQ(ss.size(), s.size()); - CHECK_EQ(ss.capacity(), s.capacity()); - CHECK_EQ(ss.data(), arr); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss.rtrim(5); - CHECK_EQ(ss.size(), s.size() - 5); - CHECK_EQ(ss.capacity(), s.capacity()); - CHECK_EQ(ss.data(), arr); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); -} - -//----------------------------------------------------------------------------- -TEST_CASE("span.ltrim") -{ - int arr[10]; - span s(arr); - auto ss = s; - - ss.ltrim(0); - CHECK_EQ(ss.size(), s.size()); - CHECK_EQ(ss.capacity(), s.capacity()); - CHECK_EQ(ss.data(), arr); - - ss.ltrim(5); - CHECK_EQ(ss.size(), s.size() - 5); - CHECK_EQ(ss.capacity(), s.capacity() - 5); - CHECK_EQ(ss.data(), arr + 5); -} -TEST_CASE("spanrs.ltrim") -{ - int arr[10]; - spanrs s(arr); - auto ss = s; - - ss.ltrim(0); - CHECK_EQ(ss.size(), s.size()); - CHECK_EQ(ss.capacity(), ss.capacity()); - CHECK_EQ(ss.data(), arr); - - ss.ltrim(5); - CHECK_EQ(ss.size(), s.size() - 5); - CHECK_EQ(ss.capacity(), s.size() - 5); - CHECK_EQ(ss.data(), arr + 5); -} -TEST_CASE("spanrsl.ltrim") -{ - int arr[10]; - spanrsl s(arr); - auto ss = s; - - ss.ltrim(0); - CHECK_EQ(ss.size(), s.size()); - CHECK_EQ(ss.capacity(), ss.capacity()); - CHECK_EQ(ss.data(), arr); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); - - ss.ltrim(5); - CHECK_EQ(ss.size(), s.size() - 5); - CHECK_EQ(ss.capacity(), s.size() - 5); - CHECK_EQ(ss.data(), arr + 5); - - ss = ss.original(); - CHECK_EQ(ss.size(), 10); - CHECK_EQ(ss.capacity(), 10); - CHECK_EQ(ss.data(), arr); -} - -//----------------------------------------------------------------------------- -const char larrc[11] = "0123456789"; -const char rarrc[11] = "1234567890"; -const int larri[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; -const int rarri[10] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 0}; -TEST_CASE("span.reverse_iter") -{ - cspan s(larri); - REQUIRE_EQ(s.data(), larri); - REQUIRE_EQ(s.begin(), std::begin(larri)); - REQUIRE_EQ(s.end(), std::end(larri)); - REQUIRE_EQ(&*s.rbegin(), s.end()-1); - using rit = cspan::const_reverse_iterator; - int pos = szconv(s.size()) - 1; - size_t count = 0; - //for(rit b = s.rbegin(), e = s.rend(); b != e; ++b) // BUG: b != e is never true on arm-eabi-g++-7 - for(rit b = s.rbegin(), e = s.rend(); b < e; ++b) - { - CHECK_EQ(&(*b), s.data() + pos); - auto spos = szconv(pos); - REQUIRE_EQ(&*b, &s[spos]); - REQUIRE_GE(pos, 0); - REQUIRE_LT(pos, szconv(s.size())); - REQUIRE_LT(count, s.size()); - CHECK_EQ(*b, s[spos]); - --pos; - ++count; - } - CHECK_EQ(pos, -1); - CHECK_EQ(count, s.size()); -} - -//----------------------------------------------------------------------------- -TEST_CASE("span_impl.eq") -{ - CHECK_EQ(cspan (larrc), cspan (larrc)); - CHECK_EQ(cspanrs(larrc), cspan (larrc)); - CHECK_EQ(cspan (larrc), cspanrs(larrc)); - CHECK_EQ(cspanrs(larrc), cspanrs(larrc)); - - CHECK_EQ(cspan (larri) , cspan (larri)); - CHECK_EQ(cspanrs(larri) , cspan (larri)); - CHECK_EQ(cspan (larri) , cspanrs(larri)); - CHECK_EQ(cspanrs(larri) , cspanrs(larri)); -} - -TEST_CASE("span_impl.lt") -{ - CHECK_LT(cspan (larrc), cspan (rarrc)); - CHECK_LT(cspanrs(larrc), cspan (rarrc)); - CHECK_LT(cspan (larrc), cspanrs(rarrc)); - CHECK_LT(cspanrs(larrc), cspanrs(rarrc)); - - CHECK_LT(cspan (larri) , cspan (rarri)); - CHECK_LT(cspanrs(larri) , cspan (rarri)); - CHECK_LT(cspan (larri) , cspanrs(rarri)); - CHECK_LT(cspanrs(larri) , cspanrs(rarri)); -} -TEST_CASE("span_impl.gt") -{ - CHECK_GT(cspan (rarrc), cspan (larrc)); - CHECK_GT(cspan (rarrc), cspanrs(larrc)); - CHECK_GT(cspanrs(rarrc), cspan (larrc)); - CHECK_GT(cspanrs(rarrc), cspanrs(larrc)); - - CHECK_GT(cspan (rarri) , cspan (larri)); - CHECK_GT(cspan (rarri) , cspanrs(larri)); - CHECK_GT(cspanrs(rarri) , cspan (larri)); - CHECK_GT(cspanrs(rarri) , cspanrs(larri)); -} - -TEST_CASE("span_impl.ge") -{ - CHECK_GE(cspan (rarrc), cspan (larrc)); - CHECK_GE(cspan (rarrc), cspanrs(larrc)); - CHECK_GE(cspanrs(rarrc), cspan (larrc)); - CHECK_GE(cspanrs(rarrc), cspanrs(larrc)); - CHECK_GE(cspan (larrc), cspan (larrc)); - CHECK_GE(cspan (larrc), cspanrs(larrc)); - CHECK_GE(cspanrs(larrc), cspan (larrc)); - CHECK_GE(cspanrs(larrc), cspanrs(larrc)); - CHECK_GE(cspan (rarri) , cspan (larri)); - CHECK_GE(cspan (rarri) , cspanrs(larri)); - CHECK_GE(cspanrs(rarri) , cspan (larri)); - CHECK_GE(cspanrs(rarri) , cspanrs(larri)); - CHECK_GE(cspan (larri) , cspan (larri)); - CHECK_GE(cspan (larri) , cspanrs(larri)); - CHECK_GE(cspanrs(larri) , cspan (larri)); - CHECK_GE(cspanrs(larri) , cspanrs(larri)); -} -TEST_CASE("span_impl.le") -{ - CHECK_LE(cspan (larrc), cspan (rarrc)); - CHECK_LE(cspanrs(larrc), cspan (rarrc)); - CHECK_LE(cspan (larrc), cspanrs(rarrc)); - CHECK_LE(cspanrs(larrc), cspanrs(rarrc)); - CHECK_LE(cspan (larrc), cspan (larrc)); - CHECK_LE(cspanrs(larrc), cspan (larrc)); - CHECK_LE(cspan (larrc), cspanrs(larrc)); - CHECK_LE(cspanrs(larrc), cspanrs(larrc)); - CHECK_LE(cspan (larri) , cspan (rarri)); - CHECK_LE(cspanrs(larri) , cspan (rarri)); - CHECK_LE(cspan (larri) , cspanrs(rarri)); - CHECK_LE(cspanrs(larri) , cspanrs(rarri)); - CHECK_LE(cspan (larri) , cspan (larri)); - CHECK_LE(cspanrs(larri) , cspan (larri)); - CHECK_LE(cspan (larri) , cspanrs(larri)); - CHECK_LE(cspanrs(larri) , cspanrs(larri)); -} - -} // namespace c4 - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_std_string.cpp b/thirdparty/ryml/ext/c4core/test/test_std_string.cpp deleted file mode 100644 index de5b3739b..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_std_string.cpp +++ /dev/null @@ -1,125 +0,0 @@ -#include "c4/test.hpp" -#ifndef C4CORE_SINGLE_HEADER -#include "c4/std/string_fwd.hpp" -#include "c4/std/string.hpp" -#endif - -namespace c4 { - -TEST_CASE("std_string.to_substr") -{ - std::string s("barnabe"); - substr ss = to_substr(s); - CHECK_EQ(ss.str, s.data()); - CHECK_EQ(ss.len, s.size()); - s[0] = 'B'; - CHECK_EQ(ss[0], 'B'); - ss[0] = 'b'; - CHECK_EQ(s[0], 'b'); -} - -TEST_CASE("std_string.to_csubstr") -{ - std::string s("barnabe"); - csubstr ss = to_csubstr(s); - CHECK_EQ(ss.str, s.data()); - CHECK_EQ(ss.len, s.size()); - s[0] = 'B'; - CHECK_EQ(ss[0], 'B'); -} - -TEST_CASE("std_string.compare_csubstr") -{ - std::string s0 = "000"; - std::string s1 = "111"; - csubstr ss0 = "000"; - csubstr ss1 = "111"; - CHECK_NE(s0.data(), ss0.data()); - CHECK_NE(s1.data(), ss1.data()); - // - CHECK_EQ(s0, ss0); - CHECK_EQ(s1, ss1); - CHECK_EQ(ss0, s0); - CHECK_EQ(ss1, s1); - // - CHECK_NE(s1, ss0); - CHECK_NE(s0, ss1); - CHECK_NE(ss1, s0); - CHECK_NE(ss0, s1); - // - CHECK_GE(s0, ss0); - CHECK_LE(s1, ss1); - CHECK_GE(ss0, s0); - CHECK_LE(ss1, s1); - CHECK_GE(s1, ss0); - CHECK_LE(s0, ss1); - CHECK_GE(ss1, s0); - CHECK_LE(ss0, s1); - // - CHECK_GT(s1, ss0); - CHECK_LT(s0, ss1); - CHECK_GT(ss1, s0); - CHECK_LT(ss0, s1); -} - -TEST_CASE("std_string.compare_substr") -{ - std::string s0 = "000"; - std::string s1 = "111"; - char buf0[] = "000"; - char buf1[] = "111"; - substr ss0 = buf0; - substr ss1 = buf1; - CHECK_NE(s0.data(), ss0.data()); - CHECK_NE(s1.data(), ss1.data()); - // - CHECK_EQ(s0, ss0); - CHECK_EQ(s1, ss1); - CHECK_EQ(ss0, s0); - CHECK_EQ(ss1, s1); - // - CHECK_NE(s1, ss0); - CHECK_NE(s0, ss1); - CHECK_NE(ss1, s0); - CHECK_NE(ss0, s1); - // - CHECK_GE(s0, ss0); - CHECK_LE(s1, ss1); - CHECK_GE(ss0, s0); - CHECK_LE(ss1, s1); - CHECK_GE(s1, ss0); - CHECK_LE(s0, ss1); - CHECK_GE(ss1, s0); - CHECK_LE(ss0, s1); - // - CHECK_GT(s1, ss0); - CHECK_LT(s0, ss1); - CHECK_GT(ss1, s0); - CHECK_LT(ss0, s1); -} - -TEST_CASE("std_string.to_chars") -{ - const std::string s0 = "000"; - char buf_[100] = {}; - substr buf = buf_; - CHECK_NE(buf.data(), s0.data()); - size_t ret = to_chars({}, s0); - CHECK_EQ(ret, s0.size()); - CHECK_NE(buf.first(ret), s0); - ret = to_chars(buf, s0); - CHECK_EQ(ret, s0.size()); - CHECK_EQ(buf.first(ret), s0); -} - -TEST_CASE("std_string.from_chars") -{ - std::string s0; - csubstr buf = "0123456798"; - CHECK_NE(buf.data(), s0.data()); - bool ok = from_chars(buf, &s0); - CHECK(ok); - CHECK_EQ(buf, s0); -} - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/test/test_std_vector.cpp b/thirdparty/ryml/ext/c4core/test/test_std_vector.cpp deleted file mode 100644 index 1cfe54f94..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_std_vector.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include "c4/test.hpp" -#ifndef C4CORE_SINGLE_HEADER -#include "c4/std/vector_fwd.hpp" -#include "c4/std/vector.hpp" -#endif - -namespace c4 { - -template -std::vector ctor(const char (&s)[N]) -{ - return std::vector(s, s+N-1); -} - -TEST_CASE("std_vector.to_csubstr") -{ - std::vector s = ctor("barnabe"); - csubstr ss = to_csubstr(s); - CHECK_EQ(ss, csubstr("barnabe")); - CHECK_EQ(ss.str, s.data()); - CHECK_EQ(ss.len, s.size()); -} - -TEST_CASE("std_vector.to_substr") -{ - std::vector s = ctor("barnabe"); - substr ss = to_substr(s); - CHECK_EQ(ss, csubstr("barnabe")); - CHECK_EQ(ss.str, s.data()); - CHECK_EQ(ss.len, s.size()); - // - CHECK_EQ(s[0], 'b'); - ss[0] = 'B'; - CHECK_EQ(s[0], 'B'); - ss[0] = 'A'; - CHECK_EQ(s[0], 'A'); -} - -TEST_CASE("std_vector.compare_csubstr") -{ - std::vector s0 = ctor("000"); - std::vector s1 = ctor("111"); - csubstr ss0 = "000"; - csubstr ss1 = "111"; - CHECK_NE(s0.data(), ss0.data()); - CHECK_NE(s1.data(), ss1.data()); - // - CHECK_EQ(s0, ss0); - CHECK_EQ(s1, ss1); - CHECK_EQ(ss0, s0); - CHECK_EQ(ss1, s1); - // - CHECK_NE(s1, ss0); - CHECK_NE(s0, ss1); - CHECK_NE(ss1, s0); - CHECK_NE(ss0, s1); - // - CHECK_GE(s0, ss0); - CHECK_LE(s1, ss1); - CHECK_GE(ss0, s0); - CHECK_LE(ss1, s1); - CHECK_GE(s1, ss0); - CHECK_LE(s0, ss1); - CHECK_GE(ss1, s0); - CHECK_LE(ss0, s1); - // - CHECK_GT(s1, ss0); - CHECK_LT(s0, ss1); - CHECK_GT(ss1, s0); - CHECK_LT(ss0, s1); -} - -TEST_CASE("std_vector.compare_substr") -{ - std::vector s0 = ctor("000"); - std::vector s1 = ctor("111"); - char buf0[] = "000"; - char buf1[] = "111"; - substr ss0 = buf0; - substr ss1 = buf1; - CHECK_NE(s0.data(), ss0.data()); - CHECK_NE(s1.data(), ss1.data()); - // - CHECK_EQ(s0, ss0); - CHECK_EQ(s1, ss1); - CHECK_EQ(ss0, s0); - CHECK_EQ(ss1, s1); - // - CHECK_NE(s1, ss0); - CHECK_NE(s0, ss1); - CHECK_NE(ss1, s0); - CHECK_NE(ss0, s1); - // - CHECK_GE(s0, ss0); - CHECK_LE(s1, ss1); - CHECK_GE(ss0, s0); - CHECK_LE(ss1, s1); - CHECK_GE(s1, ss0); - CHECK_LE(s0, ss1); - CHECK_GE(ss1, s0); - CHECK_LE(ss0, s1); - // - CHECK_GT(s1, ss0); - CHECK_LT(s0, ss1); - CHECK_GT(ss1, s0); - CHECK_LT(ss0, s1); -} - -TEST_CASE("std_vector.to_chars") -{ - const std::vector s0 = ctor("000"); - char buf_[100] = {}; - substr buf = buf_; - CHECK_NE(buf.data(), s0.data()); - size_t ret = to_chars({}, s0); - CHECK_EQ(ret, s0.size()); - CHECK_NE(buf.first(ret), s0); - ret = to_chars(buf, s0); - CHECK_EQ(ret, s0.size()); - CHECK_EQ(buf.first(ret), s0); -} - -TEST_CASE("std_vector.from_chars") -{ - std::vector s0; - csubstr buf = "0123456798"; - CHECK_NE(buf.data(), s0.data()); - bool ok = from_chars(buf, &s0); - CHECK(ok); - CHECK_EQ(buf, s0); -} - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/test/test_substr.cpp b/thirdparty/ryml/ext/c4core/test/test_substr.cpp deleted file mode 100644 index 777b4b4b0..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_substr.cpp +++ /dev/null @@ -1,4507 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/std/std.hpp" -#include "c4/substr.hpp" -#endif - -#include - -#include "c4/libtest/supprwarn_push.hpp" -#include - -namespace c4 { - -TEST_CASE("substr.ctor_from_char") -{ - char buf1[] = "{foo: 1}"; - char buf2[] = "{foo: 2}"; - substr s(buf1); - CHECK_EQ(s, "{foo: 1}"); - s = buf2; - CHECK_EQ(s, "{foo: 2}"); -} - -TEST_CASE("csubstr.ctor_from_char") -{ - char buf1[] = "{foo: 1}"; - char buf2[] = "{foo: 2}"; - csubstr s(buf1); - CHECK_EQ(s, "{foo: 1}"); - s = buf2; - CHECK_EQ(s, "{foo: 2}"); -} - -TEST_CASE("csubstr.empty_vs_null") -{ - csubstr s; - CHECK_UNARY(s.empty()); - CHECK_UNARY(s.len == 0); - CHECK_UNARY(s.str == nullptr); - CHECK_UNARY(s == nullptr); - - s = ""; - CHECK_UNARY(s.empty()); - CHECK_UNARY(s.len == 0); - CHECK_UNARY(s.str != nullptr); - CHECK_UNARY(s != nullptr); - - s = nullptr; - CHECK_UNARY(s.empty()); - CHECK_UNARY(s.len == 0); - CHECK_UNARY(s.str == nullptr); - CHECK_UNARY(s == nullptr); - - s = ""; - CHECK_UNARY(s.empty()); - CHECK_UNARY(s.len == 0); - CHECK_UNARY(s.str != nullptr); - CHECK_UNARY(s != nullptr); - - s = {}; - CHECK_UNARY(s.empty()); - CHECK_UNARY(s.len == 0); - CHECK_UNARY(s.str == nullptr); - CHECK_UNARY(s == nullptr); - - csubstr pp(nullptr); - CHECK_UNARY(pp.empty()); - CHECK_UNARY(pp.len == 0); - CHECK_UNARY(pp.str == nullptr); - CHECK_UNARY(pp == nullptr); -} - -TEST_CASE("substr.is_sub") -{ - csubstr buf = "0123456789"; - - // ref - csubstr s; - csubstr ref = buf.select("345"); - CHECK_EQ(ref, "345"); - CHECK_UNARY(buf.is_super(ref)); - CHECK_UNARY(ref.is_sub(buf)); - CHECK_FALSE(ref.is_super(buf)); - CHECK_FALSE(buf.is_sub(ref)); - - buf.clear(); - ref.clear(); - CHECK_FALSE(buf.is_super(ref)); - CHECK_FALSE(ref.is_super(buf)); - CHECK_FALSE(ref.is_sub(buf)); - CHECK_FALSE(buf.is_sub(ref)); - - buf = ""; - ref = buf; - CHECK_FALSE(buf.is_super("a")); - CHECK_UNARY(buf.is_super(ref)); -} - -TEST_CASE("substr.overlaps") -{ - csubstr buf = "0123456789"; - - // ref - csubstr s; - csubstr ref = buf.select("345"); - CHECK_EQ(ref.len, 3); - CHECK_EQ(ref, "345"); - - // all_left - s = buf.sub(0, 2); - CHECK_EQ(s, "01"); - CHECK_FALSE(ref.overlaps(s)); - CHECK_FALSE(s.overlaps(ref)); - - // all_left_tight - s = buf.sub(0, 3); - CHECK_EQ(s, "012"); - CHECK_FALSE(ref.overlaps(s)); - CHECK_FALSE(s.overlaps(ref)); - - // overlap_left - s = buf.sub(0, 4); - CHECK_EQ(s, "0123"); - CHECK_UNARY(ref.overlaps(s)); - CHECK_UNARY(s.overlaps(ref)); - - // inside_tight_left - s = buf.sub(3, 1); - CHECK_EQ(s, "3"); - CHECK_UNARY(ref.overlaps(s)); - CHECK_UNARY(s.overlaps(ref)); - s = buf.sub(3, 2); - CHECK_EQ(s, "34"); - CHECK_UNARY(ref.overlaps(s)); - CHECK_UNARY(s.overlaps(ref)); - - // all_inside_tight - s = buf.sub(4, 1); - CHECK_EQ(s, "4"); - CHECK_UNARY(ref.overlaps(s)); - CHECK_UNARY(s.overlaps(ref)); - s = buf.sub(3, 3); - CHECK_EQ(s, "345"); - CHECK_UNARY(ref.overlaps(s)); - CHECK_UNARY(s.overlaps(ref)); - - // inside_tight_right - s = buf.sub(4, 2); - CHECK_EQ(s, "45"); - CHECK_UNARY(ref.overlaps(s)); - CHECK_UNARY(s.overlaps(ref)); - s = buf.sub(5, 1); - CHECK_EQ(s, "5"); - CHECK_UNARY(ref.overlaps(s)); - CHECK_UNARY(s.overlaps(ref)); - - // overlap_right - s = buf.sub(5, 2); - CHECK_EQ(s, "56"); - CHECK_UNARY(ref.overlaps(s)); - CHECK_UNARY(s.overlaps(ref)); - s = buf.sub(5, 3); - CHECK_EQ(s, "567"); - CHECK_UNARY(ref.overlaps(s)); - CHECK_UNARY(s.overlaps(ref)); - - // all_right_tight - s = buf.sub(6, 1); - CHECK_EQ(s, "6"); - CHECK_FALSE(ref.overlaps(s)); - CHECK_FALSE(s.overlaps(ref)); - s = buf.sub(6, 2); - CHECK_EQ(s, "67"); - CHECK_FALSE(ref.overlaps(s)); - CHECK_FALSE(s.overlaps(ref)); - - // all_right - s = buf.sub(7, 1); - CHECK_EQ(s, "7"); - CHECK_FALSE(ref.overlaps(s)); - CHECK_FALSE(s.overlaps(ref)); - s = buf.sub(7, 2); - CHECK_EQ(s, "78"); - CHECK_FALSE(ref.overlaps(s)); - CHECK_FALSE(s.overlaps(ref)); - - // null vs null - csubstr n1, n2; - CHECK_EQ(n1.str, nullptr); - CHECK_EQ(n2.str, nullptr); - CHECK_EQ(n1.len, 0); - CHECK_EQ(n2.len, 0); - CHECK_FALSE(n1.overlaps(n2)); - CHECK_FALSE(n2.overlaps(n1)); -} - -TEST_CASE("substr.sub") -{ - CHECK_EQ(csubstr("10]").sub(0, 2), "10"); -} - -TEST_CASE("substr.range") -{ - csubstr s = "0123456789"; - CHECK_EQ(s.range(0, 10), "0123456789"); - CHECK_EQ(s.range(0 ), "0123456789"); - CHECK_EQ(s.range(1, 10), "123456789"); - CHECK_EQ(s.range(1 ), "123456789"); - CHECK_EQ(s.range(2, 10), "23456789"); - CHECK_EQ(s.range(2 ), "23456789"); - CHECK_EQ(s.range(3, 10), "3456789"); - CHECK_EQ(s.range(3 ), "3456789"); - CHECK_EQ(s.range(4, 10), "456789"); - CHECK_EQ(s.range(4 ), "456789"); - CHECK_EQ(s.range(5, 10), "56789"); - CHECK_EQ(s.range(5 ), "56789"); - CHECK_EQ(s.range(6, 10), "6789"); - CHECK_EQ(s.range(6 ), "6789"); - CHECK_EQ(s.range(7, 10), "789"); - CHECK_EQ(s.range(7 ), "789"); - CHECK_EQ(s.range(8, 10), "89"); - CHECK_EQ(s.range(8 ), "89"); - CHECK_EQ(s.range(9, 10), "9"); - CHECK_EQ(s.range(9 ), "9"); - CHECK_EQ(s.range(10, 10), ""); - CHECK_EQ(s.range(10 ), ""); - - CHECK_EQ(s.range(0 , 9), "012345678"); - CHECK_EQ(s.range(1 , 9), "12345678"); - CHECK_EQ(s.range(2 , 9), "2345678"); - CHECK_EQ(s.range(3 , 9), "345678"); - CHECK_EQ(s.range(4 , 9), "45678"); - CHECK_EQ(s.range(5 , 9), "5678"); - CHECK_EQ(s.range(6 , 9), "678"); - CHECK_EQ(s.range(7 , 9), "78"); - CHECK_EQ(s.range(8 , 9), "8"); - CHECK_EQ(s.range(9 , 9), ""); - - CHECK_EQ(s.range(0 , 7), "0123456"); - CHECK_EQ(s.range(1 , 7), "123456"); - CHECK_EQ(s.range(2 , 7), "23456"); - CHECK_EQ(s.range(3 , 7), "3456"); - CHECK_EQ(s.range(4 , 7), "456"); - CHECK_EQ(s.range(5 , 7), "56"); - CHECK_EQ(s.range(6 , 7), "6"); - CHECK_EQ(s.range(7 , 7), ""); - - CHECK_EQ(s.range(0 , 5), "01234"); - CHECK_EQ(s.range(1 , 5), "1234"); - CHECK_EQ(s.range(2 , 5), "234"); - CHECK_EQ(s.range(3 , 5), "34"); - CHECK_EQ(s.range(4 , 5), "4"); - CHECK_EQ(s.range(5 , 5), ""); - - CHECK_EQ(s.range(0 , 3), "012"); - CHECK_EQ(s.range(1 , 3), "12"); - CHECK_EQ(s.range(2 , 3), "2"); - CHECK_EQ(s.range(3 , 3), ""); - - CHECK_EQ(s.range(0 , 2), "01"); - CHECK_EQ(s.range(1 , 2), "1"); - CHECK_EQ(s.range(2 , 2), ""); - - CHECK_EQ(s.range(0 , 1), "0"); - CHECK_EQ(s.range(1 , 1), ""); -} - -TEST_CASE("substr.first") -{ - csubstr s = "0123456789"; - - CHECK_EQ(s.first(csubstr::npos), s); - - CHECK_EQ(s.first(10), "0123456789"); - CHECK_EQ(s.first(9), "012345678"); - CHECK_EQ(s.first(8), "01234567"); - CHECK_EQ(s.first(7), "0123456"); - CHECK_EQ(s.first(6), "012345"); - CHECK_EQ(s.first(5), "01234"); - CHECK_EQ(s.first(4), "0123"); - CHECK_EQ(s.first(3), "012"); - CHECK_EQ(s.first(2), "01"); - CHECK_EQ(s.first(1), "0"); - CHECK_EQ(s.first(0), ""); -} - -TEST_CASE("substr.last") -{ - csubstr s = "0123456789"; - - CHECK_EQ(s.last(csubstr::npos), s); - - CHECK_EQ(s.last(10), "0123456789"); - CHECK_EQ(s.last(9), "123456789"); - CHECK_EQ(s.last(8), "23456789"); - CHECK_EQ(s.last(7), "3456789"); - CHECK_EQ(s.last(6), "456789"); - CHECK_EQ(s.last(5), "56789"); - CHECK_EQ(s.last(4), "6789"); - CHECK_EQ(s.last(3), "789"); - CHECK_EQ(s.last(2), "89"); - CHECK_EQ(s.last(1), "9"); - CHECK_EQ(s.last(0), ""); -} - -TEST_CASE("substr.offs") -{ - csubstr s = "0123456789"; - - CHECK_EQ(s.offs(0, 0), s); - - CHECK_EQ(s.offs(1, 0), "123456789"); - CHECK_EQ(s.offs(0, 1), "012345678"); - CHECK_EQ(s.offs(1, 1), "12345678"); - - CHECK_EQ(s.offs(1, 2), "1234567"); - CHECK_EQ(s.offs(2, 1), "2345678"); - CHECK_EQ(s.offs(2, 2), "234567"); - - CHECK_EQ(s.offs(2, 3), "23456"); - CHECK_EQ(s.offs(3, 2), "34567"); - CHECK_EQ(s.offs(3, 3), "3456"); - - CHECK_EQ(s.offs(3, 4), "345"); - CHECK_EQ(s.offs(4, 3), "456"); - CHECK_EQ(s.offs(4, 4), "45"); - - CHECK_EQ(s.offs(4, 5), "4"); - CHECK_EQ(s.offs(5, 4), "5"); - CHECK_EQ(s.offs(5, 5), ""); -} - -TEST_CASE("substr.count") -{ - csubstr buf = "0123456789"; - - CHECK_EQ(buf.count('0'), 1); - CHECK_EQ(buf.count('0', 0), 1); - CHECK_EQ(buf.count('0', 1), 0); - CHECK_EQ(buf.count('0', buf.len), 0); - - CHECK_EQ(buf.count("01"), 1); - CHECK_EQ(buf.count("01", 0), 1); - CHECK_EQ(buf.count("01", 1), 0); - CHECK_EQ(buf.count("01", buf.len), 0); - - CHECK_EQ(buf.count('1'), 1); - CHECK_EQ(buf.count('1', 0), 1); - CHECK_EQ(buf.count('1', 1), 1); - CHECK_EQ(buf.count('1', 2), 0); - CHECK_EQ(buf.count('1', buf.len), 0); - - CHECK_EQ(buf.count("12"), 1); - CHECK_EQ(buf.count("12", 0), 1); - CHECK_EQ(buf.count("12", 1), 1); - CHECK_EQ(buf.count("12", 2), 0); - CHECK_EQ(buf.count("12", buf.len), 0); - - CHECK_EQ(buf.count('2'), 1); - CHECK_EQ(buf.count('2', 0), 1); - CHECK_EQ(buf.count('2', 1), 1); - CHECK_EQ(buf.count('2', 2), 1); - CHECK_EQ(buf.count('2', 3), 0); - CHECK_EQ(buf.count('2', buf.len), 0); - - CHECK_EQ(buf.count("23"), 1); - CHECK_EQ(buf.count("23", 0), 1); - CHECK_EQ(buf.count("23", 1), 1); - CHECK_EQ(buf.count("23", 2), 1); - CHECK_EQ(buf.count("23", 3), 0); - CHECK_EQ(buf.count("23", buf.len), 0); - - CHECK_EQ(buf.count('3'), 1); - CHECK_EQ(buf.count('3', 0), 1); - CHECK_EQ(buf.count('3', 1), 1); - CHECK_EQ(buf.count('3', 2), 1); - CHECK_EQ(buf.count('3', 3), 1); - CHECK_EQ(buf.count('3', 4), 0); - CHECK_EQ(buf.count('3', buf.len), 0); - - CHECK_EQ(buf.count("34"), 1); - CHECK_EQ(buf.count("34", 0), 1); - CHECK_EQ(buf.count("34", 1), 1); - CHECK_EQ(buf.count("34", 2), 1); - CHECK_EQ(buf.count("34", 3), 1); - CHECK_EQ(buf.count("34", 4), 0); - CHECK_EQ(buf.count("34", buf.len), 0); - - CHECK_EQ(buf.count('4'), 1); - CHECK_EQ(buf.count('4', 0), 1); - CHECK_EQ(buf.count('4', 1), 1); - CHECK_EQ(buf.count('4', 2), 1); - CHECK_EQ(buf.count('4', 3), 1); - CHECK_EQ(buf.count('4', 4), 1); - CHECK_EQ(buf.count('4', 5), 0); - CHECK_EQ(buf.count('4', buf.len), 0); - - CHECK_EQ(buf.count("45"), 1); - CHECK_EQ(buf.count("45", 0), 1); - CHECK_EQ(buf.count("45", 1), 1); - CHECK_EQ(buf.count("45", 2), 1); - CHECK_EQ(buf.count("45", 3), 1); - CHECK_EQ(buf.count("45", 4), 1); - CHECK_EQ(buf.count("45", 5), 0); - CHECK_EQ(buf.count("45", buf.len), 0); - - CHECK_EQ(buf.count('5'), 1); - CHECK_EQ(buf.count('5', 0), 1); - CHECK_EQ(buf.count('5', 1), 1); - CHECK_EQ(buf.count('5', 2), 1); - CHECK_EQ(buf.count('5', 3), 1); - CHECK_EQ(buf.count('5', 4), 1); - CHECK_EQ(buf.count('5', 5), 1); - CHECK_EQ(buf.count('5', 6), 0); - CHECK_EQ(buf.count('5', buf.len), 0); - - CHECK_EQ(buf.count("56"), 1); - CHECK_EQ(buf.count("56", 0), 1); - CHECK_EQ(buf.count("56", 1), 1); - CHECK_EQ(buf.count("56", 2), 1); - CHECK_EQ(buf.count("56", 3), 1); - CHECK_EQ(buf.count("56", 4), 1); - CHECK_EQ(buf.count("56", 5), 1); - CHECK_EQ(buf.count("56", 6), 0); - CHECK_EQ(buf.count("56", buf.len), 0); - - CHECK_EQ(buf.count('a'), 0); - CHECK_EQ(buf.count('a', 0), 0); - CHECK_EQ(buf.count('a', 1), 0); - CHECK_EQ(buf.count('a', 2), 0); - CHECK_EQ(buf.count('a', 3), 0); - CHECK_EQ(buf.count('a', 4), 0); - CHECK_EQ(buf.count('a', 5), 0); - CHECK_EQ(buf.count('a', 6), 0); - CHECK_EQ(buf.count('a', buf.len), 0); - - CHECK_EQ(buf.count("ab"), 0); - CHECK_EQ(buf.count("ab", 0), 0); - CHECK_EQ(buf.count("ab", 1), 0); - CHECK_EQ(buf.count("ab", 2), 0); - CHECK_EQ(buf.count("ab", 3), 0); - CHECK_EQ(buf.count("ab", 4), 0); - CHECK_EQ(buf.count("ab", 5), 0); - CHECK_EQ(buf.count("ab", 6), 0); - CHECK_EQ(buf.count("ab", buf.len), 0); - - buf = "00110022003300440055"; - CHECK_EQ(buf.count('0', 0), 10); - CHECK_EQ(buf.count('0', 1), 9); - CHECK_EQ(buf.count('0', 2), 8); - CHECK_EQ(buf.count('0', 3), 8); - CHECK_EQ(buf.count('0', 4), 8); - CHECK_EQ(buf.count('0', 5), 7); - CHECK_EQ(buf.count('0', 6), 6); - CHECK_EQ(buf.count('0', 7), 6); - CHECK_EQ(buf.count('0', 8), 6); - CHECK_EQ(buf.count('0', 9), 5); - CHECK_EQ(buf.count('0', 10), 4); - CHECK_EQ(buf.count('0', 11), 4); - CHECK_EQ(buf.count('0', 12), 4); - CHECK_EQ(buf.count('0', 13), 3); - CHECK_EQ(buf.count('0', 14), 2); - CHECK_EQ(buf.count('0', 15), 2); - CHECK_EQ(buf.count('0', 16), 2); - CHECK_EQ(buf.count('0', 17), 1); - CHECK_EQ(buf.count('0', 18), 0); - CHECK_EQ(buf.count('0', 19), 0); - CHECK_EQ(buf.count('0', 20), 0); - - CHECK_EQ(buf.count('1', 0), 2); - CHECK_EQ(buf.count('1', 1), 2); - CHECK_EQ(buf.count('1', 2), 2); - CHECK_EQ(buf.count('1', 3), 1); - CHECK_EQ(buf.count('1', 4), 0); - CHECK_EQ(buf.count('1', 5), 0); - - CHECK_EQ(buf.count("01" ), 1); - CHECK_EQ(buf.count("01", 2), 0); - CHECK_EQ(buf.count("10" ), 1); - CHECK_EQ(buf.count("10", 4), 0); - CHECK_EQ(buf.count("00", 0), 5); - CHECK_EQ(buf.count("00", 1), 4); - CHECK_EQ(buf.count("00", 2), 4); - CHECK_EQ(buf.count("00", 3), 4); - CHECK_EQ(buf.count("00", 4), 4); - CHECK_EQ(buf.count("00", 5), 3); - CHECK_EQ(buf.count("00", 6), 3); - CHECK_EQ(buf.count("00", 7), 3); - CHECK_EQ(buf.count("00", 8), 3); - CHECK_EQ(buf.count("00", 9), 2); - CHECK_EQ(buf.count("00", 10), 2); - CHECK_EQ(buf.count("00", 11), 2); - CHECK_EQ(buf.count("00", 12), 2); - CHECK_EQ(buf.count("00", 13), 1); - CHECK_EQ(buf.count("00", 14), 1); - CHECK_EQ(buf.count("00", 15), 1); - CHECK_EQ(buf.count("00", 16), 1); - CHECK_EQ(buf.count("00", 17), 0); - CHECK_EQ(buf.count("00", 18), 0); - CHECK_EQ(buf.count("00", 19), 0); - CHECK_EQ(buf.count("00", 20), 0); -} - -TEST_CASE("substr.select") -{ - csubstr buf = "0123456789"; - - CHECK_EQ(buf.select('0'), "0"); - CHECK_EQ(buf.select('1'), "1"); - CHECK_EQ(buf.select('2'), "2"); - CHECK_EQ(buf.select('8'), "8"); - CHECK_EQ(buf.select('9'), "9"); - - CHECK_EQ(buf.select('a').str, nullptr); - CHECK_EQ(buf.select('a').len, 0); - CHECK_EQ(buf.select('a'), ""); - - CHECK_EQ(buf.select("a").str, nullptr); - CHECK_EQ(buf.select("a").len, 0); - CHECK_EQ(buf.select("a"), ""); - - CHECK_EQ(buf.select("0"), "0"); - CHECK_EQ(buf.select("0").str, buf.str+0); - CHECK_EQ(buf.select("0").len, 1); - - CHECK_EQ(buf.select("1"), "1"); - CHECK_EQ(buf.select("1").str, buf.str+1); - CHECK_EQ(buf.select("1").len, 1); - - CHECK_EQ(buf.select("2"), "2"); - CHECK_EQ(buf.select("2").str, buf.str+2); - CHECK_EQ(buf.select("2").len, 1); - - CHECK_EQ(buf.select("9"), "9"); - CHECK_EQ(buf.select("9").str, buf.str+9); - CHECK_EQ(buf.select("9").len, 1); - - CHECK_EQ(buf.select("012"), "012"); - CHECK_EQ(buf.select("012").str, buf.str+0); - CHECK_EQ(buf.select("012").len, 3); - - CHECK_EQ(buf.select("345"), "345"); - CHECK_EQ(buf.select("345").str, buf.str+3); - CHECK_EQ(buf.select("345").len, 3); - - CHECK_EQ(buf.select("789"), "789"); - CHECK_EQ(buf.select("789").str, buf.str+7); - CHECK_EQ(buf.select("789").len, 3); - - CHECK_EQ(buf.select("89a"), ""); - CHECK_EQ(buf.select("89a").str, nullptr); - CHECK_EQ(buf.select("89a").len, 0); -} - -TEST_CASE("substr.begins_with") -{ - CHECK (csubstr(": ").begins_with(":" )); - CHECK (csubstr(": ").begins_with(':' )); - CHECK_FALSE(csubstr(":") .begins_with(": ")); - - CHECK (csubstr( "1234").begins_with('0', 0)); - CHECK (csubstr( "01234").begins_with('0', 1)); - CHECK_FALSE(csubstr( "01234").begins_with('0', 2)); - CHECK (csubstr( "001234").begins_with('0', 1)); - CHECK (csubstr( "001234").begins_with('0', 2)); - CHECK_FALSE(csubstr( "001234").begins_with('0', 3)); - CHECK (csubstr( "0001234").begins_with('0', 1)); - CHECK (csubstr( "0001234").begins_with('0', 2)); - CHECK (csubstr( "0001234").begins_with('0', 3)); - CHECK_FALSE(csubstr( "0001234").begins_with('0', 4)); - CHECK (csubstr("00001234").begins_with('0', 1)); - CHECK (csubstr("00001234").begins_with('0', 2)); - CHECK (csubstr("00001234").begins_with('0', 3)); - CHECK (csubstr("00001234").begins_with('0', 4)); - CHECK_FALSE(csubstr("00001234").begins_with('0', 5)); -} - -TEST_CASE("substr.ends_with") -{ - CHECK_UNARY(csubstr("{% if foo %}bar{% endif %}").ends_with("{% endif %}")); - - CHECK (csubstr("1234" ).ends_with('0', 0)); - CHECK (csubstr("12340" ).ends_with('0', 1)); - CHECK_FALSE(csubstr("12340" ).ends_with('0', 2)); - CHECK (csubstr("123400" ).ends_with('0', 1)); - CHECK (csubstr("123400" ).ends_with('0', 2)); - CHECK_FALSE(csubstr("123400" ).ends_with('0', 3)); - CHECK (csubstr("1234000" ).ends_with('0', 1)); - CHECK (csubstr("1234000" ).ends_with('0', 2)); - CHECK (csubstr("1234000" ).ends_with('0', 3)); - CHECK_FALSE(csubstr("1234000" ).ends_with('0', 4)); - CHECK (csubstr("12340000").ends_with('0', 1)); - CHECK (csubstr("12340000").ends_with('0', 2)); - CHECK (csubstr("12340000").ends_with('0', 3)); - CHECK (csubstr("12340000").ends_with('0', 4)); - CHECK_FALSE(csubstr("12340000").ends_with('0', 5)); -} - -TEST_CASE("substr.find") -{ - csubstr s012345 = "012345"; - CHECK(s012345.find('a') == csubstr::npos); - CHECK(s012345.find('0' ) == 0u); - CHECK(s012345.find('0', 1u) == csubstr::npos); - CHECK(s012345.find('1' ) == 1u); - CHECK(s012345.find('1', 2u) == csubstr::npos); - CHECK(s012345.find('2' ) == 2u); - CHECK(s012345.find('2', 3u) == csubstr::npos); - CHECK(s012345.find('3' ) == 3u); - CHECK(s012345.find('3', 4u) == csubstr::npos); - CHECK(s012345.find("ab" ) == csubstr::npos); - CHECK(s012345.find("01" ) == 0u); - CHECK(s012345.find("01", 1u) == csubstr::npos); - CHECK(s012345.find("12" ) == 1u); - CHECK(s012345.find("12", 2u) == csubstr::npos); - CHECK(s012345.find("23" ) == 2u); - CHECK(s012345.find("23", 3u) == csubstr::npos); -} - -TEST_CASE("substr.first_of") -{ - size_t npos = csubstr::npos; - - CHECK_EQ(csubstr("012345").first_of('a'), npos); - CHECK_EQ(csubstr("012345").first_of("ab"), npos); - - CHECK_EQ(csubstr("012345").first_of('0'), 0u); - CHECK_EQ(csubstr("012345").first_of("0"), 0u); - CHECK_EQ(csubstr("012345").first_of("01"), 0u); - CHECK_EQ(csubstr("012345").first_of("10"), 0u); - CHECK_EQ(csubstr("012345").first_of("012"), 0u); - CHECK_EQ(csubstr("012345").first_of("210"), 0u); - CHECK_EQ(csubstr("012345").first_of("0123"), 0u); - CHECK_EQ(csubstr("012345").first_of("3210"), 0u); - CHECK_EQ(csubstr("012345").first_of("01234"), 0u); - CHECK_EQ(csubstr("012345").first_of("43210"), 0u); - CHECK_EQ(csubstr("012345").first_of("012345"), 0u); - CHECK_EQ(csubstr("012345").first_of("543210"), 0u); - - CHECK_EQ(csubstr("012345").first_of('0', 2u), npos); - CHECK_EQ(csubstr("012345").first_of("0", 2u), npos); - CHECK_EQ(csubstr("012345").first_of("01", 2u), npos); - CHECK_EQ(csubstr("012345").first_of("10", 2u), npos); - CHECK_EQ(csubstr("012345").first_of("012", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("210", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("0123", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("3210", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("01234", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("43210", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("012345", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("543210", 2u), 2u); - - CHECK_EQ(csubstr("012345").first_of('5'), 5u); - CHECK_EQ(csubstr("012345").first_of("5"), 5u); - CHECK_EQ(csubstr("012345").first_of("45"), 4u); - CHECK_EQ(csubstr("012345").first_of("54"), 4u); - CHECK_EQ(csubstr("012345").first_of("345"), 3u); - CHECK_EQ(csubstr("012345").first_of("543"), 3u); - CHECK_EQ(csubstr("012345").first_of("2345"), 2u); - CHECK_EQ(csubstr("012345").first_of("5432"), 2u); - CHECK_EQ(csubstr("012345").first_of("12345"), 1u); - CHECK_EQ(csubstr("012345").first_of("54321"), 1u); - CHECK_EQ(csubstr("012345").first_of("012345"), 0u); - CHECK_EQ(csubstr("012345").first_of("543210"), 0u); - - CHECK_EQ(csubstr("012345").first_of('5', 2u), 5u); - CHECK_EQ(csubstr("012345").first_of("5", 2u), 5u); - CHECK_EQ(csubstr("012345").first_of("45", 2u), 4u); - CHECK_EQ(csubstr("012345").first_of("54", 2u), 4u); - CHECK_EQ(csubstr("012345").first_of("345", 2u), 3u); - CHECK_EQ(csubstr("012345").first_of("543", 2u), 3u); - CHECK_EQ(csubstr("012345").first_of("2345", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("5432", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("12345", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("54321", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("012345", 2u), 2u); - CHECK_EQ(csubstr("012345").first_of("543210", 2u), 2u); - - CHECK_EQ(csubstr{}.first_of('0'), npos); - CHECK_EQ(csubstr{}.first_of('0', 0u), npos); - CHECK_EQ(csubstr("012345").first_of('0', 6u), npos); - CHECK_EQ(csubstr("012345").first_of('5', 6u), npos); - CHECK_EQ(csubstr("012345").first_of("012345", 6u), npos); -} - -TEST_CASE("substr.last_of") -{ - size_t npos = csubstr::npos; - - CHECK_EQ(csubstr("012345").last_of('a'), npos); - CHECK_EQ(csubstr("012345").last_of("ab"), npos); - - CHECK_EQ(csubstr("012345").last_of('0'), 0u); - CHECK_EQ(csubstr("012345").last_of("0"), 0u); - CHECK_EQ(csubstr("012345").last_of("01"), 1u); - CHECK_EQ(csubstr("012345").last_of("10"), 1u); - CHECK_EQ(csubstr("012345").last_of("012"), 2u); - CHECK_EQ(csubstr("012345").last_of("210"), 2u); - CHECK_EQ(csubstr("012345").last_of("0123"), 3u); - CHECK_EQ(csubstr("012345").last_of("3210"), 3u); - CHECK_EQ(csubstr("012345").last_of("01234"), 4u); - CHECK_EQ(csubstr("012345").last_of("43210"), 4u); - CHECK_EQ(csubstr("012345").last_of("012345"), 5u); - CHECK_EQ(csubstr("012345").last_of("543210"), 5u); - - CHECK_EQ(csubstr("012345").last_of('0', 2u), 0u); - CHECK_EQ(csubstr("012345").last_of("0", 2u), 0u); - CHECK_EQ(csubstr("012345").last_of("01", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("10", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("012", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("210", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("0123", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("3210", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("01234", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("43210", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("012345", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("543210", 2u), 1u); - - CHECK_EQ(csubstr("012345").last_of('5'), 5u); - CHECK_EQ(csubstr("012345").last_of("5"), 5u); - CHECK_EQ(csubstr("012345").last_of("45"), 5u); - CHECK_EQ(csubstr("012345").last_of("54"), 5u); - CHECK_EQ(csubstr("012345").last_of("345"), 5u); - CHECK_EQ(csubstr("012345").last_of("543"), 5u); - CHECK_EQ(csubstr("012345").last_of("2345"), 5u); - CHECK_EQ(csubstr("012345").last_of("5432"), 5u); - CHECK_EQ(csubstr("012345").last_of("12345"), 5u); - CHECK_EQ(csubstr("012345").last_of("54321"), 5u); - CHECK_EQ(csubstr("012345").last_of("012345"), 5u); - CHECK_EQ(csubstr("012345").last_of("543210"), 5u); - - CHECK_EQ(csubstr("012345").last_of('5', 2u), npos); - CHECK_EQ(csubstr("012345").last_of("5", 2u), npos); - CHECK_EQ(csubstr("012345").last_of("45", 2u), npos); - CHECK_EQ(csubstr("012345").last_of("54", 2u), npos); - CHECK_EQ(csubstr("012345").last_of("345", 2u), npos); - CHECK_EQ(csubstr("012345").last_of("543", 2u), npos); - CHECK_EQ(csubstr("012345").last_of("2345", 2u), npos); - CHECK_EQ(csubstr("012345").last_of("5432", 2u), npos); - CHECK_EQ(csubstr("012345").last_of("12345", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("54321", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("012345", 2u), 1u); - CHECK_EQ(csubstr("012345").last_of("543210", 2u), 1u); - - CHECK_EQ(csubstr{}.last_of('?'), npos); - CHECK_EQ(csubstr("012345").last_of('0', 6u), 0u); - CHECK_EQ(csubstr("012345").last_of('5', 6u), 5u); - CHECK_EQ(csubstr("012345").last_of("012345", 6u), 5u); -} - -TEST_CASE("substr.first_not_of") -{ - size_t npos = csubstr::npos; - - CHECK_EQ(csubstr("012345").first_not_of('a'), 0u); - CHECK_EQ(csubstr("012345").first_not_of("ab"), 0u); - - CHECK_EQ(csubstr("012345").first_not_of('0'), 1u); - CHECK_EQ(csubstr("012345").first_not_of("0"), 1u); - CHECK_EQ(csubstr("012345").first_not_of("01"), 2u); - CHECK_EQ(csubstr("012345").first_not_of("10"), 2u); - CHECK_EQ(csubstr("012345").first_not_of("012"), 3u); - CHECK_EQ(csubstr("012345").first_not_of("210"), 3u); - CHECK_EQ(csubstr("012345").first_not_of("0123"), 4u); - CHECK_EQ(csubstr("012345").first_not_of("3210"), 4u); - CHECK_EQ(csubstr("012345").first_not_of("01234"), 5u); - CHECK_EQ(csubstr("012345").first_not_of("43210"), 5u); - CHECK_EQ(csubstr("012345").first_not_of("012345"), npos); - CHECK_EQ(csubstr("012345").first_not_of("543210"), npos); - - CHECK_EQ(csubstr("012345").first_not_of('0', 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("0", 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("01", 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("10", 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("012", 2u), 3u); - CHECK_EQ(csubstr("012345").first_not_of("210", 2u), 3u); - CHECK_EQ(csubstr("012345").first_not_of("0123", 2u), 4u); - CHECK_EQ(csubstr("012345").first_not_of("3210", 2u), 4u); - CHECK_EQ(csubstr("012345").first_not_of("01234", 2u), 5u); - CHECK_EQ(csubstr("012345").first_not_of("43210", 2u), 5u); - CHECK_EQ(csubstr("012345").first_not_of("012345", 2u), npos); - CHECK_EQ(csubstr("012345").first_not_of("543210", 2u), npos); - - CHECK_EQ(csubstr("012345").first_not_of('5'), 0u); - CHECK_EQ(csubstr("012345").first_not_of("5"), 0u); - CHECK_EQ(csubstr("012345").first_not_of("45"), 0u); - CHECK_EQ(csubstr("012345").first_not_of("54"), 0u); - CHECK_EQ(csubstr("012345").first_not_of("345"), 0u); - CHECK_EQ(csubstr("012345").first_not_of("543"), 0u); - CHECK_EQ(csubstr("012345").first_not_of("2345"), 0u); - CHECK_EQ(csubstr("012345").first_not_of("5432"), 0u); - CHECK_EQ(csubstr("012345").first_not_of("12345"), 0u); - CHECK_EQ(csubstr("012345").first_not_of("54321"), 0u); - CHECK_EQ(csubstr("012345").first_not_of("012345"), npos); - CHECK_EQ(csubstr("012345").first_not_of("543210"), npos); - - CHECK_EQ(csubstr("012345").first_not_of('5', 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("5", 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("45", 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("54", 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("345", 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("543", 2u), 2u); - CHECK_EQ(csubstr("012345").first_not_of("2345", 2u), npos); - CHECK_EQ(csubstr("012345").first_not_of("5432", 2u), npos); - CHECK_EQ(csubstr("012345").first_not_of("12345", 2u), npos); - CHECK_EQ(csubstr("012345").first_not_of("54321", 2u), npos); - CHECK_EQ(csubstr("012345").first_not_of("012345", 2u), npos); - CHECK_EQ(csubstr("012345").first_not_of("543210", 2u), npos); - - CHECK_EQ(csubstr("").first_not_of('0', 0u), npos); - CHECK_EQ(csubstr("012345").first_not_of('5', 6u), npos); - CHECK_EQ(csubstr("012345").first_not_of("012345", 6u), npos); -} - -TEST_CASE("substr.last_not_of") -{ - size_t npos = csubstr::npos; - - CHECK_EQ(csubstr("012345").last_not_of('a'), 5u); - CHECK_EQ(csubstr("012345").last_not_of("ab"), 5u); - - CHECK_EQ(csubstr("012345").last_not_of('5'), 4u); - CHECK_EQ(csubstr("012345").last_not_of("5"), 4u); - CHECK_EQ(csubstr("012345").last_not_of("45"), 3u); - CHECK_EQ(csubstr("012345").last_not_of("54"), 3u); - CHECK_EQ(csubstr("012345").last_not_of("345"), 2u); - CHECK_EQ(csubstr("012345").last_not_of("543"), 2u); - CHECK_EQ(csubstr("012345").last_not_of("2345"), 1u); - CHECK_EQ(csubstr("012345").last_not_of("5432"), 1u); - CHECK_EQ(csubstr("012345").last_not_of("12345"), 0u); - CHECK_EQ(csubstr("012345").last_not_of("54321"), 0u); - CHECK_EQ(csubstr("012345").last_not_of("012345"), npos); - CHECK_EQ(csubstr("012345").last_not_of("543210"), npos); - - CHECK_EQ(csubstr("012345").last_not_of('5', 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("5", 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("45", 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("54", 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("345", 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("543", 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("2345", 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("5432", 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("12345", 2u), 0u); - CHECK_EQ(csubstr("012345").last_not_of("54321", 2u), 0u); - CHECK_EQ(csubstr("012345").last_not_of("012345", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("543210", 2u), npos); - - CHECK_EQ(csubstr("012345").last_not_of('0'), 5u); - CHECK_EQ(csubstr("012345").last_not_of("0"), 5u); - CHECK_EQ(csubstr("012345").last_not_of("01"), 5u); - CHECK_EQ(csubstr("012345").last_not_of("10"), 5u); - CHECK_EQ(csubstr("012345").last_not_of("012"), 5u); - CHECK_EQ(csubstr("012345").last_not_of("210"), 5u); - CHECK_EQ(csubstr("012345").last_not_of("0123"), 5u); - CHECK_EQ(csubstr("012345").last_not_of("3210"), 5u); - CHECK_EQ(csubstr("012345").last_not_of("01234"), 5u); - CHECK_EQ(csubstr("012345").last_not_of("43210"), 5u); - CHECK_EQ(csubstr("012345").last_not_of("012345"), npos); - CHECK_EQ(csubstr("012345").last_not_of("543210"), npos); - - CHECK_EQ(csubstr("012345").last_not_of('0', 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("0", 2u), 1u); - CHECK_EQ(csubstr("012345").last_not_of("01", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("10", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("012", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("210", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("0123", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("3210", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("01234", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("43210", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("012345", 2u), npos); - CHECK_EQ(csubstr("012345").last_not_of("543210", 2u), npos); - - CHECK_EQ(csubstr("").last_not_of('0', 0u), npos); - CHECK_EQ(csubstr("012345").last_not_of('5', 6u), 4u); -} - -TEST_CASE("substr.left_of") -{ - csubstr s = "012345"; - - CHECK_EQ(s.left_of(csubstr::npos), s); - CHECK_EQ(s.left_of(csubstr::npos, /*include_pos*/false), s); - CHECK_EQ(s.left_of(csubstr::npos, /*include_pos*/true), s); - - - CHECK_EQ(s.left_of(0), ""); - CHECK_EQ(s.left_of(1), "0"); - CHECK_EQ(s.left_of(2), "01"); - CHECK_EQ(s.left_of(3), "012"); - CHECK_EQ(s.left_of(4), "0123"); - CHECK_EQ(s.left_of(5), "01234"); - CHECK_EQ(s.left_of(6), "012345"); - - CHECK_EQ(s.left_of(0, /*include_pos*/false), ""); - CHECK_EQ(s.left_of(1, /*include_pos*/false), "0"); - CHECK_EQ(s.left_of(2, /*include_pos*/false), "01"); - CHECK_EQ(s.left_of(3, /*include_pos*/false), "012"); - CHECK_EQ(s.left_of(4, /*include_pos*/false), "0123"); - CHECK_EQ(s.left_of(5, /*include_pos*/false), "01234"); - CHECK_EQ(s.left_of(6, /*include_pos*/false), "012345"); - - CHECK_UNARY(s.is_super(s.left_of(0, /*include_pos*/false))); - CHECK_UNARY(s.is_super(s.left_of(1, /*include_pos*/false))); - CHECK_UNARY(s.is_super(s.left_of(2, /*include_pos*/false))); - CHECK_UNARY(s.is_super(s.left_of(3, /*include_pos*/false))); - CHECK_UNARY(s.is_super(s.left_of(4, /*include_pos*/false))); - - - CHECK_EQ(s.left_of(0, /*include_pos*/true), "0"); - CHECK_EQ(s.left_of(1, /*include_pos*/true), "01"); - CHECK_EQ(s.left_of(2, /*include_pos*/true), "012"); - CHECK_EQ(s.left_of(3, /*include_pos*/true), "0123"); - CHECK_EQ(s.left_of(4, /*include_pos*/true), "01234"); - CHECK_EQ(s.left_of(5, /*include_pos*/true), "012345"); - - CHECK_UNARY(s.is_super(s.left_of(0, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.left_of(1, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.left_of(2, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.left_of(3, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.left_of(4, /*include_pos*/true))); - - - CHECK_EQ(s.sub(5), "5"); - CHECK_EQ(s.sub(4), "45"); - CHECK_EQ(s.sub(3), "345"); - CHECK_EQ(s.sub(2), "2345"); - CHECK_EQ(s.sub(1), "12345"); - CHECK_EQ(s.sub(0), "012345"); - - CHECK_EQ(s.left_of(s.sub(5)), "01234"); - CHECK_EQ(s.left_of(s.sub(4)), "0123"); - CHECK_EQ(s.left_of(s.sub(3)), "012"); - CHECK_EQ(s.left_of(s.sub(2)), "01"); - CHECK_EQ(s.left_of(s.sub(1)), "0"); - CHECK_EQ(s.left_of(s.sub(0)), ""); - - CHECK_UNARY(s.is_super(s.left_of(s.sub(5)))); - CHECK_UNARY(s.is_super(s.left_of(s.sub(4)))); - CHECK_UNARY(s.is_super(s.left_of(s.sub(3)))); - CHECK_UNARY(s.is_super(s.left_of(s.sub(2)))); - CHECK_UNARY(s.is_super(s.left_of(s.sub(1)))); - CHECK_UNARY(s.is_super(s.left_of(s.sub(0)))); -} - -TEST_CASE("substr.right_of") -{ - csubstr s = "012345"; - - CHECK_EQ(s.right_of(csubstr::npos), ""); - CHECK_EQ(s.right_of(csubstr::npos), ""); - - CHECK_EQ(s.right_of(csubstr::npos, /*include_pos*/false), ""); - CHECK_EQ(s.right_of(csubstr::npos, /*include_pos*/true), ""); - - - CHECK_EQ(s.right_of(0), "12345"); - CHECK_EQ(s.right_of(1), "2345"); - CHECK_EQ(s.right_of(2), "345"); - CHECK_EQ(s.right_of(3), "45"); - CHECK_EQ(s.right_of(4), "5"); - CHECK_EQ(s.right_of(5), ""); - - CHECK_EQ(s.right_of(0, /*include_pos*/false), "12345"); - CHECK_EQ(s.right_of(1, /*include_pos*/false), "2345"); - CHECK_EQ(s.right_of(2, /*include_pos*/false), "345"); - CHECK_EQ(s.right_of(3, /*include_pos*/false), "45"); - CHECK_EQ(s.right_of(4, /*include_pos*/false), "5"); - CHECK_EQ(s.right_of(5, /*include_pos*/false), ""); - - CHECK_UNARY(s.is_super(s.right_of(0))); - CHECK_UNARY(s.is_super(s.right_of(1))); - CHECK_UNARY(s.is_super(s.right_of(2))); - CHECK_UNARY(s.is_super(s.right_of(3))); - CHECK_UNARY(s.is_super(s.right_of(4))); - CHECK_UNARY(s.is_super(s.right_of(5))); - - CHECK_UNARY(s.is_super(s.right_of(0, /*include_pos*/false))); - CHECK_UNARY(s.is_super(s.right_of(1, /*include_pos*/false))); - CHECK_UNARY(s.is_super(s.right_of(2, /*include_pos*/false))); - CHECK_UNARY(s.is_super(s.right_of(3, /*include_pos*/false))); - CHECK_UNARY(s.is_super(s.right_of(4, /*include_pos*/false))); - CHECK_UNARY(s.is_super(s.right_of(5, /*include_pos*/false))); - - - CHECK_EQ(s.right_of(0, /*include_pos*/true), "012345"); - CHECK_EQ(s.right_of(1, /*include_pos*/true), "12345"); - CHECK_EQ(s.right_of(2, /*include_pos*/true), "2345"); - CHECK_EQ(s.right_of(3, /*include_pos*/true), "345"); - CHECK_EQ(s.right_of(4, /*include_pos*/true), "45"); - CHECK_EQ(s.right_of(5, /*include_pos*/true), "5"); - CHECK_EQ(s.right_of(6, /*include_pos*/true), ""); - - CHECK_UNARY(s.is_super(s.right_of(0, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.right_of(1, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.right_of(2, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.right_of(3, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.right_of(4, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.right_of(5, /*include_pos*/true))); - CHECK_UNARY(s.is_super(s.right_of(6, /*include_pos*/true))); - - - CHECK_EQ(s.sub(0, 0), ""); - CHECK_EQ(s.sub(0, 1), "0"); - CHECK_EQ(s.sub(0, 2), "01"); - CHECK_EQ(s.sub(0, 3), "012"); - CHECK_EQ(s.sub(0, 4), "0123"); - CHECK_EQ(s.sub(0, 5), "01234"); - CHECK_EQ(s.sub(0, 6), "012345"); - - CHECK_EQ(s.right_of(s.sub(0, 0)), "012345"); - CHECK_EQ(s.right_of(s.sub(0, 1)), "12345"); - CHECK_EQ(s.right_of(s.sub(0, 2)), "2345"); - CHECK_EQ(s.right_of(s.sub(0, 3)), "345"); - CHECK_EQ(s.right_of(s.sub(0, 4)), "45"); - CHECK_EQ(s.right_of(s.sub(0, 5)), "5"); - CHECK_EQ(s.right_of(s.sub(0, 6)), ""); - - CHECK_UNARY(s.is_super(s.right_of(s.sub(0, 0)))); - CHECK_UNARY(s.is_super(s.right_of(s.sub(0, 1)))); - CHECK_UNARY(s.is_super(s.right_of(s.sub(0, 2)))); - CHECK_UNARY(s.is_super(s.right_of(s.sub(0, 3)))); - CHECK_UNARY(s.is_super(s.right_of(s.sub(0, 4)))); - CHECK_UNARY(s.is_super(s.right_of(s.sub(0, 5)))); - CHECK_UNARY(s.is_super(s.right_of(s.sub(0, 6)))); -} - -TEST_CASE("substr.compare_different_length") -{ - const char s1[] = "one empty doc"; - const char s2[] = "one empty doc, explicit termination"; - csubstr c1(s1), c2(s2); - CHECK_NE(c1, c2); - CHECK_NE(c1, s2); - CHECK_NE(s1, c2); - CHECK_LT(c1, c2); - CHECK_LT(c1, s2); - CHECK_LT(s1, c2); - CHECK_GT(c2, c1); - CHECK_GT(c2, s1); - CHECK_GT(s2, c1); - CHECK_NE((c1 > c2), (c1 < c2)); - CHECK_NE((c1 > s2), (c1 < s2)); - CHECK_NE((s1 > c2), (s1 < c2)); - CHECK_NE((c2 > c1), (c2 < c1)); - CHECK_NE((c2 > s1), (c2 < s1)); - CHECK_NE((s2 > c1), (s2 < c1)); - CHECK_NE((c1 == c2), (c1 != c2)); - CHECK_NE((c1 == s2), (c1 != s2)); - CHECK_NE((s1 == c2), (s1 != c2)); - CHECK_NE((c2 == c1), (c2 != c1)); - CHECK_NE((c2 == s1), (c2 != s1)); - CHECK_NE((s2 == c1), (s2 != c1)); -} - -TEST_CASE("substr.compare_null") -{ - csubstr s1, s2, sp(" "); - CHECK_EQ(s1, ""); - CHECK_EQ(s1, s2); - CHECK(!(s1 > s2)); - CHECK(!(s1 < s2)); - CHECK((s1 <= s2)); - CHECK((s1 >= s2)); - CHECK(!(s1 != s2)); - CHECK_EQ(s1.compare('-'), -1); - CHECK_EQ(sp.compare(' '), 0); - CHECK_EQ(s1.compare("-", 1u), -1); - CHECK_EQ(s1.compare("-", 0u), 0); - CHECK_EQ(s1.compare((const char*)0, 0u), 0); - CHECK_EQ(sp.compare((const char*)0, 0u), 1); - CHECK_EQ(sp.compare(" ", 0u), 1); - CHECK_EQ(sp.compare(" ", 1u), 0); -} - -TEST_CASE("substr.compare_vs_char") -{ - CHECK_EQ(csubstr().compare('1'), -1); // str==null, len==0 - CHECK_EQ(csubstr("0123").first(0).compare('1'), -1); // str!=null, len==0 - CHECK_EQ(csubstr("0123").first(1).compare('1'), -1); - - CHECK_EQ(csubstr("-"), '-'); - CHECK_NE(csubstr("+"), '-'); - - CHECK_NE(csubstr("---"), '-'); - CHECK_NE(csubstr("---"), "-"); - - CHECK_NE(csubstr("aaa"), 'a'); - CHECK_NE(csubstr("aaa"), "a"); - - CHECK_NE(csubstr("aaa"), 'b'); - CHECK_NE(csubstr("aaa"), "b"); - - CHECK_LT(csubstr("aaa"), 'b'); - CHECK_LT(csubstr("aaa"), "b"); - - CHECK_LE(csubstr("aaa"), 'b'); - CHECK_LE(csubstr("aaa"), "b"); - - CHECK_NE(csubstr("bbb"), 'a'); - CHECK_NE(csubstr("bbb"), "a"); - - CHECK_GT(csubstr("bbb"), 'a'); - CHECK_GT(csubstr("bbb"), "a"); - - CHECK_GE(csubstr("bbb"), 'a'); - CHECK_GE(csubstr("bbb"), "a"); -} - -TEST_CASE("substr.mixed_cmp") -{ - // c++20 introduced new comparison rules and clang10 fails: - // - // error: ISO C++20 considers use of overloaded operator '==' (with operand - // types 'const c4::basic_substring' and 'const - // c4::basic_substring') to be ambiguous despite there being a unique - // best viable function [-Werror,-Wambiguous-reversed-operator] - - char sa_[] = "a"; - char sb_[] = "b"; - csubstr csa = "a"; substr sa = sa_; - csubstr csb = "b"; substr sb = sb_; - - CHECK_EQ(csa, csa); - CHECK_EQ(sa, sa); // this fails - CHECK_EQ(csa, sa); - CHECK_EQ(sa, csa); - - CHECK_NE(sa, sb); - CHECK_NE(csa, csb); - CHECK_NE(csa, sb); - CHECK_NE(sa, csb); - - CHECK_LT(sa, sb); - CHECK_LT(csa, csb); - CHECK_LT(csa, sb); - CHECK_LT(sa, csb); - - CHECK_LE(sa, sb); - CHECK_LE(csa, csb); - CHECK_LE(csa, sb); - CHECK_LE(sa, csb); - - CHECK_LE(sa, sa); - CHECK_LE(csa, csa); - CHECK_LE(csa, sa); - CHECK_LE(sa, csa); - - CHECK_GT(sb, sa); - CHECK_GT(csb, csa); - CHECK_GT(csb, sa); - CHECK_GT( sb, csa); - - CHECK_GE(sb, sa); - CHECK_GE(csb, csa); - CHECK_GE(csb, sa); - CHECK_GE( sb, csa); - - CHECK_GE(sb, sb); - CHECK_GE(csb, csb); - CHECK_GE(csb, sb); - CHECK_GE( sb, csb); -} - -TEST_CASE("substr.eqne") -{ - char buf[128]; - for(size_t i = 0; i < 5; ++i) buf[i] = (char)('0' + i); - csubstr cmp(buf, 5); - - CHECK_EQ(csubstr("01234"), cmp); - CHECK_EQ( "01234" , cmp); - CHECK_EQ( cmp, "01234"); - CHECK_NE(csubstr("0123"), cmp); - CHECK_NE( "0123" , cmp); - CHECK_NE( cmp, "0123"); - CHECK_NE(csubstr("012345"), cmp); - CHECK_NE( "012345" , cmp); - CHECK_NE( cmp, "012345"); -} - -TEST_CASE("substr.substr2csubstr") -{ - char b[] = "some string"; - substr s(b); - csubstr sc = s; - CHECK_EQ(sc, s); - const substr cs(b); - const csubstr csc(b); -} - -template -void test_first_of_any(csubstr input, bool true_or_false, size_t which, size_t pos, Args... args) -{ - csubstr::first_of_any_result r = input.first_of_any(to_csubstr(args)...); - //std::cout << input << ": " << (bool(r) ? "true" : "false") << "/which:" << r.which << "/pos:" << r.pos << "\n"; - CHECK_EQ(r, true_or_false); - if(true_or_false) - { - CHECK_UNARY(r); - } - else - { - CHECK_FALSE(r); - } - CHECK_EQ(r.which, which); - CHECK_EQ(r.pos, pos); -} - -TEST_CASE("substr.first_of_any") -{ - size_t NONE = csubstr::NONE; - size_t npos = csubstr::npos; - - test_first_of_any("foobar" , true , 0u , 3u, "bar", "barbell", "bark", "barff"); - test_first_of_any("foobar" , false, NONE, npos, "barbell", "bark", "barff"); - test_first_of_any("foobart" , false, NONE, npos, "barbell", "bark", "barff"); - - test_first_of_any("10" , false, NONE, npos, "0x", "0X", "-0x", "-0X"); - test_first_of_any("10]" , false, NONE, npos, "0x", "0X", "-0x", "-0X"); - test_first_of_any(csubstr("10]").first(2), false, NONE, npos, "0x", "0X", "-0x", "-0X"); - - - test_first_of_any("baz{% endif %}", true, 0u, 3u, "{% endif %}", "{% if " , "{% elif bar %}" , "{% else %}" ); - test_first_of_any("baz{% endif %}", true, 1u, 3u, "{% if " , "{% endif %}" , "{% elif bar %}" , "{% else %}" ); - test_first_of_any("baz{% endif %}", true, 2u, 3u, "{% if " , "{% elif bar %}" , "{% endif %}" , "{% else %}" ); - test_first_of_any("baz{% endif %}", true, 3u, 3u, "{% if " , "{% elif bar %}" , "{% else %}" , "{% endif %}"); - - test_first_of_any("baz{% e..if %}", false, NONE, npos, "{% endif %}", "{% if " , "{% elif bar %}" , "{% else %}" ); - test_first_of_any("baz{% e..if %}", false, NONE, npos, "{% if " , "{% endif %}" , "{% elif bar %}" , "{% else %}" ); - test_first_of_any("baz{% e..if %}", false, NONE, npos, "{% if " , "{% elif bar %}" , "{% endif %}" , "{% else %}" ); - test_first_of_any("baz{% e..if %}", false, NONE, npos, "{% if " , "{% elif bar %}" , "{% else %}" , "{% endif %}"); - - - test_first_of_any("bar{% else %}baz{% endif %}", true, 0u, 3u, "{% else %}" , "{% if " , "{% elif bar %}" , "{% endif %}"); - test_first_of_any("bar{% else %}baz{% endif %}", true, 1u, 3u, "{% if " , "{% else %}" , "{% elif bar %}" , "{% endif %}"); - test_first_of_any("bar{% else %}baz{% endif %}", true, 2u, 3u, "{% if " , "{% elif bar %}" , "{% else %}" , "{% endif %}"); - test_first_of_any("bar{% else %}baz{% endif %}", true, 3u, 3u, "{% if " , "{% elif bar %}" , "{% endif %}" , "{% else %}" ); - - test_first_of_any("bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% else %}" , "{% if " , "{% elif bar %}" , "{% endif %}"); - test_first_of_any("bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% if " , "{% else %}" , "{% elif bar %}" , "{% endif %}"); - test_first_of_any("bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% if " , "{% elif bar %}" , "{% else %}" , "{% endif %}"); - test_first_of_any("bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% if " , "{% elif bar %}" , "{% endif %}" , "{% else %}" ); - - - test_first_of_any("foo{% elif bar %}bar{% else %}baz{% endif %}", true, 0u, 3u, "{% elif bar %}" , "{% if " , "{% else %}" , "{% endif %}" ); - test_first_of_any("foo{% elif bar %}bar{% else %}baz{% endif %}", true, 1u, 3u, "{% if " , "{% elif bar %}" , "{% else %}" , "{% endif %}" ); - test_first_of_any("foo{% elif bar %}bar{% else %}baz{% endif %}", true, 2u, 3u, "{% if " , "{% else %}" , "{% elif bar %}" , "{% endif %}" ); - test_first_of_any("foo{% elif bar %}bar{% else %}baz{% endif %}", true, 3u, 3u, "{% if " , "{% else %}" , "{% endif %}" , "{% elif bar %}"); - - test_first_of_any("foo{% e..f bar %}bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% elif bar %}" , "{% if " , "{% else %}" , "{% endif %}" ); - test_first_of_any("foo{% e..f bar %}bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% if " , "{% elif bar %}" , "{% else %}" , "{% endif %}" ); - test_first_of_any("foo{% e..f bar %}bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% if " , "{% else %}" , "{% elif bar %}" , "{% endif %}" ); - test_first_of_any("foo{% e..f bar %}bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% if " , "{% else %}" , "{% endif %}" , "{% elif bar %}"); - - - test_first_of_any("{% if foo %}foo{% elif bar %}bar{% else %}baz{% endif %}", true, 0u, 0u, "{% if " , "{% elif bar %}" , "{% else %}" , "{% endif %}" ); - test_first_of_any("{% if foo %}foo{% elif bar %}bar{% else %}baz{% endif %}", true, 1u, 0u, "{% elif bar %}" , "{% if " , "{% else %}" , "{% endif %}" ); - test_first_of_any("{% if foo %}foo{% elif bar %}bar{% else %}baz{% endif %}", true, 2u, 0u, "{% elif bar %}" , "{% else %}" , "{% if " , "{% endif %}" ); - test_first_of_any("{% if foo %}foo{% elif bar %}bar{% else %}baz{% endif %}", true, 3u, 0u, "{% elif bar %}" , "{% else %}" , "{% endif %}", "{% if " ); - - test_first_of_any("{% .. foo %}foo{% e..f bar %}bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% if " , "{% elif bar %}" , "{% else %}" , "{% endif %}" ); - test_first_of_any("{% .. foo %}foo{% e..f bar %}bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% elif bar %}" , "{% if " , "{% else %}" , "{% endif %}" ); - test_first_of_any("{% .. foo %}foo{% e..f bar %}bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% elif bar %}" , "{% else %}" , "{% if " , "{% endif %}" ); - test_first_of_any("{% .. foo %}foo{% e..f bar %}bar{% e..e %}baz{% e..if %}", false, NONE, npos, "{% elif bar %}" , "{% else %}" , "{% endif %}", "{% if " ); -} - - -TEST_CASE("substr.pair_range_esc") -{ - const char q = '\''; - CHECK_EQ(csubstr("").pair_range_esc(q), ""); - CHECK_EQ(csubstr("'").pair_range_esc(q), ""); - CHECK_EQ(csubstr("''").pair_range_esc(q), "''"); - CHECK_EQ(csubstr("'\\'\\''").pair_range_esc(q), "'\\'\\''"); - CHECK_EQ(csubstr("asdasdasd''asdasd").pair_range_esc(q), "''"); - CHECK_EQ(csubstr("asdasdasd'abc'asdasda").pair_range_esc(q), "'abc'"); -} - -TEST_CASE("substr.pair_range") -{ - CHECK_EQ(csubstr("").pair_range('{', '}'), ""); - CHECK_EQ(csubstr("{").pair_range('{', '}'), ""); - CHECK_EQ(csubstr("}").pair_range('{', '}'), ""); - CHECK_EQ(csubstr("{}").pair_range('{', '}'), "{}"); - CHECK_EQ(csubstr("{abc}").pair_range('{', '}'), "{abc}"); - CHECK_EQ(csubstr("123{abc}456").pair_range('{', '}'), "{abc}"); -} - -TEST_CASE("substr.pair_range_nested") -{ - CHECK_EQ(csubstr("").pair_range_nested('{', '}'), ""); - CHECK_EQ(csubstr("{").pair_range_nested('{', '}'), ""); - CHECK_EQ(csubstr("}").pair_range_nested('{', '}'), ""); - CHECK_EQ(csubstr("{}").pair_range_nested('{', '}'), "{}"); - CHECK_EQ(csubstr("{abc}").pair_range_nested('{', '}'), "{abc}"); - CHECK_EQ(csubstr("123{abc}456").pair_range_nested('{', '}'), "{abc}"); - CHECK_EQ(csubstr("123{abc}456{def}").pair_range_nested('{', '}'), "{abc}"); - CHECK_EQ(csubstr( "{{}}").pair_range_nested('{', '}'), "{{}}"); - CHECK_EQ(csubstr("123{{}}456").pair_range_nested('{', '}'), "{{}}"); - CHECK_EQ(csubstr( "{a{}b{}c}").pair_range_nested('{', '}'), "{a{}b{}c}"); - CHECK_EQ(csubstr("123{a{}b{}c}456").pair_range_nested('{', '}'), "{a{}b{}c}"); - CHECK_EQ(csubstr( "{a{{}}b{{}}c}").pair_range_nested('{', '}'), "{a{{}}b{{}}c}"); - CHECK_EQ(csubstr("123{a{{}}b{{}}c}456").pair_range_nested('{', '}'), "{a{{}}b{{}}c}"); - CHECK_EQ(csubstr( "{{{}}a{{}}b{{}}c{{}}}").pair_range_nested('{', '}'), "{{{}}a{{}}b{{}}c{{}}}"); - CHECK_EQ(csubstr("123{{{}}a{{}}b{{}}c{{}}}456").pair_range_nested('{', '}'), "{{{}}a{{}}b{{}}c{{}}}"); -} - -TEST_CASE("substr.unquoted") -{ - CHECK_EQ(csubstr("").unquoted(), ""); - - CHECK_EQ(csubstr("''").unquoted(), ""); - CHECK_EQ(csubstr("\"\"").unquoted(), ""); - - CHECK_EQ(csubstr("'\''").unquoted(), "'"); - - CHECK_EQ(csubstr("aa").unquoted(), "aa"); - CHECK_EQ(csubstr("'aa'").unquoted(), "aa"); - CHECK_EQ(csubstr("\"aa\"").unquoted(), "aa"); - CHECK_EQ(csubstr("'aa\''").unquoted(), "aa'"); -} - - -TEST_CASE("substr.first_non_empty_span") -{ - CHECK_EQ(csubstr("foo bar").first_non_empty_span(), "foo"); - CHECK_EQ(csubstr(" foo bar").first_non_empty_span(), "foo"); - CHECK_EQ(csubstr("\n \r \t foo bar").first_non_empty_span(), "foo"); - CHECK_EQ(csubstr("\n \r \t foo\n\r\t bar").first_non_empty_span(), "foo"); - CHECK_EQ(csubstr("\n \r \t foo\n\r\t bar").first_non_empty_span(), "foo"); - CHECK_EQ(csubstr(",\n \r \t foo\n\r\t bar").first_non_empty_span(), ","); -} - -TEST_CASE("substr.first_uint_span") -{ - CHECK_EQ(csubstr("1234").first_uint_span(), "1234"); - CHECK_EQ(csubstr("+1234").first_uint_span(), "+1234"); - CHECK_EQ(csubstr("-1234").first_uint_span(), ""); - CHECK_EQ(csubstr("1234 asdkjh").first_uint_span(), "1234"); - CHECK_EQ(csubstr("1234\rasdkjh").first_uint_span(), "1234"); - CHECK_EQ(csubstr("1234\tasdkjh").first_uint_span(), "1234"); - CHECK_EQ(csubstr("1234\nasdkjh").first_uint_span(), "1234"); - CHECK_EQ(csubstr("1234]asdkjh").first_uint_span(), "1234"); - CHECK_EQ(csubstr("1234)asdkjh").first_uint_span(), "1234"); - CHECK_EQ(csubstr("1234gasdkjh").first_uint_span(), ""); - CHECK_EQ(csubstr("1").first_uint_span(), "1"); - CHECK_EQ(csubstr("+1").first_uint_span(), "+1"); - CHECK_EQ(csubstr("-1").first_uint_span(), ""); - CHECK_EQ(csubstr("-0").first_uint_span(), ""); - CHECK_EQ(csubstr("0").first_uint_span(), "0"); - CHECK_EQ(csubstr("+0").first_uint_span(), "+0"); - CHECK_EQ(csubstr("-0").first_uint_span(), ""); - CHECK_EQ(csubstr("1234 abc").first_uint_span(), "1234"); - CHECK_EQ(csubstr("abc 1234 abc").first_uint_span(), ""); - CHECK_EQ(csubstr("+0x1234 abc").first_uint_span(), "+0x1234"); - CHECK_EQ(csubstr("-0x1234 abc").first_uint_span(), ""); - CHECK_EQ(csubstr("0x1234 abc").first_uint_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234\rabc").first_uint_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234\nabc").first_uint_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234\tabc").first_uint_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234]abc").first_uint_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234)abc").first_uint_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234g").first_uint_span(), ""); - CHECK_EQ(csubstr("0b01").first_uint_span(), "0b01"); - CHECK_EQ(csubstr("+0b01").first_uint_span(), "+0b01"); - CHECK_EQ(csubstr("-0b01").first_uint_span(), ""); - CHECK_EQ(csubstr("0b01 asdasd").first_uint_span(), "0b01"); - CHECK_EQ(csubstr("0b01\rasdasd").first_uint_span(), "0b01"); - CHECK_EQ(csubstr("0b01\tasdasd").first_uint_span(), "0b01"); - CHECK_EQ(csubstr("0b01\nasdasd").first_uint_span(), "0b01"); - CHECK_EQ(csubstr("0b01]asdasd").first_uint_span(), "0b01"); - CHECK_EQ(csubstr("0b01)asdasd").first_uint_span(), "0b01"); - CHECK_EQ(csubstr("0b01hasdasd").first_uint_span(), ""); - CHECK_EQ(csubstr("+").first_uint_span(), ""); - CHECK_EQ(csubstr("-").first_uint_span(), ""); -} - -TEST_CASE("substr.first_int_span") -{ - CHECK_EQ(csubstr("1234").first_int_span(), "1234"); - CHECK_EQ(csubstr("+1234").first_int_span(), "+1234"); - CHECK_EQ(csubstr("-1234").first_int_span(), "-1234"); - CHECK_EQ(csubstr("-1234 asdkjh").first_int_span(), "-1234"); - CHECK_EQ(csubstr("-1234\rasdkjh").first_int_span(), "-1234"); - CHECK_EQ(csubstr("-1234\tasdkjh").first_int_span(), "-1234"); - CHECK_EQ(csubstr("-1234\nasdkjh").first_int_span(), "-1234"); - CHECK_EQ(csubstr("-1234]asdkjh").first_int_span(), "-1234"); - CHECK_EQ(csubstr("-1234)asdkjh").first_int_span(), "-1234"); - CHECK_EQ(csubstr("-1234gasdkjh").first_int_span(), ""); - CHECK_EQ(csubstr("1").first_int_span(), "1"); - CHECK_EQ(csubstr("+1").first_int_span(), "+1"); - CHECK_EQ(csubstr("-1").first_int_span(), "-1"); - CHECK_EQ(csubstr("-0").first_int_span(), "-0"); - CHECK_EQ(csubstr("0").first_int_span(), "0"); - CHECK_EQ(csubstr("+0").first_int_span(), "+0"); - CHECK_EQ(csubstr("-0").first_int_span(), "-0"); - CHECK_EQ(csubstr("1234 abc").first_int_span(), "1234"); - CHECK_EQ(csubstr("abc 1234 abc").first_int_span(), ""); - CHECK_EQ(csubstr("+0x1234 abc").first_int_span(), "+0x1234"); - CHECK_EQ(csubstr("-0x1234 abc").first_int_span(), "-0x1234"); - CHECK_EQ(csubstr("0x1234 abc").first_int_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234\rabc").first_int_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234\nabc").first_int_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234\tabc").first_int_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234]abc").first_int_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234)abc").first_int_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234gabc").first_int_span(), ""); - CHECK_EQ(csubstr("0b01").first_int_span(), "0b01"); - CHECK_EQ(csubstr("+0b01").first_int_span(), "+0b01"); - CHECK_EQ(csubstr("-0b01").first_int_span(), "-0b01"); - CHECK_EQ(csubstr("0b01 asdasd").first_int_span(), "0b01"); - CHECK_EQ(csubstr("0b01\rasdasd").first_int_span(), "0b01"); - CHECK_EQ(csubstr("0b01\tasdasd").first_int_span(), "0b01"); - CHECK_EQ(csubstr("0b01\nasdasd").first_int_span(), "0b01"); - CHECK_EQ(csubstr("0b01]asdasd").first_int_span(), "0b01"); - CHECK_EQ(csubstr("0b01)asdasd").first_int_span(), "0b01"); - CHECK_EQ(csubstr("0b01gasdasd").first_int_span(), ""); -} - -TEST_CASE("substr.first_real_span") -{ - // all integers are reals - CHECK_EQ(csubstr("1234").first_real_span(), "1234"); - CHECK_EQ(csubstr("+1234").first_real_span(), "+1234"); - CHECK_EQ(csubstr("-1234").first_real_span(), "-1234"); - CHECK_EQ(csubstr("-1234 asdkjh").first_real_span(), "-1234"); - CHECK_EQ(csubstr("-1234\rasdkjh").first_real_span(), "-1234"); - CHECK_EQ(csubstr("-1234\tasdkjh").first_real_span(), "-1234"); - CHECK_EQ(csubstr("-1234\nasdkjh").first_real_span(), "-1234"); - CHECK_EQ(csubstr("-1234]asdkjh").first_real_span(), "-1234"); - CHECK_EQ(csubstr("-1234)asdkjh").first_real_span(), "-1234"); - CHECK_EQ(csubstr("-1234gasdkjh").first_real_span(), ""); - CHECK_EQ(csubstr("1").first_real_span(), "1"); - CHECK_EQ(csubstr("+1").first_real_span(), "+1"); - CHECK_EQ(csubstr("-1").first_real_span(), "-1"); - CHECK_EQ(csubstr("-0").first_real_span(), "-0"); - CHECK_EQ(csubstr("0").first_real_span(), "0"); - CHECK_EQ(csubstr("+0").first_real_span(), "+0"); - CHECK_EQ(csubstr("-0").first_real_span(), "-0"); - CHECK_EQ(csubstr("1234 abc").first_real_span(), "1234"); - CHECK_EQ(csubstr("abc 1234 abc").first_real_span(), ""); - CHECK_EQ(csubstr("+0x1234 abc").first_real_span(), "+0x1234"); - CHECK_EQ(csubstr("-0x1234 abc").first_real_span(), "-0x1234"); - CHECK_EQ(csubstr("0x1234 abc").first_real_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234\rabc").first_real_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234\nabc").first_real_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234\tabc").first_real_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234]abc").first_real_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234)abc").first_real_span(), "0x1234"); - CHECK_EQ(csubstr("0x1234gabc").first_real_span(), ""); - CHECK_EQ(csubstr("0b01").first_real_span(), "0b01"); - CHECK_EQ(csubstr("+0b01").first_real_span(), "+0b01"); - CHECK_EQ(csubstr("-0b01").first_real_span(), "-0b01"); - CHECK_EQ(csubstr("0b01 asdasd").first_real_span(), "0b01"); - CHECK_EQ(csubstr("0b01\rasdasd").first_real_span(), "0b01"); - CHECK_EQ(csubstr("0b01\tasdasd").first_real_span(), "0b01"); - CHECK_EQ(csubstr("0b01\nasdasd").first_real_span(), "0b01"); - CHECK_EQ(csubstr("0b01]asdasd").first_real_span(), "0b01"); - CHECK_EQ(csubstr("0b01)asdasd").first_real_span(), "0b01"); - CHECK_EQ(csubstr("0b01gasdasd").first_real_span(), ""); - CHECK_EQ(csubstr("0b1.01 asdasd").first_real_span(), "0b1.01"); - CHECK_EQ(csubstr("0b1.01\rasdasd").first_real_span(), "0b1.01"); - CHECK_EQ(csubstr("0b1.01\tasdasd").first_real_span(), "0b1.01"); - CHECK_EQ(csubstr("0b1.01\nasdasd").first_real_span(), "0b1.01"); - CHECK_EQ(csubstr("0b1.01]asdasd").first_real_span(), "0b1.01"); - CHECK_EQ(csubstr("0b1.01)asdasd").first_real_span(), "0b1.01"); - CHECK_EQ(csubstr("0b1.01gasdasd").first_real_span(), ""); - CHECK_EQ(csubstr("0b1.02 asdasd").first_real_span(), ""); - CHECK_EQ(csubstr("0b1.02\rasdasd").first_real_span(), ""); - CHECK_EQ(csubstr("0b1.02\tasdasd").first_real_span(), ""); - CHECK_EQ(csubstr("0b1.02\nasdasd").first_real_span(), ""); - CHECK_EQ(csubstr("0b1.02]asdasd").first_real_span(), ""); - CHECK_EQ(csubstr("0b1.02)asdasd").first_real_span(), ""); - CHECK_EQ(csubstr("0b1.02gasdasd").first_real_span(), ""); - CHECK_EQ(csubstr("+").first_real_span(), ""); - CHECK_EQ(csubstr("-").first_real_span(), ""); - CHECK_EQ(csubstr("+0x").first_real_span(), ""); - CHECK_EQ(csubstr("-0x").first_real_span(), ""); - CHECK_EQ(csubstr("+0b").first_real_span(), ""); - CHECK_EQ(csubstr("-0b").first_real_span(), ""); - CHECK_EQ(csubstr("+0o").first_real_span(), ""); - CHECK_EQ(csubstr("-0o").first_real_span(), ""); - CHECK_EQ(csubstr("-1.234 asdkjh").first_real_span(), "-1.234"); -// CHECK_EQ(csubstr("-1.234e5 asdkjh").first_real_span(), "-1.234e5"); - CHECK_EQ(csubstr("-1.234e+5 asdkjh").first_real_span(), "-1.234e+5"); - CHECK_EQ(csubstr("-1.234e-5 asdkjh").first_real_span(), "-1.234e-5"); - CHECK_EQ(csubstr("0x1.e8480p+19 asdkjh").first_real_span(), "0x1.e8480p+19"); - CHECK_EQ(csubstr("0x1.e8480p-19 asdkjh").first_real_span(), "0x1.e8480p-19"); - CHECK_EQ(csubstr("-0x1.e8480p+19 asdkjh").first_real_span(), "-0x1.e8480p+19"); - CHECK_EQ(csubstr("-0x1.e8480p-19 asdkjh").first_real_span(), "-0x1.e8480p-19"); - CHECK_EQ(csubstr("+0x1.e8480p+19 asdkjh").first_real_span(), "+0x1.e8480p+19"); - CHECK_EQ(csubstr("+0x1.e8480p-19 asdkjh").first_real_span(), "+0x1.e8480p-19"); - CHECK_EQ(csubstr("infinity").first_real_span(), "infinity"); - CHECK_EQ(csubstr(" infinity").first_real_span(), "infinity"); - CHECK_EQ(csubstr("-infinity").first_real_span(), "-infinity"); - CHECK_EQ(csubstr(" -infinity").first_real_span(), "-infinity"); - CHECK_EQ(csubstr("+infinity").first_real_span(), "+infinity"); - CHECK_EQ(csubstr(" +infinity").first_real_span(), "+infinity"); - CHECK_EQ(csubstr("infinity ").first_real_span(), "infinity"); - CHECK_EQ(csubstr(" infinity ").first_real_span(), "infinity"); - CHECK_EQ(csubstr("-infinity ").first_real_span(), "-infinity"); - CHECK_EQ(csubstr(" -infinity ").first_real_span(), "-infinity"); - CHECK_EQ(csubstr("+infinity ").first_real_span(), "+infinity"); - CHECK_EQ(csubstr(" +infinity ").first_real_span(), "+infinity"); - CHECK_EQ(csubstr("infinity1").first_real_span(), ""); - CHECK_EQ(csubstr(" infinity1").first_real_span(), ""); - CHECK_EQ(csubstr("-infinity1").first_real_span(), ""); - CHECK_EQ(csubstr(" -infinity1").first_real_span(), ""); - CHECK_EQ(csubstr("+infinity1").first_real_span(), ""); - CHECK_EQ(csubstr(" +infinity1").first_real_span(), ""); - CHECK_EQ(csubstr("infin").first_real_span(), ""); - CHECK_EQ(csubstr(" infin").first_real_span(), ""); - CHECK_EQ(csubstr("-infin").first_real_span(), ""); - CHECK_EQ(csubstr(" -infin").first_real_span(), ""); - CHECK_EQ(csubstr("+infin").first_real_span(), ""); - CHECK_EQ(csubstr(" +infin").first_real_span(), ""); - CHECK_EQ(csubstr("inflated").first_real_span(), ""); - CHECK_EQ(csubstr(" inflated").first_real_span(), ""); - CHECK_EQ(csubstr("-inflated").first_real_span(), ""); - CHECK_EQ(csubstr(" -inflated").first_real_span(), ""); - CHECK_EQ(csubstr("+inflated").first_real_span(), ""); - CHECK_EQ(csubstr(" +inflated").first_real_span(), ""); - CHECK_EQ(csubstr("inf").first_real_span(), "inf"); - CHECK_EQ(csubstr(" inf").first_real_span(), "inf"); - CHECK_EQ(csubstr("-inf").first_real_span(), "-inf"); - CHECK_EQ(csubstr(" -inf").first_real_span(), "-inf"); - CHECK_EQ(csubstr("+inf").first_real_span(), "+inf"); - CHECK_EQ(csubstr(" +inf").first_real_span(), "+inf"); - CHECK_EQ(csubstr("inf ").first_real_span(), "inf"); - CHECK_EQ(csubstr(" inf ").first_real_span(), "inf"); - CHECK_EQ(csubstr("-inf ").first_real_span(), "-inf"); - CHECK_EQ(csubstr(" -inf ").first_real_span(), "-inf"); - CHECK_EQ(csubstr("+inf ").first_real_span(), "+inf"); - CHECK_EQ(csubstr(" +inf ").first_real_span(), "+inf"); - CHECK_EQ(csubstr("inf1").first_real_span(), ""); - CHECK_EQ(csubstr(" inf1").first_real_span(), ""); - CHECK_EQ(csubstr("-inf1").first_real_span(), ""); - CHECK_EQ(csubstr(" -inf1").first_real_span(), ""); - CHECK_EQ(csubstr("+inf1").first_real_span(), ""); - CHECK_EQ(csubstr(" +inf1").first_real_span(), ""); - CHECK_EQ(csubstr("nan").first_real_span(), "nan"); - CHECK_EQ(csubstr(" nan").first_real_span(), "nan"); - CHECK_EQ(csubstr("-nan").first_real_span(), "-nan"); - CHECK_EQ(csubstr(" -nan").first_real_span(), "-nan"); - CHECK_EQ(csubstr("+nan").first_real_span(), "+nan"); - CHECK_EQ(csubstr(" +nan").first_real_span(), "+nan"); - CHECK_EQ(csubstr("nan ").first_real_span(), "nan"); - CHECK_EQ(csubstr(" nan ").first_real_span(), "nan"); - CHECK_EQ(csubstr("-nan ").first_real_span(), "-nan"); - CHECK_EQ(csubstr(" -nan ").first_real_span(), "-nan"); - CHECK_EQ(csubstr("+nan ").first_real_span(), "+nan"); - CHECK_EQ(csubstr(" +nan ").first_real_span(), "+nan"); - CHECK_EQ(csubstr("nan1").first_real_span(), ""); - CHECK_EQ(csubstr(" nan1").first_real_span(), ""); - CHECK_EQ(csubstr("-nan1").first_real_span(), ""); - CHECK_EQ(csubstr(" -nan1").first_real_span(), ""); - CHECK_EQ(csubstr("+nan1").first_real_span(), ""); - CHECK_EQ(csubstr(" +nan1").first_real_span(), ""); -} - -// start with some obvious direct tests -TEST_CASE("substr.is_unsigned_integer") -{ - SUBCASE("empty_string") - { - CHECK_FALSE(csubstr().is_unsigned_integer()); - CHECK_FALSE(csubstr("").is_unsigned_integer()); - } - SUBCASE("signs") - { - CHECK_FALSE(csubstr("-").is_unsigned_integer()); - CHECK_FALSE(csubstr("+").is_unsigned_integer()); - CHECK_FALSE(csubstr("-1").is_unsigned_integer()); - CHECK_UNARY(csubstr("+1").is_unsigned_integer()); - } - SUBCASE("whitespace_before") - { - CHECK_FALSE(csubstr(" 0").is_unsigned_integer()); - CHECK_FALSE(csubstr(" 1").is_unsigned_integer()); - CHECK_FALSE(csubstr(" -1").is_unsigned_integer()); - CHECK_FALSE(csubstr(" 0.1").is_unsigned_integer()); - CHECK_FALSE(csubstr(" -0.1").is_unsigned_integer()); - } - SUBCASE("whitespace_after") - { - CHECK_FALSE(csubstr("0 ").is_unsigned_integer()); - CHECK_FALSE(csubstr("1 ").is_unsigned_integer()); - CHECK_FALSE(csubstr("-1 ").is_unsigned_integer()); - CHECK_FALSE(csubstr("0.1 ").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0.1 ").is_unsigned_integer()); - } - SUBCASE("whitespace_only") - { - CHECK_FALSE(csubstr(" ").is_unsigned_integer()); - CHECK_FALSE(csubstr("\t\t\t\t").is_unsigned_integer()); - CHECK_FALSE(csubstr("\n\n\n\n").is_unsigned_integer()); - CHECK_FALSE(csubstr("\r\r\r\r").is_unsigned_integer()); - } - SUBCASE("decimal") - { - CHECK_UNARY(csubstr("0").is_unsigned_integer()); - CHECK_UNARY(csubstr("1").is_unsigned_integer()); - CHECK_FALSE(csubstr("-1").is_unsigned_integer()); - CHECK_FALSE(csubstr("0.1").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0.1").is_unsigned_integer()); - } - SUBCASE("hexadecimal") - { - CHECK_FALSE(csubstr("0x").is_unsigned_integer()); - CHECK_FALSE(csubstr("0X").is_unsigned_integer()); - CHECK_UNARY(csubstr("0x1").is_unsigned_integer()); - CHECK_UNARY(csubstr("0X1").is_unsigned_integer()); - CHECK_UNARY(csubstr("0x0").is_unsigned_integer()); - CHECK_UNARY(csubstr("0X0").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xa").is_unsigned_integer()); - CHECK_UNARY(csubstr("0Xa").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xb").is_unsigned_integer()); - CHECK_UNARY(csubstr("0Xb").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xc").is_unsigned_integer()); - CHECK_UNARY(csubstr("0Xc").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xd").is_unsigned_integer()); - CHECK_UNARY(csubstr("0Xd").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xe").is_unsigned_integer()); - CHECK_UNARY(csubstr("0Xe").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xf").is_unsigned_integer()); - CHECK_UNARY(csubstr("0Xf").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xA").is_unsigned_integer()); - CHECK_UNARY(csubstr("0XA").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xB").is_unsigned_integer()); - CHECK_UNARY(csubstr("0XB").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xC").is_unsigned_integer()); - CHECK_UNARY(csubstr("0XC").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xD").is_unsigned_integer()); - CHECK_UNARY(csubstr("0XD").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xE").is_unsigned_integer()); - CHECK_UNARY(csubstr("0XE").is_unsigned_integer()); - CHECK_UNARY(csubstr("0xF").is_unsigned_integer()); - CHECK_UNARY(csubstr("0XF").is_unsigned_integer()); - CHECK_FALSE(csubstr("0xg").is_unsigned_integer()); - CHECK_FALSE(csubstr("0Xg").is_unsigned_integer()); - CHECK_FALSE(csubstr("0xG").is_unsigned_integer()); - CHECK_FALSE(csubstr("0XG").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0x1").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0X1").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0x0").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0X0").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xa").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0Xa").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xb").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0Xb").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xc").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0Xc").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xd").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0Xd").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xe").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0Xe").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xf").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0Xf").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xA").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0XA").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xB").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0XB").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xC").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0XC").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xD").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0XD").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xE").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0XE").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xF").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0XF").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xg").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0Xg").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0xG").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0XG").is_unsigned_integer()); - } - SUBCASE("binary") - { - CHECK_FALSE(csubstr("0b").is_unsigned_integer()); - CHECK_FALSE(csubstr("0B").is_unsigned_integer()); - CHECK_UNARY(csubstr("0b0").is_unsigned_integer()); - CHECK_UNARY(csubstr("0B0").is_unsigned_integer()); - CHECK_UNARY(csubstr("0b1").is_unsigned_integer()); - CHECK_UNARY(csubstr("0B1").is_unsigned_integer()); - CHECK_FALSE(csubstr("0b2").is_unsigned_integer()); - CHECK_FALSE(csubstr("0B2").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0b0").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0B0").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0b1").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0B1").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0b2").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0B2").is_unsigned_integer()); - } - SUBCASE("octal") - { - CHECK_FALSE(csubstr("0o").is_unsigned_integer()); - CHECK_FALSE(csubstr("0O").is_unsigned_integer()); - CHECK_UNARY(csubstr("0o0").is_unsigned_integer()); - CHECK_UNARY(csubstr("0O0").is_unsigned_integer()); - CHECK_UNARY(csubstr("0o1").is_unsigned_integer()); - CHECK_UNARY(csubstr("0O1").is_unsigned_integer()); - CHECK_UNARY(csubstr("0o6").is_unsigned_integer()); - CHECK_UNARY(csubstr("0O6").is_unsigned_integer()); - CHECK_UNARY(csubstr("0o6").is_unsigned_integer()); - CHECK_UNARY(csubstr("0O6").is_unsigned_integer()); - CHECK_UNARY(csubstr("0o7").is_unsigned_integer()); - CHECK_UNARY(csubstr("0O7").is_unsigned_integer()); - CHECK_FALSE(csubstr("0o8").is_unsigned_integer()); - CHECK_FALSE(csubstr("0O8").is_unsigned_integer()); - CHECK_FALSE(csubstr("0o9").is_unsigned_integer()); - CHECK_FALSE(csubstr("0O9").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0o0").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0O0").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0o1").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0O1").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0o6").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0O6").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0o6").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0O6").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0o7").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0O7").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0o8").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0O8").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0o9").is_unsigned_integer()); - CHECK_FALSE(csubstr("-0O9").is_unsigned_integer()); - } -} - -TEST_CASE("substr.is_integer") -{ - SUBCASE("empty_string") - { - CHECK_FALSE(csubstr().is_integer()); - CHECK_FALSE(csubstr("").is_integer()); - } - SUBCASE("signs") - { - CHECK_FALSE(csubstr("-").is_integer()); - CHECK_FALSE(csubstr("+").is_integer()); - CHECK_UNARY(csubstr("-1").is_integer()); - CHECK_UNARY(csubstr("+1").is_integer()); - } - SUBCASE("whitespace_before") - { - CHECK_FALSE(csubstr(" 0").is_integer()); - CHECK_FALSE(csubstr(" 1").is_integer()); - CHECK_FALSE(csubstr(" -1").is_integer()); - CHECK_FALSE(csubstr(" 0.1").is_integer()); - CHECK_FALSE(csubstr(" -0.1").is_integer()); - } - SUBCASE("whitespace_after") - { - CHECK_FALSE(csubstr("0 ").is_integer()); - CHECK_FALSE(csubstr("1 ").is_integer()); - CHECK_FALSE(csubstr("-1 ").is_integer()); - CHECK_FALSE(csubstr("0.1 ").is_integer()); - CHECK_FALSE(csubstr("-0.1 ").is_integer()); - } - SUBCASE("whitespace_only") - { - CHECK_FALSE(csubstr(" ").is_integer()); - CHECK_FALSE(csubstr("\t\t\t\t").is_integer()); - CHECK_FALSE(csubstr("\n\n\n\n").is_integer()); - CHECK_FALSE(csubstr("\r\r\r\r").is_integer()); - } - SUBCASE("decimal") - { - CHECK_UNARY(csubstr("0").is_integer()); - CHECK_UNARY(csubstr("1").is_integer()); - CHECK_UNARY(csubstr("-1").is_integer()); - CHECK_FALSE(csubstr("0.1").is_integer()); - CHECK_FALSE(csubstr("-0.1").is_integer()); - } - SUBCASE("hexadecimal") - { - CHECK_FALSE(csubstr("0x").is_integer()); - CHECK_FALSE(csubstr("0X").is_integer()); - CHECK_UNARY(csubstr("0x1").is_integer()); - CHECK_UNARY(csubstr("0X1").is_integer()); - CHECK_UNARY(csubstr("0x0").is_integer()); - CHECK_UNARY(csubstr("0X0").is_integer()); - CHECK_UNARY(csubstr("0xa").is_integer()); - CHECK_UNARY(csubstr("0Xa").is_integer()); - CHECK_UNARY(csubstr("0xb").is_integer()); - CHECK_UNARY(csubstr("0Xb").is_integer()); - CHECK_UNARY(csubstr("0xc").is_integer()); - CHECK_UNARY(csubstr("0Xc").is_integer()); - CHECK_UNARY(csubstr("0xd").is_integer()); - CHECK_UNARY(csubstr("0Xd").is_integer()); - CHECK_UNARY(csubstr("0xe").is_integer()); - CHECK_UNARY(csubstr("0Xe").is_integer()); - CHECK_UNARY(csubstr("0xf").is_integer()); - CHECK_UNARY(csubstr("0Xf").is_integer()); - CHECK_UNARY(csubstr("0xA").is_integer()); - CHECK_UNARY(csubstr("0XA").is_integer()); - CHECK_UNARY(csubstr("0xB").is_integer()); - CHECK_UNARY(csubstr("0XB").is_integer()); - CHECK_UNARY(csubstr("0xC").is_integer()); - CHECK_UNARY(csubstr("0XC").is_integer()); - CHECK_UNARY(csubstr("0xD").is_integer()); - CHECK_UNARY(csubstr("0XD").is_integer()); - CHECK_UNARY(csubstr("0xE").is_integer()); - CHECK_UNARY(csubstr("0XE").is_integer()); - CHECK_UNARY(csubstr("0xF").is_integer()); - CHECK_UNARY(csubstr("0XF").is_integer()); - CHECK_FALSE(csubstr("0xg").is_integer()); - CHECK_FALSE(csubstr("0Xg").is_integer()); - CHECK_FALSE(csubstr("0xG").is_integer()); - CHECK_FALSE(csubstr("0XG").is_integer()); - CHECK_UNARY(csubstr("-0x1").is_integer()); - CHECK_UNARY(csubstr("-0X1").is_integer()); - CHECK_UNARY(csubstr("-0x0").is_integer()); - CHECK_UNARY(csubstr("-0X0").is_integer()); - CHECK_UNARY(csubstr("-0xa").is_integer()); - CHECK_UNARY(csubstr("-0Xa").is_integer()); - CHECK_UNARY(csubstr("-0xb").is_integer()); - CHECK_UNARY(csubstr("-0Xb").is_integer()); - CHECK_UNARY(csubstr("-0xc").is_integer()); - CHECK_UNARY(csubstr("-0Xc").is_integer()); - CHECK_UNARY(csubstr("-0xd").is_integer()); - CHECK_UNARY(csubstr("-0Xd").is_integer()); - CHECK_UNARY(csubstr("-0xe").is_integer()); - CHECK_UNARY(csubstr("-0Xe").is_integer()); - CHECK_UNARY(csubstr("-0xf").is_integer()); - CHECK_UNARY(csubstr("-0Xf").is_integer()); - CHECK_UNARY(csubstr("-0xA").is_integer()); - CHECK_UNARY(csubstr("-0XA").is_integer()); - CHECK_UNARY(csubstr("-0xB").is_integer()); - CHECK_UNARY(csubstr("-0XB").is_integer()); - CHECK_UNARY(csubstr("-0xC").is_integer()); - CHECK_UNARY(csubstr("-0XC").is_integer()); - CHECK_UNARY(csubstr("-0xD").is_integer()); - CHECK_UNARY(csubstr("-0XD").is_integer()); - CHECK_UNARY(csubstr("-0xE").is_integer()); - CHECK_UNARY(csubstr("-0XE").is_integer()); - CHECK_UNARY(csubstr("-0xF").is_integer()); - CHECK_UNARY(csubstr("-0XF").is_integer()); - CHECK_FALSE(csubstr("-0xg").is_integer()); - CHECK_FALSE(csubstr("-0Xg").is_integer()); - CHECK_FALSE(csubstr("-0xG").is_integer()); - CHECK_FALSE(csubstr("-0XG").is_integer()); - } - SUBCASE("binary") - { - CHECK_FALSE(csubstr("0b").is_integer()); - CHECK_FALSE(csubstr("0B").is_integer()); - CHECK_UNARY(csubstr("0b0").is_integer()); - CHECK_UNARY(csubstr("0B0").is_integer()); - CHECK_UNARY(csubstr("0b1").is_integer()); - CHECK_UNARY(csubstr("0B1").is_integer()); - CHECK_FALSE(csubstr("0b2").is_integer()); - CHECK_FALSE(csubstr("0B2").is_integer()); - CHECK_UNARY(csubstr("-0b0").is_integer()); - CHECK_UNARY(csubstr("-0B0").is_integer()); - CHECK_UNARY(csubstr("-0b1").is_integer()); - CHECK_UNARY(csubstr("-0B1").is_integer()); - CHECK_FALSE(csubstr("-0b2").is_integer()); - CHECK_FALSE(csubstr("-0B2").is_integer()); - } - SUBCASE("octal") - { - CHECK_FALSE(csubstr("0o").is_integer()); - CHECK_FALSE(csubstr("0O").is_integer()); - CHECK_UNARY(csubstr("0o0").is_integer()); - CHECK_UNARY(csubstr("0O0").is_integer()); - CHECK_UNARY(csubstr("0o1").is_integer()); - CHECK_UNARY(csubstr("0O1").is_integer()); - CHECK_UNARY(csubstr("0o6").is_integer()); - CHECK_UNARY(csubstr("0O6").is_integer()); - CHECK_UNARY(csubstr("0o6").is_integer()); - CHECK_UNARY(csubstr("0O6").is_integer()); - CHECK_UNARY(csubstr("0o7").is_integer()); - CHECK_UNARY(csubstr("0O7").is_integer()); - CHECK_FALSE(csubstr("0o8").is_integer()); - CHECK_FALSE(csubstr("0O8").is_integer()); - CHECK_FALSE(csubstr("0o9").is_integer()); - CHECK_FALSE(csubstr("0O9").is_integer()); - CHECK_UNARY(csubstr("-0o0").is_integer()); - CHECK_UNARY(csubstr("-0O0").is_integer()); - CHECK_UNARY(csubstr("-0o1").is_integer()); - CHECK_UNARY(csubstr("-0O1").is_integer()); - CHECK_UNARY(csubstr("-0o6").is_integer()); - CHECK_UNARY(csubstr("-0O6").is_integer()); - CHECK_UNARY(csubstr("-0o6").is_integer()); - CHECK_UNARY(csubstr("-0O6").is_integer()); - CHECK_UNARY(csubstr("-0o7").is_integer()); - CHECK_UNARY(csubstr("-0O7").is_integer()); - CHECK_FALSE(csubstr("-0o8").is_integer()); - CHECK_FALSE(csubstr("-0O8").is_integer()); - CHECK_FALSE(csubstr("-0o9").is_integer()); - CHECK_FALSE(csubstr("-0O9").is_integer()); - } -} - -TEST_CASE("substr.is_real") -{ - SUBCASE("empty_string") - { - CHECK_FALSE(csubstr().is_real()); - CHECK_FALSE(csubstr("").is_real()); - } - SUBCASE("signs") - { - CHECK_FALSE(csubstr("-").is_real()); - CHECK_FALSE(csubstr("+").is_real()); - CHECK_UNARY(csubstr("-1").is_real()); - CHECK_UNARY(csubstr("+1").is_real()); - } - SUBCASE("whitespace_before") - { - CHECK_FALSE(csubstr(" 0").is_real()); - CHECK_FALSE(csubstr(" 1").is_real()); - CHECK_FALSE(csubstr(" -1").is_real()); - CHECK_FALSE(csubstr(" 0.1").is_real()); - CHECK_FALSE(csubstr(" -0.1").is_real()); - } - SUBCASE("whitespace_after") - { - CHECK_FALSE(csubstr("0 ").is_real()); - CHECK_FALSE(csubstr("1 ").is_real()); - CHECK_FALSE(csubstr("-1 ").is_real()); - CHECK_FALSE(csubstr("0.1 ").is_real()); - CHECK_FALSE(csubstr("-0.1 ").is_real()); - } - SUBCASE("whitespace_only") - { - CHECK_FALSE(csubstr(" ").is_real()); - CHECK_FALSE(csubstr("\t\t\t\t").is_real()); - CHECK_FALSE(csubstr("\n\n\n\n").is_real()); - CHECK_FALSE(csubstr("\r\r\r\r").is_real()); - } - SUBCASE("decimal") - { - CHECK_UNARY(csubstr("0").is_real()); - CHECK_UNARY(csubstr("1").is_real()); - CHECK_UNARY(csubstr("-1").is_real()); - CHECK_UNARY(csubstr("0.1").is_real()); - CHECK_UNARY(csubstr("-0.1").is_real()); - } - SUBCASE("hexadecimal") - { - CHECK_FALSE(csubstr("0x").is_real()); - CHECK_FALSE(csubstr("0X").is_real()); - CHECK_UNARY(csubstr("0x1").is_real()); - CHECK_UNARY(csubstr("0X1").is_real()); - CHECK_UNARY(csubstr("0x0").is_real()); - CHECK_UNARY(csubstr("0X0").is_real()); - CHECK_UNARY(csubstr("0xa").is_real()); - CHECK_UNARY(csubstr("0Xa").is_real()); - CHECK_UNARY(csubstr("0xb").is_real()); - CHECK_UNARY(csubstr("0Xb").is_real()); - CHECK_UNARY(csubstr("0xc").is_real()); - CHECK_UNARY(csubstr("0Xc").is_real()); - CHECK_UNARY(csubstr("0xd").is_real()); - CHECK_UNARY(csubstr("0Xd").is_real()); - CHECK_UNARY(csubstr("0xe").is_real()); - CHECK_UNARY(csubstr("0Xe").is_real()); - CHECK_UNARY(csubstr("0xf").is_real()); - CHECK_UNARY(csubstr("0Xf").is_real()); - CHECK_UNARY(csubstr("0xA").is_real()); - CHECK_UNARY(csubstr("0XA").is_real()); - CHECK_UNARY(csubstr("0xB").is_real()); - CHECK_UNARY(csubstr("0XB").is_real()); - CHECK_UNARY(csubstr("0xC").is_real()); - CHECK_UNARY(csubstr("0XC").is_real()); - CHECK_UNARY(csubstr("0xD").is_real()); - CHECK_UNARY(csubstr("0XD").is_real()); - CHECK_UNARY(csubstr("0xE").is_real()); - CHECK_UNARY(csubstr("0XE").is_real()); - CHECK_UNARY(csubstr("0xF").is_real()); - CHECK_UNARY(csubstr("0XF").is_real()); - CHECK_FALSE(csubstr("0xg").is_real()); - CHECK_FALSE(csubstr("0Xg").is_real()); - CHECK_FALSE(csubstr("0xG").is_real()); - CHECK_FALSE(csubstr("0XG").is_real()); - CHECK_UNARY(csubstr("-0x1").is_real()); - CHECK_UNARY(csubstr("-0X1").is_real()); - CHECK_UNARY(csubstr("-0x0").is_real()); - CHECK_UNARY(csubstr("-0X0").is_real()); - CHECK_UNARY(csubstr("-0xa").is_real()); - CHECK_UNARY(csubstr("-0Xa").is_real()); - CHECK_UNARY(csubstr("-0xb").is_real()); - CHECK_UNARY(csubstr("-0Xb").is_real()); - CHECK_UNARY(csubstr("-0xc").is_real()); - CHECK_UNARY(csubstr("-0Xc").is_real()); - CHECK_UNARY(csubstr("-0xd").is_real()); - CHECK_UNARY(csubstr("-0Xd").is_real()); - CHECK_UNARY(csubstr("-0xe").is_real()); - CHECK_UNARY(csubstr("-0Xe").is_real()); - CHECK_UNARY(csubstr("-0xf").is_real()); - CHECK_UNARY(csubstr("-0Xf").is_real()); - CHECK_UNARY(csubstr("-0xA").is_real()); - CHECK_UNARY(csubstr("-0XA").is_real()); - CHECK_UNARY(csubstr("-0xB").is_real()); - CHECK_UNARY(csubstr("-0XB").is_real()); - CHECK_UNARY(csubstr("-0xC").is_real()); - CHECK_UNARY(csubstr("-0XC").is_real()); - CHECK_UNARY(csubstr("-0xD").is_real()); - CHECK_UNARY(csubstr("-0XD").is_real()); - CHECK_UNARY(csubstr("-0xE").is_real()); - CHECK_UNARY(csubstr("-0XE").is_real()); - CHECK_UNARY(csubstr("-0xF").is_real()); - CHECK_UNARY(csubstr("-0XF").is_real()); - CHECK_FALSE(csubstr("-0xg").is_real()); - CHECK_FALSE(csubstr("-0Xg").is_real()); - CHECK_FALSE(csubstr("-0xG").is_real()); - CHECK_FALSE(csubstr("-0XG").is_real()); - CHECK_UNARY(csubstr("0x1.e8480p+19").is_real()); - CHECK_UNARY(csubstr("0X1.e8480P+19").is_real()); - CHECK_UNARY(csubstr("0x1.e8480P+19").is_real()); - CHECK_UNARY(csubstr("0X1.e8480p+19").is_real()); - CHECK_UNARY(csubstr("-0x1.e8480p+19").is_real()); - CHECK_UNARY(csubstr("-0X1.e8480P+19").is_real()); - CHECK_UNARY(csubstr("-0x1.e8480P+19").is_real()); - CHECK_UNARY(csubstr("-0X1.e8480p+19").is_real()); - } - SUBCASE("binary") - { - CHECK_FALSE(csubstr("0b").is_real()); - CHECK_FALSE(csubstr("0B").is_real()); - CHECK_UNARY(csubstr("0b0").is_real()); - CHECK_UNARY(csubstr("0B0").is_real()); - CHECK_UNARY(csubstr("0b1").is_real()); - CHECK_UNARY(csubstr("0B1").is_real()); - CHECK_FALSE(csubstr("0b2").is_real()); - CHECK_FALSE(csubstr("0B2").is_real()); - CHECK_UNARY(csubstr("-0b0").is_real()); - CHECK_UNARY(csubstr("-0B0").is_real()); - CHECK_UNARY(csubstr("-0b1").is_real()); - CHECK_UNARY(csubstr("-0B1").is_real()); - CHECK_FALSE(csubstr("-0b2").is_real()); - CHECK_FALSE(csubstr("-0B2").is_real()); - } - SUBCASE("octal") - { - CHECK_FALSE(csubstr("0o").is_real()); - CHECK_FALSE(csubstr("0O").is_real()); - CHECK_UNARY(csubstr("0o0").is_real()); - CHECK_UNARY(csubstr("0O0").is_real()); - CHECK_UNARY(csubstr("0o1").is_real()); - CHECK_UNARY(csubstr("0O1").is_real()); - CHECK_UNARY(csubstr("0o6").is_real()); - CHECK_UNARY(csubstr("0O6").is_real()); - CHECK_UNARY(csubstr("0o6").is_real()); - CHECK_UNARY(csubstr("0O6").is_real()); - CHECK_UNARY(csubstr("0o7").is_real()); - CHECK_UNARY(csubstr("0O7").is_real()); - CHECK_FALSE(csubstr("0o8").is_real()); - CHECK_FALSE(csubstr("0O8").is_real()); - CHECK_FALSE(csubstr("0o9").is_real()); - CHECK_FALSE(csubstr("0O9").is_real()); - CHECK_UNARY(csubstr("-0o0").is_real()); - CHECK_UNARY(csubstr("-0O0").is_real()); - CHECK_UNARY(csubstr("-0o1").is_real()); - CHECK_UNARY(csubstr("-0O1").is_real()); - CHECK_UNARY(csubstr("-0o6").is_real()); - CHECK_UNARY(csubstr("-0O6").is_real()); - CHECK_UNARY(csubstr("-0o6").is_real()); - CHECK_UNARY(csubstr("-0O6").is_real()); - CHECK_UNARY(csubstr("-0o7").is_real()); - CHECK_UNARY(csubstr("-0O7").is_real()); - CHECK_FALSE(csubstr("-0o8").is_real()); - CHECK_FALSE(csubstr("-0O8").is_real()); - CHECK_FALSE(csubstr("-0o9").is_real()); - CHECK_FALSE(csubstr("-0O9").is_real()); - } - SUBCASE("infinity") - { - CHECK_UNARY(csubstr("infinity").is_real()); - CHECK_FALSE(csubstr(" infinity").is_real()); - CHECK_UNARY(csubstr("-infinity").is_real()); - CHECK_FALSE(csubstr(" -infinity").is_real()); - CHECK_UNARY(csubstr("+infinity").is_real()); - CHECK_FALSE(csubstr(" +infinity").is_real()); - CHECK_FALSE(csubstr("infinity ").is_real()); - CHECK_FALSE(csubstr(" infinity ").is_real()); - CHECK_FALSE(csubstr("-infinity ").is_real()); - CHECK_FALSE(csubstr(" -infinity ").is_real()); - CHECK_FALSE(csubstr("+infinity ").is_real()); - CHECK_FALSE(csubstr(" +infinity ").is_real()); - CHECK_FALSE(csubstr("infinity1").is_real()); - CHECK_FALSE(csubstr(" infinity1").is_real()); - CHECK_FALSE(csubstr("-infinity1").is_real()); - CHECK_FALSE(csubstr(" -infinity1").is_real()); - CHECK_FALSE(csubstr("+infinity1").is_real()); - CHECK_FALSE(csubstr(" +infinity1").is_real()); - CHECK_FALSE(csubstr("infin").is_real()); - CHECK_FALSE(csubstr(" infin").is_real()); - CHECK_FALSE(csubstr("-infin").is_real()); - CHECK_FALSE(csubstr(" -infin").is_real()); - CHECK_FALSE(csubstr("+infin").is_real()); - CHECK_FALSE(csubstr(" +infin").is_real()); - CHECK_FALSE(csubstr("inflated").is_real()); - CHECK_FALSE(csubstr(" inflated").is_real()); - CHECK_FALSE(csubstr("-inflated").is_real()); - CHECK_FALSE(csubstr(" -inflated").is_real()); - CHECK_FALSE(csubstr("+inflated").is_real()); - CHECK_FALSE(csubstr(" +inflated").is_real()); - } - SUBCASE("inf") - { - CHECK_UNARY(csubstr("inf").is_real()); - CHECK_FALSE(csubstr(" inf").is_real()); - CHECK_UNARY(csubstr("-inf").is_real()); - CHECK_FALSE(csubstr(" -inf").is_real()); - CHECK_UNARY(csubstr("+inf").is_real()); - CHECK_FALSE(csubstr(" +inf").is_real()); - CHECK_FALSE(csubstr("inf ").is_real()); - CHECK_FALSE(csubstr(" inf ").is_real()); - CHECK_FALSE(csubstr("-inf ").is_real()); - CHECK_FALSE(csubstr(" -inf ").is_real()); - CHECK_FALSE(csubstr("+inf ").is_real()); - CHECK_FALSE(csubstr(" +inf ").is_real()); - CHECK_FALSE(csubstr("inf1").is_real()); - CHECK_FALSE(csubstr(" inf1").is_real()); - CHECK_FALSE(csubstr("-inf1").is_real()); - CHECK_FALSE(csubstr(" -inf1").is_real()); - CHECK_FALSE(csubstr("+inf1").is_real()); - CHECK_FALSE(csubstr(" +inf1").is_real()); - } - SUBCASE("nan") - { - CHECK_UNARY(csubstr("nan").is_real()); - CHECK_FALSE(csubstr(" nan").is_real()); - CHECK_UNARY(csubstr("-nan").is_real()); - CHECK_FALSE(csubstr(" -nan").is_real()); - CHECK_UNARY(csubstr("+nan").is_real()); - CHECK_FALSE(csubstr(" +nan").is_real()); - CHECK_FALSE(csubstr("nan ").is_real()); - CHECK_FALSE(csubstr(" nan ").is_real()); - CHECK_FALSE(csubstr("-nan ").is_real()); - CHECK_FALSE(csubstr(" -nan ").is_real()); - CHECK_FALSE(csubstr("+nan ").is_real()); - CHECK_FALSE(csubstr(" +nan ").is_real()); - CHECK_FALSE(csubstr("nan1").is_real()); - CHECK_FALSE(csubstr(" nan1").is_real()); - CHECK_FALSE(csubstr("-nan1").is_real()); - CHECK_FALSE(csubstr(" -nan1").is_real()); - CHECK_FALSE(csubstr("+nan1").is_real()); - CHECK_FALSE(csubstr(" +nan1").is_real()); - } -} - -typedef enum : uint8_t { kIsNone = 0, kIsUint = 1, kIsInt = 3, kIsReal = 7 } NumberClass; -struct number -{ - csubstr num; - NumberClass cls; - - template - number(const char (&n)[N], NumberClass c) : num(n), cls(c) {} - number(csubstr n, NumberClass c) : num(n), cls(c) {} - - void test(csubstr ref={}) - { - if(ref.empty()) - ref = num; - INFO("num=" << num); - INFO("ref=" << ref); - switch(cls) - { - case kIsUint: - { - INFO("uint"); - CHECK_EQ(num.first_uint_span(), ref); - CHECK_EQ(num.first_int_span(), ref); - CHECK_EQ(num.first_real_span(), ref); - CHECK_UNARY(num.first_uint_span().is_unsigned_integer()); - CHECK_UNARY(num.first_uint_span().is_integer()); - CHECK_UNARY(num.first_uint_span().is_number()); - break; - } - case kIsInt: - { - INFO("int"); - CHECK_EQ(num.first_uint_span(), ""); - CHECK_EQ(num.first_int_span(), ref); - CHECK_EQ(num.first_real_span(), ref); - CHECK_FALSE(num.first_int_span().is_unsigned_integer()); - CHECK_UNARY(num.first_int_span().is_integer()); - CHECK_UNARY(num.first_int_span().is_number()); - break; - } - case kIsReal: - { - INFO("real"); - CHECK_EQ(num.first_uint_span(), ""); - CHECK_EQ(num.first_int_span(), ""); - CHECK_EQ(num.first_real_span(), ref); - CHECK_FALSE(num.first_real_span().is_unsigned_integer()); - CHECK_FALSE(num.first_real_span().is_integer()); - CHECK_UNARY(num .first_real_span().is_number()); - break; - } - case kIsNone: - { - INFO("none"); - CHECK_EQ(num.first_uint_span(), ""); - CHECK_EQ(num.first_int_span(), ""); - CHECK_EQ(num.first_real_span(), ""); - CHECK_FALSE(num.is_unsigned_integer()); - CHECK_FALSE(num.is_integer()); - CHECK_FALSE(num.is_number()); - break; - } - default: - { - CHECK_UNARY(false);//FAIL(); - break; - } - } - } -}; - -const number numbers[] = { - {"", kIsNone}, - {"0x", kIsNone}, - {"0b", kIsNone}, - {"0o", kIsNone}, - {".", kIsNone}, - {"0x.", kIsNone}, - {"0b.", kIsNone}, - {"0o.", kIsNone}, - {"-", kIsNone}, - {"0x-", kIsNone}, - {"0b-", kIsNone}, - {"0o-", kIsNone}, - {"+", kIsNone}, - {"0x+", kIsNone}, - {"0b+", kIsNone}, - {"0o+", kIsNone}, - {".e", kIsNone}, - {".e+", kIsNone}, - {".e-", kIsNone}, - {"0x.p", kIsNone}, - {"0x.p+", kIsNone}, - {"0x.p-", kIsNone}, - {"0b.p", kIsNone}, - {"0b.p+", kIsNone}, - {"0b.p-", kIsNone}, - {"0o.p", kIsNone}, - {"0o.p+", kIsNone}, - {"0o.p-", kIsNone}, - {"0x.p+", kIsNone}, - {"0x0.p+", kIsNone}, - {"0x.0p+", kIsNone}, - {"0x.p+0", kIsNone}, - {"0x.p+00", kIsNone}, - {"0x0.0p+", kIsNone}, - {"0x.0p+0", kIsReal}, - {"0x0.p+0", kIsReal}, - {"0x0.0p+0", kIsReal}, - {"0x0.0p+00", kIsReal}, - {"0x00.00p+00", kIsReal}, - {"0x0p0.00p+00", kIsNone}, - {"0x00.0p0p+00", kIsNone}, - {"0x00.00p+0p0", kIsNone}, - {"0x00.00p+00p", kIsNone}, - {"0b.p+", kIsNone}, - {"0b0.p+", kIsNone}, - {"0b.0p+", kIsNone}, - {"0b.p+0", kIsNone}, - {"0b.p+00", kIsNone}, - {"0b0.0p+", kIsNone}, - {"0b.0p+0", kIsReal}, - {"0b0.p+0", kIsReal}, - {"0b0.0p+0", kIsReal}, - {"0b0.0p+00", kIsReal}, - {"0b00.00p+00", kIsReal}, - {"0b0p0.00p+00", kIsNone}, - {"0b00.0p0p+00", kIsNone}, - {"0b00.00p+0p0", kIsNone}, - {"0b00.00p+00p", kIsNone}, - {"0o.p+", kIsNone}, - {"0o0.p+", kIsNone}, - {"0o.0p+", kIsNone}, - {"0o.p+0", kIsNone}, - {"0o.p+00", kIsNone}, - {"0o0.0p+", kIsNone}, - {"0o.0p+0", kIsReal}, - {"0o0.p+0", kIsReal}, - {"0o0.0p+0", kIsReal}, - {"0o0.0p+00", kIsReal}, - {"0o00.00p+00", kIsReal}, - {"0o0p0.00p+00", kIsNone}, - {"0o00.0p0p+00", kIsNone}, - {"0o00.00p+0p0", kIsNone}, - {"0o00.00p+00p", kIsNone}, - {".e+", kIsNone}, - {"0.e+", kIsNone}, - {".0e+", kIsNone}, - {".e+0", kIsNone}, - {".e+00", kIsNone}, - {"0.0e+", kIsNone}, - {".0e+0", kIsReal}, - {"0.e+0", kIsReal}, - {"0.0e+0", kIsReal}, - {"0.0e+00", kIsReal}, - {"00.00e+00", kIsReal}, - {"0e0.00e+00", kIsNone}, - {"00.0e0e+00", kIsNone}, - {"00.00e+0e0", kIsNone}, - {"00.00e+00e", kIsNone}, - {"0x1234.", kIsReal}, - {"+0x1234.", kIsReal}, - {"-0x1234.", kIsReal}, - {"0x.1234", kIsReal}, - {"0x0.1.2.3", kIsNone}, - {"0o0.1.2.3", kIsNone}, - {"0b0.1.0.1", kIsNone}, - {"a", kIsNone}, - {"b", kIsNone}, - {"?", kIsNone}, - {"0.0.1", kIsNone}, - {"0.0.9", kIsNone}, - {"0.0.10", kIsNone}, - {"0.1.0", kIsNone}, - {"0.9.0", kIsNone}, - {"0.10.0", kIsNone}, - {"1.0.0", kIsNone}, - {"9.0.0", kIsNone}, - {"10.0.0", kIsNone}, - {".0", kIsReal}, - {"0.", kIsReal}, - {"0.0", kIsReal}, - {"1234", kIsUint}, - {"+1234", kIsUint}, - {"-1234", kIsInt}, - {"1234.0", kIsReal}, - {"+1234.0", kIsReal}, - {"-1234.0", kIsReal}, - {"0000", kIsUint}, - {"0123", kIsUint}, - {"0", kIsUint}, - {"1", kIsUint}, - {"1.", kIsReal}, - {".1", kIsReal}, - {"0x1234", kIsUint}, - {"+0x1234", kIsUint}, - {"-0x1234", kIsInt}, - {"0b01", kIsUint}, - {"1e+1", kIsReal}, - {"1e-1", kIsReal}, - {"1.e-1", kIsReal}, - {"1.e+1", kIsReal}, - {"1.0e-1", kIsReal}, - {"1.0e+1", kIsReal}, - {"1e+123", kIsReal}, - {"1e-123", kIsReal}, - {"1.e-123", kIsReal}, - {"1.e+123", kIsReal}, - {"1.0e123", kIsReal}, - {"1.0e-123", kIsReal}, - {"1.0e+123", kIsReal}, - {"0x1.e8480p+19", kIsReal}, - {"0x1.g8480p+19", kIsNone}, - {"0xg.e8480p+19", kIsNone}, - {"0b101.011p+19", kIsReal}, - {"0b101.012p+19", kIsNone}, - {"0b102.011p+19", kIsNone}, - {"0o173.045p+19", kIsReal}, - {"0o173.048p+19", kIsNone}, - {"0o178.045p+19", kIsNone}, - {"infinity", kIsReal}, - {"inf", kIsReal}, - {"nan", kIsReal}, -}; - -TEST_CASE("substr.is_number") -{ - SUBCASE("basic.hex") - { - CHECK_EQ(csubstr("0x.0p+0").first_real_span(), csubstr("0x.0p+0")); - CHECK_EQ(csubstr("0x)sdkjhsdfkju").first_int_span(), csubstr{}); - CHECK_EQ(csubstr("0x)sdkjhsdfkju").first_uint_span(), csubstr{}); - CHECK_EQ(csubstr("0x)sdkjhsdfkju").first_real_span(), csubstr{}); - CHECK_EQ(csubstr("0x0)sdkjhsdfkju").first_int_span(), csubstr("0x0")); - CHECK_EQ(csubstr("0x0)sdkjhsdfkju").first_uint_span(), csubstr("0x0")); - CHECK_EQ(csubstr("0x0)sdkjhsdfkju").first_real_span(), csubstr("0x0")); - } - SUBCASE("basic.dec") - { - CHECK_EQ(csubstr("+infinity").first_real_span(), csubstr("+infinity")); - CHECK_EQ(csubstr("-infinity").first_real_span(), csubstr("-infinity")); - CHECK_EQ(csubstr("+inf").first_real_span(), csubstr("+inf")); - CHECK_EQ(csubstr("-inf").first_real_span(), csubstr("-inf")); - CHECK_EQ(csubstr("0.e+0").first_real_span(), csubstr("0.e+0")); - CHECK_EQ(csubstr("0.e-0").first_real_span(), csubstr("0.e-0")); - } - SUBCASE("plain") - { - for(number n : numbers) - { - n.test(); - } - } - char buf[128]; - SUBCASE("leading+") - { - for(number n : numbers) - { - INFO("orig=" << n.num); - substr withplus = cat_sub(buf, '+', n.num); - NumberClass cls = n.cls; - if(withplus.begins_with("+-") || withplus.begins_with("++")) - cls = kIsNone; - number cp(withplus, cls); - cp.test(); - } - } - SUBCASE("leading-") - { - for(number n : numbers) - { - INFO("orig=" << n.num); - substr withminus = cat_sub(buf, '-', n.num); - NumberClass cls = n.cls; - if(cls == kIsUint) - cls = kIsInt; - if(withminus.begins_with("--") || withminus.begins_with("-+")) - cls = kIsNone; - number cp(withminus, cls); - cp.test(); - } - } - SUBCASE("capital_e") - { - for(number n : numbers) - { - INFO("orig=" << n.num); - substr replaced = cat_sub(buf, n.num); - replaced.replace('e', 'E'); - number cp(replaced, n.cls); - cp.test(); - } - } - SUBCASE("capital_p") - { - for(number n : numbers) - { - INFO("orig=" << n.num); - substr replaced = cat_sub(buf, n.num); - replaced.replace('p', 'P'); - number cp(replaced, n.cls); - cp.test(); - } - } - SUBCASE("capital_0x") - { - for(number n : numbers) - { - INFO("orig=" << n.num); - substr replaced = cat_sub(buf, n.num); - replaced.replace('x', 'X'); - number cp(replaced, n.cls); - cp.test(); - } - } - SUBCASE("capital_0b") - { - for(number n : numbers) - { - INFO("orig=" << n.num); - substr replaced = cat_sub(buf, n.num); - replaced.replace('b', 'B'); - number cp(replaced, n.cls); - cp.test(); - } - } - SUBCASE("capital_0o") - { - for(number n : numbers) - { - INFO("orig=" << n.num); - substr replaced = cat_sub(buf, n.num); - replaced.replace('o', 'O'); - number cp(replaced, n.cls); - cp.test(); - } - } - SUBCASE("numbers before") - { - char numbuf_[16] = {}; - substr numbuf = numbuf_; - for(number n : numbers) - { - INFO("orig=" << n.num); - for(char c : {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}) - { - numbuf.fill(c); - substr result = cat_sub(buf, numbuf, n.num); - number cp(result.sub(numbuf.len), n.cls); - cp.test(); - } - } - } - SUBCASE("numbers after") - { - char numbuf_[16] = {}; - substr numbuf = numbuf_; - for(number n : numbers) - { - INFO("orig=" << n.num); - for(char c : {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'}) - { - numbuf.fill(c); - substr result = cat_sub(buf, n.num, numbuf); - number cp(result.first(n.num.len), n.cls); - cp.test(); - } - } - } - SUBCASE("delimiter after") - { - for(number n : numbers) - { - INFO("orig=" << n.num); - for(char c : {' ', '\n', ']', ')', '}', ',', ';', '\r', '\t','\0'}) - { - INFO("delimiter='" << c << "'"); - substr result = cat_sub(buf, n.num, c); - number cp(result, n.cls); - cp.test(n.num); - } - } - } - csubstr garbage = "sdkjhsdfkju"; - SUBCASE("prepend") - { - // adding anything before the number will make it not be a number - for(number n : numbers) - { - if(n.num.empty()) - continue; - INFO("orig=" << n.num); - for(int i = 0; i < 127; ++i) - { - char c = (char)i; - csubstr fmtd = cat_sub(buf, garbage, c, n.num); - number cp(fmtd, kIsNone); - cp.test(); - } - } - } - SUBCASE("append") - { - // adding after may or may not make it a number - for(number const& n : numbers) - { - INFO("orig=" << n.num); - for(int i = 0; i < 127; ++i) - { - number cp = n; - char c = (char)i; - cp.num = cat_sub(buf, n.num, c, garbage); - if(!csubstr::_is_delim_char(c)) - { - cp.cls = kIsNone; - } - cp.test(n.num); - } - } - } -} - -TEST_CASE("substr.triml") -{ - using S = csubstr; - - CHECK_EQ(S("aaabbb" ).triml('a' ), "bbb"); - CHECK_EQ(S("aaabbb" ).triml('b' ), "aaabbb"); - CHECK_EQ(S("aaabbb" ).triml('c' ), "aaabbb"); - CHECK_EQ(S("aaabbb" ).triml("ab"), ""); - CHECK_EQ(S("aaabbb" ).triml("ba"), ""); - CHECK_EQ(S("aaabbb" ).triml("cd"), "aaabbb"); - CHECK_EQ(S("aaa...bbb").triml('a' ), "...bbb"); - CHECK_EQ(S("aaa...bbb").triml('b' ), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").triml('c' ), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").triml("ab"), "...bbb"); - CHECK_EQ(S("aaa...bbb").triml("ba"), "...bbb"); - CHECK_EQ(S("aaa...bbb").triml("ab."), ""); - CHECK_EQ(S("aaa...bbb").triml("a."), "bbb"); - CHECK_EQ(S("aaa...bbb").triml(".a"), "bbb"); - CHECK_EQ(S("aaa...bbb").triml("b."), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").triml(".b"), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").triml("cd"), "aaa...bbb"); - - CHECK_EQ(S("ab" ).triml('a' ), "b"); - CHECK_EQ(S("ab" ).triml('b' ), "ab"); - CHECK_EQ(S("ab" ).triml('c' ), "ab"); - CHECK_EQ(S("ab" ).triml("ab"), ""); - CHECK_EQ(S("ab" ).triml("ba"), ""); - CHECK_EQ(S("ab" ).triml("cd"), "ab"); - CHECK_EQ(S("a...b").triml('a' ), "...b"); - CHECK_EQ(S("a...b").triml('b' ), "a...b"); - CHECK_EQ(S("a...b").triml('c' ), "a...b"); - CHECK_EQ(S("a...b").triml("ab"), "...b"); - CHECK_EQ(S("a...b").triml("ba"), "...b"); - CHECK_EQ(S("a...b").triml("ab."), ""); - CHECK_EQ(S("a...b").triml("a."), "b"); - CHECK_EQ(S("a...b").triml(".a"), "b"); - CHECK_EQ(S("a...b").triml("b."), "a...b"); - CHECK_EQ(S("a...b").triml(".b"), "a...b"); - CHECK_EQ(S("a...b").triml("cd"), "a...b"); -} - -TEST_CASE("substr.trimr") -{ - using S = csubstr; - - CHECK_EQ(S("aaabbb" ).trimr('a' ), "aaabbb"); - CHECK_EQ(S("aaabbb" ).trimr('b' ), "aaa"); - CHECK_EQ(S("aaabbb" ).trimr('c' ), "aaabbb"); - CHECK_EQ(S("aaabbb" ).trimr("ab"), ""); - CHECK_EQ(S("aaabbb" ).trimr("ba"), ""); - CHECK_EQ(S("aaabbb" ).trimr("cd"), "aaabbb"); - CHECK_EQ(S("aaa...bbb").trimr('a' ), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").trimr('b' ), "aaa..."); - CHECK_EQ(S("aaa...bbb").trimr('c' ), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").trimr("ab"), "aaa..."); - CHECK_EQ(S("aaa...bbb").trimr("ba"), "aaa..."); - CHECK_EQ(S("aaa...bbb").trimr("ab."), ""); - CHECK_EQ(S("aaa...bbb").trimr("a."), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").trimr(".a"), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").trimr("b."), "aaa"); - CHECK_EQ(S("aaa...bbb").trimr(".b"), "aaa"); - CHECK_EQ(S("aaa...bbb").trimr("cd"), "aaa...bbb"); - - CHECK_EQ(S("ab" ).trimr('a' ), "ab"); - CHECK_EQ(S("ab" ).trimr('b' ), "a"); - CHECK_EQ(S("ab" ).trimr('c' ), "ab"); - CHECK_EQ(S("ab" ).trimr("ab"), ""); - CHECK_EQ(S("ab" ).trimr("ba"), ""); - CHECK_EQ(S("ab" ).trimr("cd"), "ab"); - CHECK_EQ(S("a...b").trimr('a' ), "a...b"); - CHECK_EQ(S("a...b").trimr('b' ), "a..."); - CHECK_EQ(S("a...b").trimr('c' ), "a...b"); - CHECK_EQ(S("a...b").trimr("ab"), "a..."); - CHECK_EQ(S("a...b").trimr("ba"), "a..."); - CHECK_EQ(S("a...b").trimr("ab."), ""); - CHECK_EQ(S("a...b").trimr("a."), "a...b"); - CHECK_EQ(S("a...b").trimr(".a"), "a...b"); - CHECK_EQ(S("a...b").trimr("b."), "a"); - CHECK_EQ(S("a...b").trimr(".b"), "a"); - CHECK_EQ(S("a...b").trimr("cd"), "a...b"); -} - -TEST_CASE("substr.trim") -{ - using S = csubstr; - - CHECK_EQ(S("aaabbb" ).trim('a' ), "bbb"); - CHECK_EQ(S("aaabbb" ).trim('b' ), "aaa"); - CHECK_EQ(S("aaabbb" ).trim('c' ), "aaabbb"); - CHECK_EQ(S("aaabbb" ).trim("ab"), ""); - CHECK_EQ(S("aaabbb" ).trim("ba"), ""); - CHECK_EQ(S("aaabbb" ).trim("cd"), "aaabbb"); - CHECK_EQ(S("aaa...bbb").trim('a' ), "...bbb"); - CHECK_EQ(S("aaa...bbb").trim('b' ), "aaa..."); - CHECK_EQ(S("aaa...bbb").trim('c' ), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").trim("ab"), "..."); - CHECK_EQ(S("aaa...bbb").trim("ba"), "..."); - CHECK_EQ(S("aaa...bbb").trim('c' ), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").trim("ab."), ""); - CHECK_EQ(S("aaa...bbb").trim("." ), "aaa...bbb"); - CHECK_EQ(S("aaa...bbb").trim("a."), "bbb"); - CHECK_EQ(S("aaa...bbb").trim(".a"), "bbb"); - CHECK_EQ(S("aaa...bbb").trim("b."), "aaa"); - CHECK_EQ(S("aaa...bbb").trim(".b"), "aaa"); - CHECK_EQ(S("aaa...bbb").trim("cd"), "aaa...bbb"); - - CHECK_EQ(S("ab" ).trim('a' ), "b"); - CHECK_EQ(S("ab" ).trim('b' ), "a"); - CHECK_EQ(S("ab" ).trim('c' ), "ab"); - CHECK_EQ(S("ab" ).trim("ab"), ""); - CHECK_EQ(S("ab" ).trim("ba"), ""); - CHECK_EQ(S("ab" ).trim("cd"), "ab"); - CHECK_EQ(S("a...b").trim('a' ), "...b"); - CHECK_EQ(S("a...b").trim('b' ), "a..."); - CHECK_EQ(S("a...b").trim('c' ), "a...b"); - CHECK_EQ(S("a...b").trim("ab"), "..."); - CHECK_EQ(S("a...b").trim("ba"), "..."); - CHECK_EQ(S("a...b").trim('c' ), "a...b"); - CHECK_EQ(S("a...b").trim("ab."), ""); - CHECK_EQ(S("a...b").trim("." ), "a...b"); - CHECK_EQ(S("a...b").trim("a."), "b"); - CHECK_EQ(S("a...b").trim(".a"), "b"); - CHECK_EQ(S("a...b").trim("b."), "a"); - CHECK_EQ(S("a...b").trim(".b"), "a"); - CHECK_EQ(S("a...b").trim("cd"), "a...b"); -} - -TEST_CASE("substr.pop_right") -{ - using S = csubstr; - - CHECK_EQ(S("0/1/2" ).pop_right('/' ), "2"); - CHECK_EQ(S("0/1/2" ).pop_right('/', true), "2"); - CHECK_EQ(S("0/1/2/" ).pop_right('/' ), ""); - CHECK_EQ(S("0/1/2/" ).pop_right('/', true), "2/"); - CHECK_EQ(S("0/1/2///" ).pop_right('/' ), ""); - CHECK_EQ(S("0/1/2///" ).pop_right('/', true), "2///"); - - CHECK_EQ(S("0/1//2" ).pop_right('/' ), "2"); - CHECK_EQ(S("0/1//2" ).pop_right('/', true), "2"); - CHECK_EQ(S("0/1//2/" ).pop_right('/' ), ""); - CHECK_EQ(S("0/1//2/" ).pop_right('/', true), "2/"); - CHECK_EQ(S("0/1//2///" ).pop_right('/' ), ""); - CHECK_EQ(S("0/1//2///" ).pop_right('/', true), "2///"); - - CHECK_EQ(S("0/1///2" ).pop_right('/' ), "2"); - CHECK_EQ(S("0/1///2" ).pop_right('/', true), "2"); - CHECK_EQ(S("0/1///2/" ).pop_right('/' ), ""); - CHECK_EQ(S("0/1///2/" ).pop_right('/', true), "2/"); - CHECK_EQ(S("0/1///2///" ).pop_right('/' ), ""); - CHECK_EQ(S("0/1///2///" ).pop_right('/', true), "2///"); - - CHECK_EQ(S("/0/1/2" ).pop_right('/' ), "2"); - CHECK_EQ(S("/0/1/2" ).pop_right('/', true), "2"); - CHECK_EQ(S("/0/1/2/" ).pop_right('/' ), ""); - CHECK_EQ(S("/0/1/2/" ).pop_right('/', true), "2/"); - CHECK_EQ(S("/0/1/2///").pop_right('/' ), ""); - CHECK_EQ(S("/0/1/2///").pop_right('/', true), "2///"); - - CHECK_EQ(S("0" ).pop_right('/' ), "0"); - CHECK_EQ(S("0" ).pop_right('/', true), "0"); - CHECK_EQ(S("0/" ).pop_right('/' ), ""); - CHECK_EQ(S("0/" ).pop_right('/', true), "0/"); - CHECK_EQ(S("0///" ).pop_right('/' ), ""); - CHECK_EQ(S("0///" ).pop_right('/', true), "0///"); - - CHECK_EQ(S("/0" ).pop_right('/' ), "0"); - CHECK_EQ(S("/0" ).pop_right('/', true), "0"); - CHECK_EQ(S("/0/" ).pop_right('/' ), ""); - CHECK_EQ(S("/0/" ).pop_right('/', true), "0/"); - CHECK_EQ(S("/0///" ).pop_right('/' ), ""); - CHECK_EQ(S("/0///" ).pop_right('/', true), "0///"); - - CHECK_EQ(S("/" ).pop_right('/' ), ""); - CHECK_EQ(S("/" ).pop_right('/', true), ""); - CHECK_EQ(S("///" ).pop_right('/' ), ""); - CHECK_EQ(S("///" ).pop_right('/', true), ""); - - CHECK_EQ(S("" ).pop_right('/' ), ""); - CHECK_EQ(S("" ).pop_right('/', true), ""); - - CHECK_EQ(S("0-1-2" ).pop_right('-' ), "2"); - CHECK_EQ(S("0-1-2" ).pop_right('-', true), "2"); - CHECK_EQ(S("0-1-2-" ).pop_right('-' ), ""); - CHECK_EQ(S("0-1-2-" ).pop_right('-', true), "2-"); - CHECK_EQ(S("0-1-2---" ).pop_right('-' ), ""); - CHECK_EQ(S("0-1-2---" ).pop_right('-', true), "2---"); - - CHECK_EQ(S("0-1--2" ).pop_right('-' ), "2"); - CHECK_EQ(S("0-1--2" ).pop_right('-', true), "2"); - CHECK_EQ(S("0-1--2-" ).pop_right('-' ), ""); - CHECK_EQ(S("0-1--2-" ).pop_right('-', true), "2-"); - CHECK_EQ(S("0-1--2---" ).pop_right('-' ), ""); - CHECK_EQ(S("0-1--2---" ).pop_right('-', true), "2---"); - - CHECK_EQ(S("0-1---2" ).pop_right('-' ), "2"); - CHECK_EQ(S("0-1---2" ).pop_right('-', true), "2"); - CHECK_EQ(S("0-1---2-" ).pop_right('-' ), ""); - CHECK_EQ(S("0-1---2-" ).pop_right('-', true), "2-"); - CHECK_EQ(S("0-1---2---" ).pop_right('-' ), ""); - CHECK_EQ(S("0-1---2---" ).pop_right('-', true), "2---"); - - CHECK_EQ(S("-0-1-2" ).pop_right('-' ), "2"); - CHECK_EQ(S("-0-1-2" ).pop_right('-', true), "2"); - CHECK_EQ(S("-0-1-2-" ).pop_right('-' ), ""); - CHECK_EQ(S("-0-1-2-" ).pop_right('-', true), "2-"); - CHECK_EQ(S("-0-1-2---").pop_right('-' ), ""); - CHECK_EQ(S("-0-1-2---").pop_right('-', true), "2---"); - - CHECK_EQ(S("0" ).pop_right('-' ), "0"); - CHECK_EQ(S("0" ).pop_right('-', true), "0"); - CHECK_EQ(S("0-" ).pop_right('-' ), ""); - CHECK_EQ(S("0-" ).pop_right('-', true), "0-"); - CHECK_EQ(S("0---" ).pop_right('-' ), ""); - CHECK_EQ(S("0---" ).pop_right('-', true), "0---"); - - CHECK_EQ(S("-0" ).pop_right('-' ), "0"); - CHECK_EQ(S("-0" ).pop_right('-', true), "0"); - CHECK_EQ(S("-0-" ).pop_right('-' ), ""); - CHECK_EQ(S("-0-" ).pop_right('-', true), "0-"); - CHECK_EQ(S("-0---" ).pop_right('-' ), ""); - CHECK_EQ(S("-0---" ).pop_right('-', true), "0---"); - - CHECK_EQ(S("-" ).pop_right('-' ), ""); - CHECK_EQ(S("-" ).pop_right('-', true), ""); - CHECK_EQ(S("---" ).pop_right('-' ), ""); - CHECK_EQ(S("---" ).pop_right('-', true), ""); - - CHECK_EQ(S("" ).pop_right('-' ), ""); - CHECK_EQ(S("" ).pop_right('-', true), ""); -} - -TEST_CASE("substr.pop_left") -{ - using S = csubstr; - - CHECK_EQ(S("0/1/2" ).pop_left('/' ), "0"); - CHECK_EQ(S("0/1/2" ).pop_left('/', true), "0"); - CHECK_EQ(S("0/1/2/" ).pop_left('/' ), "0"); - CHECK_EQ(S("0/1/2/" ).pop_left('/', true), "0"); - CHECK_EQ(S("0/1/2///" ).pop_left('/' ), "0"); - CHECK_EQ(S("0/1/2///" ).pop_left('/', true), "0"); - - CHECK_EQ(S("0//1/2" ).pop_left('/' ), "0"); - CHECK_EQ(S("0//1/2" ).pop_left('/', true), "0"); - CHECK_EQ(S("0//1/2/" ).pop_left('/' ), "0"); - CHECK_EQ(S("0//1/2/" ).pop_left('/', true), "0"); - CHECK_EQ(S("0//1/2///" ).pop_left('/' ), "0"); - CHECK_EQ(S("0//1/2///" ).pop_left('/', true), "0"); - - CHECK_EQ(S("0///1/2" ).pop_left('/' ), "0"); - CHECK_EQ(S("0///1/2" ).pop_left('/', true), "0"); - CHECK_EQ(S("0///1/2/" ).pop_left('/' ), "0"); - CHECK_EQ(S("0///1/2/" ).pop_left('/', true), "0"); - CHECK_EQ(S("0///1/2///" ).pop_left('/' ), "0"); - CHECK_EQ(S("0///1/2///" ).pop_left('/', true), "0"); - - CHECK_EQ(S("/0/1/2" ).pop_left('/' ), ""); - CHECK_EQ(S("/0/1/2" ).pop_left('/', true), "/0"); - CHECK_EQ(S("/0/1/2/" ).pop_left('/' ), ""); - CHECK_EQ(S("/0/1/2/" ).pop_left('/', true), "/0"); - CHECK_EQ(S("/0/1/2///").pop_left('/' ), ""); - CHECK_EQ(S("/0/1/2///").pop_left('/', true), "/0"); - CHECK_EQ(S("///0/1/2" ).pop_left('/' ), ""); - CHECK_EQ(S("///0/1/2" ).pop_left('/', true), "///0"); - CHECK_EQ(S("///0/1/2/").pop_left('/' ), ""); - CHECK_EQ(S("///0/1/2/").pop_left('/', true), "///0"); - CHECK_EQ(S("///0/1/2/").pop_left('/' ), ""); - CHECK_EQ(S("///0/1/2/").pop_left('/', true), "///0"); - - CHECK_EQ(S("0" ).pop_left('/' ), "0"); - CHECK_EQ(S("0" ).pop_left('/', true), "0"); - CHECK_EQ(S("0/" ).pop_left('/' ), "0"); - CHECK_EQ(S("0/" ).pop_left('/', true), "0"); - CHECK_EQ(S("0///" ).pop_left('/' ), "0"); - CHECK_EQ(S("0///" ).pop_left('/', true), "0"); - - CHECK_EQ(S("/0" ).pop_left('/' ), ""); - CHECK_EQ(S("/0" ).pop_left('/', true), "/0"); - CHECK_EQ(S("/0/" ).pop_left('/' ), ""); - CHECK_EQ(S("/0/" ).pop_left('/', true), "/0"); - CHECK_EQ(S("/0///" ).pop_left('/' ), ""); - CHECK_EQ(S("/0///" ).pop_left('/', true), "/0"); - CHECK_EQ(S("///0///" ).pop_left('/' ), ""); - CHECK_EQ(S("///0///" ).pop_left('/', true), "///0"); - - CHECK_EQ(S("/" ).pop_left('/' ), ""); - CHECK_EQ(S("/" ).pop_left('/', true), ""); - CHECK_EQ(S("///" ).pop_left('/' ), ""); - CHECK_EQ(S("///" ).pop_left('/', true), ""); - - CHECK_EQ(S("" ).pop_left('/' ), ""); - CHECK_EQ(S("" ).pop_left('/', true), ""); - - CHECK_EQ(S("0-1-2" ).pop_left('-' ), "0"); - CHECK_EQ(S("0-1-2" ).pop_left('-', true), "0"); - CHECK_EQ(S("0-1-2-" ).pop_left('-' ), "0"); - CHECK_EQ(S("0-1-2-" ).pop_left('-', true), "0"); - CHECK_EQ(S("0-1-2---" ).pop_left('-' ), "0"); - CHECK_EQ(S("0-1-2---" ).pop_left('-', true), "0"); - - CHECK_EQ(S("0--1-2" ).pop_left('-' ), "0"); - CHECK_EQ(S("0--1-2" ).pop_left('-', true), "0"); - CHECK_EQ(S("0--1-2-" ).pop_left('-' ), "0"); - CHECK_EQ(S("0--1-2-" ).pop_left('-', true), "0"); - CHECK_EQ(S("0--1-2---" ).pop_left('-' ), "0"); - CHECK_EQ(S("0--1-2---" ).pop_left('-', true), "0"); - - CHECK_EQ(S("0---1-2" ).pop_left('-' ), "0"); - CHECK_EQ(S("0---1-2" ).pop_left('-', true), "0"); - CHECK_EQ(S("0---1-2-" ).pop_left('-' ), "0"); - CHECK_EQ(S("0---1-2-" ).pop_left('-', true), "0"); - CHECK_EQ(S("0---1-2---" ).pop_left('-' ), "0"); - CHECK_EQ(S("0---1-2---" ).pop_left('-', true), "0"); - - CHECK_EQ(S("-0-1-2" ).pop_left('-' ), ""); - CHECK_EQ(S("-0-1-2" ).pop_left('-', true), "-0"); - CHECK_EQ(S("-0-1-2-" ).pop_left('-' ), ""); - CHECK_EQ(S("-0-1-2-" ).pop_left('-', true), "-0"); - CHECK_EQ(S("-0-1-2---").pop_left('-' ), ""); - CHECK_EQ(S("-0-1-2---").pop_left('-', true), "-0"); - CHECK_EQ(S("---0-1-2" ).pop_left('-' ), ""); - CHECK_EQ(S("---0-1-2" ).pop_left('-', true), "---0"); - CHECK_EQ(S("---0-1-2-").pop_left('-' ), ""); - CHECK_EQ(S("---0-1-2-").pop_left('-', true), "---0"); - CHECK_EQ(S("---0-1-2-").pop_left('-' ), ""); - CHECK_EQ(S("---0-1-2-").pop_left('-', true), "---0"); - - CHECK_EQ(S("0" ).pop_left('-' ), "0"); - CHECK_EQ(S("0" ).pop_left('-', true), "0"); - CHECK_EQ(S("0-" ).pop_left('-' ), "0"); - CHECK_EQ(S("0-" ).pop_left('-', true), "0"); - CHECK_EQ(S("0---" ).pop_left('-' ), "0"); - CHECK_EQ(S("0---" ).pop_left('-', true), "0"); - - CHECK_EQ(S("-0" ).pop_left('-' ), ""); - CHECK_EQ(S("-0" ).pop_left('-', true), "-0"); - CHECK_EQ(S("-0-" ).pop_left('-' ), ""); - CHECK_EQ(S("-0-" ).pop_left('-', true), "-0"); - CHECK_EQ(S("-0---" ).pop_left('-' ), ""); - CHECK_EQ(S("-0---" ).pop_left('-', true), "-0"); - CHECK_EQ(S("---0---" ).pop_left('-' ), ""); - CHECK_EQ(S("---0---" ).pop_left('-', true), "---0"); - - CHECK_EQ(S("-" ).pop_left('-' ), ""); - CHECK_EQ(S("-" ).pop_left('-', true), ""); - CHECK_EQ(S("---" ).pop_left('-' ), ""); - CHECK_EQ(S("---" ).pop_left('-', true), ""); - - CHECK_EQ(S("" ).pop_left('-' ), ""); - CHECK_EQ(S("" ).pop_left('-', true), ""); -} - -TEST_CASE("substr.gpop_left") -{ - using S = csubstr; - - CHECK_EQ(S("0/1/2" ).gpop_left('/' ), "0/1"); - CHECK_EQ(S("0/1/2" ).gpop_left('/', true), "0/1"); - CHECK_EQ(S("0/1/2/" ).gpop_left('/' ), "0/1/2"); - CHECK_EQ(S("0/1/2/" ).gpop_left('/', true), "0/1"); - CHECK_EQ(S("0/1/2//" ).gpop_left('/' ), "0/1/2/"); - CHECK_EQ(S("0/1/2//" ).gpop_left('/', true), "0/1"); - CHECK_EQ(S("0/1/2///" ).gpop_left('/' ), "0/1/2//"); - CHECK_EQ(S("0/1/2///" ).gpop_left('/', true), "0/1"); - - CHECK_EQ(S("0/1//2" ).gpop_left('/' ), "0/1/"); - CHECK_EQ(S("0/1//2" ).gpop_left('/', true), "0/1"); - CHECK_EQ(S("0/1//2/" ).gpop_left('/' ), "0/1//2"); - CHECK_EQ(S("0/1//2/" ).gpop_left('/', true), "0/1"); - CHECK_EQ(S("0/1//2//" ).gpop_left('/' ), "0/1//2/"); - CHECK_EQ(S("0/1//2//" ).gpop_left('/', true), "0/1"); - CHECK_EQ(S("0/1//2///" ).gpop_left('/' ), "0/1//2//"); - CHECK_EQ(S("0/1//2///" ).gpop_left('/', true), "0/1"); - - CHECK_EQ(S("0/1///2" ).gpop_left('/' ), "0/1//"); - CHECK_EQ(S("0/1///2" ).gpop_left('/', true), "0/1"); - CHECK_EQ(S("0/1///2/" ).gpop_left('/' ), "0/1///2"); - CHECK_EQ(S("0/1///2/" ).gpop_left('/', true), "0/1"); - CHECK_EQ(S("0/1///2//" ).gpop_left('/' ), "0/1///2/"); - CHECK_EQ(S("0/1///2//" ).gpop_left('/', true), "0/1"); - CHECK_EQ(S("0/1///2///" ).gpop_left('/' ), "0/1///2//"); - CHECK_EQ(S("0/1///2///" ).gpop_left('/', true), "0/1"); - - CHECK_EQ(S("/0/1/2" ).gpop_left('/' ), "/0/1"); - CHECK_EQ(S("/0/1/2" ).gpop_left('/', true), "/0/1"); - CHECK_EQ(S("/0/1/2/" ).gpop_left('/' ), "/0/1/2"); - CHECK_EQ(S("/0/1/2/" ).gpop_left('/', true), "/0/1"); - CHECK_EQ(S("/0/1/2//" ).gpop_left('/' ), "/0/1/2/"); - CHECK_EQ(S("/0/1/2//" ).gpop_left('/', true), "/0/1"); - CHECK_EQ(S("/0/1/2///" ).gpop_left('/' ), "/0/1/2//"); - CHECK_EQ(S("/0/1/2///" ).gpop_left('/', true), "/0/1"); - - CHECK_EQ(S("//0/1/2" ).gpop_left('/' ), "//0/1"); - CHECK_EQ(S("//0/1/2" ).gpop_left('/', true), "//0/1"); - CHECK_EQ(S("//0/1/2/" ).gpop_left('/' ), "//0/1/2"); - CHECK_EQ(S("//0/1/2/" ).gpop_left('/', true), "//0/1"); - CHECK_EQ(S("//0/1/2//" ).gpop_left('/' ), "//0/1/2/"); - CHECK_EQ(S("//0/1/2//" ).gpop_left('/', true), "//0/1"); - CHECK_EQ(S("//0/1/2///" ).gpop_left('/' ), "//0/1/2//"); - CHECK_EQ(S("//0/1/2///" ).gpop_left('/', true), "//0/1"); - - CHECK_EQ(S("///0/1/2" ).gpop_left('/' ), "///0/1"); - CHECK_EQ(S("///0/1/2" ).gpop_left('/', true), "///0/1"); - CHECK_EQ(S("///0/1/2/" ).gpop_left('/' ), "///0/1/2"); - CHECK_EQ(S("///0/1/2/" ).gpop_left('/', true), "///0/1"); - CHECK_EQ(S("///0/1/2//" ).gpop_left('/' ), "///0/1/2/"); - CHECK_EQ(S("///0/1/2//" ).gpop_left('/', true), "///0/1"); - CHECK_EQ(S("///0/1/2///").gpop_left('/' ), "///0/1/2//"); - CHECK_EQ(S("///0/1/2///").gpop_left('/', true), "///0/1"); - - - CHECK_EQ(S("0/1" ).gpop_left('/' ), "0"); - CHECK_EQ(S("0/1" ).gpop_left('/', true), "0"); - CHECK_EQ(S("0/1/" ).gpop_left('/' ), "0/1"); - CHECK_EQ(S("0/1/" ).gpop_left('/', true), "0"); - CHECK_EQ(S("0/1//" ).gpop_left('/' ), "0/1/"); - CHECK_EQ(S("0/1//" ).gpop_left('/', true), "0"); - CHECK_EQ(S("0/1///" ).gpop_left('/' ), "0/1//"); - CHECK_EQ(S("0/1///" ).gpop_left('/', true), "0"); - - CHECK_EQ(S("0//1" ).gpop_left('/' ), "0/"); - CHECK_EQ(S("0//1" ).gpop_left('/', true), "0"); - CHECK_EQ(S("0//1/" ).gpop_left('/' ), "0//1"); - CHECK_EQ(S("0//1/" ).gpop_left('/', true), "0"); - CHECK_EQ(S("0//1//" ).gpop_left('/' ), "0//1/"); - CHECK_EQ(S("0//1//" ).gpop_left('/', true), "0"); - CHECK_EQ(S("0//1///" ).gpop_left('/' ), "0//1//"); - CHECK_EQ(S("0//1///" ).gpop_left('/', true), "0"); - - CHECK_EQ(S("0///1" ).gpop_left('/' ), "0//"); - CHECK_EQ(S("0///1" ).gpop_left('/', true), "0"); - CHECK_EQ(S("0///1/" ).gpop_left('/' ), "0///1"); - CHECK_EQ(S("0///1/" ).gpop_left('/', true), "0"); - CHECK_EQ(S("0///1//" ).gpop_left('/' ), "0///1/"); - CHECK_EQ(S("0///1//" ).gpop_left('/', true), "0"); - CHECK_EQ(S("0///1///" ).gpop_left('/' ), "0///1//"); - CHECK_EQ(S("0///1///" ).gpop_left('/', true), "0"); - - CHECK_EQ(S("/0/1" ).gpop_left('/' ), "/0"); - CHECK_EQ(S("/0/1" ).gpop_left('/', true), "/0"); - CHECK_EQ(S("/0/1/" ).gpop_left('/' ), "/0/1"); - CHECK_EQ(S("/0/1/" ).gpop_left('/', true), "/0"); - CHECK_EQ(S("/0/1//" ).gpop_left('/' ), "/0/1/"); - CHECK_EQ(S("/0/1//" ).gpop_left('/', true), "/0"); - CHECK_EQ(S("/0/1///" ).gpop_left('/' ), "/0/1//"); - CHECK_EQ(S("/0/1///" ).gpop_left('/', true), "/0"); - - CHECK_EQ(S("/0//1" ).gpop_left('/' ), "/0/"); - CHECK_EQ(S("/0//1" ).gpop_left('/', true), "/0"); - CHECK_EQ(S("/0//1/" ).gpop_left('/' ), "/0//1"); - CHECK_EQ(S("/0//1/" ).gpop_left('/', true), "/0"); - CHECK_EQ(S("/0//1//" ).gpop_left('/' ), "/0//1/"); - CHECK_EQ(S("/0//1//" ).gpop_left('/', true), "/0"); - CHECK_EQ(S("/0//1///" ).gpop_left('/' ), "/0//1//"); - CHECK_EQ(S("/0//1///" ).gpop_left('/', true), "/0"); - - CHECK_EQ(S("/0///1" ).gpop_left('/' ), "/0//"); - CHECK_EQ(S("/0///1" ).gpop_left('/', true), "/0"); - CHECK_EQ(S("/0///1/" ).gpop_left('/' ), "/0///1"); - CHECK_EQ(S("/0///1/" ).gpop_left('/', true), "/0"); - CHECK_EQ(S("/0///1//" ).gpop_left('/' ), "/0///1/"); - CHECK_EQ(S("/0///1//" ).gpop_left('/', true), "/0"); - CHECK_EQ(S("/0///1///" ).gpop_left('/' ), "/0///1//"); - CHECK_EQ(S("/0///1///" ).gpop_left('/', true), "/0"); - - CHECK_EQ(S("//0/1" ).gpop_left('/' ), "//0"); - CHECK_EQ(S("//0/1" ).gpop_left('/', true), "//0"); - CHECK_EQ(S("//0/1/" ).gpop_left('/' ), "//0/1"); - CHECK_EQ(S("//0/1/" ).gpop_left('/', true), "//0"); - CHECK_EQ(S("//0/1//" ).gpop_left('/' ), "//0/1/"); - CHECK_EQ(S("//0/1//" ).gpop_left('/', true), "//0"); - CHECK_EQ(S("//0/1///" ).gpop_left('/' ), "//0/1//"); - CHECK_EQ(S("//0/1///" ).gpop_left('/', true), "//0"); - - CHECK_EQ(S("//0//1" ).gpop_left('/' ), "//0/"); - CHECK_EQ(S("//0//1" ).gpop_left('/', true), "//0"); - CHECK_EQ(S("//0//1/" ).gpop_left('/' ), "//0//1"); - CHECK_EQ(S("//0//1/" ).gpop_left('/', true), "//0"); - CHECK_EQ(S("//0//1//" ).gpop_left('/' ), "//0//1/"); - CHECK_EQ(S("//0//1//" ).gpop_left('/', true), "//0"); - CHECK_EQ(S("//0//1///" ).gpop_left('/' ), "//0//1//"); - CHECK_EQ(S("//0//1///" ).gpop_left('/', true), "//0"); - - CHECK_EQ(S("//0///1" ).gpop_left('/' ), "//0//"); - CHECK_EQ(S("//0///1" ).gpop_left('/', true), "//0"); - CHECK_EQ(S("//0///1/" ).gpop_left('/' ), "//0///1"); - CHECK_EQ(S("//0///1/" ).gpop_left('/', true), "//0"); - CHECK_EQ(S("//0///1//" ).gpop_left('/' ), "//0///1/"); - CHECK_EQ(S("//0///1//" ).gpop_left('/', true), "//0"); - CHECK_EQ(S("//0///1///" ).gpop_left('/' ), "//0///1//"); - CHECK_EQ(S("//0///1///" ).gpop_left('/', true), "//0"); - - CHECK_EQ(S("0" ).gpop_left('/' ), ""); - CHECK_EQ(S("0" ).gpop_left('/', true), ""); - CHECK_EQ(S("0/" ).gpop_left('/' ), "0"); - CHECK_EQ(S("0/" ).gpop_left('/', true), ""); - CHECK_EQ(S("0//" ).gpop_left('/' ), "0/"); - CHECK_EQ(S("0//" ).gpop_left('/', true), ""); - CHECK_EQ(S("0///" ).gpop_left('/' ), "0//"); - CHECK_EQ(S("0///" ).gpop_left('/', true), ""); - - CHECK_EQ(S("/0" ).gpop_left('/' ), ""); - CHECK_EQ(S("/0" ).gpop_left('/', true), ""); - CHECK_EQ(S("/0/" ).gpop_left('/' ), "/0"); - CHECK_EQ(S("/0/" ).gpop_left('/', true), ""); - CHECK_EQ(S("/0//" ).gpop_left('/' ), "/0/"); - CHECK_EQ(S("/0//" ).gpop_left('/', true), ""); - CHECK_EQ(S("/0///" ).gpop_left('/' ), "/0//"); - CHECK_EQ(S("/0///" ).gpop_left('/', true), ""); - - CHECK_EQ(S("//0" ).gpop_left('/' ), "/"); - CHECK_EQ(S("//0" ).gpop_left('/', true), ""); - CHECK_EQ(S("//0/" ).gpop_left('/' ), "//0"); - CHECK_EQ(S("//0/" ).gpop_left('/', true), ""); - CHECK_EQ(S("//0//" ).gpop_left('/' ), "//0/"); - CHECK_EQ(S("//0//" ).gpop_left('/', true), ""); - CHECK_EQ(S("//0///" ).gpop_left('/' ), "//0//"); - CHECK_EQ(S("//0///" ).gpop_left('/', true), ""); - - CHECK_EQ(S("///0" ).gpop_left('/' ), "//"); - CHECK_EQ(S("///0" ).gpop_left('/', true), ""); - CHECK_EQ(S("///0/" ).gpop_left('/' ), "///0"); - CHECK_EQ(S("///0/" ).gpop_left('/', true), ""); - CHECK_EQ(S("///0//" ).gpop_left('/' ), "///0/"); - CHECK_EQ(S("///0//" ).gpop_left('/', true), ""); - CHECK_EQ(S("///0///" ).gpop_left('/' ), "///0//"); - CHECK_EQ(S("///0///" ).gpop_left('/', true), ""); - - CHECK_EQ(S("/" ).gpop_left('/' ), ""); - CHECK_EQ(S("/" ).gpop_left('/', true), ""); - CHECK_EQ(S("//" ).gpop_left('/' ), "/"); - CHECK_EQ(S("//" ).gpop_left('/', true), ""); - CHECK_EQ(S("///" ).gpop_left('/' ), "//"); - CHECK_EQ(S("///" ).gpop_left('/', true), ""); - - CHECK_EQ(S("" ).gpop_left('/' ), ""); - CHECK_EQ(S("" ).gpop_left('/', true), ""); -} - -TEST_CASE("substr.gpop_right") -{ - using S = csubstr; - - CHECK_EQ(S("0/1/2" ).gpop_right('/' ), "1/2"); - CHECK_EQ(S("0/1/2" ).gpop_right('/', true), "1/2"); - CHECK_EQ(S("0/1/2/" ).gpop_right('/' ), "1/2/"); - CHECK_EQ(S("0/1/2/" ).gpop_right('/', true), "1/2/"); - CHECK_EQ(S("0/1/2//" ).gpop_right('/' ), "1/2//"); - CHECK_EQ(S("0/1/2//" ).gpop_right('/', true), "1/2//"); - CHECK_EQ(S("0/1/2///" ).gpop_right('/' ), "1/2///"); - CHECK_EQ(S("0/1/2///" ).gpop_right('/', true), "1/2///"); - - CHECK_EQ(S("0//1/2" ).gpop_right('/' ), "/1/2"); - CHECK_EQ(S("0//1/2" ).gpop_right('/', true), "1/2"); - CHECK_EQ(S("0//1/2/" ).gpop_right('/' ), "/1/2/"); - CHECK_EQ(S("0//1/2/" ).gpop_right('/', true), "1/2/"); - CHECK_EQ(S("0//1/2//" ).gpop_right('/' ), "/1/2//"); - CHECK_EQ(S("0//1/2//" ).gpop_right('/', true), "1/2//"); - CHECK_EQ(S("0//1/2///" ).gpop_right('/' ), "/1/2///"); - CHECK_EQ(S("0//1/2///" ).gpop_right('/', true), "1/2///"); - - CHECK_EQ(S("0///1/2" ).gpop_right('/' ), "//1/2"); - CHECK_EQ(S("0///1/2" ).gpop_right('/', true), "1/2"); - CHECK_EQ(S("0///1/2/" ).gpop_right('/' ), "//1/2/"); - CHECK_EQ(S("0///1/2/" ).gpop_right('/', true), "1/2/"); - CHECK_EQ(S("0///1/2//" ).gpop_right('/' ), "//1/2//"); - CHECK_EQ(S("0///1/2//" ).gpop_right('/', true), "1/2//"); - CHECK_EQ(S("0///1/2///" ).gpop_right('/' ), "//1/2///"); - CHECK_EQ(S("0///1/2///" ).gpop_right('/', true), "1/2///"); - - - CHECK_EQ(S("/0/1/2" ).gpop_right('/' ), "0/1/2"); - CHECK_EQ(S("/0/1/2" ).gpop_right('/', true), "1/2"); - CHECK_EQ(S("/0/1/2/" ).gpop_right('/' ), "0/1/2/"); - CHECK_EQ(S("/0/1/2/" ).gpop_right('/', true), "1/2/"); - CHECK_EQ(S("/0/1/2//" ).gpop_right('/' ), "0/1/2//"); - CHECK_EQ(S("/0/1/2//" ).gpop_right('/', true), "1/2//"); - CHECK_EQ(S("/0/1/2///" ).gpop_right('/' ), "0/1/2///"); - CHECK_EQ(S("/0/1/2///" ).gpop_right('/', true), "1/2///"); - - CHECK_EQ(S("/0//1/2" ).gpop_right('/' ), "0//1/2"); - CHECK_EQ(S("/0//1/2" ).gpop_right('/', true), "1/2"); - CHECK_EQ(S("/0//1/2/" ).gpop_right('/' ), "0//1/2/"); - CHECK_EQ(S("/0//1/2/" ).gpop_right('/', true), "1/2/"); - CHECK_EQ(S("/0//1/2//" ).gpop_right('/' ), "0//1/2//"); - CHECK_EQ(S("/0//1/2//" ).gpop_right('/', true), "1/2//"); - CHECK_EQ(S("/0//1/2///" ).gpop_right('/' ), "0//1/2///"); - CHECK_EQ(S("/0//1/2///" ).gpop_right('/', true), "1/2///"); - - CHECK_EQ(S("/0///1/2" ).gpop_right('/' ), "0///1/2"); - CHECK_EQ(S("/0///1/2" ).gpop_right('/', true), "1/2"); - CHECK_EQ(S("/0///1/2/" ).gpop_right('/' ), "0///1/2/"); - CHECK_EQ(S("/0///1/2/" ).gpop_right('/', true), "1/2/"); - CHECK_EQ(S("/0///1/2//" ).gpop_right('/' ), "0///1/2//"); - CHECK_EQ(S("/0///1/2//" ).gpop_right('/', true), "1/2//"); - CHECK_EQ(S("/0///1/2///" ).gpop_right('/' ), "0///1/2///"); - CHECK_EQ(S("/0///1/2///" ).gpop_right('/', true), "1/2///"); - - - CHECK_EQ(S("//0/1/2" ).gpop_right('/' ), "/0/1/2"); - CHECK_EQ(S("//0/1/2" ).gpop_right('/', true), "1/2"); - CHECK_EQ(S("//0/1/2/" ).gpop_right('/' ), "/0/1/2/"); - CHECK_EQ(S("//0/1/2/" ).gpop_right('/', true), "1/2/"); - CHECK_EQ(S("//0/1/2//" ).gpop_right('/' ), "/0/1/2//"); - CHECK_EQ(S("//0/1/2//" ).gpop_right('/', true), "1/2//"); - CHECK_EQ(S("//0/1/2///" ).gpop_right('/' ), "/0/1/2///"); - CHECK_EQ(S("//0/1/2///" ).gpop_right('/', true), "1/2///"); - - CHECK_EQ(S("//0//1/2" ).gpop_right('/' ), "/0//1/2"); - CHECK_EQ(S("//0//1/2" ).gpop_right('/', true), "1/2"); - CHECK_EQ(S("//0//1/2/" ).gpop_right('/' ), "/0//1/2/"); - CHECK_EQ(S("//0//1/2/" ).gpop_right('/', true), "1/2/"); - CHECK_EQ(S("//0//1/2//" ).gpop_right('/' ), "/0//1/2//"); - CHECK_EQ(S("//0//1/2//" ).gpop_right('/', true), "1/2//"); - CHECK_EQ(S("//0//1/2///" ).gpop_right('/' ), "/0//1/2///"); - CHECK_EQ(S("//0//1/2///" ).gpop_right('/', true), "1/2///"); - - CHECK_EQ(S("//0///1/2" ).gpop_right('/' ), "/0///1/2"); - CHECK_EQ(S("//0///1/2" ).gpop_right('/', true), "1/2"); - CHECK_EQ(S("//0///1/2/" ).gpop_right('/' ), "/0///1/2/"); - CHECK_EQ(S("//0///1/2/" ).gpop_right('/', true), "1/2/"); - CHECK_EQ(S("//0///1/2//" ).gpop_right('/' ), "/0///1/2//"); - CHECK_EQ(S("//0///1/2//" ).gpop_right('/', true), "1/2//"); - CHECK_EQ(S("//0///1/2///" ).gpop_right('/' ), "/0///1/2///"); - CHECK_EQ(S("//0///1/2///" ).gpop_right('/', true), "1/2///"); - - - CHECK_EQ(S("0/1" ).gpop_right('/' ), "1"); - CHECK_EQ(S("0/1" ).gpop_right('/', true), "1"); - CHECK_EQ(S("0/1/" ).gpop_right('/' ), "1/"); - CHECK_EQ(S("0/1/" ).gpop_right('/', true), "1/"); - CHECK_EQ(S("0/1//" ).gpop_right('/' ), "1//"); - CHECK_EQ(S("0/1//" ).gpop_right('/', true), "1//"); - CHECK_EQ(S("0/1///" ).gpop_right('/' ), "1///"); - CHECK_EQ(S("0/1///" ).gpop_right('/', true), "1///"); - - CHECK_EQ(S("0//1" ).gpop_right('/' ), "/1"); - CHECK_EQ(S("0//1" ).gpop_right('/', true), "1"); - CHECK_EQ(S("0//1/" ).gpop_right('/' ), "/1/"); - CHECK_EQ(S("0//1/" ).gpop_right('/', true), "1/"); - CHECK_EQ(S("0//1//" ).gpop_right('/' ), "/1//"); - CHECK_EQ(S("0//1//" ).gpop_right('/', true), "1//"); - CHECK_EQ(S("0//1///" ).gpop_right('/' ), "/1///"); - CHECK_EQ(S("0//1///" ).gpop_right('/', true), "1///"); - - CHECK_EQ(S("0///1" ).gpop_right('/' ), "//1"); - CHECK_EQ(S("0///1" ).gpop_right('/', true), "1"); - CHECK_EQ(S("0///1/" ).gpop_right('/' ), "//1/"); - CHECK_EQ(S("0///1/" ).gpop_right('/', true), "1/"); - CHECK_EQ(S("0///1//" ).gpop_right('/' ), "//1//"); - CHECK_EQ(S("0///1//" ).gpop_right('/', true), "1//"); - CHECK_EQ(S("0///1///" ).gpop_right('/' ), "//1///"); - CHECK_EQ(S("0///1///" ).gpop_right('/', true), "1///"); - - - CHECK_EQ(S("/0/1" ).gpop_right('/' ), "0/1"); - CHECK_EQ(S("/0/1" ).gpop_right('/', true), "1"); - CHECK_EQ(S("/0/1/" ).gpop_right('/' ), "0/1/"); - CHECK_EQ(S("/0/1/" ).gpop_right('/', true), "1/"); - CHECK_EQ(S("/0/1//" ).gpop_right('/' ), "0/1//"); - CHECK_EQ(S("/0/1//" ).gpop_right('/', true), "1//"); - CHECK_EQ(S("/0/1///" ).gpop_right('/' ), "0/1///"); - CHECK_EQ(S("/0/1///" ).gpop_right('/', true), "1///"); - - CHECK_EQ(S("/0//1" ).gpop_right('/' ), "0//1"); - CHECK_EQ(S("/0//1" ).gpop_right('/', true), "1"); - CHECK_EQ(S("/0//1/" ).gpop_right('/' ), "0//1/"); - CHECK_EQ(S("/0//1/" ).gpop_right('/', true), "1/"); - CHECK_EQ(S("/0//1//" ).gpop_right('/' ), "0//1//"); - CHECK_EQ(S("/0//1//" ).gpop_right('/', true), "1//"); - CHECK_EQ(S("/0//1///" ).gpop_right('/' ), "0//1///"); - CHECK_EQ(S("/0//1///" ).gpop_right('/', true), "1///"); - - CHECK_EQ(S("/0///1" ).gpop_right('/' ), "0///1"); - CHECK_EQ(S("/0///1" ).gpop_right('/', true), "1"); - CHECK_EQ(S("/0///1/" ).gpop_right('/' ), "0///1/"); - CHECK_EQ(S("/0///1/" ).gpop_right('/', true), "1/"); - CHECK_EQ(S("/0///1//" ).gpop_right('/' ), "0///1//"); - CHECK_EQ(S("/0///1//" ).gpop_right('/', true), "1//"); - CHECK_EQ(S("/0///1///" ).gpop_right('/' ), "0///1///"); - CHECK_EQ(S("/0///1///" ).gpop_right('/', true), "1///"); - - - CHECK_EQ(S("//0/1" ).gpop_right('/' ), "/0/1"); - CHECK_EQ(S("//0/1" ).gpop_right('/', true), "1"); - CHECK_EQ(S("//0/1/" ).gpop_right('/' ), "/0/1/"); - CHECK_EQ(S("//0/1/" ).gpop_right('/', true), "1/"); - CHECK_EQ(S("//0/1//" ).gpop_right('/' ), "/0/1//"); - CHECK_EQ(S("//0/1//" ).gpop_right('/', true), "1//"); - CHECK_EQ(S("//0/1///" ).gpop_right('/' ), "/0/1///"); - CHECK_EQ(S("//0/1///" ).gpop_right('/', true), "1///"); - - CHECK_EQ(S("//0//1" ).gpop_right('/' ), "/0//1"); - CHECK_EQ(S("//0//1" ).gpop_right('/', true), "1"); - CHECK_EQ(S("//0//1/" ).gpop_right('/' ), "/0//1/"); - CHECK_EQ(S("//0//1/" ).gpop_right('/', true), "1/"); - CHECK_EQ(S("//0//1//" ).gpop_right('/' ), "/0//1//"); - CHECK_EQ(S("//0//1//" ).gpop_right('/', true), "1//"); - CHECK_EQ(S("//0//1///" ).gpop_right('/' ), "/0//1///"); - CHECK_EQ(S("//0//1///" ).gpop_right('/', true), "1///"); - - CHECK_EQ(S("//0///1" ).gpop_right('/' ), "/0///1"); - CHECK_EQ(S("//0///1" ).gpop_right('/', true), "1"); - CHECK_EQ(S("//0///1/" ).gpop_right('/' ), "/0///1/"); - CHECK_EQ(S("//0///1/" ).gpop_right('/', true), "1/"); - CHECK_EQ(S("//0///1//" ).gpop_right('/' ), "/0///1//"); - CHECK_EQ(S("//0///1//" ).gpop_right('/', true), "1//"); - CHECK_EQ(S("//0///1///" ).gpop_right('/' ), "/0///1///"); - CHECK_EQ(S("//0///1///" ).gpop_right('/', true), "1///"); - - - CHECK_EQ(S("0" ).gpop_right('/' ), ""); - CHECK_EQ(S("0" ).gpop_right('/', true), ""); - CHECK_EQ(S("0/" ).gpop_right('/' ), ""); - CHECK_EQ(S("0/" ).gpop_right('/', true), ""); - CHECK_EQ(S("0//" ).gpop_right('/' ), "/"); - CHECK_EQ(S("0//" ).gpop_right('/', true), ""); - CHECK_EQ(S("0///" ).gpop_right('/' ), "//"); - CHECK_EQ(S("0///" ).gpop_right('/', true), ""); - - CHECK_EQ(S("/0" ).gpop_right('/' ), "0"); - CHECK_EQ(S("/0" ).gpop_right('/', true), ""); - CHECK_EQ(S("/0/" ).gpop_right('/' ), "0/"); - CHECK_EQ(S("/0/" ).gpop_right('/', true), ""); - CHECK_EQ(S("/0//" ).gpop_right('/' ), "0//"); - CHECK_EQ(S("/0//" ).gpop_right('/', true), ""); - CHECK_EQ(S("/0///" ).gpop_right('/' ), "0///"); - CHECK_EQ(S("/0///" ).gpop_right('/', true), ""); - - CHECK_EQ(S("//0" ).gpop_right('/' ), "/0"); - CHECK_EQ(S("//0" ).gpop_right('/', true), ""); - CHECK_EQ(S("//0/" ).gpop_right('/' ), "/0/"); - CHECK_EQ(S("//0/" ).gpop_right('/', true), ""); - CHECK_EQ(S("//0//" ).gpop_right('/' ), "/0//"); - CHECK_EQ(S("//0//" ).gpop_right('/', true), ""); - CHECK_EQ(S("//0///" ).gpop_right('/' ), "/0///"); - CHECK_EQ(S("//0///" ).gpop_right('/', true), ""); - - CHECK_EQ(S("///0" ).gpop_right('/' ), "//0"); - CHECK_EQ(S("///0" ).gpop_right('/', true), ""); - CHECK_EQ(S("///0/" ).gpop_right('/' ), "//0/"); - CHECK_EQ(S("///0/" ).gpop_right('/', true), ""); - CHECK_EQ(S("///0//" ).gpop_right('/' ), "//0//"); - CHECK_EQ(S("///0//" ).gpop_right('/', true), ""); - CHECK_EQ(S("///0///" ).gpop_right('/' ), "//0///"); - CHECK_EQ(S("///0///" ).gpop_right('/', true), ""); - - CHECK_EQ(S("/" ).gpop_right('/' ), ""); - CHECK_EQ(S("/" ).gpop_right('/', true), ""); - CHECK_EQ(S("//" ).gpop_right('/' ), "/"); - CHECK_EQ(S("//" ).gpop_right('/', true), ""); - CHECK_EQ(S("///" ).gpop_right('/' ), "//"); - CHECK_EQ(S("///" ).gpop_right('/', true), ""); - - CHECK_EQ(S("" ).gpop_right('/' ), ""); - CHECK_EQ(S("" ).gpop_right('/', true), ""); -} - -TEST_CASE("substr.basename") -{ - using S = csubstr; - CHECK_EQ(S("0/1/2").basename(), "2"); - CHECK_EQ(S("0/1/2/").basename(), "2"); - CHECK_EQ(S("0/1/2///").basename(), "2"); - CHECK_EQ(S("/0/1/2").basename(), "2"); - CHECK_EQ(S("/0/1/2/").basename(), "2"); - CHECK_EQ(S("/0/1/2///").basename(), "2"); - CHECK_EQ(S("///0/1/2").basename(), "2"); - CHECK_EQ(S("///0/1/2/").basename(), "2"); - CHECK_EQ(S("///0/1/2///").basename(), "2"); - CHECK_EQ(S("/").basename(), ""); - CHECK_EQ(S("//").basename(), ""); - CHECK_EQ(S("///").basename(), ""); - CHECK_EQ(S("////").basename(), ""); - CHECK_EQ(S("").basename(), ""); -} - -TEST_CASE("substr.dirname") -{ - using S = csubstr; - CHECK_EQ(S("0/1/2").dirname(), "0/1/"); - CHECK_EQ(S("0/1/2/").dirname(), "0/1/"); - CHECK_EQ(S("/0/1/2").dirname(), "/0/1/"); - CHECK_EQ(S("/0/1/2/").dirname(), "/0/1/"); - CHECK_EQ(S("///0/1/2").dirname(), "///0/1/"); - CHECK_EQ(S("///0/1/2/").dirname(), "///0/1/"); - CHECK_EQ(S("/0").dirname(), "/"); - CHECK_EQ(S("/").dirname(), "/"); - CHECK_EQ(S("//").dirname(), "//"); - CHECK_EQ(S("///").dirname(), "///"); - CHECK_EQ(S("////").dirname(), "////"); - CHECK_EQ(S("").dirname(), ""); -} - -TEST_CASE("substr.extshort") -{ - using S = csubstr; - CHECK_EQ(S("filename.with.ext").extshort(), "ext"); - CHECK_EQ(S("filename.with.ext.").extshort(), ""); - CHECK_EQ(S(".a.b").extshort(), "b"); - CHECK_EQ(S(".a.b.").extshort(), ""); - CHECK_EQ(S(".b..").extshort(), ""); - CHECK_EQ(S("..b.").extshort(), ""); -} - -TEST_CASE("substr.extlong") -{ - using S = csubstr; - CHECK_EQ(S("filename.with.ext").extlong(), "with.ext"); - CHECK_EQ(S("filename.with.ext.").extlong(), "with.ext."); - CHECK_EQ(S(".a.b").extlong(), "a.b"); - CHECK_EQ(S(".a.b.").extlong(), "a.b."); - CHECK_EQ(S(".b..").extlong(), "b.."); - CHECK_EQ(S("..b.").extlong(), ".b."); -} - -TEST_CASE("substr.next_split") -{ - using S = csubstr; - - { - S const n; - typename S::size_type pos = 0; - S ss; - CHECK_EQ(n.next_split(':', &pos, &ss), false); - CHECK_EQ(ss.empty(), true); - CHECK_EQ(n.next_split(':', &pos, &ss), false); - CHECK_EQ(ss.empty(), true); - pos = 0; - CHECK_EQ(n.next_split(',', &pos, &ss), false); - CHECK_EQ(ss.empty(), true); - CHECK_EQ(n.next_split(',', &pos, &ss), false); - CHECK_EQ(ss.empty(), true); - } - - { - S const n("0"); - typename S::size_type pos = 0; - S ss; - CHECK_EQ(n.next_split(':', &pos, &ss), true); - CHECK_EQ(ss.empty(), false); - CHECK_EQ(n.next_split(':', &pos, &ss), false); - CHECK_EQ(ss.empty(), true); - CHECK_EQ(n.next_split(':', &pos, &ss), false); - CHECK_EQ(ss.empty(), true); - pos = 0; - CHECK_EQ(n.next_split(',', &pos, &ss), true); - CHECK_EQ(ss.empty(), false); - CHECK_EQ(n.next_split(',', &pos, &ss), false); - CHECK_EQ(ss.empty(), true); - CHECK_EQ(n.next_split(',', &pos, &ss), false); - CHECK_EQ(ss.empty(), true); - } - - { - S const n; - typename S::size_type pos = 0; - typename S::size_type count = 0; - S ss; - while(n.next_split(':', &pos, &ss)) - { - ++count; - } - CHECK_EQ(count, 0); - } - - { - S const n("0123456"); - typename S::size_type pos = 0; - typename S::size_type count = 0; - S ss; - while(n.next_split(':', &pos, &ss)) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), n.size()); - CHECK_EQ(ss.empty(), false); - break; - default: - CHECK_UNARY(false);//GTEST_FAIL(); - break; - } - ++count; - } - CHECK_EQ(count, 1); - } - - { - S const n("0123456:"); - typename S::size_type pos = 0; - typename S::size_type count = 0; - S ss; - while(n.next_split(':', &pos, &ss)) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), n.size()-1); - CHECK_EQ(ss.empty(), false); - break; - case 1: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - default: - CHECK_UNARY(false);//GTEST_FAIL(); - break; - } - ++count; - } - CHECK_EQ(count, 2); - } - - { - S const n(":0123456:"); - typename S::size_type pos = 0; - typename S::size_type count = 0; - S ss; - while(n.next_split(':', &pos, &ss)) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - case 1: - CHECK_EQ(ss.size(), n.size()-2); - CHECK_EQ(ss.empty(), false); - break; - case 2: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - default: - CHECK_UNARY(false);//GTEST_FAIL(); - break; - } - ++count; - } - CHECK_EQ(count, 3); - } - - { - S const n(":"); - typename S::size_type pos = 0; - typename S::size_type count = 0; - S ss; - while(n.next_split(':', &pos, &ss)) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - case 1: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - default: - CHECK_UNARY(false);//GTEST_FAIL(); - break; - } - ++count; - } - CHECK_EQ(count, 2); - } - - { - S const n("01:23:45:67"); - typename S::size_type pos = 0; - typename S::size_type count = 0; - S ss; - while(n.next_split(':', &pos, &ss)) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "01"); - CHECK_NE(ss, "01:"); - CHECK_NE(ss, ":01:"); - CHECK_NE(ss, ":01"); - break; - case 1: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "23"); - CHECK_NE(ss, "23:"); - CHECK_NE(ss, ":23:"); - CHECK_NE(ss, ":23"); - break; - case 2: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "45"); - CHECK_NE(ss, "45:"); - CHECK_NE(ss, ":45:"); - CHECK_NE(ss, ":45"); - break; - case 3: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "67"); - CHECK_NE(ss, "67:"); - CHECK_NE(ss, ":67:"); - CHECK_NE(ss, ":67"); - break; - default: - CHECK_UNARY(false);//GTEST_FAIL(); - break; - } - count++; - } - CHECK_EQ(count, 4); - } - - { - const S n(":01:23:45:67:"); - typename S::size_type pos = 0; - typename S::size_type count = 0; - S ss; - while(n.next_split(':', &pos, &ss)) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - case 1: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "01"); - CHECK_NE(ss, "01:"); - CHECK_NE(ss, ":01:"); - CHECK_NE(ss, ":01"); - break; - case 2: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "23"); - CHECK_NE(ss, "23:"); - CHECK_NE(ss, ":23:"); - CHECK_NE(ss, ":23"); - break; - case 3: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "45"); - CHECK_NE(ss, "45:"); - CHECK_NE(ss, ":45:"); - CHECK_NE(ss, ":45"); - break; - case 4: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "67"); - CHECK_NE(ss, "67:"); - CHECK_NE(ss, ":67:"); - CHECK_NE(ss, ":67"); - break; - case 5: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - default: - CHECK_UNARY(false);//GTEST_FAIL(); - break; - } - count++; - } - CHECK_EQ(count, 6); - } - - { - const S n("::::01:23:45:67::::"); - typename S::size_type pos = 0; - typename S::size_type count = 0; - S ss; - while(n.next_split(':', &pos, &ss)) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 1: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 2: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 3: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 4: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "01"); - CHECK_NE(ss, "01:"); - CHECK_NE(ss, ":01:"); - CHECK_NE(ss, ":01"); - break; - case 5: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "23"); - CHECK_NE(ss, "23:"); - CHECK_NE(ss, ":23:"); - CHECK_NE(ss, ":23"); - break; - case 6: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "45"); - CHECK_NE(ss, "45:"); - CHECK_NE(ss, ":45:"); - CHECK_NE(ss, ":45"); - break; - case 7: - CHECK_EQ(ss.size(), 2); - CHECK_EQ(ss, "67"); - CHECK_NE(ss, "67:"); - CHECK_NE(ss, ":67:"); - CHECK_NE(ss, ":67"); - break; - case 8: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 9: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 10: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 11: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - default: - CHECK_UNARY(false);//GTEST_FAIL(); - break; - } - count++; - } - CHECK_EQ(count, 12); - } -} - -TEST_CASE("substr.split") -{ - using S = csubstr; - - { - S const n; - { - auto spl = n.split(':'); - auto beg = spl.begin(); - auto end = spl.end(); - CHECK_UNARY(beg == end); - } - } - - { - S const n("foo:bar:baz"); - auto spl = n.split(':'); - auto beg = spl.begin(); - auto end = spl.end(); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(end->size(), 0); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - auto it = beg; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "foo"); - CHECK_UNARY(it != end); - CHECK_UNARY(it == beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - ++it; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "bar"); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - ++it; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "baz"); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - ++it; - CHECK_EQ(it->size(), 0); - CHECK_UNARY(it == end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - it = beg; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "foo"); - CHECK_UNARY(it != end); - CHECK_UNARY(it == beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - it++; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "bar"); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - it++; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "baz"); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - it++; - CHECK_EQ(it->size(), 0); - CHECK_UNARY(it == end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - } - - { - S const n("foo:bar:baz:"); - auto spl = n.split(':'); - auto beg = spl.begin(); - auto end = spl.end(); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(end->size(), 0); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - auto it = beg; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "foo"); - CHECK_UNARY(it != end); - CHECK_UNARY(it == beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - ++it; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "bar"); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - ++it; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "baz"); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - ++it; - CHECK_EQ(it->size(), 0); - CHECK_EQ(*it, ""); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - ++it; - CHECK_EQ(it->size(), 0); - CHECK_UNARY(it == end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - //-------------------------- - it = beg; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "foo"); - CHECK_UNARY(it != end); - CHECK_UNARY(it == beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - it++; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "bar"); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - it++; - CHECK_EQ(it->size(), 3); - CHECK_EQ(*it, "baz"); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - it++; - CHECK_EQ(it->size(), 0); - CHECK_EQ(*it, ""); - CHECK_UNARY(it != end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - it++; - CHECK_EQ(it->size(), 0); - CHECK_UNARY(it == end); - CHECK_UNARY(it != beg); - CHECK_EQ(beg->size(), 3); - CHECK_EQ(*beg, "foo"); - CHECK_UNARY(beg != end); - } - - { - S const n; - auto s = n.split(':'); - // check that multiple calls to begin() always yield the same result - CHECK_EQ(*s.begin(), ""); - CHECK_EQ(*s.begin(), ""); - CHECK_EQ(*s.begin(), ""); - // check that multiple calls to end() always yield the same result - auto e = s.end(); - CHECK_UNARY(s.end() == e); - CHECK_UNARY(s.end() == e); - // - auto it = s.begin(); - CHECK_EQ(*it, ""); - CHECK_EQ(it->empty(), true); - CHECK_EQ(it->size(), 0); - ++it; - CHECK_UNARY(it == e); - } - - { - S const n("01:23:45:67"); - auto s = n.split(':'); - // check that multiple calls to begin() always yield the same result - CHECK_EQ(*s.begin(), "01"); - CHECK_EQ(*s.begin(), "01"); - CHECK_EQ(*s.begin(), "01"); - // check that multiple calls to end() always yield the same result - auto e = s.end(); - CHECK_UNARY(s.end() == e); - CHECK_UNARY(s.end() == e); - CHECK_UNARY(s.end() == e); - // - auto it = s.begin(); - CHECK_EQ(*it, "01"); - CHECK_EQ(it->size(), 2); - ++it; - CHECK_EQ(*it, "23"); - CHECK_EQ(it->size(), 2); - ++it; - CHECK_EQ(*it, "45"); - CHECK_EQ(it->size(), 2); - ++it; - CHECK_EQ(*it, "67"); - CHECK_EQ(it->size(), 2); - ++it; - CHECK_UNARY(it == s.end()); - } - - { - S const n; - typename S::size_type count = 0; - for(auto &ss : n.split(':')) - { - ++count; - } - CHECK_EQ(count, 0); - } - - { - S const n("0123456"); - { - auto spl = n.split(':'); - auto beg = spl.begin(); - auto end = spl.end(); - CHECK_EQ(beg->size(), n.size()); - CHECK_EQ(end->size(), 0); - } - typename S::size_type count = 0; - for(auto &ss : n.split(':')) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), n.size()); - CHECK_EQ(ss.empty(), false); - break; - } - ++count; - } - CHECK_EQ(count, 1); - } - - { - S const n("foo:bar"); - typename S::size_type count = 0; - for(auto &ss : n.split(':')) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 3); - CHECK_EQ(ss.empty(), false); - CHECK_EQ(ss, "foo"); - break; - case 1: - CHECK_EQ(ss.size(), 3); - CHECK_EQ(ss.empty(), false); - CHECK_EQ(ss, "bar"); - break; - } - ++count; - } - CHECK_EQ(count, 2); - } - - { - S const n("0123456:"); - typename S::size_type count = 0; - for(auto &ss : n.split(':')) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), n.size()-1); - CHECK_EQ(ss.empty(), false); - break; - case 1: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - } - ++count; - } - CHECK_EQ(count, 2); - } - - { - S const n(":0123456:"); - typename S::size_type count = 0; - for(auto &ss : n.split(':')) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - case 1: - CHECK_EQ(ss.size(), n.size()-2); - CHECK_EQ(ss.empty(), false); - break; - case 2: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - } - ++count; - } - CHECK_EQ(count, 3); - } - - { - S const n(":"); - typename S::size_type count = 0; - for(auto &ss : n.split(':')) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - case 1: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - } - ++count; - } - CHECK_EQ(count, 2); - } - - { - S const n("01:23:45:67"); - typename S::size_type count = 0; - for(auto &ss : n.split(':')) - { - switch(count) - { - case 0: - CHECK_EQ(ss, "01"); - CHECK_NE(ss, "01:"); - CHECK_NE(ss, ":01:"); - CHECK_NE(ss, ":01"); - break; - case 1: - CHECK_EQ(ss, "23"); - CHECK_NE(ss, "23:"); - CHECK_NE(ss, ":23:"); - CHECK_NE(ss, ":23"); - break; - case 2: - CHECK_EQ(ss, "45"); - CHECK_NE(ss, "45:"); - CHECK_NE(ss, ":45:"); - CHECK_NE(ss, ":45"); - break; - case 3: - CHECK_EQ(ss, "67"); - CHECK_NE(ss, "67:"); - CHECK_NE(ss, ":67:"); - CHECK_NE(ss, ":67"); - break; - } - count++; - } - CHECK_EQ(count, 4); - } - - { - const S n(":01:23:45:67:"); - typename S::size_type count = 0; - for(auto &ss : n.split(':')) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - case 1: - CHECK_EQ(ss, "01"); - CHECK_NE(ss, "01:"); - CHECK_NE(ss, ":01:"); - CHECK_NE(ss, ":01"); - break; - case 2: - CHECK_EQ(ss, "23"); - CHECK_NE(ss, "23:"); - CHECK_NE(ss, ":23:"); - CHECK_NE(ss, ":23"); - break; - case 3: - CHECK_EQ(ss, "45"); - CHECK_NE(ss, "45:"); - CHECK_NE(ss, ":45:"); - CHECK_NE(ss, ":45"); - break; - case 4: - CHECK_EQ(ss, "67"); - CHECK_NE(ss, "67:"); - CHECK_NE(ss, ":67:"); - CHECK_NE(ss, ":67"); - break; - case 5: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - break; - } - count++; - } - CHECK_EQ(count, 6); - } - - { - const S n("::::01:23:45:67::::"); - typename S::size_type count = 0; - for(auto &ss : n.split(':')) - { - switch(count) - { - case 0: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 1: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 2: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 3: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 4: - CHECK_EQ(ss, "01"); - CHECK_NE(ss, "01:"); - CHECK_NE(ss, ":01:"); - CHECK_NE(ss, ":01"); - break; - case 5: - CHECK_EQ(ss, "23"); - CHECK_NE(ss, "23:"); - CHECK_NE(ss, ":23:"); - CHECK_NE(ss, ":23"); - break; - case 6: - CHECK_EQ(ss, "45"); - CHECK_NE(ss, "45:"); - CHECK_NE(ss, ":45:"); - CHECK_NE(ss, ":45"); - break; - case 7: - CHECK_EQ(ss, "67"); - CHECK_NE(ss, "67:"); - CHECK_NE(ss, ":67:"); - CHECK_NE(ss, ":67"); - break; - case 8: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 9: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 10: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - case 11: - CHECK_EQ(ss.size(), 0); - CHECK_EQ(ss.empty(), true); - CHECK_NE(ss, "::"); - break; - } - count++; - } - CHECK_EQ(count, 12); - } -} - - -//----------------------------------------------------------------------------- -TEST_CASE("substr.copy_from") -{ - char buf[128] = {0}; - substr s = buf; - CHECK_EQ(s.size(), sizeof(buf)-1); - CHECK_NE(s.first(3), "123"); - s.copy_from("123"); - CHECK_EQ(s.first(3), "123"); - CHECK_EQ(s.first(6), "123\0\0\0"); - s.copy_from("+++", 3); - CHECK_EQ(s.first(6), "123+++"); - CHECK_EQ(s.first(9), "123+++\0\0\0"); - s.copy_from("456", 6); - CHECK_EQ(s.first(9), "123+++456"); - CHECK_EQ(s.first(12), "123+++456\0\0\0"); - s.copy_from("***", 3); - CHECK_EQ(s.first(9), "123***456"); - CHECK_EQ(s.first(12), "123***456\0\0\0"); - - // make sure that it's safe to pass source strings that don't fit - // in the remaining destination space - substr ss = s.first(9); - ss.copy_from("987654321", 9); // should be a no-op - CHECK_EQ(s.first(12), "123***456\0\0\0"); - ss.copy_from("987654321", 6); - CHECK_EQ(s.first(12), "123***987\0\0\0"); - ss.copy_from("987654321", 3); - CHECK_EQ(s.first(12), "123987654\0\0\0"); - ss.first(3).copy_from("987654321"); - CHECK_EQ(s.first(12), "987987654\0\0\0"); -} - - -//----------------------------------------------------------------------------- -void do_test_reverse(substr s, csubstr orig, csubstr expected) -{ - CHECK_EQ(s, orig); - s.reverse(); - CHECK_EQ(s, expected); - s.reverse(); - CHECK_EQ(s, orig); - // - CHECK_EQ(s, orig); - s.reverse_sub(0, s.len); - CHECK_EQ(s, expected); - s.reverse_sub(0, s.len); - CHECK_EQ(s, orig); - // - CHECK_EQ(s, orig); - s.reverse_range(0, s.len); - CHECK_EQ(s, expected); - s.reverse_range(0, s.len); - CHECK_EQ(s, orig); -} - -TEST_CASE("substr.reverse") -{ - char buf[] = "0123456789"; - do_test_reverse(buf, "0123456789", "9876543210"); - do_test_reverse(buf, "0123456789", "9876543210"); - - // in the middle - substr s = buf; - s.sub(2, 2).reverse(); - CHECK_EQ(s, "0132456789"); - s.sub(2, 2).reverse(); - CHECK_EQ(s, "0123456789"); - - s.sub(4, 2).reverse(); - CHECK_EQ(s, "0123546789"); - s.sub(4, 2).reverse(); - CHECK_EQ(s, "0123456789"); - - // at the beginning - s.first(3).reverse(); - CHECK_EQ(s, "2103456789"); - s.first(3).reverse(); - CHECK_EQ(s, "0123456789"); - - // at the end - s.last(3).reverse(); - CHECK_EQ(s, "0123456987"); - s.last(3).reverse(); - CHECK_EQ(s, "0123456789"); -} - - -TEST_CASE("substr.reverse_sub") -{ - char buf[] = "0123456789"; - substr s = buf; - - s.reverse_sub(0, s.len); - CHECK_EQ(s, "9876543210"); - s.reverse_sub(0, s.len); - CHECK_EQ(s, "0123456789"); - - s.reverse_sub(0, 0); - CHECK_EQ(s, "0123456789"); - s.reverse_sub(s.len, 0); - CHECK_EQ(s, "0123456789"); - - s.reverse_sub(1, 3); - CHECK_EQ(s, "0321456789"); - s.reverse_sub(1, 3); - CHECK_EQ(s, "0123456789"); -} - -TEST_CASE("substr.reverse_range") -{ - char buf[] = "0123456789"; - substr s = buf; - - s.reverse_range(0, s.len); - CHECK_EQ(s, "9876543210"); - s.reverse_range(0, s.len); - CHECK_EQ(s, "0123456789"); - - s.reverse_range(0, 0); - CHECK_EQ(s, "0123456789"); - s.reverse_range(s.len, s.len); - CHECK_EQ(s, "0123456789"); - - s.reverse_range(1, 3); - CHECK_EQ(s, "0213456789"); - s.reverse_range(1, 3); - CHECK_EQ(s, "0123456789"); -} - - -//----------------------------------------------------------------------------- -TEST_CASE("substr.erase") -{ - char buf[] = "0123456789"; - - substr s = buf; - CHECK_EQ(s.len, s.size()); - CHECK_EQ(s.len, 10); - CHECK_EQ(s, "0123456789"); - - substr ss = s.first(6); - CHECK_EQ(ss.len, 6); - for(size_t i = 0; i <= ss.len; ++i) - { - ss.erase(i, 0); // must be a no-op - CHECK_EQ(s, "0123456789"); - ss.erase_range(i, i); // must be a no-op - CHECK_EQ(s, "0123456789"); - ss.erase(ss.len-i, i); // must be a no-op - CHECK_EQ(s, "0123456789"); - } - - substr r; - ss = ss.erase(0, 1); - CHECK_EQ(ss.len, 5); - CHECK_EQ(ss, "12345"); - CHECK_EQ(s, "1234556789"); - ss = ss.erase(0, 2); - CHECK_EQ(ss.len, 3); - CHECK_EQ(ss, "345"); - CHECK_EQ(s, "3454556789"); - - csubstr s55 = s.sub(4, 2); - ss = s.erase(s55); - CHECK_EQ(s, "3454678989"); -} - - -//----------------------------------------------------------------------------- -TEST_CASE("substr.replace") -{ - char buf[] = "0.1.2.3.4.5.6.7.8.9"; - - substr s = buf; - - auto ret = s.replace('+', '.'); - CHECK_EQ(ret, 0); - - ret = s.replace('.', '.', s.len); - CHECK_EQ(s, "0.1.2.3.4.5.6.7.8.9"); - CHECK_EQ(ret, 0); - ret = s.replace('.', '.'); - CHECK_EQ(ret, 9); - CHECK_EQ(s, "0.1.2.3.4.5.6.7.8.9"); - - ret = s.replace('.', '+', s.len); - CHECK_EQ(s, "0.1.2.3.4.5.6.7.8.9"); - CHECK_EQ(ret, 0); - ret = s.replace('.', '+'); - CHECK_EQ(ret, 9); - CHECK_EQ(s, "0+1+2+3+4+5+6+7+8+9"); - - ret = s.replace("16", '.', s.len); - CHECK_EQ(s, "0+1+2+3+4+5+6+7+8+9"); - CHECK_EQ(ret, 0); - ret = s.replace("16", '.'); - CHECK_EQ(ret, 2); - CHECK_EQ(s, "0+.+2+3+4+5+.+7+8+9"); - ret = s.replace("3+2", '_'); - CHECK_EQ(ret, 11); - CHECK_EQ(s, "0_._____4_5_._7_8_9"); - - // must accept empty string - ret = s.sub(0, 0).replace('0', '1'); - CHECK_EQ(ret, 0); - CHECK_EQ(s, "0_._____4_5_._7_8_9"); - ret = s.sub(0, 0).replace("0", '1'); - CHECK_EQ(ret, 0); - CHECK_EQ(s, "0_._____4_5_._7_8_9"); -} - -TEST_CASE("substr.replace_all") -{ - char buf[] = "0.1.2.3.4.5.6.7.8.9"; - std::string tmp, out("0+1+2+3+4+5+6+7+8+9"); - - // must accept empty string - substr(buf).sub(0, 0).replace_all(to_substr(tmp), "0", "X"); - CHECK_EQ(csubstr(buf), "0.1.2.3.4.5.6.7.8.9"); - - substr r; - auto replall = [&](csubstr pattern, csubstr repl) -> substr { - tmp = out; - csubstr rtmp = to_csubstr(tmp); - out.resize(128); - substr dst = to_substr(out); - size_t sz = rtmp.replace_all(dst, pattern, repl); - CHECK_LE(sz, out.size()); - out.resize(sz); - return dst.first(sz); - }; - r = replall("0+1", "0+++++1"); - // the result must be a view of out - CHECK_FALSE(r.empty()); - CHECK_FALSE(out.empty()); - CHECK_EQ(r.size(), out.size()); - CHECK_EQ(r.front(), out.front()); - CHECK_EQ(r.back(), out.back()); - CHECK_EQ(r, "0+++++1+2+3+4+5+6+7+8+9"); - - r = replall("+", ""); - CHECK_EQ(r, "0123456789"); - - r = replall("+", ""); - CHECK_EQ(r, "0123456789"); // must not change - - r = replall("0123456789", "9876543210"); - CHECK_EQ(r, "9876543210"); - - r = replall("987", "."); - CHECK_EQ(r, ".6543210"); - - r = replall("210", "."); - CHECK_EQ(r, ".6543."); - - r = replall("6543", ":"); - CHECK_EQ(r, ".:."); - - r = replall(".:.", ""); - CHECK_EQ(r, ""); -} - -TEST_CASE("substr.short_integer") -{ - char buf[] = "-"; - CHECK_FALSE(substr(buf).is_integer()); - CHECK_FALSE(csubstr("-").is_integer()); - CHECK_FALSE(csubstr("+").is_integer()); -} - -} // namespace c4 - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_szconv.cpp b/thirdparty/ryml/ext/c4core/test/test_szconv.cpp deleted file mode 100644 index d54bf48a0..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_szconv.cpp +++ /dev/null @@ -1,166 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/szconv.hpp" -#endif - -#include "c4/libtest/supprwarn_push.hpp" -#include "c4/test.hpp" - -#define C4_EXPECT_NARROWER(yes_or_no, ty_out, ty_in) \ - CHECK_UNARY(( yes_or_no (is_narrower_size::value))); - -namespace c4 { - -TEST_CASE("is_narrower_size.signed_types") -{ - C4_EXPECT_NARROWER( ! , int8_t , int8_t ); - C4_EXPECT_NARROWER( , int8_t , int16_t); - C4_EXPECT_NARROWER( , int8_t , int32_t); - C4_EXPECT_NARROWER( , int8_t , int64_t); - C4_EXPECT_NARROWER( , int8_t , uint8_t ); - C4_EXPECT_NARROWER( , int8_t , uint16_t); - C4_EXPECT_NARROWER( , int8_t , uint32_t); - C4_EXPECT_NARROWER( , int8_t , uint64_t); - C4_EXPECT_NARROWER( ! , int16_t , int8_t ); - C4_EXPECT_NARROWER( ! , int16_t , int16_t); - C4_EXPECT_NARROWER( , int16_t , int32_t); - C4_EXPECT_NARROWER( , int16_t , int64_t); - C4_EXPECT_NARROWER( ! , int16_t , uint8_t ); - C4_EXPECT_NARROWER( , int16_t , uint16_t); - C4_EXPECT_NARROWER( , int16_t , uint32_t); - C4_EXPECT_NARROWER( , int16_t , uint64_t); - C4_EXPECT_NARROWER( ! , int32_t , int8_t ); - C4_EXPECT_NARROWER( ! , int32_t , int16_t); - C4_EXPECT_NARROWER( ! , int32_t , int32_t); - C4_EXPECT_NARROWER( , int32_t , int64_t); - C4_EXPECT_NARROWER( ! , int32_t , uint8_t ); - C4_EXPECT_NARROWER( ! , int32_t , uint16_t); - C4_EXPECT_NARROWER( , int32_t , uint32_t); - C4_EXPECT_NARROWER( , int32_t , uint64_t); - C4_EXPECT_NARROWER( ! , int64_t , int8_t ); - C4_EXPECT_NARROWER( ! , int64_t , int16_t); - C4_EXPECT_NARROWER( ! , int64_t , int32_t); - C4_EXPECT_NARROWER( ! , int64_t , int64_t); - C4_EXPECT_NARROWER( ! , int64_t , uint8_t ); - C4_EXPECT_NARROWER( ! , int64_t , uint16_t); - C4_EXPECT_NARROWER( ! , int64_t , uint32_t); - C4_EXPECT_NARROWER( , int64_t , uint64_t); -} - -TEST_CASE("is_narrower_size.unsigned_types") -{ - C4_EXPECT_NARROWER( ! , uint8_t , int8_t ); - C4_EXPECT_NARROWER( , uint8_t , int16_t); - C4_EXPECT_NARROWER( , uint8_t , int32_t); - C4_EXPECT_NARROWER( , uint8_t , int64_t); - C4_EXPECT_NARROWER( ! , uint8_t , uint8_t ); - C4_EXPECT_NARROWER( , uint8_t , uint16_t); - C4_EXPECT_NARROWER( , uint8_t , uint32_t); - C4_EXPECT_NARROWER( , uint8_t , uint64_t); - C4_EXPECT_NARROWER( ! , uint16_t , int8_t ); - C4_EXPECT_NARROWER( ! , uint16_t , int16_t); - C4_EXPECT_NARROWER( , uint16_t , int32_t); - C4_EXPECT_NARROWER( , uint16_t , int64_t); - C4_EXPECT_NARROWER( ! , uint16_t , uint8_t ); - C4_EXPECT_NARROWER( ! , uint16_t , uint16_t); - C4_EXPECT_NARROWER( , uint16_t , uint32_t); - C4_EXPECT_NARROWER( , uint16_t , uint64_t); - C4_EXPECT_NARROWER( ! , uint32_t , int8_t ); - C4_EXPECT_NARROWER( ! , uint32_t , int16_t); - C4_EXPECT_NARROWER( ! , uint32_t , int32_t); - C4_EXPECT_NARROWER( , uint32_t , int64_t); - C4_EXPECT_NARROWER( ! , uint32_t , uint8_t ); - C4_EXPECT_NARROWER( ! , uint32_t , uint16_t); - C4_EXPECT_NARROWER( ! , uint32_t , uint32_t); - C4_EXPECT_NARROWER( , uint32_t , uint64_t); - C4_EXPECT_NARROWER( ! , uint64_t , int8_t ); - C4_EXPECT_NARROWER( ! , uint64_t , int16_t); - C4_EXPECT_NARROWER( ! , uint64_t , int32_t); - C4_EXPECT_NARROWER( ! , uint64_t , int64_t); - C4_EXPECT_NARROWER( ! , uint64_t , uint8_t ); - C4_EXPECT_NARROWER( ! , uint64_t , uint16_t); - C4_EXPECT_NARROWER( ! , uint64_t , uint32_t); - C4_EXPECT_NARROWER( ! , uint64_t , uint64_t); -} - -template -typename std::enable_if::value, void>::type -test_szconv() -{ - // nothing to do here -} - -template -typename std::enable_if< ! std::is_same::value, void>::type -test_szconv() -{ - C4_STATIC_ASSERT(std::is_integral::value); - C4_STATIC_ASSERT(std::is_integral::value); - - const I imax = std::numeric_limits::max(); - const I imin = std::numeric_limits::min(); - const O omax = std::numeric_limits::max(); - const O omin = std::numeric_limits::min(); - - CHECK_EQ(szconv(I(0)), O(0)); - CHECK_EQ(szconv(I(0)), I(0)); - -#if C4_USE_XASSERT - if((uint64_t)omax < (uint64_t)imax) - { - C4_EXPECT_ERROR_OCCURS(); - O out = szconv(imax); - } - else if((uint64_t)omax > (uint64_t)imax) - { - C4_EXPECT_ERROR_OCCURS(); - I out = szconv(omax); - } -#endif -} - -#define DO_TEST_SZCONV(ty) \ - TEST_CASE("szconv." #ty "_to_int8") \ - { \ - test_szconv(); \ - } \ - TEST_CASE("zconv." #ty "_to_uint8") \ - { \ - test_szconv(); \ - } \ - TEST_CASE("zconv." #ty "_to_int16") \ - { \ - test_szconv(); \ - } \ - TEST_CASE("zconv." #ty "_to_uint16") \ - { \ - test_szconv(); \ - } \ - TEST_CASE("zconv." #ty "_to_int32") \ - { \ - test_szconv(); \ - } \ - TEST_CASE("zconv." #ty "_to_uint32") \ - { \ - test_szconv(); \ - } \ - TEST_CASE("zconv." #ty "_to_int64") \ - { \ - test_szconv(); \ - } \ - TEST_CASE("szconv." #ty "_to_uint64") \ - { \ - test_szconv(); \ - } - -DO_TEST_SZCONV(int8) -DO_TEST_SZCONV(uint8) -DO_TEST_SZCONV(int16) -DO_TEST_SZCONV(uint16) -DO_TEST_SZCONV(int32) -DO_TEST_SZCONV(uint32) -DO_TEST_SZCONV(int64) -DO_TEST_SZCONV(uint64) - -} // namespace c4 - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_type_name.cpp b/thirdparty/ryml/ext/c4core/test/test_type_name.cpp deleted file mode 100644 index 422676bcf..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_type_name.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "c4/test.hpp" - -#ifndef C4CORE_SINGLE_HEADER -#include "c4/type_name.hpp" -#endif - -class SomeTypeName {}; -struct SomeStructName {}; - -namespace c4 { - -class SomeTypeNameInsideANamespace {}; -struct SomeStructNameInsideANamespace {}; - -template -cspan cstr(const char (&s)[N]) -{ - cspan o(s, N-1); - return o; -} - -TEST_CASE("type_name.intrinsic_types") -{ - CHECK_EQ(type_name(), cstr("int")); - CHECK_EQ(type_name(), cstr("float")); - CHECK_EQ(type_name(), cstr("double")); -} -TEST_CASE("type_name.classes") -{ - CHECK_EQ(type_name(), cstr("SomeTypeName")); - CHECK_EQ(type_name<::SomeTypeName>(), cstr("SomeTypeName")); -} -TEST_CASE("type_name.structs") -{ - CHECK_EQ(type_name(), cstr("SomeStructName")); - CHECK_EQ(type_name<::SomeStructName>(), cstr("SomeStructName")); -} -TEST_CASE("type_name.inside_namespace") -{ - CHECK_EQ(type_name(), cstr("c4::SomeTypeNameInsideANamespace")); - CHECK_EQ(type_name(), cstr("c4::SomeTypeNameInsideANamespace")); - CHECK_EQ(type_name<::c4::SomeTypeNameInsideANamespace>(), cstr("c4::SomeTypeNameInsideANamespace")); - - CHECK_EQ(type_name(), cstr("c4::SomeStructNameInsideANamespace")); - CHECK_EQ(type_name(), cstr("c4::SomeStructNameInsideANamespace")); - CHECK_EQ(type_name<::c4::SomeStructNameInsideANamespace>(), cstr("c4::SomeStructNameInsideANamespace")); -} - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/test/test_types.cpp b/thirdparty/ryml/ext/c4core/test/test_types.cpp deleted file mode 100644 index 697ce289d..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_types.cpp +++ /dev/null @@ -1,81 +0,0 @@ -#ifndef C4CORE_SINGLE_HEADER -#include "c4/config.hpp" -#endif -#include - -#include "c4/libtest/supprwarn_push.hpp" -#include "c4/test.hpp" - -namespace c4 { - -C4_STATIC_ASSERT((std::is_same< fastcref< char >, char >::value)); -C4_STATIC_ASSERT((std::is_same< fastcref< i8 >, i8 >::value)); -C4_STATIC_ASSERT((std::is_same< fastcref< u8 >, u8 >::value)); -C4_STATIC_ASSERT((std::is_same< fastcref< i16 >, i16 >::value)); -C4_STATIC_ASSERT((std::is_same< fastcref< u16 >, u16 >::value)); -C4_STATIC_ASSERT((std::is_same< fastcref< i32 >, i32 >::value)); -C4_STATIC_ASSERT((std::is_same< fastcref< u32 >, u32 >::value)); -C4_STATIC_ASSERT((std::is_same< fastcref< i64 >, i64 >::value)); -C4_STATIC_ASSERT((std::is_same< fastcref< u64 >, u64 >::value)); - -using carr64 = char[64]; -C4_STATIC_ASSERT((std::is_same< fastcref< carr64 >, carr64 const& >::value)); -C4_STATIC_ASSERT((std::is_same< fastcref< std::string >, std::string const& >::value)); - -//----------------------------------------------------------------------------- - -C4_BEGIN_HIDDEN_NAMESPACE -template< class T > struct ufonix { T a; }; -using F = ufonix< uint32_t >; -C4_END_HIDDEN_NAMESPACE - -TEST_CASE("TestSizeStructs.min_remainder") -{ - CHECK_EQ(min_remainder(4, 6), 2); - CHECK_EQ(min_remainder(6, 6), 0); - CHECK_EQ(min_remainder(8, 6), 0); -} - -TEST_CASE("TestSizeStructs.mult_remainder") -{ - CHECK_EQ(mult_remainder(6, 1), 0); - CHECK_EQ(mult_remainder(6, 2), 0); - CHECK_EQ(mult_remainder(6, 3), 0); - CHECK_EQ(mult_remainder(6, 4), 2); - CHECK_EQ(mult_remainder(6, 5), 4); - CHECK_EQ(mult_remainder(6, 6), 0); - CHECK_EQ(mult_remainder(6, 7), 1); -} -TEST_CASE("TestSizeStructs.Padded") -{ - CHECK_EQ(sizeof(F), sizeof(uint32_t)); - CHECK_EQ((sizeof(Padded< F, 0 >)), sizeof(F)); - CHECK_EQ((sizeof(Padded< F, 1 >)), sizeof(F)+1); - CHECK_EQ((sizeof(Padded< F, 2 >)), sizeof(F)+2); - CHECK_EQ((sizeof(Padded< F, 3 >)), sizeof(F)+3); -} -TEST_CASE("TestSizeStructs.MinSized") -{ - CHECK_EQ((sizeof(MinSized< F, 14 >)), 14); - CHECK_EQ((sizeof(MinSized< F, 15 >)), 15); - CHECK_EQ((sizeof(MinSized< F, 16 >)), 16); - CHECK_EQ((sizeof(MinSized< F, 17 >)), 17); -} -TEST_CASE("TestSizeStructs.MultSized") -{ - using G = ufonix< char[8] >; - CHECK_EQ((sizeof(MultSized< G, 7 >)), 14); - CHECK_EQ((sizeof(MultSized< G, 6 >)), 12); - CHECK_EQ((sizeof(MultSized< G, 5 >)), 10); - CHECK_EQ((sizeof(MultSized< G, 4 >)), 8); -} -TEST_CASE("TestSizeStructs.UbufSized") -{ - CHECK_EQ((sizeof(UbufSized>)), 64); - CHECK_EQ((sizeof(UbufSized>)), 64); - CHECK_EQ((sizeof(UbufSized>)), 80); -} - -} // namespace c4 - -#include "c4/libtest/supprwarn_pop.hpp" diff --git a/thirdparty/ryml/ext/c4core/test/test_utf.cpp b/thirdparty/ryml/ext/c4core/test/test_utf.cpp deleted file mode 100644 index ffa4fb338..000000000 --- a/thirdparty/ryml/ext/c4core/test/test_utf.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include "c4/test.hpp" -#ifndef C4CORE_SINGLE_HEADER -#include "c4/std/string.hpp" -#include "c4/std/vector.hpp" -#include "c4/format.hpp" -#include "c4/utf.hpp" -#endif - -#include "c4/libtest/supprwarn_push.hpp" - -#include - -namespace c4 { - -struct utft -{ - csubstr code_point; - csubstr character; - uint32_t character_val; - csubstr character_val_hex; -}; -constexpr const utft utf_chars[] = { -#include "./utfchars.inc" -}; - -TEST_CASE("utf.decode_code_point") -{ - size_t i = 0; - char decoded_buf[64]; - for(auto uc : utf_chars) - { - INFO("utfchars[", i, "]: codepoint=", uc.code_point, ' ', - "character=", uc.character.empty() ? csubstr{} : uc.character, ' ', - "val=", uc.character_val_hex, '(', uc.character_val, ')'); - i++; - csubstr cpstr = uc.code_point.sub(2).triml('0'); - if(cpstr.empty()) - continue; - csubstr decoded = decode_code_point(decoded_buf, cpstr); - CHECK_UNARY(uc.code_point.begins_with("U+")); - if(uc.character.empty()) - continue; - CHECK_EQ(decoded.len, uc.character.len); - CHECK_EQ(decoded, uc.character); - } -} - -} // namespace c4 diff --git a/thirdparty/ryml/ext/c4core/test/utfchars.inc b/thirdparty/ryml/ext/c4core/test/utfchars.inc deleted file mode 100644 index b0b23d4c0..000000000 --- a/thirdparty/ryml/ext/c4core/test/utfchars.inc +++ /dev/null @@ -1,6274 +0,0 @@ -// https://www.utf8-chartable.de/unicode-utf8-table.pl -#define _c(cp, wysiwyg, hex) utft{csubstr{#cp}, csubstr{wysiwyg}, UINT32_C(0x##hex), csubstr{"0x" #hex}} -_c(U+0000, "\0", 00), -_c(U+0001, "", 01), -_c(U+0002, "", 02), -_c(U+0003, "", 03), -_c(U+0004, "", 04), -_c(U+0005, "", 05), -_c(U+0006, "", 06), -_c(U+0007, "", 07), -_c(U+0008, "\b", 08), -_c(U+0009, "\t", 09), -_c(U+000A, "\n", 0a), -_c(U+000B, "", 0b), -_c(U+000C, "", 0c), -_c(U+000D, "\r", 0d), -_c(U+000E, "", 0e), -_c(U+000F, "", 0f), -_c(U+0010, "", 10), -_c(U+0011, "", 11), -_c(U+0012, "", 12), -_c(U+0013, "", 13), -_c(U+0014, "", 14), -_c(U+0015, "", 15), -_c(U+0016, "", 16), -_c(U+0017, "", 17), -_c(U+0018, "", 18), -_c(U+0019, "", 19), -_c(U+001A, "", 1a), -_c(U+001B, "", 1b), -_c(U+001C, "", 1c), -_c(U+001D, "", 1d), -_c(U+001E, "", 1e), -_c(U+001F, "", 1f), -_c(U+0020, " ", 20), -_c(U+0021, "!", 21), -_c(U+0022, "\"", 22), -_c(U+0023, "#", 23), -_c(U+0024, "$", 24), -_c(U+0025, "%", 25), -_c(U+0026, "&", 26), -_c(U+0027, "'", 27), -_c(U+0028, "(", 28), -_c(U+0029, ")", 29), -_c(U+002A, "*", 2a), -_c(U+002B, "+", 2b), -_c(U+002C, ",", 2c), -_c(U+002D, "-", 2d), -_c(U+002E, ".", 2e), -_c(U+002F, "/", 2f), -_c(U+0030, "0", 30), -_c(U+0031, "1", 31), -_c(U+0032, "2", 32), -_c(U+0033, "3", 33), -_c(U+0034, "4", 34), -_c(U+0035, "5", 35), -_c(U+0036, "6", 36), -_c(U+0037, "7", 37), -_c(U+0038, "8", 38), -_c(U+0039, "9", 39), -_c(U+003A, ":", 3a), -_c(U+003B, ";", 3b), -_c(U+003C, "<", 3c), -_c(U+003D, "=", 3d), -_c(U+003E, ">", 3e), -_c(U+003F, "?", 3f), -_c(U+0040, "@", 40), -_c(U+0041, "A", 41), -_c(U+0042, "B", 42), -_c(U+0043, "C", 43), -_c(U+0044, "D", 44), -_c(U+0045, "E", 45), -_c(U+0046, "F", 46), -_c(U+0047, "G", 47), -_c(U+0048, "H", 48), -_c(U+0049, "I", 49), -_c(U+004A, "J", 4a), -_c(U+004B, "K", 4b), -_c(U+004C, "L", 4c), -_c(U+004D, "M", 4d), -_c(U+004E, "N", 4e), -_c(U+004F, "O", 4f), -_c(U+0050, "P", 50), -_c(U+0051, "Q", 51), -_c(U+0052, "R", 52), -_c(U+0053, "S", 53), -_c(U+0054, "T", 54), -_c(U+0055, "U", 55), -_c(U+0056, "V", 56), -_c(U+0057, "W", 57), -_c(U+0058, "X", 58), -_c(U+0059, "Y", 59), -_c(U+005A, "Z", 5a), -_c(U+005B, "[", 5b), -_c(U+005C, "\\", 5c), -_c(U+005D, "]", 5d), -_c(U+005E, "^", 5e), -_c(U+005F, "_", 5f), -_c(U+0060, "`", 60), -_c(U+0061, "a", 61), -_c(U+0062, "b", 62), -_c(U+0063, "c", 63), -_c(U+0064, "d", 64), -_c(U+0065, "e", 65), -_c(U+0066, "f", 66), -_c(U+0067, "g", 67), -_c(U+0068, "h", 68), -_c(U+0069, "i", 69), -_c(U+006A, "j", 6a), -_c(U+006B, "k", 6b), -_c(U+006C, "l", 6c), -_c(U+006D, "m", 6d), -_c(U+006E, "n", 6e), -_c(U+006F, "o", 6f), -_c(U+0070, "p", 70), -_c(U+0071, "q", 71), -_c(U+0072, "r", 72), -_c(U+0073, "s", 73), -_c(U+0074, "t", 74), -_c(U+0075, "u", 75), -_c(U+0076, "v", 76), -_c(U+0077, "w", 77), -_c(U+0078, "x", 78), -_c(U+0079, "y", 79), -_c(U+007A, "z", 7a), -_c(U+007B, "", 7b), -_c(U+007C, "|", 7c), -_c(U+007D, "}", 7d), -_c(U+007E, "~", 7e), -_c(U+007F, "", 7f), // del -_c(U+0080, "", c280), -_c(U+0081, "", c281), -_c(U+0082, "", c282), -_c(U+0083, "", c283), -_c(U+0084, "", c284), -_c(U+0085, "", c285), -_c(U+0086, "", c286), -_c(U+0087, "", c287), -_c(U+0088, "", c288), -_c(U+0089, "", c289), -_c(U+008A, "", c28a), -_c(U+008B, "", c28b), -_c(U+008C, "", c28c), -_c(U+008D, "", c28d), -_c(U+008E, "", c28e), -_c(U+008F, "", c28f), -_c(U+0090, "", c290), -_c(U+0091, "", c291), -_c(U+0092, "", c292), -_c(U+0093, "", c293), -_c(U+0094, "", c294), -_c(U+0095, "", c295), -_c(U+0096, "", c296), -_c(U+0097, "", c297), -_c(U+0098, "", c298), -_c(U+0099, "", c299), -_c(U+009A, "", c29a), -_c(U+009B, "", c29b), -_c(U+009C, "", c29c), -_c(U+009D, "", c29d), -_c(U+009E, "", c29e), -_c(U+009F, "", c29f), -_c(U+00A0, "", c2a0), -_c(U+00A1, "¡", c2a1), -_c(U+00A2, "¢", c2a2), -_c(U+00A3, "£", c2a3), -_c(U+00A4, "¤", c2a4), -_c(U+00A5, "¥", c2a5), -_c(U+00A6, "¦", c2a6), -_c(U+00A7, "§", c2a7), -_c(U+00A8, "¨", c2a8), -_c(U+00A9, "©", c2a9), -_c(U+00AA, "ª", c2aa), -_c(U+00AB, "«", c2ab), -_c(U+00AC, "¬", c2ac), -_c(U+00AD, "­", c2ad), -_c(U+00AE, "®", c2ae), -_c(U+00AF, "¯", c2af), -_c(U+00B0, "°", c2b0), -_c(U+00B1, "±", c2b1), -_c(U+00B2, "²", c2b2), -_c(U+00B3, "³", c2b3), -_c(U+00B4, "´", c2b4), -_c(U+00B5, "µ", c2b5), -_c(U+00B6, "¶", c2b6), -_c(U+00B7, "·", c2b7), -_c(U+00B8, "¸", c2b8), -_c(U+00B9, "¹", c2b9), -_c(U+00BA, "º", c2ba), -_c(U+00BB, "»", c2bb), -_c(U+00BC, "¼", c2bc), -_c(U+00BD, "½", c2bd), -_c(U+00BE, "¾", c2be), -_c(U+00BF, "¿", c2bf), -_c(U+00C0, "À", c380), -_c(U+00C1, "Á", c381), -_c(U+00C2, "Â", c382), -_c(U+00C3, "Ã", c383), -_c(U+00C4, "Ä", c384), -_c(U+00C5, "Å", c385), -_c(U+00C6, "Æ", c386), -_c(U+00C7, "Ç", c387), -_c(U+00C8, "È", c388), -_c(U+00C9, "É", c389), -_c(U+00CA, "Ê", c38a), -_c(U+00CB, "Ë", c38b), -_c(U+00CC, "Ì", c38c), -_c(U+00CD, "Í", c38d), -_c(U+00CE, "Î", c38e), -_c(U+00CF, "Ï", c38f), -_c(U+00D0, "Ð", c390), -_c(U+00D1, "Ñ", c391), -_c(U+00D2, "Ò", c392), -_c(U+00D3, "Ó", c393), -_c(U+00D4, "Ô", c394), -_c(U+00D5, "Õ", c395), -_c(U+00D6, "Ö", c396), -_c(U+00D7, "×", c397), -_c(U+00D8, "Ø", c398), -_c(U+00D9, "Ù", c399), -_c(U+00DA, "Ú", c39a), -_c(U+00DB, "Û", c39b), -_c(U+00DC, "Ü", c39c), -_c(U+00DD, "Ý", c39d), -_c(U+00DE, "Þ", c39e), -_c(U+00DF, "ß", c39f), -_c(U+00E0, "à", c3a0), -_c(U+00E1, "á", c3a1), -_c(U+00E2, "â", c3a2), -_c(U+00E3, "ã", c3a3), -_c(U+00E4, "ä", c3a4), -_c(U+00E5, "å", c3a5), -_c(U+00E6, "æ", c3a6), -_c(U+00E7, "ç", c3a7), -_c(U+00E8, "è", c3a8), -_c(U+00E9, "é", c3a9), -_c(U+00EA, "ê", c3aa), -_c(U+00EB, "ë", c3ab), -_c(U+00EC, "ì", c3ac), -_c(U+00ED, "í", c3ad), -_c(U+00EE, "î", c3ae), -_c(U+00EF, "ï", c3af), -_c(U+00F0, "ð", c3b0), -_c(U+00F1, "ñ", c3b1), -_c(U+00F2, "ò", c3b2), -_c(U+00F3, "ó", c3b3), -_c(U+00F4, "ô", c3b4), -_c(U+00F5, "õ", c3b5), -_c(U+00F6, "ö", c3b6), -_c(U+00F7, "÷", c3b7), -_c(U+00F8, "ø", c3b8), -_c(U+00F9, "ù", c3b9), -_c(U+00FA, "ú", c3ba), -_c(U+00FB, "û", c3bb), -_c(U+00FC, "ü", c3bc), -_c(U+00FD, "ý", c3bd), -_c(U+00FE, "þ", c3be), -_c(U+00FF, "ÿ", c3bf), -_c(U+0100, "Ā", c480), -_c(U+0101, "ā", c481), -_c(U+0102, "Ă", c482), -_c(U+0103, "ă", c483), -_c(U+0104, "Ą", c484), -_c(U+0105, "ą", c485), -_c(U+0106, "Ć", c486), -_c(U+0107, "ć", c487), -_c(U+0108, "Ĉ", c488), -_c(U+0109, "ĉ", c489), -_c(U+010A, "Ċ", c48a), -_c(U+010B, "ċ", c48b), -_c(U+010C, "Č", c48c), -_c(U+010D, "č", c48d), -_c(U+010E, "Ď", c48e), -_c(U+010F, "ď", c48f), -_c(U+0110, "Đ", c490), -_c(U+0111, "đ", c491), -_c(U+0112, "Ē", c492), -_c(U+0113, "ē", c493), -_c(U+0114, "Ĕ", c494), -_c(U+0115, "ĕ", c495), -_c(U+0116, "Ė", c496), -_c(U+0117, "ė", c497), -_c(U+0118, "Ę", c498), -_c(U+0119, "ę", c499), -_c(U+011A, "Ě", c49a), -_c(U+011B, "ě", c49b), -_c(U+011C, "Ĝ", c49c), -_c(U+011D, "ĝ", c49d), -_c(U+011E, "Ğ", c49e), -_c(U+011F, "ğ", c49f), -_c(U+0120, "Ġ", c4a0), -_c(U+0121, "ġ", c4a1), -_c(U+0122, "Ģ", c4a2), -_c(U+0123, "ģ", c4a3), -_c(U+0124, "Ĥ", c4a4), -_c(U+0125, "ĥ", c4a5), -_c(U+0126, "Ħ", c4a6), -_c(U+0127, "ħ", c4a7), -_c(U+0128, "Ĩ", c4a8), -_c(U+0129, "ĩ", c4a9), -_c(U+012A, "Ī", c4aa), -_c(U+012B, "ī", c4ab), -_c(U+012C, "Ĭ", c4ac), -_c(U+012D, "ĭ", c4ad), -_c(U+012E, "Į", c4ae), -_c(U+012F, "į", c4af), -_c(U+0130, "İ", c4b0), -_c(U+0131, "ı", c4b1), -_c(U+0132, "IJ", c4b2), -_c(U+0133, "ij", c4b3), -_c(U+0134, "Ĵ", c4b4), -_c(U+0135, "ĵ", c4b5), -_c(U+0136, "Ķ", c4b6), -_c(U+0137, "ķ", c4b7), -_c(U+0138, "ĸ", c4b8), -_c(U+0139, "Ĺ", c4b9), -_c(U+013A, "ĺ", c4ba), -_c(U+013B, "Ļ", c4bb), -_c(U+013C, "ļ", c4bc), -_c(U+013D, "Ľ", c4bd), -_c(U+013E, "ľ", c4be), -_c(U+013F, "Ŀ", c4bf), -_c(U+0140, "ŀ", c580), -_c(U+0141, "Ł", c581), -_c(U+0142, "ł", c582), -_c(U+0143, "Ń", c583), -_c(U+0144, "ń", c584), -_c(U+0145, "Ņ", c585), -_c(U+0146, "ņ", c586), -_c(U+0147, "Ň", c587), -_c(U+0148, "ň", c588), -_c(U+0149, "ʼn", c589), -_c(U+014A, "Ŋ", c58a), -_c(U+014B, "ŋ", c58b), -_c(U+014C, "Ō", c58c), -_c(U+014D, "ō", c58d), -_c(U+014E, "Ŏ", c58e), -_c(U+014F, "ŏ", c58f), -_c(U+0150, "Ő", c590), -_c(U+0151, "ő", c591), -_c(U+0152, "Œ", c592), -_c(U+0153, "œ", c593), -_c(U+0154, "Ŕ", c594), -_c(U+0155, "ŕ", c595), -_c(U+0156, "Ŗ", c596), -_c(U+0157, "ŗ", c597), -_c(U+0158, "Ř", c598), -_c(U+0159, "ř", c599), -_c(U+015A, "Ś", c59a), -_c(U+015B, "ś", c59b), -_c(U+015C, "Ŝ", c59c), -_c(U+015D, "ŝ", c59d), -_c(U+015E, "Ş", c59e), -_c(U+015F, "ş", c59f), -_c(U+0160, "Š", c5a0), -_c(U+0161, "š", c5a1), -_c(U+0162, "Ţ", c5a2), -_c(U+0163, "ţ", c5a3), -_c(U+0164, "Ť", c5a4), -_c(U+0165, "ť", c5a5), -_c(U+0166, "Ŧ", c5a6), -_c(U+0167, "ŧ", c5a7), -_c(U+0168, "Ũ", c5a8), -_c(U+0169, "ũ", c5a9), -_c(U+016A, "Ū", c5aa), -_c(U+016B, "ū", c5ab), -_c(U+016C, "Ŭ", c5ac), -_c(U+016D, "ŭ", c5ad), -_c(U+016E, "Ů", c5ae), -_c(U+016F, "ů", c5af), -_c(U+0170, "Ű", c5b0), -_c(U+0171, "ű", c5b1), -_c(U+0172, "Ų", c5b2), -_c(U+0173, "ų", c5b3), -_c(U+0174, "Ŵ", c5b4), -_c(U+0175, "ŵ", c5b5), -_c(U+0176, "Ŷ", c5b6), -_c(U+0177, "ŷ", c5b7), -_c(U+0178, "Ÿ", c5b8), -_c(U+0179, "Ź", c5b9), -_c(U+017A, "ź", c5ba), -_c(U+017B, "Ż", c5bb), -_c(U+017C, "ż", c5bc), -_c(U+017D, "Ž", c5bd), -_c(U+017E, "ž", c5be), -_c(U+017F, "ſ", c5bf), -_c(U+0180, "ƀ", c680), -_c(U+0181, "Ɓ", c681), -_c(U+0182, "Ƃ", c682), -_c(U+0183, "ƃ", c683), -_c(U+0184, "Ƅ", c684), -_c(U+0185, "ƅ", c685), -_c(U+0186, "Ɔ", c686), -_c(U+0187, "Ƈ", c687), -_c(U+0188, "ƈ", c688), -_c(U+0189, "Ɖ", c689), -_c(U+018A, "Ɗ", c68a), -_c(U+018B, "Ƌ", c68b), -_c(U+018C, "ƌ", c68c), -_c(U+018D, "ƍ", c68d), -_c(U+018E, "Ǝ", c68e), -_c(U+018F, "Ə", c68f), -_c(U+0190, "Ɛ", c690), -_c(U+0191, "Ƒ", c691), -_c(U+0192, "ƒ", c692), -_c(U+0193, "Ɠ", c693), -_c(U+0194, "Ɣ", c694), -_c(U+0195, "ƕ", c695), -_c(U+0196, "Ɩ", c696), -_c(U+0197, "Ɨ", c697), -_c(U+0198, "Ƙ", c698), -_c(U+0199, "ƙ", c699), -_c(U+019A, "ƚ", c69a), -_c(U+019B, "ƛ", c69b), -_c(U+019C, "Ɯ", c69c), -_c(U+019D, "Ɲ", c69d), -_c(U+019E, "ƞ", c69e), -_c(U+019F, "Ɵ", c69f), -_c(U+01A0, "Ơ", c6a0), -_c(U+01A1, "ơ", c6a1), -_c(U+01A2, "Ƣ", c6a2), -_c(U+01A3, "ƣ", c6a3), -_c(U+01A4, "Ƥ", c6a4), -_c(U+01A5, "ƥ", c6a5), -_c(U+01A6, "Ʀ", c6a6), -_c(U+01A7, "Ƨ", c6a7), -_c(U+01A8, "ƨ", c6a8), -_c(U+01A9, "Ʃ", c6a9), -_c(U+01AA, "ƪ", c6aa), -_c(U+01AB, "ƫ", c6ab), -_c(U+01AC, "Ƭ", c6ac), -_c(U+01AD, "ƭ", c6ad), -_c(U+01AE, "Ʈ", c6ae), -_c(U+01AF, "Ư", c6af), -_c(U+01B0, "ư", c6b0), -_c(U+01B1, "Ʊ", c6b1), -_c(U+01B2, "Ʋ", c6b2), -_c(U+01B3, "Ƴ", c6b3), -_c(U+01B4, "ƴ", c6b4), -_c(U+01B5, "Ƶ", c6b5), -_c(U+01B6, "ƶ", c6b6), -_c(U+01B7, "Ʒ", c6b7), -_c(U+01B8, "Ƹ", c6b8), -_c(U+01B9, "ƹ", c6b9), -_c(U+01BA, "ƺ", c6ba), -_c(U+01BB, "ƻ", c6bb), -_c(U+01BC, "Ƽ", c6bc), -_c(U+01BD, "ƽ", c6bd), -_c(U+01BE, "ƾ", c6be), -_c(U+01BF, "ƿ", c6bf), -_c(U+01C0, "ǀ", c780), -_c(U+01C1, "ǁ", c781), -_c(U+01C2, "ǂ", c782), -_c(U+01C3, "ǃ", c783), -_c(U+01C4, "DŽ", c784), -_c(U+01C5, "Dž", c785), -_c(U+01C6, "dž", c786), -_c(U+01C7, "LJ", c787), -_c(U+01C8, "Lj", c788), -_c(U+01C9, "lj", c789), -_c(U+01CA, "NJ", c78a), -_c(U+01CB, "Nj", c78b), -_c(U+01CC, "nj", c78c), -_c(U+01CD, "Ǎ", c78d), -_c(U+01CE, "ǎ", c78e), -_c(U+01CF, "Ǐ", c78f), -_c(U+01D0, "ǐ", c790), -_c(U+01D1, "Ǒ", c791), -_c(U+01D2, "ǒ", c792), -_c(U+01D3, "Ǔ", c793), -_c(U+01D4, "ǔ", c794), -_c(U+01D5, "Ǖ", c795), -_c(U+01D6, "ǖ", c796), -_c(U+01D7, "Ǘ", c797), -_c(U+01D8, "ǘ", c798), -_c(U+01D9, "Ǚ", c799), -_c(U+01DA, "ǚ", c79a), -_c(U+01DB, "Ǜ", c79b), -_c(U+01DC, "ǜ", c79c), -_c(U+01DD, "ǝ", c79d), -_c(U+01DE, "Ǟ", c79e), -_c(U+01DF, "ǟ", c79f), -_c(U+01E0, "Ǡ", c7a0), -_c(U+01E1, "ǡ", c7a1), -_c(U+01E2, "Ǣ", c7a2), -_c(U+01E3, "ǣ", c7a3), -_c(U+01E4, "Ǥ", c7a4), -_c(U+01E5, "ǥ", c7a5), -_c(U+01E6, "Ǧ", c7a6), -_c(U+01E7, "ǧ", c7a7), -_c(U+01E8, "Ǩ", c7a8), -_c(U+01E9, "ǩ", c7a9), -_c(U+01EA, "Ǫ", c7aa), -_c(U+01EB, "ǫ", c7ab), -_c(U+01EC, "Ǭ", c7ac), -_c(U+01ED, "ǭ", c7ad), -_c(U+01EE, "Ǯ", c7ae), -_c(U+01EF, "ǯ", c7af), -_c(U+01F0, "ǰ", c7b0), -_c(U+01F1, "DZ", c7b1), -_c(U+01F2, "Dz", c7b2), -_c(U+01F3, "dz", c7b3), -_c(U+01F4, "Ǵ", c7b4), -_c(U+01F5, "ǵ", c7b5), -_c(U+01F6, "Ƕ", c7b6), -_c(U+01F7, "Ƿ", c7b7), -_c(U+01F8, "Ǹ", c7b8), -_c(U+01F9, "ǹ", c7b9), -_c(U+01FA, "Ǻ", c7ba), -_c(U+01FB, "ǻ", c7bb), -_c(U+01FC, "Ǽ", c7bc), -_c(U+01FD, "ǽ", c7bd), -_c(U+01FE, "Ǿ", c7be), -_c(U+01FF, "ǿ", c7bf), -_c(U+0200, "Ȁ", c880), -_c(U+0201, "ȁ", c881), -_c(U+0202, "Ȃ", c882), -_c(U+0203, "ȃ", c883), -_c(U+0204, "Ȅ", c884), -_c(U+0205, "ȅ", c885), -_c(U+0206, "Ȇ", c886), -_c(U+0207, "ȇ", c887), -_c(U+0208, "Ȉ", c888), -_c(U+0209, "ȉ", c889), -_c(U+020A, "Ȋ", c88a), -_c(U+020B, "ȋ", c88b), -_c(U+020C, "Ȍ", c88c), -_c(U+020D, "ȍ", c88d), -_c(U+020E, "Ȏ", c88e), -_c(U+020F, "ȏ", c88f), -_c(U+0210, "Ȑ", c890), -_c(U+0211, "ȑ", c891), -_c(U+0212, "Ȓ", c892), -_c(U+0213, "ȓ", c893), -_c(U+0214, "Ȕ", c894), -_c(U+0215, "ȕ", c895), -_c(U+0216, "Ȗ", c896), -_c(U+0217, "ȗ", c897), -_c(U+0218, "Ș", c898), -_c(U+0219, "ș", c899), -_c(U+021A, "Ț", c89a), -_c(U+021B, "ț", c89b), -_c(U+021C, "Ȝ", c89c), -_c(U+021D, "ȝ", c89d), -_c(U+021E, "Ȟ", c89e), -_c(U+021F, "ȟ", c89f), -_c(U+0220, "Ƞ", c8a0), -_c(U+0221, "ȡ", c8a1), -_c(U+0222, "Ȣ", c8a2), -_c(U+0223, "ȣ", c8a3), -_c(U+0224, "Ȥ", c8a4), -_c(U+0225, "ȥ", c8a5), -_c(U+0226, "Ȧ", c8a6), -_c(U+0227, "ȧ", c8a7), -_c(U+0228, "Ȩ", c8a8), -_c(U+0229, "ȩ", c8a9), -_c(U+022A, "Ȫ", c8aa), -_c(U+022B, "ȫ", c8ab), -_c(U+022C, "Ȭ", c8ac), -_c(U+022D, "ȭ", c8ad), -_c(U+022E, "Ȯ", c8ae), -_c(U+022F, "ȯ", c8af), -_c(U+0230, "Ȱ", c8b0), -_c(U+0231, "ȱ", c8b1), -_c(U+0232, "Ȳ", c8b2), -_c(U+0233, "ȳ", c8b3), -_c(U+0234, "ȴ", c8b4), -_c(U+0235, "ȵ", c8b5), -_c(U+0236, "ȶ", c8b6), -_c(U+0237, "ȷ", c8b7), -_c(U+0238, "ȸ", c8b8), -_c(U+0239, "ȹ", c8b9), -_c(U+023A, "Ⱥ", c8ba), -_c(U+023B, "Ȼ", c8bb), -_c(U+023C, "ȼ", c8bc), -_c(U+023D, "Ƚ", c8bd), -_c(U+023E, "Ⱦ", c8be), -_c(U+023F, "ȿ", c8bf), -_c(U+0240, "ɀ", c980), -_c(U+0241, "Ɂ", c981), -_c(U+0242, "ɂ", c982), -_c(U+0243, "Ƀ", c983), -_c(U+0244, "Ʉ", c984), -_c(U+0245, "Ʌ", c985), -_c(U+0246, "Ɇ", c986), -_c(U+0247, "ɇ", c987), -_c(U+0248, "Ɉ", c988), -_c(U+0249, "ɉ", c989), -_c(U+024A, "Ɋ", c98a), -_c(U+024B, "ɋ", c98b), -_c(U+024C, "Ɍ", c98c), -_c(U+024D, "ɍ", c98d), -_c(U+024E, "Ɏ", c98e), -_c(U+024F, "ɏ", c98f), -_c(U+0250, "ɐ", c990), -_c(U+0251, "ɑ", c991), -_c(U+0252, "ɒ", c992), -_c(U+0253, "ɓ", c993), -_c(U+0254, "ɔ", c994), -_c(U+0255, "ɕ", c995), -_c(U+0256, "ɖ", c996), -_c(U+0257, "ɗ", c997), -_c(U+0258, "ɘ", c998), -_c(U+0259, "ə", c999), -_c(U+025A, "ɚ", c99a), -_c(U+025B, "ɛ", c99b), -_c(U+025C, "ɜ", c99c), -_c(U+025D, "ɝ", c99d), -_c(U+025E, "ɞ", c99e), -_c(U+025F, "ɟ", c99f), -_c(U+0260, "ɠ", c9a0), -_c(U+0261, "ɡ", c9a1), -_c(U+0262, "ɢ", c9a2), -_c(U+0263, "ɣ", c9a3), -_c(U+0264, "ɤ", c9a4), -_c(U+0265, "ɥ", c9a5), -_c(U+0266, "ɦ", c9a6), -_c(U+0267, "ɧ", c9a7), -_c(U+0268, "ɨ", c9a8), -_c(U+0269, "ɩ", c9a9), -_c(U+026A, "ɪ", c9aa), -_c(U+026B, "ɫ", c9ab), -_c(U+026C, "ɬ", c9ac), -_c(U+026D, "ɭ", c9ad), -_c(U+026E, "ɮ", c9ae), -_c(U+026F, "ɯ", c9af), -_c(U+0270, "ɰ", c9b0), -_c(U+0271, "ɱ", c9b1), -_c(U+0272, "ɲ", c9b2), -_c(U+0273, "ɳ", c9b3), -_c(U+0274, "ɴ", c9b4), -_c(U+0275, "ɵ", c9b5), -_c(U+0276, "ɶ", c9b6), -_c(U+0277, "ɷ", c9b7), -_c(U+0278, "ɸ", c9b8), -_c(U+0279, "ɹ", c9b9), -_c(U+027A, "ɺ", c9ba), -_c(U+027B, "ɻ", c9bb), -_c(U+027C, "ɼ", c9bc), -_c(U+027D, "ɽ", c9bd), -_c(U+027E, "ɾ", c9be), -_c(U+027F, "ɿ", c9bf), -_c(U+0280, "ʀ", ca80), -_c(U+0281, "ʁ", ca81), -_c(U+0282, "ʂ", ca82), -_c(U+0283, "ʃ", ca83), -_c(U+0284, "ʄ", ca84), -_c(U+0285, "ʅ", ca85), -_c(U+0286, "ʆ", ca86), -_c(U+0287, "ʇ", ca87), -_c(U+0288, "ʈ", ca88), -_c(U+0289, "ʉ", ca89), -_c(U+028A, "ʊ", ca8a), -_c(U+028B, "ʋ", ca8b), -_c(U+028C, "ʌ", ca8c), -_c(U+028D, "ʍ", ca8d), -_c(U+028E, "ʎ", ca8e), -_c(U+028F, "ʏ", ca8f), -_c(U+0290, "ʐ", ca90), -_c(U+0291, "ʑ", ca91), -_c(U+0292, "ʒ", ca92), -_c(U+0293, "ʓ", ca93), -_c(U+0294, "ʔ", ca94), -_c(U+0295, "ʕ", ca95), -_c(U+0296, "ʖ", ca96), -_c(U+0297, "ʗ", ca97), -_c(U+0298, "ʘ", ca98), -_c(U+0299, "ʙ", ca99), -_c(U+029A, "ʚ", ca9a), -_c(U+029B, "ʛ", ca9b), -_c(U+029C, "ʜ", ca9c), -_c(U+029D, "ʝ", ca9d), -_c(U+029E, "ʞ", ca9e), -_c(U+029F, "ʟ", ca9f), -_c(U+02A0, "ʠ", caa0), -_c(U+02A1, "ʡ", caa1), -_c(U+02A2, "ʢ", caa2), -_c(U+02A3, "ʣ", caa3), -_c(U+02A4, "ʤ", caa4), -_c(U+02A5, "ʥ", caa5), -_c(U+02A6, "ʦ", caa6), -_c(U+02A7, "ʧ", caa7), -_c(U+02A8, "ʨ", caa8), -_c(U+02A9, "ʩ", caa9), -_c(U+02AA, "ʪ", caaa), -_c(U+02AB, "ʫ", caab), -_c(U+02AC, "ʬ", caac), -_c(U+02AD, "ʭ", caad), -_c(U+02AE, "ʮ", caae), -_c(U+02AF, "ʯ", caaf), -_c(U+02B0, "ʰ", cab0), -_c(U+02B1, "ʱ", cab1), -_c(U+02B2, "ʲ", cab2), -_c(U+02B3, "ʳ", cab3), -_c(U+02B4, "ʴ", cab4), -_c(U+02B5, "ʵ", cab5), -_c(U+02B6, "ʶ", cab6), -_c(U+02B7, "ʷ", cab7), -_c(U+02B8, "ʸ", cab8), -_c(U+02B9, "ʹ", cab9), -_c(U+02BA, "ʺ", caba), -_c(U+02BB, "ʻ", cabb), -_c(U+02BC, "ʼ", cabc), -_c(U+02BD, "ʽ", cabd), -_c(U+02BE, "ʾ", cabe), -_c(U+02BF, "ʿ", cabf), -_c(U+02C0, "ˀ", cb80), -_c(U+02C1, "ˁ", cb81), -_c(U+02C2, "˂", cb82), -_c(U+02C3, "˃", cb83), -_c(U+02C4, "˄", cb84), -_c(U+02C5, "˅", cb85), -_c(U+02C6, "ˆ", cb86), -_c(U+02C7, "ˇ", cb87), -_c(U+02C8, "ˈ", cb88), -_c(U+02C9, "ˉ", cb89), -_c(U+02CA, "ˊ", cb8a), -_c(U+02CB, "ˋ", cb8b), -_c(U+02CC, "ˌ", cb8c), -_c(U+02CD, "ˍ", cb8d), -_c(U+02CE, "ˎ", cb8e), -_c(U+02CF, "ˏ", cb8f), -_c(U+02D0, "ː", cb90), -_c(U+02D1, "ˑ", cb91), -_c(U+02D2, "˒", cb92), -_c(U+02D3, "˓", cb93), -_c(U+02D4, "˔", cb94), -_c(U+02D5, "˕", cb95), -_c(U+02D6, "˖", cb96), -_c(U+02D7, "˗", cb97), -_c(U+02D8, "˘", cb98), -_c(U+02D9, "˙", cb99), -_c(U+02DA, "˚", cb9a), -_c(U+02DB, "˛", cb9b), -_c(U+02DC, "˜", cb9c), -_c(U+02DD, "˝", cb9d), -_c(U+02DE, "˞", cb9e), -_c(U+02DF, "˟", cb9f), -_c(U+02E0, "ˠ", cba0), -_c(U+02E1, "ˡ", cba1), -_c(U+02E2, "ˢ", cba2), -_c(U+02E3, "ˣ", cba3), -_c(U+02E4, "ˤ", cba4), -_c(U+02E5, "˥", cba5), -_c(U+02E6, "˦", cba6), -_c(U+02E7, "˧", cba7), -_c(U+02E8, "˨", cba8), -_c(U+02E9, "˩", cba9), -_c(U+02EA, "˪", cbaa), -_c(U+02EB, "˫", cbab), -_c(U+02EC, "ˬ", cbac), -_c(U+02ED, "˭", cbad), -_c(U+02EE, "ˮ", cbae), -_c(U+02EF, "˯", cbaf), -_c(U+02F0, "˰", cbb0), -_c(U+02F1, "˱", cbb1), -_c(U+02F2, "˲", cbb2), -_c(U+02F3, "˳", cbb3), -_c(U+02F4, "˴", cbb4), -_c(U+02F5, "˵", cbb5), -_c(U+02F6, "˶", cbb6), -_c(U+02F7, "˷", cbb7), -_c(U+02F8, "˸", cbb8), -_c(U+02F9, "˹", cbb9), -_c(U+02FA, "˺", cbba), -_c(U+02FB, "˻", cbbb), -_c(U+02FC, "˼", cbbc), -_c(U+02FD, "˽", cbbd), -_c(U+02FE, "˾", cbbe), -//_c(U+02FF, "˿", cbbf), -//_c(U+0300, ̀"̀ ", cc80), -//_c(U+0301, ́" ", cc81), -//_c(U+0302, ̂" ", cc82), -//_c(U+0303, ̃" ", cc83), -//_c(U+0304, ̄" ", cc84), -//_c(U+0305, "̅" , cc85), -//_c(U+0306, ̆" ", cc86), -//_c(U+0307, ̇" ", cc87), -//_c(U+0308, ̈" ", cc88), -//_c(U+0309, ̉" ", cc89), -//_c(U+030A, ̊" ", cc8a), -//_c(U+030B, ̋" ", cc8b), -//_c(U+030C, ̌" ", cc8c), -//_c(U+030D, "̍" , cc8d), -//_c(U+030E, "̎" , cc8e), -//_c(U+030F, ̏" ", cc8f), -//_c(U+0310, "̐" , cc90), -//_c(U+0311, ̑" ", cc91), -//_c(U+0312, ̒" ", cc92), -//_c(U+0313, "̓" , cc93), -//_c(U+0314, "̔" , cc94), -//_c(U+0315, "̕" , cc95), -//_c(U+0316, "̖" , cc96), -//_c(U+0317, "̗" , cc97), -//_c(U+0318, "̘" , cc98), -//_c(U+0319, "̙" , cc99), -//_c(U+031A, "̚" , cc9a), -//_c(U+031B, ̛" ", cc9b), -//_c(U+031C, "̜" , cc9c), -//_c(U+031D, "̝" , cc9d), -//_c(U+031E, "̞" , cc9e), -_c(U+031F, "̟" , cc9f), -_c(U+0320, "̠" , cca0), -_c(U+0321, "̡" , cca1), -_c(U+0322, "̢" , cca2), -//_c(U+0323, ̣" ", cca3), -//_c(U+0324, ̤" ", cca4), -_c(U+0325, "̥" , cca5), -//_c(U+0326, ̦" ", cca6), -//_c(U+0327, ̧" ", cca7), -//_c(U+0328, ̨" ", cca8), -//_c(U+0329, "̩" , cca9), -_c(U+032A, "̪" , ccaa), -_c(U+032B, "̫" , ccab), -_c(U+032C, "̬" , ccac), -_c(U+032D, "̭" , ccad), -//_c(U+032E, ̮" ", ccae), -_c(U+032F, "̯" , ccaf), -_c(U+0330, "̰" , ccb0), -//_c(U+0331, ̱" ", ccb1), -_c(U+0332, "̲" , ccb2), -_c(U+0333, "̳" , ccb3), -_c(U+0334, "̴" , ccb4), -//_c(U+0335, ̵" ", ccb5), -//_c(U+0336, ̶" ", ccb6), -_c(U+0337, "̷" , ccb7), -_c(U+0338, "̸" , ccb8), -_c(U+0339, "̹" , ccb9), -_c(U+033A, "̺" , ccba), -_c(U+033B, "̻" , ccbb), -_c(U+033C, "̼" , ccbc), -_c(U+033D, "̽" , ccbd), -_c(U+033E, "̾" , ccbe), -_c(U+033F, "̿" , ccbf), -_c(U+0340, "̀" , cd80), -_c(U+0341, "́" , cd81), -_c(U+0342, "͂" , cd82), -_c(U+0343, "̓" , cd83), -_c(U+0344, "̈́" , cd84), -_c(U+0345, "ͅ" , cd85), -_c(U+0346, "͆" , cd86), -_c(U+0347, "͇" , cd87), -_c(U+0348, "͈" , cd88), -_c(U+0349, "͉" , cd89), -_c(U+034A, "͊" , cd8a), -_c(U+034B, "͋" , cd8b), -_c(U+034C, "͌" , cd8c), -_c(U+034D, "͍" , cd8d), -_c(U+034E, "͎" , cd8e), -_c(U+034F, "͏" , cd8f), -_c(U+0350, "͐" , cd90), -_c(U+0351, "͑" , cd91), -_c(U+0352, "͒" , cd92), -_c(U+0353, "͓" , cd93), -_c(U+0354, "͔" , cd94), -_c(U+0355, "͕" , cd95), -_c(U+0356, "͖" , cd96), -_c(U+0357, "͗" , cd97), -_c(U+0358, "͘" , cd98), -_c(U+0359, "͙" , cd99), -_c(U+035A, "͚" , cd9a), -_c(U+035B, "͛" , cd9b), -_c(U+035C, "͜" , cd9c), -_c(U+035D, "͝" , cd9d), -_c(U+035E, "͞" , cd9e), -_c(U+035F, "͟" , cd9f), -_c(U+0360, "͠" , cda0), -_c(U+0361, "͡" , cda1), -_c(U+0362, "͢" , cda2), -_c(U+0363, "ͣ" , cda3), -_c(U+0364, "ͤ" , cda4), -_c(U+0365, "ͥ" , cda5), -_c(U+0366, "ͦ" , cda6), -_c(U+0367, "ͧ" , cda7), -_c(U+0368, "ͨ" , cda8), -_c(U+0369, "ͩ" , cda9), -_c(U+036A, "ͪ" , cdaa), -_c(U+036B, "ͫ" , cdab), -_c(U+036C, "ͬ" , cdac), -_c(U+036D, "ͭ" , cdad), -_c(U+036E, "ͮ" , cdae), -_c(U+036F, "ͯ" , cdaf), -_c(U+0370, "Ͱ", cdb0), -_c(U+0371, "ͱ", cdb1), -_c(U+0372, "Ͳ", cdb2), -_c(U+0373, "ͳ", cdb3), -_c(U+0374, "ʹ", cdb4), -_c(U+0375, "͵", cdb5), -_c(U+0376, "Ͷ", cdb6), -_c(U+0377, "ͷ", cdb7), -_c(U+0378, "͸", cdb8), -_c(U+0379, "͹", cdb9), -_c(U+037A, "ͺ", cdba), -_c(U+037B, "ͻ", cdbb), -_c(U+037C, "ͼ", cdbc), -_c(U+037D, "ͽ", cdbd), -_c(U+037E, ";", cdbe), -_c(U+037F, "Ϳ", cdbf), -_c(U+0380, "΀", ce80), -_c(U+0381, "΁", ce81), -_c(U+0382, "΂", ce82), -_c(U+0383, "΃", ce83), -_c(U+0384, "΄", ce84), -_c(U+0385, "΅", ce85), -_c(U+0386, "Ά", ce86), -_c(U+0387, "·", ce87), -_c(U+0388, "Έ", ce88), -_c(U+0389, "Ή", ce89), -_c(U+038A, "Ί", ce8a), -_c(U+038B, "΋", ce8b), -_c(U+038C, "Ό", ce8c), -_c(U+038D, "΍", ce8d), -_c(U+038E, "Ύ", ce8e), -_c(U+038F, "Ώ", ce8f), -_c(U+0390, "ΐ", ce90), -_c(U+0391, "Α", ce91), -_c(U+0392, "Β", ce92), -_c(U+0393, "Γ", ce93), -_c(U+0394, "Δ", ce94), -_c(U+0395, "Ε", ce95), -_c(U+0396, "Ζ", ce96), -_c(U+0397, "Η", ce97), -_c(U+0398, "Θ", ce98), -_c(U+0399, "Ι", ce99), -_c(U+039A, "Κ", ce9a), -_c(U+039B, "Λ", ce9b), -_c(U+039C, "Μ", ce9c), -_c(U+039D, "Ν", ce9d), -_c(U+039E, "Ξ", ce9e), -_c(U+039F, "Ο", ce9f), -_c(U+03A0, "Π", cea0), -_c(U+03A1, "Ρ", cea1), -_c(U+03A2, "΢", cea2), -_c(U+03A3, "Σ", cea3), -_c(U+03A4, "Τ", cea4), -_c(U+03A5, "Υ", cea5), -_c(U+03A6, "Φ", cea6), -_c(U+03A7, "Χ", cea7), -_c(U+03A8, "Ψ", cea8), -_c(U+03A9, "Ω", cea9), -_c(U+03AA, "Ϊ", ceaa), -_c(U+03AB, "Ϋ", ceab), -_c(U+03AC, "ά", ceac), -_c(U+03AD, "έ", cead), -_c(U+03AE, "ή", ceae), -_c(U+03AF, "ί", ceaf), -_c(U+03B0, "ΰ", ceb0), -_c(U+03B1, "α", ceb1), -_c(U+03B2, "β", ceb2), -_c(U+03B3, "γ", ceb3), -_c(U+03B4, "δ", ceb4), -_c(U+03B5, "ε", ceb5), -_c(U+03B6, "ζ", ceb6), -_c(U+03B7, "η", ceb7), -_c(U+03B8, "θ", ceb8), -_c(U+03B9, "ι", ceb9), -_c(U+03BA, "κ", ceba), -_c(U+03BB, "λ", cebb), -_c(U+03BC, "μ", cebc), -_c(U+03BD, "ν", cebd), -_c(U+03BE, "ξ", cebe), -_c(U+03BF, "ο", cebf), -_c(U+03C0, "π", cf80), -_c(U+03C1, "ρ", cf81), -_c(U+03C2, "ς", cf82), -_c(U+03C3, "σ", cf83), -_c(U+03C4, "τ", cf84), -_c(U+03C5, "υ", cf85), -_c(U+03C6, "φ", cf86), -_c(U+03C7, "χ", cf87), -_c(U+03C8, "ψ", cf88), -_c(U+03C9, "ω", cf89), -_c(U+03CA, "ϊ", cf8a), -_c(U+03CB, "ϋ", cf8b), -_c(U+03CC, "ό", cf8c), -_c(U+03CD, "ύ", cf8d), -_c(U+03CE, "ώ", cf8e), -_c(U+03CF, "Ϗ", cf8f), -_c(U+03D0, "ϐ", cf90), -_c(U+03D1, "ϑ", cf91), -_c(U+03D2, "ϒ", cf92), -_c(U+03D3, "ϓ", cf93), -_c(U+03D4, "ϔ", cf94), -_c(U+03D5, "ϕ", cf95), -_c(U+03D6, "ϖ", cf96), -_c(U+03D7, "ϗ", cf97), -_c(U+03D8, "Ϙ", cf98), -_c(U+03D9, "ϙ", cf99), -_c(U+03DA, "Ϛ", cf9a), -_c(U+03DB, "ϛ", cf9b), -_c(U+03DC, "Ϝ", cf9c), -_c(U+03DD, "ϝ", cf9d), -_c(U+03DE, "Ϟ", cf9e), -_c(U+03DF, "ϟ", cf9f), -_c(U+03E0, "Ϡ", cfa0), -_c(U+03E1, "ϡ", cfa1), -_c(U+03E2, "Ϣ", cfa2), -_c(U+03E3, "ϣ", cfa3), -_c(U+03E4, "Ϥ", cfa4), -_c(U+03E5, "ϥ", cfa5), -_c(U+03E6, "Ϧ", cfa6), -_c(U+03E7, "ϧ", cfa7), -_c(U+03E8, "Ϩ", cfa8), -_c(U+03E9, "ϩ", cfa9), -_c(U+03EA, "Ϫ", cfaa), -_c(U+03EB, "ϫ", cfab), -_c(U+03EC, "Ϭ", cfac), -_c(U+03ED, "ϭ", cfad), -_c(U+03EE, "Ϯ", cfae), -_c(U+03EF, "ϯ", cfaf), -_c(U+03F0, "ϰ", cfb0), -_c(U+03F1, "ϱ", cfb1), -_c(U+03F2, "ϲ", cfb2), -_c(U+03F3, "ϳ", cfb3), -_c(U+03F4, "ϴ", cfb4), -_c(U+03F5, "ϵ", cfb5), -_c(U+03F6, "϶", cfb6), -_c(U+03F7, "Ϸ", cfb7), -_c(U+03F8, "ϸ", cfb8), -_c(U+03F9, "Ϲ", cfb9), -_c(U+03FA, "Ϻ", cfba), -_c(U+03FB, "ϻ", cfbb), -_c(U+03FC, "ϼ", cfbc), -_c(U+03FD, "Ͻ", cfbd), -_c(U+03FE, "Ͼ", cfbe), -_c(U+03FF, "Ͽ", cfbf), -_c(U+0400, "Ѐ", d080), -_c(U+0401, "Ё", d081), -_c(U+0402, "Ђ", d082), -_c(U+0403, "Ѓ", d083), -_c(U+0404, "Є", d084), -_c(U+0405, "Ѕ", d085), -_c(U+0406, "І", d086), -_c(U+0407, "Ї", d087), -_c(U+0408, "Ј", d088), -_c(U+0409, "Љ", d089), -_c(U+040A, "Њ", d08a), -_c(U+040B, "Ћ", d08b), -_c(U+040C, "Ќ", d08c), -_c(U+040D, "Ѝ", d08d), -_c(U+040E, "Ў", d08e), -_c(U+040F, "Џ", d08f), -_c(U+0410, "А", d090), -_c(U+0411, "Б", d091), -_c(U+0412, "В", d092), -_c(U+0413, "Г", d093), -_c(U+0414, "Д", d094), -_c(U+0415, "Е", d095), -_c(U+0416, "Ж", d096), -_c(U+0417, "З", d097), -_c(U+0418, "И", d098), -_c(U+0419, "Й", d099), -_c(U+041A, "К", d09a), -_c(U+041B, "Л", d09b), -_c(U+041C, "М", d09c), -_c(U+041D, "Н", d09d), -_c(U+041E, "О", d09e), -_c(U+041F, "П", d09f), -_c(U+0420, "Р", d0a0), -_c(U+0421, "С", d0a1), -_c(U+0422, "Т", d0a2), -_c(U+0423, "У", d0a3), -_c(U+0424, "Ф", d0a4), -_c(U+0425, "Х", d0a5), -_c(U+0426, "Ц", d0a6), -_c(U+0427, "Ч", d0a7), -_c(U+0428, "Ш", d0a8), -_c(U+0429, "Щ", d0a9), -_c(U+042A, "Ъ", d0aa), -_c(U+042B, "Ы", d0ab), -_c(U+042C, "Ь", d0ac), -_c(U+042D, "Э", d0ad), -_c(U+042E, "Ю", d0ae), -_c(U+042F, "Я", d0af), -_c(U+0430, "а", d0b0), -_c(U+0431, "б", d0b1), -_c(U+0432, "в", d0b2), -_c(U+0433, "г", d0b3), -_c(U+0434, "д", d0b4), -_c(U+0435, "е", d0b5), -_c(U+0436, "ж", d0b6), -_c(U+0437, "з", d0b7), -_c(U+0438, "и", d0b8), -_c(U+0439, "й", d0b9), -_c(U+043A, "к", d0ba), -_c(U+043B, "л", d0bb), -_c(U+043C, "м", d0bc), -_c(U+043D, "н", d0bd), -_c(U+043E, "о", d0be), -_c(U+043F, "п", d0bf), -_c(U+0440, "р", d180), -_c(U+0441, "с", d181), -_c(U+0442, "т", d182), -_c(U+0443, "у", d183), -_c(U+0444, "ф", d184), -_c(U+0445, "х", d185), -_c(U+0446, "ц", d186), -_c(U+0447, "ч", d187), -_c(U+0448, "ш", d188), -_c(U+0449, "щ", d189), -_c(U+044A, "ъ", d18a), -_c(U+044B, "ы", d18b), -_c(U+044C, "ь", d18c), -_c(U+044D, "э", d18d), -_c(U+044E, "ю", d18e), -_c(U+044F, "я", d18f), -_c(U+0450, "ѐ", d190), -_c(U+0451, "ё", d191), -_c(U+0452, "ђ", d192), -_c(U+0453, "ѓ", d193), -_c(U+0454, "є", d194), -_c(U+0455, "ѕ", d195), -_c(U+0456, "і", d196), -_c(U+0457, "ї", d197), -_c(U+0458, "ј", d198), -_c(U+0459, "љ", d199), -_c(U+045A, "њ", d19a), -_c(U+045B, "ћ", d19b), -_c(U+045C, "ќ", d19c), -_c(U+045D, "ѝ", d19d), -_c(U+045E, "ў", d19e), -_c(U+045F, "џ", d19f), -_c(U+0460, "Ѡ", d1a0), -_c(U+0461, "ѡ", d1a1), -_c(U+0462, "Ѣ", d1a2), -_c(U+0463, "ѣ", d1a3), -_c(U+0464, "Ѥ", d1a4), -_c(U+0465, "ѥ", d1a5), -_c(U+0466, "Ѧ", d1a6), -_c(U+0467, "ѧ", d1a7), -_c(U+0468, "Ѩ", d1a8), -_c(U+0469, "ѩ", d1a9), -_c(U+046A, "Ѫ", d1aa), -_c(U+046B, "ѫ", d1ab), -_c(U+046C, "Ѭ", d1ac), -_c(U+046D, "ѭ", d1ad), -_c(U+046E, "Ѯ", d1ae), -_c(U+046F, "ѯ", d1af), -_c(U+0470, "Ѱ", d1b0), -_c(U+0471, "ѱ", d1b1), -_c(U+0472, "Ѳ", d1b2), -_c(U+0473, "ѳ", d1b3), -_c(U+0474, "Ѵ", d1b4), -_c(U+0475, "ѵ", d1b5), -_c(U+0476, "Ѷ", d1b6), -_c(U+0477, "ѷ", d1b7), -_c(U+0478, "Ѹ", d1b8), -_c(U+0479, "ѹ", d1b9), -_c(U+047A, "Ѻ", d1ba), -_c(U+047B, "ѻ", d1bb), -_c(U+047C, "Ѽ", d1bc), -_c(U+047D, "ѽ", d1bd), -_c(U+047E, "Ѿ", d1be), -_c(U+047F, "ѿ", d1bf), -_c(U+20A0, "₠", e282a0), -_c(U+20A1, "₡", e282a1), -_c(U+20A2, "₢", e282a2), -_c(U+20A3, "₣", e282a3), -_c(U+20A4, "₤", e282a4), -_c(U+20A5, "₥", e282a5), -_c(U+20A6, "₦", e282a6), -_c(U+20A7, "₧", e282a7), -_c(U+20A8, "₨", e282a8), -_c(U+20A9, "₩", e282a9), -_c(U+20AA, "₪", e282aa), -_c(U+20AB, "₫", e282ab), -_c(U+20AC, "€", e282ac), -_c(U+20AD, "₭", e282ad), -_c(U+20AE, "₮", e282ae), -_c(U+20AF, "₯", e282af), -_c(U+20B0, "₰", e282b0), -_c(U+20B1, "₱", e282b1), -_c(U+20B2, "₲", e282b2), -_c(U+20B3, "₳", e282b3), -_c(U+20B4, "₴", e282b4), -_c(U+20B5, "₵", e282b5), -_c(U+20B6, "₶", e282b6), -_c(U+20B7, "₷", e282b7), -_c(U+20B8, "₸", e282b8), -_c(U+20B9, "₹", e282b9), -_c(U+20BA, "₺", e282ba), -_c(U+20BB, "₻", e282bb), -_c(U+20BC, "₼", e282bc), -_c(U+20BD, "₽", e282bd), -_c(U+20BE, "₾", e282be), -_c(U+20BF, "₿", e282bf), -_c(U+20C0, "⃀", e28380), -_c(U+20C1, "⃁", e28381), -_c(U+20C2, "⃂", e28382), -_c(U+20C3, "⃃", e28383), -_c(U+20C4, "⃄", e28384), -_c(U+20C5, "⃅", e28385), -_c(U+20C6, "⃆", e28386), -_c(U+20C7, "⃇", e28387), -_c(U+20C8, "⃈", e28388), -_c(U+20C9, "⃉", e28389), -_c(U+20CA, "⃊", e2838a), -_c(U+20CB, "⃋", e2838b), -_c(U+20CC, "⃌", e2838c), -_c(U+20CD, "⃍", e2838d), -_c(U+20CE, "⃎", e2838e), -_c(U+20CF, "⃏", e2838f), -_c(U+20D0, "⃐", e28390), -_c(U+20D1, "⃑", e28391), -_c(U+20D2, "⃒", e28392), -_c(U+20D3, "⃓", e28393), -_c(U+20D4, "⃔", e28394), -_c(U+20D5, "⃕", e28395), -_c(U+20D6, "⃖", e28396), -_c(U+20D7, "⃗", e28397), -_c(U+20D8, "⃘", e28398), -_c(U+20D9, "⃙", e28399), -_c(U+20DA, "⃚", e2839a), -_c(U+20DB, "⃛", e2839b), -_c(U+20DC, "⃜", e2839c), -_c(U+20DD, "⃝", e2839d), -_c(U+20DE, "⃞", e2839e), -_c(U+20DF, "⃟", e2839f), -_c(U+20E0, "⃠", e283a0), -_c(U+20E1, "⃡", e283a1), -_c(U+20E2, "⃢", e283a2), -_c(U+20E3, "⃣", e283a3), -_c(U+20E4, "⃤", e283a4), -_c(U+20E5, "⃥", e283a5), -_c(U+20E6, "⃦", e283a6), -_c(U+20E7, "⃧", e283a7), -_c(U+20E8, "⃨", e283a8), -_c(U+20E9, "⃩", e283a9), -_c(U+20EA, "⃪", e283aa), -_c(U+20EB, "⃫", e283ab), -_c(U+20EC, "⃬", e283ac), -_c(U+20ED, "⃭", e283ad), -_c(U+20EE, "⃮", e283ae), -_c(U+20EF, "⃯", e283af), -_c(U+20F0, "⃰", e283b0), -_c(U+20F1, "⃱", e283b1), -_c(U+20F2, "⃲", e283b2), -_c(U+20F3, "⃳", e283b3), -_c(U+20F4, "⃴", e283b4), -_c(U+20F5, "⃵", e283b5), -_c(U+20F6, "⃶", e283b6), -_c(U+20F7, "⃷", e283b7), -_c(U+20F8, "⃸", e283b8), -_c(U+20F9, "⃹", e283b9), -_c(U+20FA, "⃺", e283ba), -_c(U+20FB, "⃻", e283bb), -_c(U+20FC, "⃼", e283bc), -_c(U+20FD, "⃽", e283bd), -_c(U+20FE, "⃾", e283be), -_c(U+20FF, "⃿", e283bf), -_c(U+2100, "℀", e28480), -_c(U+2101, "℁", e28481), -_c(U+2102, "ℂ", e28482), -_c(U+2103, "℃", e28483), -_c(U+2104, "℄", e28484), -_c(U+2105, "℅", e28485), -_c(U+2106, "℆", e28486), -_c(U+2107, "ℇ", e28487), -_c(U+2108, "℈", e28488), -_c(U+2109, "℉", e28489), -_c(U+210A, "ℊ", e2848a), -_c(U+210B, "ℋ", e2848b), -_c(U+210C, "ℌ", e2848c), -_c(U+210D, "ℍ", e2848d), -_c(U+210E, "ℎ", e2848e), -_c(U+210F, "ℏ", e2848f), -_c(U+2110, "ℐ", e28490), -_c(U+2111, "ℑ", e28491), -_c(U+2112, "ℒ", e28492), -_c(U+2113, "ℓ", e28493), -_c(U+2114, "℔", e28494), -_c(U+2115, "ℕ", e28495), -_c(U+2116, "№", e28496), -_c(U+2117, "℗", e28497), -_c(U+2118, "℘", e28498), -_c(U+2119, "ℙ", e28499), -_c(U+211A, "ℚ", e2849a), -_c(U+211B, "ℛ", e2849b), -_c(U+211C, "ℜ", e2849c), -_c(U+211D, "ℝ", e2849d), -_c(U+211E, "℞", e2849e), -_c(U+211F, "℟", e2849f), -_c(U+2120, "℠", e284a0), -_c(U+2121, "℡", e284a1), -_c(U+2122, "™", e284a2), -_c(U+2123, "℣", e284a3), -_c(U+2124, "ℤ", e284a4), -_c(U+2125, "℥", e284a5), -_c(U+2126, "Ω", e284a6), -_c(U+2127, "℧", e284a7), -_c(U+2128, "ℨ", e284a8), -_c(U+2129, "℩", e284a9), -_c(U+212A, "K", e284aa), -_c(U+212B, "Å", e284ab), -_c(U+212C, "ℬ", e284ac), -_c(U+212D, "ℭ", e284ad), -_c(U+212E, "℮", e284ae), -_c(U+212F, "ℯ", e284af), -_c(U+2130, "ℰ", e284b0), -_c(U+2131, "ℱ", e284b1), -_c(U+2132, "Ⅎ", e284b2), -_c(U+2133, "ℳ", e284b3), -_c(U+2134, "ℴ", e284b4), -_c(U+2135, "ℵ", e284b5), -_c(U+2136, "ℶ", e284b6), -_c(U+2137, "ℷ", e284b7), -_c(U+2138, "ℸ", e284b8), -_c(U+2139, "ℹ", e284b9), -_c(U+213A, "℺", e284ba), -_c(U+213B, "℻", e284bb), -_c(U+213C, "ℼ", e284bc), -_c(U+213D, "ℽ", e284bd), -_c(U+213E, "ℾ", e284be), -_c(U+213F, "ℿ", e284bf), -_c(U+2140, "⅀", e28580), -_c(U+2141, "⅁", e28581), -_c(U+2142, "⅂", e28582), -_c(U+2143, "⅃", e28583), -_c(U+2144, "⅄", e28584), -_c(U+2145, "ⅅ", e28585), -_c(U+2146, "ⅆ", e28586), -_c(U+2147, "ⅇ", e28587), -_c(U+2148, "ⅈ", e28588), -_c(U+2149, "ⅉ", e28589), -_c(U+214A, "⅊", e2858a), -_c(U+214B, "⅋", e2858b), -_c(U+214C, "⅌", e2858c), -_c(U+214D, "⅍", e2858d), -_c(U+214E, "ⅎ", e2858e), -_c(U+214F, "⅏", e2858f), -_c(U+2150, "⅐", e28590), -_c(U+2151, "⅑", e28591), -_c(U+2152, "⅒", e28592), -_c(U+2153, "⅓", e28593), -_c(U+2154, "⅔", e28594), -_c(U+2155, "⅕", e28595), -_c(U+2156, "⅖", e28596), -_c(U+2157, "⅗", e28597), -_c(U+2158, "⅘", e28598), -_c(U+2159, "⅙", e28599), -_c(U+215A, "⅚", e2859a), -_c(U+215B, "⅛", e2859b), -_c(U+215C, "⅜", e2859c), -_c(U+215D, "⅝", e2859d), -_c(U+215E, "⅞", e2859e), -_c(U+215F, "⅟", e2859f), -_c(U+2160, "Ⅰ", e285a0), -_c(U+2161, "Ⅱ", e285a1), -_c(U+2162, "Ⅲ", e285a2), -_c(U+2163, "Ⅳ", e285a3), -_c(U+2164, "Ⅴ", e285a4), -_c(U+2165, "Ⅵ", e285a5), -_c(U+2166, "Ⅶ", e285a6), -_c(U+2167, "Ⅷ", e285a7), -_c(U+2168, "Ⅸ", e285a8), -_c(U+2169, "Ⅹ", e285a9), -_c(U+216A, "Ⅺ", e285aa), -_c(U+216B, "Ⅻ", e285ab), -_c(U+216C, "Ⅼ", e285ac), -_c(U+216D, "Ⅽ", e285ad), -_c(U+216E, "Ⅾ", e285ae), -_c(U+216F, "Ⅿ", e285af), -_c(U+2170, "ⅰ", e285b0), -_c(U+2171, "ⅱ", e285b1), -_c(U+2172, "ⅲ", e285b2), -_c(U+2173, "ⅳ", e285b3), -_c(U+2174, "ⅴ", e285b4), -_c(U+2175, "ⅵ", e285b5), -_c(U+2176, "ⅶ", e285b6), -_c(U+2177, "ⅷ", e285b7), -_c(U+2178, "ⅸ", e285b8), -_c(U+2179, "ⅹ", e285b9), -_c(U+217A, "ⅺ", e285ba), -_c(U+217B, "ⅻ", e285bb), -_c(U+217C, "ⅼ", e285bc), -_c(U+217D, "ⅽ", e285bd), -_c(U+217E, "ⅾ", e285be), -_c(U+217F, "ⅿ", e285bf), -_c(U+2180, "ↀ", e28680), -_c(U+2181, "ↁ", e28681), -_c(U+2182, "ↂ", e28682), -_c(U+2183, "Ↄ", e28683), -_c(U+2184, "ↄ", e28684), -_c(U+2185, "ↅ", e28685), -_c(U+2186, "ↆ", e28686), -_c(U+2187, "ↇ", e28687), -_c(U+2188, "ↈ", e28688), -_c(U+2189, "↉", e28689), -_c(U+218A, "↊", e2868a), -_c(U+218B, "↋", e2868b), -_c(U+218C, "↌", e2868c), -_c(U+218D, "↍", e2868d), -_c(U+218E, "↎", e2868e), -_c(U+218F, "↏", e2868f), -_c(U+2190, "←", e28690), -_c(U+2191, "↑", e28691), -_c(U+2192, "→", e28692), -_c(U+2193, "↓", e28693), -_c(U+2194, "↔", e28694), -_c(U+2195, "↕", e28695), -_c(U+2196, "↖", e28696), -_c(U+2197, "↗", e28697), -_c(U+2198, "↘", e28698), -_c(U+2199, "↙", e28699), -_c(U+219A, "↚", e2869a), -_c(U+219B, "↛", e2869b), -_c(U+219C, "↜", e2869c), -_c(U+219D, "↝", e2869d), -_c(U+219E, "↞", e2869e), -_c(U+219F, "↟", e2869f), -_c(U+21A0, "↠", e286a0), -_c(U+21A1, "↡", e286a1), -_c(U+21A2, "↢", e286a2), -_c(U+21A3, "↣", e286a3), -_c(U+21A4, "↤", e286a4), -_c(U+21A5, "↥", e286a5), -_c(U+21A6, "↦", e286a6), -_c(U+21A7, "↧", e286a7), -_c(U+21A8, "↨", e286a8), -_c(U+21A9, "↩", e286a9), -_c(U+21AA, "↪", e286aa), -_c(U+21AB, "↫", e286ab), -_c(U+21AC, "↬", e286ac), -_c(U+21AD, "↭", e286ad), -_c(U+21AE, "↮", e286ae), -_c(U+21AF, "↯", e286af), -_c(U+21B0, "↰", e286b0), -_c(U+21B1, "↱", e286b1), -_c(U+21B2, "↲", e286b2), -_c(U+21B3, "↳", e286b3), -_c(U+21B4, "↴", e286b4), -_c(U+21B5, "↵", e286b5), -_c(U+21B6, "↶", e286b6), -_c(U+21B7, "↷", e286b7), -_c(U+21B8, "↸", e286b8), -_c(U+21B9, "↹", e286b9), -_c(U+21BA, "↺", e286ba), -_c(U+21BB, "↻", e286bb), -_c(U+21BC, "↼", e286bc), -_c(U+21BD, "↽", e286bd), -_c(U+21BE, "↾", e286be), -_c(U+21BF, "↿", e286bf), -_c(U+21C0, "⇀", e28780), -_c(U+21C1, "⇁", e28781), -_c(U+21C2, "⇂", e28782), -_c(U+21C3, "⇃", e28783), -_c(U+21C4, "⇄", e28784), -_c(U+21C5, "⇅", e28785), -_c(U+21C6, "⇆", e28786), -_c(U+21C7, "⇇", e28787), -_c(U+21C8, "⇈", e28788), -_c(U+21C9, "⇉", e28789), -_c(U+21CA, "⇊", e2878a), -_c(U+21CB, "⇋", e2878b), -_c(U+21CC, "⇌", e2878c), -_c(U+21CD, "⇍", e2878d), -_c(U+21CE, "⇎", e2878e), -_c(U+21CF, "⇏", e2878f), -_c(U+21D0, "⇐", e28790), -_c(U+21D1, "⇑", e28791), -_c(U+21D2, "⇒", e28792), -_c(U+21D3, "⇓", e28793), -_c(U+21D4, "⇔", e28794), -_c(U+21D5, "⇕", e28795), -_c(U+21D6, "⇖", e28796), -_c(U+21D7, "⇗", e28797), -_c(U+21D8, "⇘", e28798), -_c(U+21D9, "⇙", e28799), -_c(U+21DA, "⇚", e2879a), -_c(U+21DB, "⇛", e2879b), -_c(U+21DC, "⇜", e2879c), -_c(U+21DD, "⇝", e2879d), -_c(U+21DE, "⇞", e2879e), -_c(U+21DF, "⇟", e2879f), -_c(U+21E0, "⇠", e287a0), -_c(U+21E1, "⇡", e287a1), -_c(U+21E2, "⇢", e287a2), -_c(U+21E3, "⇣", e287a3), -_c(U+21E4, "⇤", e287a4), -_c(U+21E5, "⇥", e287a5), -_c(U+21E6, "⇦", e287a6), -_c(U+21E7, "⇧", e287a7), -_c(U+21E8, "⇨", e287a8), -_c(U+21E9, "⇩", e287a9), -_c(U+21EA, "⇪", e287aa), -_c(U+21EB, "⇫", e287ab), -_c(U+21EC, "⇬", e287ac), -_c(U+21ED, "⇭", e287ad), -_c(U+21EE, "⇮", e287ae), -_c(U+21EF, "⇯", e287af), -_c(U+21F0, "⇰", e287b0), -_c(U+21F1, "⇱", e287b1), -_c(U+21F2, "⇲", e287b2), -_c(U+21F3, "⇳", e287b3), -_c(U+21F4, "⇴", e287b4), -_c(U+21F5, "⇵", e287b5), -_c(U+21F6, "⇶", e287b6), -_c(U+21F7, "⇷", e287b7), -_c(U+21F8, "⇸", e287b8), -_c(U+21F9, "⇹", e287b9), -_c(U+21FA, "⇺", e287ba), -_c(U+21FB, "⇻", e287bb), -_c(U+21FC, "⇼", e287bc), -_c(U+21FD, "⇽", e287bd), -_c(U+21FE, "⇾", e287be), -_c(U+21FF, "⇿", e287bf), -_c(U+2200, "∀", e28880), -_c(U+2201, "∁", e28881), -_c(U+2202, "∂", e28882), -_c(U+2203, "∃", e28883), -_c(U+2204, "∄", e28884), -_c(U+2205, "∅", e28885), -_c(U+2206, "∆", e28886), -_c(U+2207, "∇", e28887), -_c(U+2208, "∈", e28888), -_c(U+2209, "∉", e28889), -_c(U+220A, "∊", e2888a), -_c(U+220B, "∋", e2888b), -_c(U+220C, "∌", e2888c), -_c(U+220D, "∍", e2888d), -_c(U+220E, "∎", e2888e), -_c(U+220F, "∏", e2888f), -_c(U+2210, "∐", e28890), -_c(U+2211, "∑", e28891), -_c(U+2212, "−", e28892), -_c(U+2213, "∓", e28893), -_c(U+2214, "∔", e28894), -_c(U+2215, "∕", e28895), -_c(U+2216, "∖", e28896), -_c(U+2217, "∗", e28897), -_c(U+2218, "∘", e28898), -_c(U+2219, "∙", e28899), -_c(U+221A, "√", e2889a), -_c(U+221B, "∛", e2889b), -_c(U+221C, "∜", e2889c), -_c(U+221D, "∝", e2889d), -_c(U+221E, "∞", e2889e), -_c(U+221F, "∟", e2889f), -_c(U+2220, "∠", e288a0), -_c(U+2221, "∡", e288a1), -_c(U+2222, "∢", e288a2), -_c(U+2223, "∣", e288a3), -_c(U+2224, "∤", e288a4), -_c(U+2225, "∥", e288a5), -_c(U+2226, "∦", e288a6), -_c(U+2227, "∧", e288a7), -_c(U+2228, "∨", e288a8), -_c(U+2229, "∩", e288a9), -_c(U+222A, "∪", e288aa), -_c(U+222B, "∫", e288ab), -_c(U+222C, "∬", e288ac), -_c(U+222D, "∭", e288ad), -_c(U+222E, "∮", e288ae), -_c(U+222F, "∯", e288af), -_c(U+2230, "∰", e288b0), -_c(U+2231, "∱", e288b1), -_c(U+2232, "∲", e288b2), -_c(U+2233, "∳", e288b3), -_c(U+2234, "∴", e288b4), -_c(U+2235, "∵", e288b5), -_c(U+2236, "∶", e288b6), -_c(U+2237, "∷", e288b7), -_c(U+2238, "∸", e288b8), -_c(U+2239, "∹", e288b9), -_c(U+223A, "∺", e288ba), -_c(U+223B, "∻", e288bb), -_c(U+223C, "∼", e288bc), -_c(U+223D, "∽", e288bd), -_c(U+223E, "∾", e288be), -_c(U+223F, "∿", e288bf), -_c(U+2240, "≀", e28980), -_c(U+2241, "≁", e28981), -_c(U+2242, "≂", e28982), -_c(U+2243, "≃", e28983), -_c(U+2244, "≄", e28984), -_c(U+2245, "≅", e28985), -_c(U+2246, "≆", e28986), -_c(U+2247, "≇", e28987), -_c(U+2248, "≈", e28988), -_c(U+2249, "≉", e28989), -_c(U+224A, "≊", e2898a), -_c(U+224B, "≋", e2898b), -_c(U+224C, "≌", e2898c), -_c(U+224D, "≍", e2898d), -_c(U+224E, "≎", e2898e), -_c(U+224F, "≏", e2898f), -_c(U+2250, "≐", e28990), -_c(U+2251, "≑", e28991), -_c(U+2252, "≒", e28992), -_c(U+2253, "≓", e28993), -_c(U+2254, "≔", e28994), -_c(U+2255, "≕", e28995), -_c(U+2256, "≖", e28996), -_c(U+2257, "≗", e28997), -_c(U+2258, "≘", e28998), -_c(U+2259, "≙", e28999), -_c(U+225A, "≚", e2899a), -_c(U+225B, "≛", e2899b), -_c(U+225C, "≜", e2899c), -_c(U+225D, "≝", e2899d), -_c(U+225E, "≞", e2899e), -_c(U+225F, "≟", e2899f), -_c(U+2260, "≠", e289a0), -_c(U+2261, "≡", e289a1), -_c(U+2262, "≢", e289a2), -_c(U+2263, "≣", e289a3), -_c(U+2264, "≤", e289a4), -_c(U+2265, "≥", e289a5), -_c(U+2266, "≦", e289a6), -_c(U+2267, "≧", e289a7), -_c(U+2268, "≨", e289a8), -_c(U+2269, "≩", e289a9), -_c(U+226A, "≪", e289aa), -_c(U+226B, "≫", e289ab), -_c(U+226C, "≬", e289ac), -_c(U+226D, "≭", e289ad), -_c(U+226E, "≮", e289ae), -_c(U+226F, "≯", e289af), -_c(U+2270, "≰", e289b0), -_c(U+2271, "≱", e289b1), -_c(U+2272, "≲", e289b2), -_c(U+2273, "≳", e289b3), -_c(U+2274, "≴", e289b4), -_c(U+2275, "≵", e289b5), -_c(U+2276, "≶", e289b6), -_c(U+2277, "≷", e289b7), -_c(U+2278, "≸", e289b8), -_c(U+2279, "≹", e289b9), -_c(U+227A, "≺", e289ba), -_c(U+227B, "≻", e289bb), -_c(U+227C, "≼", e289bc), -_c(U+227D, "≽", e289bd), -_c(U+227E, "≾", e289be), -_c(U+227F, "≿", e289bf), -_c(U+2280, "⊀", e28a80), -_c(U+2281, "⊁", e28a81), -_c(U+2282, "⊂", e28a82), -_c(U+2283, "⊃", e28a83), -_c(U+2284, "⊄", e28a84), -_c(U+2285, "⊅", e28a85), -_c(U+2286, "⊆", e28a86), -_c(U+2287, "⊇", e28a87), -_c(U+2288, "⊈", e28a88), -_c(U+2289, "⊉", e28a89), -_c(U+228A, "⊊", e28a8a), -_c(U+228B, "⊋", e28a8b), -_c(U+228C, "⊌", e28a8c), -_c(U+228D, "⊍", e28a8d), -_c(U+228E, "⊎", e28a8e), -_c(U+228F, "⊏", e28a8f), -_c(U+2290, "⊐", e28a90), -_c(U+2291, "⊑", e28a91), -_c(U+2292, "⊒", e28a92), -_c(U+2293, "⊓", e28a93), -_c(U+2294, "⊔", e28a94), -_c(U+2295, "⊕", e28a95), -_c(U+2296, "⊖", e28a96), -_c(U+2297, "⊗", e28a97), -_c(U+2298, "⊘", e28a98), -_c(U+2299, "⊙", e28a99), -_c(U+229A, "⊚", e28a9a), -_c(U+229B, "⊛", e28a9b), -_c(U+229C, "⊜", e28a9c), -_c(U+229D, "⊝", e28a9d), -_c(U+229E, "⊞", e28a9e), -_c(U+229F, "⊟", e28a9f), -_c(U+22A0, "⊠", e28aa0), -_c(U+22A1, "⊡", e28aa1), -_c(U+22A2, "⊢", e28aa2), -_c(U+22A3, "⊣", e28aa3), -_c(U+22A4, "⊤", e28aa4), -_c(U+22A5, "⊥", e28aa5), -_c(U+22A6, "⊦", e28aa6), -_c(U+22A7, "⊧", e28aa7), -_c(U+22A8, "⊨", e28aa8), -_c(U+22A9, "⊩", e28aa9), -_c(U+22AA, "⊪", e28aaa), -_c(U+22AB, "⊫", e28aab), -_c(U+22AC, "⊬", e28aac), -_c(U+22AD, "⊭", e28aad), -_c(U+22AE, "⊮", e28aae), -_c(U+22AF, "⊯", e28aaf), -_c(U+22B0, "⊰", e28ab0), -_c(U+22B1, "⊱", e28ab1), -_c(U+22B2, "⊲", e28ab2), -_c(U+22B3, "⊳", e28ab3), -_c(U+22B4, "⊴", e28ab4), -_c(U+22B5, "⊵", e28ab5), -_c(U+22B6, "⊶", e28ab6), -_c(U+22B7, "⊷", e28ab7), -_c(U+22B8, "⊸", e28ab8), -_c(U+22B9, "⊹", e28ab9), -_c(U+22BA, "⊺", e28aba), -_c(U+22BB, "⊻", e28abb), -_c(U+22BC, "⊼", e28abc), -_c(U+22BD, "⊽", e28abd), -_c(U+22BE, "⊾", e28abe), -_c(U+22BF, "⊿", e28abf), -_c(U+22C0, "⋀", e28b80), -_c(U+22C1, "⋁", e28b81), -_c(U+22C2, "⋂", e28b82), -_c(U+22C3, "⋃", e28b83), -_c(U+22C4, "⋄", e28b84), -_c(U+22C5, "⋅", e28b85), -_c(U+22C6, "⋆", e28b86), -_c(U+22C7, "⋇", e28b87), -_c(U+22C8, "⋈", e28b88), -_c(U+22C9, "⋉", e28b89), -_c(U+22CA, "⋊", e28b8a), -_c(U+22CB, "⋋", e28b8b), -_c(U+22CC, "⋌", e28b8c), -_c(U+22CD, "⋍", e28b8d), -_c(U+22CE, "⋎", e28b8e), -_c(U+22CF, "⋏", e28b8f), -_c(U+22D0, "⋐", e28b90), -_c(U+22D1, "⋑", e28b91), -_c(U+22D2, "⋒", e28b92), -_c(U+22D3, "⋓", e28b93), -_c(U+22D4, "⋔", e28b94), -_c(U+22D5, "⋕", e28b95), -_c(U+22D6, "⋖", e28b96), -_c(U+22D7, "⋗", e28b97), -_c(U+22D8, "⋘", e28b98), -_c(U+22D9, "⋙", e28b99), -_c(U+22DA, "⋚", e28b9a), -_c(U+22DB, "⋛", e28b9b), -_c(U+22DC, "⋜", e28b9c), -_c(U+22DD, "⋝", e28b9d), -_c(U+22DE, "⋞", e28b9e), -_c(U+22DF, "⋟", e28b9f), -_c(U+22E0, "⋠", e28ba0), -_c(U+22E1, "⋡", e28ba1), -_c(U+22E2, "⋢", e28ba2), -_c(U+22E3, "⋣", e28ba3), -_c(U+22E4, "⋤", e28ba4), -_c(U+22E5, "⋥", e28ba5), -_c(U+22E6, "⋦", e28ba6), -_c(U+22E7, "⋧", e28ba7), -_c(U+22E8, "⋨", e28ba8), -_c(U+22E9, "⋩", e28ba9), -_c(U+22EA, "⋪", e28baa), -_c(U+22EB, "⋫", e28bab), -_c(U+22EC, "⋬", e28bac), -_c(U+22ED, "⋭", e28bad), -_c(U+22EE, "⋮", e28bae), -_c(U+22EF, "⋯", e28baf), -_c(U+22F0, "⋰", e28bb0), -_c(U+22F1, "⋱", e28bb1), -_c(U+22F2, "⋲", e28bb2), -_c(U+22F3, "⋳", e28bb3), -_c(U+22F4, "⋴", e28bb4), -_c(U+22F5, "⋵", e28bb5), -_c(U+22F6, "⋶", e28bb6), -_c(U+22F7, "⋷", e28bb7), -_c(U+22F8, "⋸", e28bb8), -_c(U+22F9, "⋹", e28bb9), -_c(U+22FA, "⋺", e28bba), -_c(U+22FB, "⋻", e28bbb), -_c(U+22FC, "⋼", e28bbc), -_c(U+22FD, "⋽", e28bbd), -_c(U+22FE, "⋾", e28bbe), -_c(U+22FF, "⋿", e28bbf), -_c(U+2300, "⌀", e28c80), -_c(U+2301, "⌁", e28c81), -_c(U+2302, "⌂", e28c82), -_c(U+2303, "⌃", e28c83), -_c(U+2304, "⌄", e28c84), -_c(U+2305, "⌅", e28c85), -_c(U+2306, "⌆", e28c86), -_c(U+2307, "⌇", e28c87), -_c(U+2308, "⌈", e28c88), -_c(U+2309, "⌉", e28c89), -_c(U+230A, "⌊", e28c8a), -_c(U+230B, "⌋", e28c8b), -_c(U+230C, "⌌", e28c8c), -_c(U+230D, "⌍", e28c8d), -_c(U+230E, "⌎", e28c8e), -_c(U+230F, "⌏", e28c8f), -_c(U+2310, "⌐", e28c90), -_c(U+2311, "⌑", e28c91), -_c(U+2312, "⌒", e28c92), -_c(U+2313, "⌓", e28c93), -_c(U+2314, "⌔", e28c94), -_c(U+2315, "⌕", e28c95), -_c(U+2316, "⌖", e28c96), -_c(U+2317, "⌗", e28c97), -_c(U+2318, "⌘", e28c98), -_c(U+2319, "⌙", e28c99), -_c(U+231A, "⌚", e28c9a), -_c(U+231B, "⌛", e28c9b), -_c(U+231C, "⌜", e28c9c), -_c(U+231D, "⌝", e28c9d), -_c(U+231E, "⌞", e28c9e), -_c(U+231F, "⌟", e28c9f), -_c(U+2320, "⌠", e28ca0), -_c(U+2321, "⌡", e28ca1), -_c(U+2322, "⌢", e28ca2), -_c(U+2323, "⌣", e28ca3), -_c(U+2324, "⌤", e28ca4), -_c(U+2325, "⌥", e28ca5), -_c(U+2326, "⌦", e28ca6), -_c(U+2327, "⌧", e28ca7), -_c(U+2328, "⌨", e28ca8), -_c(U+2329, "〈", e28ca9), -_c(U+232A, "〉", e28caa), -_c(U+232B, "⌫", e28cab), -_c(U+232C, "⌬", e28cac), -_c(U+232D, "⌭", e28cad), -_c(U+232E, "⌮", e28cae), -_c(U+232F, "⌯", e28caf), -_c(U+2330, "⌰", e28cb0), -_c(U+2331, "⌱", e28cb1), -_c(U+2332, "⌲", e28cb2), -_c(U+2333, "⌳", e28cb3), -_c(U+2334, "⌴", e28cb4), -_c(U+2335, "⌵", e28cb5), -_c(U+2336, "⌶", e28cb6), -_c(U+2337, "⌷", e28cb7), -_c(U+2338, "⌸", e28cb8), -_c(U+2339, "⌹", e28cb9), -_c(U+233A, "⌺", e28cba), -_c(U+233B, "⌻", e28cbb), -_c(U+233C, "⌼", e28cbc), -_c(U+233D, "⌽", e28cbd), -_c(U+233E, "⌾", e28cbe), -_c(U+233F, "⌿", e28cbf), -_c(U+2340, "⍀", e28d80), -_c(U+2341, "⍁", e28d81), -_c(U+2342, "⍂", e28d82), -_c(U+2343, "⍃", e28d83), -_c(U+2344, "⍄", e28d84), -_c(U+2345, "⍅", e28d85), -_c(U+2346, "⍆", e28d86), -_c(U+2347, "⍇", e28d87), -_c(U+2348, "⍈", e28d88), -_c(U+2349, "⍉", e28d89), -_c(U+234A, "⍊", e28d8a), -_c(U+234B, "⍋", e28d8b), -_c(U+234C, "⍌", e28d8c), -_c(U+234D, "⍍", e28d8d), -_c(U+234E, "⍎", e28d8e), -_c(U+234F, "⍏", e28d8f), -_c(U+2350, "⍐", e28d90), -_c(U+2351, "⍑", e28d91), -_c(U+2352, "⍒", e28d92), -_c(U+2353, "⍓", e28d93), -_c(U+2354, "⍔", e28d94), -_c(U+2355, "⍕", e28d95), -_c(U+2356, "⍖", e28d96), -_c(U+2357, "⍗", e28d97), -_c(U+2358, "⍘", e28d98), -_c(U+2359, "⍙", e28d99), -_c(U+235A, "⍚", e28d9a), -_c(U+235B, "⍛", e28d9b), -_c(U+235C, "⍜", e28d9c), -_c(U+235D, "⍝", e28d9d), -_c(U+235E, "⍞", e28d9e), -_c(U+235F, "⍟", e28d9f), -_c(U+2360, "⍠", e28da0), -_c(U+2361, "⍡", e28da1), -_c(U+2362, "⍢", e28da2), -_c(U+2363, "⍣", e28da3), -_c(U+2364, "⍤", e28da4), -_c(U+2365, "⍥", e28da5), -_c(U+2366, "⍦", e28da6), -_c(U+2367, "⍧", e28da7), -_c(U+2368, "⍨", e28da8), -_c(U+2369, "⍩", e28da9), -_c(U+236A, "⍪", e28daa), -_c(U+236B, "⍫", e28dab), -_c(U+236C, "⍬", e28dac), -_c(U+236D, "⍭", e28dad), -_c(U+236E, "⍮", e28dae), -_c(U+236F, "⍯", e28daf), -_c(U+2370, "⍰", e28db0), -_c(U+2371, "⍱", e28db1), -_c(U+2372, "⍲", e28db2), -_c(U+2373, "⍳", e28db3), -_c(U+2374, "⍴", e28db4), -_c(U+2375, "⍵", e28db5), -_c(U+2376, "⍶", e28db6), -_c(U+2377, "⍷", e28db7), -_c(U+2378, "⍸", e28db8), -_c(U+2379, "⍹", e28db9), -_c(U+237A, "⍺", e28dba), -_c(U+237B, "⍻", e28dbb), -_c(U+237C, "⍼", e28dbc), -_c(U+237D, "⍽", e28dbd), -_c(U+237E, "⍾", e28dbe), -_c(U+237F, "⍿", e28dbf), -_c(U+2380, "⎀", e28e80), -_c(U+2381, "⎁", e28e81), -_c(U+2382, "⎂", e28e82), -_c(U+2383, "⎃", e28e83), -_c(U+2384, "⎄", e28e84), -_c(U+2385, "⎅", e28e85), -_c(U+2386, "⎆", e28e86), -_c(U+2387, "⎇", e28e87), -_c(U+2388, "⎈", e28e88), -_c(U+2389, "⎉", e28e89), -_c(U+238A, "⎊", e28e8a), -_c(U+238B, "⎋", e28e8b), -_c(U+238C, "⎌", e28e8c), -_c(U+238D, "⎍", e28e8d), -_c(U+238E, "⎎", e28e8e), -_c(U+238F, "⎏", e28e8f), -_c(U+2390, "⎐", e28e90), -_c(U+2391, "⎑", e28e91), -_c(U+2392, "⎒", e28e92), -_c(U+2393, "⎓", e28e93), -_c(U+2394, "⎔", e28e94), -_c(U+2395, "⎕", e28e95), -_c(U+2396, "⎖", e28e96), -_c(U+2397, "⎗", e28e97), -_c(U+2398, "⎘", e28e98), -_c(U+2399, "⎙", e28e99), -_c(U+239A, "⎚", e28e9a), -_c(U+239B, "⎛", e28e9b), -_c(U+239C, "⎜", e28e9c), -_c(U+239D, "⎝", e28e9d), -_c(U+239E, "⎞", e28e9e), -_c(U+239F, "⎟", e28e9f), -_c(U+23A0, "⎠", e28ea0), -_c(U+23A1, "⎡", e28ea1), -_c(U+23A2, "⎢", e28ea2), -_c(U+23A3, "⎣", e28ea3), -_c(U+23A4, "⎤", e28ea4), -_c(U+23A5, "⎥", e28ea5), -_c(U+23A6, "⎦", e28ea6), -_c(U+23A7, "⎧", e28ea7), -_c(U+23A8, "⎨", e28ea8), -_c(U+23A9, "⎩", e28ea9), -_c(U+23AA, "⎪", e28eaa), -_c(U+23AB, "⎫", e28eab), -_c(U+23AC, "⎬", e28eac), -_c(U+23AD, "⎭", e28ead), -_c(U+23AE, "⎮", e28eae), -_c(U+23AF, "⎯", e28eaf), -_c(U+23B0, "⎰", e28eb0), -_c(U+23B1, "⎱", e28eb1), -_c(U+23B2, "⎲", e28eb2), -_c(U+23B3, "⎳", e28eb3), -_c(U+23B4, "⎴", e28eb4), -_c(U+23B5, "⎵", e28eb5), -_c(U+23B6, "⎶", e28eb6), -_c(U+23B7, "⎷", e28eb7), -_c(U+23B8, "⎸", e28eb8), -_c(U+23B9, "⎹", e28eb9), -_c(U+23BA, "⎺", e28eba), -_c(U+23BB, "⎻", e28ebb), -_c(U+23BC, "⎼", e28ebc), -_c(U+23BD, "⎽", e28ebd), -_c(U+23BE, "⎾", e28ebe), -_c(U+23BF, "⎿", e28ebf), -_c(U+23C0, "⏀", e28f80), -_c(U+23C1, "⏁", e28f81), -_c(U+23C2, "⏂", e28f82), -_c(U+23C3, "⏃", e28f83), -_c(U+23C4, "⏄", e28f84), -_c(U+23C5, "⏅", e28f85), -_c(U+23C6, "⏆", e28f86), -_c(U+23C7, "⏇", e28f87), -_c(U+23C8, "⏈", e28f88), -_c(U+23C9, "⏉", e28f89), -_c(U+23CA, "⏊", e28f8a), -_c(U+23CB, "⏋", e28f8b), -_c(U+23CC, "⏌", e28f8c), -_c(U+23CD, "⏍", e28f8d), -_c(U+23CE, "⏎", e28f8e), -_c(U+23CF, "⏏", e28f8f), -_c(U+23D0, "⏐", e28f90), -_c(U+23D1, "⏑", e28f91), -_c(U+23D2, "⏒", e28f92), -_c(U+23D3, "⏓", e28f93), -_c(U+23D4, "⏔", e28f94), -_c(U+23D5, "⏕", e28f95), -_c(U+23D6, "⏖", e28f96), -_c(U+23D7, "⏗", e28f97), -_c(U+23D8, "⏘", e28f98), -_c(U+23D9, "⏙", e28f99), -_c(U+23DA, "⏚", e28f9a), -_c(U+23DB, "⏛", e28f9b), -_c(U+23DC, "⏜", e28f9c), -_c(U+23DD, "⏝", e28f9d), -_c(U+23DE, "⏞", e28f9e), -_c(U+23DF, "⏟", e28f9f), -_c(U+23E0, "⏠", e28fa0), -_c(U+23E1, "⏡", e28fa1), -_c(U+23E2, "⏢", e28fa2), -_c(U+23E3, "⏣", e28fa3), -_c(U+23E4, "⏤", e28fa4), -_c(U+23E5, "⏥", e28fa5), -_c(U+23E6, "⏦", e28fa6), -_c(U+23E7, "⏧", e28fa7), -_c(U+23E8, "⏨", e28fa8), -_c(U+23E9, "⏩", e28fa9), -_c(U+23EA, "⏪", e28faa), -_c(U+23EB, "⏫", e28fab), -_c(U+23EC, "⏬", e28fac), -_c(U+23ED, "⏭", e28fad), -_c(U+23EE, "⏮", e28fae), -_c(U+23EF, "⏯", e28faf), -_c(U+23F0, "⏰", e28fb0), -_c(U+23F1, "⏱", e28fb1), -_c(U+23F2, "⏲", e28fb2), -_c(U+23F3, "⏳", e28fb3), -_c(U+23F4, "⏴", e28fb4), -_c(U+23F5, "⏵", e28fb5), -_c(U+23F6, "⏶", e28fb6), -_c(U+23F7, "⏷", e28fb7), -_c(U+23F8, "⏸", e28fb8), -_c(U+23F9, "⏹", e28fb9), -_c(U+23FA, "⏺", e28fba), -_c(U+23FB, "⏻", e28fbb), -_c(U+23FC, "⏼", e28fbc), -_c(U+23FD, "⏽", e28fbd), -_c(U+23FE, "⏾", e28fbe), -_c(U+23FF, "⏿", e28fbf), -_c(U+2400, "␀", e29080), -_c(U+2401, "␁", e29081), -_c(U+2402, "␂", e29082), -_c(U+2403, "␃", e29083), -_c(U+2404, "␄", e29084), -_c(U+2405, "␅", e29085), -_c(U+2406, "␆", e29086), -_c(U+2407, "␇", e29087), -_c(U+2408, "␈", e29088), -_c(U+2409, "␉", e29089), -_c(U+240A, "␊", e2908a), -_c(U+240B, "␋", e2908b), -_c(U+240C, "␌", e2908c), -_c(U+240D, "␍", e2908d), -_c(U+240E, "␎", e2908e), -_c(U+240F, "␏", e2908f), -_c(U+2410, "␐", e29090), -_c(U+2411, "␑", e29091), -_c(U+2412, "␒", e29092), -_c(U+2413, "␓", e29093), -_c(U+2414, "␔", e29094), -_c(U+2415, "␕", e29095), -_c(U+2416, "␖", e29096), -_c(U+2417, "␗", e29097), -_c(U+2418, "␘", e29098), -_c(U+2419, "␙", e29099), -_c(U+241A, "␚", e2909a), -_c(U+241B, "␛", e2909b), -_c(U+241C, "␜", e2909c), -_c(U+241D, "␝", e2909d), -_c(U+241E, "␞", e2909e), -_c(U+241F, "␟", e2909f), -_c(U+2420, "␠", e290a0), -_c(U+2421, "␡", e290a1), -_c(U+2422, "␢", e290a2), -_c(U+2423, "␣", e290a3), -_c(U+2424, "␤", e290a4), -_c(U+2425, "␥", e290a5), -_c(U+2426, "␦", e290a6), -_c(U+2427, "␧", e290a7), -_c(U+2428, "␨", e290a8), -_c(U+2429, "␩", e290a9), -_c(U+242A, "␪", e290aa), -_c(U+242B, "␫", e290ab), -_c(U+242C, "␬", e290ac), -_c(U+242D, "␭", e290ad), -_c(U+242E, "␮", e290ae), -_c(U+242F, "␯", e290af), -_c(U+2430, "␰", e290b0), -_c(U+2431, "␱", e290b1), -_c(U+2432, "␲", e290b2), -_c(U+2433, "␳", e290b3), -_c(U+2434, "␴", e290b4), -_c(U+2435, "␵", e290b5), -_c(U+2436, "␶", e290b6), -_c(U+2437, "␷", e290b7), -_c(U+2438, "␸", e290b8), -_c(U+2439, "␹", e290b9), -_c(U+243A, "␺", e290ba), -_c(U+243B, "␻", e290bb), -_c(U+243C, "␼", e290bc), -_c(U+243D, "␽", e290bd), -_c(U+243E, "␾", e290be), -_c(U+243F, "␿", e290bf), -_c(U+2440, "⑀", e29180), -_c(U+2441, "⑁", e29181), -_c(U+2442, "⑂", e29182), -_c(U+2443, "⑃", e29183), -_c(U+2444, "⑄", e29184), -_c(U+2445, "⑅", e29185), -_c(U+2446, "⑆", e29186), -_c(U+2447, "⑇", e29187), -_c(U+2448, "⑈", e29188), -_c(U+2449, "⑉", e29189), -_c(U+244A, "⑊", e2918a), -_c(U+244B, "⑋", e2918b), -_c(U+244C, "⑌", e2918c), -_c(U+244D, "⑍", e2918d), -_c(U+244E, "⑎", e2918e), -_c(U+244F, "⑏", e2918f), -_c(U+2450, "⑐", e29190), -_c(U+2451, "⑑", e29191), -_c(U+2452, "⑒", e29192), -_c(U+2453, "⑓", e29193), -_c(U+2454, "⑔", e29194), -_c(U+2455, "⑕", e29195), -_c(U+2456, "⑖", e29196), -_c(U+2457, "⑗", e29197), -_c(U+2458, "⑘", e29198), -_c(U+2459, "⑙", e29199), -_c(U+245A, "⑚", e2919a), -_c(U+245B, "⑛", e2919b), -_c(U+245C, "⑜", e2919c), -_c(U+245D, "⑝", e2919d), -_c(U+245E, "⑞", e2919e), -_c(U+245F, "⑟", e2919f), -_c(U+2460, "①", e291a0), -_c(U+2461, "②", e291a1), -_c(U+2462, "③", e291a2), -_c(U+2463, "④", e291a3), -_c(U+2464, "⑤", e291a4), -_c(U+2465, "⑥", e291a5), -_c(U+2466, "⑦", e291a6), -_c(U+2467, "⑧", e291a7), -_c(U+2468, "⑨", e291a8), -_c(U+2469, "⑩", e291a9), -_c(U+246A, "⑪", e291aa), -_c(U+246B, "⑫", e291ab), -_c(U+246C, "⑬", e291ac), -_c(U+246D, "⑭", e291ad), -_c(U+246E, "⑮", e291ae), -_c(U+246F, "⑯", e291af), -_c(U+2470, "⑰", e291b0), -_c(U+2471, "⑱", e291b1), -_c(U+2472, "⑲", e291b2), -_c(U+2473, "⑳", e291b3), -_c(U+2474, "⑴", e291b4), -_c(U+2475, "⑵", e291b5), -_c(U+2476, "⑶", e291b6), -_c(U+2477, "⑷", e291b7), -_c(U+2478, "⑸", e291b8), -_c(U+2479, "⑹", e291b9), -_c(U+247A, "⑺", e291ba), -_c(U+247B, "⑻", e291bb), -_c(U+247C, "⑼", e291bc), -_c(U+247D, "⑽", e291bd), -_c(U+247E, "⑾", e291be), -_c(U+247F, "⑿", e291bf), -_c(U+2480, "⒀", e29280), -_c(U+2481, "⒁", e29281), -_c(U+2482, "⒂", e29282), -_c(U+2483, "⒃", e29283), -_c(U+2484, "⒄", e29284), -_c(U+2485, "⒅", e29285), -_c(U+2486, "⒆", e29286), -_c(U+2487, "⒇", e29287), -_c(U+2488, "⒈", e29288), -_c(U+2489, "⒉", e29289), -_c(U+248A, "⒊", e2928a), -_c(U+248B, "⒋", e2928b), -_c(U+248C, "⒌", e2928c), -_c(U+248D, "⒍", e2928d), -_c(U+248E, "⒎", e2928e), -_c(U+248F, "⒏", e2928f), -_c(U+2490, "⒐", e29290), -_c(U+2491, "⒑", e29291), -_c(U+2492, "⒒", e29292), -_c(U+2493, "⒓", e29293), -_c(U+2494, "⒔", e29294), -_c(U+2495, "⒕", e29295), -_c(U+2496, "⒖", e29296), -_c(U+2497, "⒗", e29297), -_c(U+2498, "⒘", e29298), -_c(U+2499, "⒙", e29299), -_c(U+249A, "⒚", e2929a), -_c(U+249B, "⒛", e2929b), -_c(U+249C, "⒜", e2929c), -_c(U+249D, "⒝", e2929d), -_c(U+249E, "⒞", e2929e), -_c(U+249F, "⒟", e2929f), -_c(U+2601, "☁", e29881), -_c(U+2602, "☂", e29882), -_c(U+2603, "☃", e29883), -_c(U+2604, "☄", e29884), -_c(U+2605, "★", e29885), -_c(U+2606, "☆", e29886), -_c(U+2607, "☇", e29887), -_c(U+2608, "☈", e29888), -_c(U+2609, "☉", e29889), -_c(U+260A, "☊", e2988a), -_c(U+260B, "☋", e2988b), -_c(U+260C, "☌", e2988c), -_c(U+260D, "☍", e2988d), -_c(U+260E, "☎", e2988e), -_c(U+260F, "☏", e2988f), -_c(U+2610, "☐", e29890), -_c(U+2611, "☑", e29891), -_c(U+2612, "☒", e29892), -_c(U+2613, "☓", e29893), -_c(U+2614, "☔", e29894), -_c(U+2615, "☕", e29895), -_c(U+2616, "☖", e29896), -_c(U+2617, "☗", e29897), -_c(U+2618, "☘", e29898), -_c(U+2619, "☙", e29899), -_c(U+261A, "☚", e2989a), -_c(U+261B, "☛", e2989b), -_c(U+261C, "☜", e2989c), -_c(U+261D, "☝", e2989d), -_c(U+261E, "☞", e2989e), -_c(U+261F, "☟", e2989f), -_c(U+2620, "☠", e298a0), -_c(U+2621, "☡", e298a1), -_c(U+2622, "☢", e298a2), -_c(U+2623, "☣", e298a3), -_c(U+2624, "☤", e298a4), -_c(U+2625, "☥", e298a5), -_c(U+2626, "☦", e298a6), -_c(U+2627, "☧", e298a7), -_c(U+2628, "☨", e298a8), -_c(U+2629, "☩", e298a9), -_c(U+262A, "☪", e298aa), -_c(U+262B, "☫", e298ab), -_c(U+262C, "☬", e298ac), -_c(U+262D, "☭", e298ad), -_c(U+262E, "☮", e298ae), -_c(U+262F, "☯", e298af), -_c(U+2630, "☰", e298b0), -_c(U+2631, "☱", e298b1), -_c(U+2632, "☲", e298b2), -_c(U+2633, "☳", e298b3), -_c(U+2634, "☴", e298b4), -_c(U+2635, "☵", e298b5), -_c(U+2636, "☶", e298b6), -_c(U+2637, "☷", e298b7), -_c(U+2638, "☸", e298b8), -_c(U+2639, "☹", e298b9), -_c(U+263A, "☺", e298ba), -_c(U+263B, "☻", e298bb), -_c(U+263C, "☼", e298bc), -_c(U+263D, "☽", e298bd), -_c(U+263E, "☾", e298be), -_c(U+263F, "☿", e298bf), -_c(U+2640, "♀", e29980), -_c(U+2641, "♁", e29981), -_c(U+2642, "♂", e29982), -_c(U+2643, "♃", e29983), -_c(U+2644, "♄", e29984), -_c(U+2645, "♅", e29985), -_c(U+2646, "♆", e29986), -_c(U+2647, "♇", e29987), -_c(U+2648, "♈", e29988), -_c(U+2649, "♉", e29989), -_c(U+264A, "♊", e2998a), -_c(U+264B, "♋", e2998b), -_c(U+264C, "♌", e2998c), -_c(U+264D, "♍", e2998d), -_c(U+264E, "♎", e2998e), -_c(U+264F, "♏", e2998f), -_c(U+2650, "♐", e29990), -_c(U+2651, "♑", e29991), -_c(U+2652, "♒", e29992), -_c(U+2653, "♓", e29993), -_c(U+2654, "♔", e29994), -_c(U+2655, "♕", e29995), -_c(U+2656, "♖", e29996), -_c(U+2657, "♗", e29997), -_c(U+2658, "♘", e29998), -_c(U+2659, "♙", e29999), -_c(U+265A, "♚", e2999a), -_c(U+265B, "♛", e2999b), -_c(U+265C, "♜", e2999c), -_c(U+265D, "♝", e2999d), -_c(U+265E, "♞", e2999e), -_c(U+265F, "♟", e2999f), -_c(U+2660, "♠", e299a0), -_c(U+2661, "♡", e299a1), -_c(U+2662, "♢", e299a2), -_c(U+2663, "♣", e299a3), -_c(U+2664, "♤", e299a4), -_c(U+2665, "♥", e299a5), -_c(U+2666, "♦", e299a6), -_c(U+2667, "♧", e299a7), -_c(U+2668, "♨", e299a8), -_c(U+2669, "♩", e299a9), -_c(U+266A, "♪", e299aa), -_c(U+266B, "♫", e299ab), -_c(U+266C, "♬", e299ac), -_c(U+266D, "♭", e299ad), -_c(U+266E, "♮", e299ae), -_c(U+266F, "♯", e299af), -_c(U+2670, "♰", e299b0), -_c(U+2671, "♱", e299b1), -_c(U+2672, "♲", e299b2), -_c(U+2673, "♳", e299b3), -_c(U+2674, "♴", e299b4), -_c(U+2675, "♵", e299b5), -_c(U+2676, "♶", e299b6), -_c(U+2677, "♷", e299b7), -_c(U+2678, "♸", e299b8), -_c(U+2679, "♹", e299b9), -_c(U+267A, "♺", e299ba), -_c(U+267B, "♻", e299bb), -_c(U+267C, "♼", e299bc), -_c(U+267D, "♽", e299bd), -_c(U+267E, "♾", e299be), -_c(U+267F, "♿", e299bf), -_c(U+2680, "⚀", e29a80), -_c(U+2681, "⚁", e29a81), -_c(U+2682, "⚂", e29a82), -_c(U+2683, "⚃", e29a83), -_c(U+2684, "⚄", e29a84), -_c(U+2685, "⚅", e29a85), -_c(U+2686, "⚆", e29a86), -_c(U+2687, "⚇", e29a87), -_c(U+2688, "⚈", e29a88), -_c(U+2689, "⚉", e29a89), -_c(U+268A, "⚊", e29a8a), -_c(U+268B, "⚋", e29a8b), -_c(U+268C, "⚌", e29a8c), -_c(U+268D, "⚍", e29a8d), -_c(U+268E, "⚎", e29a8e), -_c(U+268F, "⚏", e29a8f), -_c(U+2690, "⚐", e29a90), -_c(U+2691, "⚑", e29a91), -_c(U+2692, "⚒", e29a92), -_c(U+2693, "⚓", e29a93), -_c(U+2694, "⚔", e29a94), -_c(U+2695, "⚕", e29a95), -_c(U+2696, "⚖", e29a96), -_c(U+2697, "⚗", e29a97), -_c(U+2698, "⚘", e29a98), -_c(U+2699, "⚙", e29a99), -_c(U+269A, "⚚", e29a9a), -_c(U+269B, "⚛", e29a9b), -_c(U+269C, "⚜", e29a9c), -_c(U+269D, "⚝", e29a9d), -_c(U+269E, "⚞", e29a9e), -_c(U+269F, "⚟", e29a9f), -_c(U+26A0, "⚠", e29aa0), -_c(U+26A1, "⚡", e29aa1), -_c(U+26A2, "⚢", e29aa2), -_c(U+26A3, "⚣", e29aa3), -_c(U+26A4, "⚤", e29aa4), -_c(U+26A5, "⚥", e29aa5), -_c(U+26A6, "⚦", e29aa6), -_c(U+26A7, "⚧", e29aa7), -_c(U+26A8, "⚨", e29aa8), -_c(U+26A9, "⚩", e29aa9), -_c(U+26AA, "⚪", e29aaa), -_c(U+26AB, "⚫", e29aab), -_c(U+26AC, "⚬", e29aac), -_c(U+26AD, "⚭", e29aad), -_c(U+26AE, "⚮", e29aae), -_c(U+26AF, "⚯", e29aaf), -_c(U+26B0, "⚰", e29ab0), -_c(U+26B1, "⚱", e29ab1), -_c(U+26B2, "⚲", e29ab2), -_c(U+26B3, "⚳", e29ab3), -_c(U+26B4, "⚴", e29ab4), -_c(U+26B5, "⚵", e29ab5), -_c(U+26B6, "⚶", e29ab6), -_c(U+26B7, "⚷", e29ab7), -_c(U+26B8, "⚸", e29ab8), -_c(U+26B9, "⚹", e29ab9), -_c(U+26BA, "⚺", e29aba), -_c(U+26BB, "⚻", e29abb), -_c(U+26BC, "⚼", e29abc), -_c(U+26BD, "⚽", e29abd), -_c(U+26BE, "⚾", e29abe), -_c(U+26BF, "⚿", e29abf), -_c(U+26C0, "⛀", e29b80), -_c(U+26C1, "⛁", e29b81), -_c(U+26C2, "⛂", e29b82), -_c(U+26C3, "⛃", e29b83), -_c(U+26C4, "⛄", e29b84), -_c(U+26C5, "⛅", e29b85), -_c(U+26C6, "⛆", e29b86), -_c(U+26C7, "⛇", e29b87), -_c(U+26C8, "⛈", e29b88), -_c(U+26C9, "⛉", e29b89), -_c(U+26CA, "⛊", e29b8a), -_c(U+26CB, "⛋", e29b8b), -_c(U+26CC, "⛌", e29b8c), -_c(U+26CD, "⛍", e29b8d), -_c(U+26CE, "⛎", e29b8e), -_c(U+26CF, "⛏", e29b8f), -_c(U+26D0, "⛐", e29b90), -_c(U+26D1, "⛑", e29b91), -_c(U+26D2, "⛒", e29b92), -_c(U+26D3, "⛓", e29b93), -_c(U+26D4, "⛔", e29b94), -_c(U+26D5, "⛕", e29b95), -_c(U+26D6, "⛖", e29b96), -_c(U+26D7, "⛗", e29b97), -_c(U+26D8, "⛘", e29b98), -_c(U+26D9, "⛙", e29b99), -_c(U+26DA, "⛚", e29b9a), -_c(U+26DB, "⛛", e29b9b), -_c(U+26DC, "⛜", e29b9c), -_c(U+26DD, "⛝", e29b9d), -_c(U+26DE, "⛞", e29b9e), -_c(U+26DF, "⛟", e29b9f), -_c(U+26E0, "⛠", e29ba0), -_c(U+26E1, "⛡", e29ba1), -_c(U+26E2, "⛢", e29ba2), -_c(U+26E3, "⛣", e29ba3), -_c(U+26E4, "⛤", e29ba4), -_c(U+26E5, "⛥", e29ba5), -_c(U+26E6, "⛦", e29ba6), -_c(U+26E7, "⛧", e29ba7), -_c(U+26E8, "⛨", e29ba8), -_c(U+26E9, "⛩", e29ba9), -_c(U+26EA, "⛪", e29baa), -_c(U+26EB, "⛫", e29bab), -_c(U+26EC, "⛬", e29bac), -_c(U+26ED, "⛭", e29bad), -_c(U+26EE, "⛮", e29bae), -_c(U+26EF, "⛯", e29baf), -_c(U+26F0, "⛰", e29bb0), -_c(U+26F1, "⛱", e29bb1), -_c(U+26F2, "⛲", e29bb2), -_c(U+26F3, "⛳", e29bb3), -_c(U+26F4, "⛴", e29bb4), -_c(U+26F5, "⛵", e29bb5), -_c(U+26F6, "⛶", e29bb6), -_c(U+26F7, "⛷", e29bb7), -_c(U+26F8, "⛸", e29bb8), -_c(U+26F9, "⛹", e29bb9), -_c(U+26FA, "⛺", e29bba), -_c(U+26FB, "⛻", e29bbb), -_c(U+26FC, "⛼", e29bbc), -_c(U+26FD, "⛽", e29bbd), -_c(U+26FE, "⛾", e29bbe), -_c(U+26FF, "⛿", e29bbf), -_c(U+2700, "✀", e29c80), -_c(U+2701, "✁", e29c81), -_c(U+2702, "✂", e29c82), -_c(U+2703, "✃", e29c83), -_c(U+2704, "✄", e29c84), -_c(U+2705, "✅", e29c85), -_c(U+2706, "✆", e29c86), -_c(U+2707, "✇", e29c87), -_c(U+2708, "✈", e29c88), -_c(U+2709, "✉", e29c89), -_c(U+270A, "✊", e29c8a), -_c(U+270B, "✋", e29c8b), -_c(U+270C, "✌", e29c8c), -_c(U+270D, "✍", e29c8d), -_c(U+270E, "✎", e29c8e), -_c(U+270F, "✏", e29c8f), -_c(U+2710, "✐", e29c90), -_c(U+2711, "✑", e29c91), -_c(U+2712, "✒", e29c92), -_c(U+2713, "✓", e29c93), -_c(U+2714, "✔", e29c94), -_c(U+2715, "✕", e29c95), -_c(U+2716, "✖", e29c96), -_c(U+2717, "✗", e29c97), -_c(U+2718, "✘", e29c98), -_c(U+2719, "✙", e29c99), -_c(U+271A, "✚", e29c9a), -_c(U+271B, "✛", e29c9b), -_c(U+271C, "✜", e29c9c), -_c(U+271D, "✝", e29c9d), -_c(U+271E, "✞", e29c9e), -_c(U+271F, "✟", e29c9f), -_c(U+2720, "✠", e29ca0), -_c(U+2721, "✡", e29ca1), -_c(U+2722, "✢", e29ca2), -_c(U+2723, "✣", e29ca3), -_c(U+2724, "✤", e29ca4), -_c(U+2725, "✥", e29ca5), -_c(U+2726, "✦", e29ca6), -_c(U+2727, "✧", e29ca7), -_c(U+2728, "✨", e29ca8), -_c(U+2729, "✩", e29ca9), -_c(U+272A, "✪", e29caa), -_c(U+272B, "✫", e29cab), -_c(U+272C, "✬", e29cac), -_c(U+272D, "✭", e29cad), -_c(U+272E, "✮", e29cae), -_c(U+272F, "✯", e29caf), -_c(U+2730, "✰", e29cb0), -_c(U+2731, "✱", e29cb1), -_c(U+2732, "✲", e29cb2), -_c(U+2733, "✳", e29cb3), -_c(U+2734, "✴", e29cb4), -_c(U+2735, "✵", e29cb5), -_c(U+2736, "✶", e29cb6), -_c(U+2737, "✷", e29cb7), -_c(U+2738, "✸", e29cb8), -_c(U+2739, "✹", e29cb9), -_c(U+273A, "✺", e29cba), -_c(U+273B, "✻", e29cbb), -_c(U+273C, "✼", e29cbc), -_c(U+273D, "✽", e29cbd), -_c(U+273E, "✾", e29cbe), -_c(U+273F, "✿", e29cbf), -_c(U+2740, "❀", e29d80), -_c(U+2741, "❁", e29d81), -_c(U+2742, "❂", e29d82), -_c(U+2743, "❃", e29d83), -_c(U+2744, "❄", e29d84), -_c(U+2745, "❅", e29d85), -_c(U+2746, "❆", e29d86), -_c(U+2747, "❇", e29d87), -_c(U+2748, "❈", e29d88), -_c(U+2749, "❉", e29d89), -_c(U+274A, "❊", e29d8a), -_c(U+274B, "❋", e29d8b), -_c(U+274C, "❌", e29d8c), -_c(U+274D, "❍", e29d8d), -_c(U+274E, "❎", e29d8e), -_c(U+274F, "❏", e29d8f), -_c(U+2750, "❐", e29d90), -_c(U+2751, "❑", e29d91), -_c(U+2752, "❒", e29d92), -_c(U+2753, "❓", e29d93), -_c(U+2754, "❔", e29d94), -_c(U+2755, "❕", e29d95), -_c(U+2756, "❖", e29d96), -_c(U+2757, "❗", e29d97), -_c(U+2758, "❘", e29d98), -_c(U+2759, "❙", e29d99), -_c(U+275A, "❚", e29d9a), -_c(U+275B, "❛", e29d9b), -_c(U+275C, "❜", e29d9c), -_c(U+275D, "❝", e29d9d), -_c(U+275E, "❞", e29d9e), -_c(U+275F, "❟", e29d9f), -_c(U+2760, "❠", e29da0), -_c(U+2761, "❡", e29da1), -_c(U+2762, "❢", e29da2), -_c(U+2763, "❣", e29da3), -_c(U+2764, "❤", e29da4), -_c(U+2765, "❥", e29da5), -_c(U+2766, "❦", e29da6), -_c(U+2767, "❧", e29da7), -_c(U+2768, "❨", e29da8), -_c(U+2769, "❩", e29da9), -_c(U+276A, "❪", e29daa), -_c(U+276B, "❫", e29dab), -_c(U+276C, "❬", e29dac), -_c(U+276D, "❭", e29dad), -_c(U+276E, "❮", e29dae), -_c(U+276F, "❯", e29daf), -_c(U+2770, "❰", e29db0), -_c(U+2771, "❱", e29db1), -_c(U+2772, "❲", e29db2), -_c(U+2773, "❳", e29db3), -_c(U+2774, "❴", e29db4), -_c(U+2775, "❵", e29db5), -_c(U+2776, "❶", e29db6), -_c(U+2777, "❷", e29db7), -_c(U+2778, "❸", e29db8), -_c(U+2779, "❹", e29db9), -_c(U+277A, "❺", e29dba), -_c(U+277B, "❻", e29dbb), -_c(U+277C, "❼", e29dbc), -_c(U+277D, "❽", e29dbd), -_c(U+277E, "❾", e29dbe), -_c(U+277F, "❿", e29dbf), -_c(U+2780, "➀", e29e80), -_c(U+2781, "➁", e29e81), -_c(U+2782, "➂", e29e82), -_c(U+2783, "➃", e29e83), -_c(U+2784, "➄", e29e84), -_c(U+2785, "➅", e29e85), -_c(U+2786, "➆", e29e86), -_c(U+2787, "➇", e29e87), -_c(U+2788, "➈", e29e88), -_c(U+2789, "➉", e29e89), -_c(U+278A, "➊", e29e8a), -_c(U+278B, "➋", e29e8b), -_c(U+278C, "➌", e29e8c), -_c(U+278D, "➍", e29e8d), -_c(U+278E, "➎", e29e8e), -_c(U+278F, "➏", e29e8f), -_c(U+2790, "➐", e29e90), -_c(U+2791, "➑", e29e91), -_c(U+2792, "➒", e29e92), -_c(U+2793, "➓", e29e93), -_c(U+2794, "➔", e29e94), -_c(U+2795, "➕", e29e95), -_c(U+2796, "➖", e29e96), -_c(U+2797, "➗", e29e97), -_c(U+2798, "➘", e29e98), -_c(U+2799, "➙", e29e99), -_c(U+279A, "➚", e29e9a), -_c(U+279B, "➛", e29e9b), -_c(U+279C, "➜", e29e9c), -_c(U+279D, "➝", e29e9d), -_c(U+279E, "➞", e29e9e), -_c(U+279F, "➟", e29e9f), -_c(U+27A0, "➠", e29ea0), -_c(U+27A1, "➡", e29ea1), -_c(U+27A2, "➢", e29ea2), -_c(U+27A3, "➣", e29ea3), -_c(U+27A4, "➤", e29ea4), -_c(U+27A5, "➥", e29ea5), -_c(U+27A6, "➦", e29ea6), -_c(U+27A7, "➧", e29ea7), -_c(U+27A8, "➨", e29ea8), -_c(U+27A9, "➩", e29ea9), -_c(U+27AA, "➪", e29eaa), -_c(U+27AB, "➫", e29eab), -_c(U+27AC, "➬", e29eac), -_c(U+27AD, "➭", e29ead), -_c(U+27AE, "➮", e29eae), -_c(U+27AF, "➯", e29eaf), -_c(U+27B0, "➰", e29eb0), -_c(U+27B1, "➱", e29eb1), -_c(U+27B2, "➲", e29eb2), -_c(U+27B3, "➳", e29eb3), -_c(U+27B4, "➴", e29eb4), -_c(U+27B5, "➵", e29eb5), -_c(U+27B6, "➶", e29eb6), -_c(U+27B7, "➷", e29eb7), -_c(U+27B8, "➸", e29eb8), -_c(U+27B9, "➹", e29eb9), -_c(U+27BA, "➺", e29eba), -_c(U+27BB, "➻", e29ebb), -_c(U+27BC, "➼", e29ebc), -_c(U+27BD, "➽", e29ebd), -_c(U+27BE, "➾", e29ebe), -_c(U+27BF, "➿", e29ebf), -_c(U+27C0, "⟀", e29f80), -_c(U+27C1, "⟁", e29f81), -_c(U+27C2, "⟂", e29f82), -_c(U+27C3, "⟃", e29f83), -_c(U+27C4, "⟄", e29f84), -_c(U+27C5, "⟅", e29f85), -_c(U+27C6, "⟆", e29f86), -_c(U+27C7, "⟇", e29f87), -_c(U+27C8, "⟈", e29f88), -_c(U+27C9, "⟉", e29f89), -_c(U+27CA, "⟊", e29f8a), -_c(U+27CB, "⟋", e29f8b), -_c(U+27CC, "⟌", e29f8c), -_c(U+27CD, "⟍", e29f8d), -_c(U+27CE, "⟎", e29f8e), -_c(U+27CF, "⟏", e29f8f), -_c(U+27D0, "⟐", e29f90), -_c(U+27D1, "⟑", e29f91), -_c(U+27D2, "⟒", e29f92), -_c(U+27D3, "⟓", e29f93), -_c(U+27D4, "⟔", e29f94), -_c(U+27D5, "⟕", e29f95), -_c(U+27D6, "⟖", e29f96), -_c(U+27D7, "⟗", e29f97), -_c(U+27D8, "⟘", e29f98), -_c(U+27D9, "⟙", e29f99), -_c(U+27DA, "⟚", e29f9a), -_c(U+27DB, "⟛", e29f9b), -_c(U+27DC, "⟜", e29f9c), -_c(U+27DD, "⟝", e29f9d), -_c(U+27DE, "⟞", e29f9e), -_c(U+27DF, "⟟", e29f9f), -_c(U+27E0, "⟠", e29fa0), -_c(U+27E1, "⟡", e29fa1), -_c(U+27E2, "⟢", e29fa2), -_c(U+27E3, "⟣", e29fa3), -_c(U+27E4, "⟤", e29fa4), -_c(U+27E5, "⟥", e29fa5), -_c(U+27E6, "⟦", e29fa6), -_c(U+27E7, "⟧", e29fa7), -_c(U+27E8, "⟨", e29fa8), -_c(U+27E9, "⟩", e29fa9), -_c(U+27EA, "⟪", e29faa), -_c(U+27EB, "⟫", e29fab), -_c(U+27EC, "⟬", e29fac), -_c(U+27ED, "⟭", e29fad), -_c(U+27EE, "⟮", e29fae), -_c(U+27EF, "⟯", e29faf), -_c(U+27F0, "⟰", e29fb0), -_c(U+27F1, "⟱", e29fb1), -_c(U+27F2, "⟲", e29fb2), -_c(U+27F3, "⟳", e29fb3), -_c(U+27F4, "⟴", e29fb4), -_c(U+27F5, "⟵", e29fb5), -_c(U+27F6, "⟶", e29fb6), -_c(U+27F7, "⟷", e29fb7), -_c(U+27F8, "⟸", e29fb8), -_c(U+27F9, "⟹", e29fb9), -_c(U+27FA, "⟺", e29fba), -_c(U+27FB, "⟻", e29fbb), -_c(U+27FC, "⟼", e29fbc), -_c(U+27FD, "⟽", e29fbd), -_c(U+27FE, "⟾", e29fbe), -_c(U+27FF, "⟿", e29fbf), -_c(U+2800, "⠀", e2a080), -_c(U+2801, "⠁", e2a081), -_c(U+2802, "⠂", e2a082), -_c(U+2803, "⠃", e2a083), -_c(U+2804, "⠄", e2a084), -_c(U+2805, "⠅", e2a085), -_c(U+2806, "⠆", e2a086), -_c(U+2807, "⠇", e2a087), -_c(U+2808, "⠈", e2a088), -_c(U+2809, "⠉", e2a089), -_c(U+280A, "⠊", e2a08a), -_c(U+280B, "⠋", e2a08b), -_c(U+280C, "⠌", e2a08c), -_c(U+280D, "⠍", e2a08d), -_c(U+280E, "⠎", e2a08e), -_c(U+280F, "⠏", e2a08f), -_c(U+2810, "⠐", e2a090), -_c(U+2811, "⠑", e2a091), -_c(U+2812, "⠒", e2a092), -_c(U+2813, "⠓", e2a093), -_c(U+2814, "⠔", e2a094), -_c(U+2815, "⠕", e2a095), -_c(U+2816, "⠖", e2a096), -_c(U+2817, "⠗", e2a097), -_c(U+2818, "⠘", e2a098), -_c(U+2819, "⠙", e2a099), -_c(U+281A, "⠚", e2a09a), -_c(U+281B, "⠛", e2a09b), -_c(U+281C, "⠜", e2a09c), -_c(U+281D, "⠝", e2a09d), -_c(U+281E, "⠞", e2a09e), -_c(U+281F, "⠟", e2a09f), -_c(U+2820, "⠠", e2a0a0), -_c(U+2821, "⠡", e2a0a1), -_c(U+2822, "⠢", e2a0a2), -_c(U+2823, "⠣", e2a0a3), -_c(U+2824, "⠤", e2a0a4), -_c(U+2825, "⠥", e2a0a5), -_c(U+2826, "⠦", e2a0a6), -_c(U+2827, "⠧", e2a0a7), -_c(U+2828, "⠨", e2a0a8), -_c(U+2829, "⠩", e2a0a9), -_c(U+282A, "⠪", e2a0aa), -_c(U+282B, "⠫", e2a0ab), -_c(U+282C, "⠬", e2a0ac), -_c(U+282D, "⠭", e2a0ad), -_c(U+282E, "⠮", e2a0ae), -_c(U+282F, "⠯", e2a0af), -_c(U+2830, "⠰", e2a0b0), -_c(U+2831, "⠱", e2a0b1), -_c(U+2832, "⠲", e2a0b2), -_c(U+2833, "⠳", e2a0b3), -_c(U+2834, "⠴", e2a0b4), -_c(U+2835, "⠵", e2a0b5), -_c(U+2836, "⠶", e2a0b6), -_c(U+2837, "⠷", e2a0b7), -_c(U+2838, "⠸", e2a0b8), -_c(U+2839, "⠹", e2a0b9), -_c(U+283A, "⠺", e2a0ba), -_c(U+283B, "⠻", e2a0bb), -_c(U+283C, "⠼", e2a0bc), -_c(U+283D, "⠽", e2a0bd), -_c(U+283E, "⠾", e2a0be), -_c(U+283F, "⠿", e2a0bf), -_c(U+2840, "⡀", e2a180), -_c(U+2841, "⡁", e2a181), -_c(U+2842, "⡂", e2a182), -_c(U+2843, "⡃", e2a183), -_c(U+2844, "⡄", e2a184), -_c(U+2845, "⡅", e2a185), -_c(U+2846, "⡆", e2a186), -_c(U+2847, "⡇", e2a187), -_c(U+2848, "⡈", e2a188), -_c(U+2849, "⡉", e2a189), -_c(U+284A, "⡊", e2a18a), -_c(U+284B, "⡋", e2a18b), -_c(U+284C, "⡌", e2a18c), -_c(U+284D, "⡍", e2a18d), -_c(U+284E, "⡎", e2a18e), -_c(U+284F, "⡏", e2a18f), -_c(U+2850, "⡐", e2a190), -_c(U+2851, "⡑", e2a191), -_c(U+2852, "⡒", e2a192), -_c(U+2853, "⡓", e2a193), -_c(U+2854, "⡔", e2a194), -_c(U+2855, "⡕", e2a195), -_c(U+2856, "⡖", e2a196), -_c(U+2857, "⡗", e2a197), -_c(U+2858, "⡘", e2a198), -_c(U+2859, "⡙", e2a199), -_c(U+285A, "⡚", e2a19a), -_c(U+285B, "⡛", e2a19b), -_c(U+285C, "⡜", e2a19c), -_c(U+285D, "⡝", e2a19d), -_c(U+285E, "⡞", e2a19e), -_c(U+285F, "⡟", e2a19f), -_c(U+2860, "⡠", e2a1a0), -_c(U+2861, "⡡", e2a1a1), -_c(U+2862, "⡢", e2a1a2), -_c(U+2863, "⡣", e2a1a3), -_c(U+2864, "⡤", e2a1a4), -_c(U+2865, "⡥", e2a1a5), -_c(U+2866, "⡦", e2a1a6), -_c(U+2867, "⡧", e2a1a7), -_c(U+2868, "⡨", e2a1a8), -_c(U+2869, "⡩", e2a1a9), -_c(U+286A, "⡪", e2a1aa), -_c(U+286B, "⡫", e2a1ab), -_c(U+286C, "⡬", e2a1ac), -_c(U+286D, "⡭", e2a1ad), -_c(U+286E, "⡮", e2a1ae), -_c(U+286F, "⡯", e2a1af), -_c(U+2870, "⡰", e2a1b0), -_c(U+2871, "⡱", e2a1b1), -_c(U+2872, "⡲", e2a1b2), -_c(U+2873, "⡳", e2a1b3), -_c(U+2874, "⡴", e2a1b4), -_c(U+2875, "⡵", e2a1b5), -_c(U+2876, "⡶", e2a1b6), -_c(U+2877, "⡷", e2a1b7), -_c(U+2878, "⡸", e2a1b8), -_c(U+2879, "⡹", e2a1b9), -_c(U+287A, "⡺", e2a1ba), -_c(U+287B, "⡻", e2a1bb), -_c(U+287C, "⡼", e2a1bc), -_c(U+287D, "⡽", e2a1bd), -_c(U+287E, "⡾", e2a1be), -_c(U+287F, "⡿", e2a1bf), -_c(U+2880, "⢀", e2a280), -_c(U+2881, "⢁", e2a281), -_c(U+2882, "⢂", e2a282), -_c(U+2883, "⢃", e2a283), -_c(U+2884, "⢄", e2a284), -_c(U+2885, "⢅", e2a285), -_c(U+2886, "⢆", e2a286), -_c(U+2887, "⢇", e2a287), -_c(U+2888, "⢈", e2a288), -_c(U+2889, "⢉", e2a289), -_c(U+288A, "⢊", e2a28a), -_c(U+288B, "⢋", e2a28b), -_c(U+288C, "⢌", e2a28c), -_c(U+288D, "⢍", e2a28d), -_c(U+288E, "⢎", e2a28e), -_c(U+288F, "⢏", e2a28f), -_c(U+2890, "⢐", e2a290), -_c(U+2891, "⢑", e2a291), -_c(U+2892, "⢒", e2a292), -_c(U+2893, "⢓", e2a293), -_c(U+2894, "⢔", e2a294), -_c(U+2895, "⢕", e2a295), -_c(U+2896, "⢖", e2a296), -_c(U+2897, "⢗", e2a297), -_c(U+2898, "⢘", e2a298), -_c(U+2899, "⢙", e2a299), -_c(U+289A, "⢚", e2a29a), -_c(U+289B, "⢛", e2a29b), -_c(U+289C, "⢜", e2a29c), -_c(U+289D, "⢝", e2a29d), -_c(U+289E, "⢞", e2a29e), -_c(U+289F, "⢟", e2a29f), -_c(U+28A0, "⢠", e2a2a0), -_c(U+28A1, "⢡", e2a2a1), -_c(U+28A2, "⢢", e2a2a2), -_c(U+28A3, "⢣", e2a2a3), -_c(U+28A4, "⢤", e2a2a4), -_c(U+28A5, "⢥", e2a2a5), -_c(U+28A6, "⢦", e2a2a6), -_c(U+28A7, "⢧", e2a2a7), -_c(U+28A8, "⢨", e2a2a8), -_c(U+28A9, "⢩", e2a2a9), -_c(U+28AA, "⢪", e2a2aa), -_c(U+28AB, "⢫", e2a2ab), -_c(U+28AC, "⢬", e2a2ac), -_c(U+28AD, "⢭", e2a2ad), -_c(U+28AE, "⢮", e2a2ae), -_c(U+28AF, "⢯", e2a2af), -_c(U+28B0, "⢰", e2a2b0), -_c(U+28B1, "⢱", e2a2b1), -_c(U+28B2, "⢲", e2a2b2), -_c(U+28B3, "⢳", e2a2b3), -_c(U+28B4, "⢴", e2a2b4), -_c(U+28B5, "⢵", e2a2b5), -_c(U+28B6, "⢶", e2a2b6), -_c(U+28B7, "⢷", e2a2b7), -_c(U+28B8, "⢸", e2a2b8), -_c(U+28B9, "⢹", e2a2b9), -_c(U+28BA, "⢺", e2a2ba), -_c(U+28BB, "⢻", e2a2bb), -_c(U+28BC, "⢼", e2a2bc), -_c(U+28BD, "⢽", e2a2bd), -_c(U+28BE, "⢾", e2a2be), -_c(U+28BF, "⢿", e2a2bf), -_c(U+28C0, "⣀", e2a380), -_c(U+28C1, "⣁", e2a381), -_c(U+28C2, "⣂", e2a382), -_c(U+28C3, "⣃", e2a383), -_c(U+28C4, "⣄", e2a384), -_c(U+28C5, "⣅", e2a385), -_c(U+28C6, "⣆", e2a386), -_c(U+28C7, "⣇", e2a387), -_c(U+28C8, "⣈", e2a388), -_c(U+28C9, "⣉", e2a389), -_c(U+28CA, "⣊", e2a38a), -_c(U+28CB, "⣋", e2a38b), -_c(U+28CC, "⣌", e2a38c), -_c(U+28CD, "⣍", e2a38d), -_c(U+28CE, "⣎", e2a38e), -_c(U+28CF, "⣏", e2a38f), -_c(U+28D0, "⣐", e2a390), -_c(U+28D1, "⣑", e2a391), -_c(U+28D2, "⣒", e2a392), -_c(U+28D3, "⣓", e2a393), -_c(U+28D4, "⣔", e2a394), -_c(U+28D5, "⣕", e2a395), -_c(U+28D6, "⣖", e2a396), -_c(U+28D7, "⣗", e2a397), -_c(U+28D8, "⣘", e2a398), -_c(U+28D9, "⣙", e2a399), -_c(U+28DA, "⣚", e2a39a), -_c(U+28DB, "⣛", e2a39b), -_c(U+28DC, "⣜", e2a39c), -_c(U+28DD, "⣝", e2a39d), -_c(U+28DE, "⣞", e2a39e), -_c(U+28DF, "⣟", e2a39f), -_c(U+28E0, "⣠", e2a3a0), -_c(U+28E1, "⣡", e2a3a1), -_c(U+28E2, "⣢", e2a3a2), -_c(U+28E3, "⣣", e2a3a3), -_c(U+28E4, "⣤", e2a3a4), -_c(U+28E5, "⣥", e2a3a5), -_c(U+28E6, "⣦", e2a3a6), -_c(U+28E7, "⣧", e2a3a7), -_c(U+28E8, "⣨", e2a3a8), -_c(U+28E9, "⣩", e2a3a9), -_c(U+28EA, "⣪", e2a3aa), -_c(U+28EB, "⣫", e2a3ab), -_c(U+28EC, "⣬", e2a3ac), -_c(U+28ED, "⣭", e2a3ad), -_c(U+28EE, "⣮", e2a3ae), -_c(U+28EF, "⣯", e2a3af), -_c(U+28F0, "⣰", e2a3b0), -_c(U+28F1, "⣱", e2a3b1), -_c(U+28F2, "⣲", e2a3b2), -_c(U+28F3, "⣳", e2a3b3), -_c(U+28F4, "⣴", e2a3b4), -_c(U+28F5, "⣵", e2a3b5), -_c(U+28F6, "⣶", e2a3b6), -_c(U+28F7, "⣷", e2a3b7), -_c(U+28F8, "⣸", e2a3b8), -_c(U+28F9, "⣹", e2a3b9), -_c(U+28FA, "⣺", e2a3ba), -_c(U+28FB, "⣻", e2a3bb), -_c(U+28FC, "⣼", e2a3bc), -_c(U+28FD, "⣽", e2a3bd), -_c(U+28FE, "⣾", e2a3be), -_c(U+28FF, "⣿", e2a3bf), -_c(U+2900, "⤀", e2a480), -_c(U+2901, "⤁", e2a481), -_c(U+2902, "⤂", e2a482), -_c(U+2903, "⤃", e2a483), -_c(U+2904, "⤄", e2a484), -_c(U+2905, "⤅", e2a485), -_c(U+2906, "⤆", e2a486), -_c(U+2907, "⤇", e2a487), -_c(U+2908, "⤈", e2a488), -_c(U+2909, "⤉", e2a489), -_c(U+290A, "⤊", e2a48a), -_c(U+290B, "⤋", e2a48b), -_c(U+290C, "⤌", e2a48c), -_c(U+290D, "⤍", e2a48d), -_c(U+290E, "⤎", e2a48e), -_c(U+290F, "⤏", e2a48f), -_c(U+2910, "⤐", e2a490), -_c(U+2911, "⤑", e2a491), -_c(U+2912, "⤒", e2a492), -_c(U+2913, "⤓", e2a493), -_c(U+2914, "⤔", e2a494), -_c(U+2915, "⤕", e2a495), -_c(U+2916, "⤖", e2a496), -_c(U+2917, "⤗", e2a497), -_c(U+2918, "⤘", e2a498), -_c(U+2919, "⤙", e2a499), -_c(U+291A, "⤚", e2a49a), -_c(U+291B, "⤛", e2a49b), -_c(U+291C, "⤜", e2a49c), -_c(U+291D, "⤝", e2a49d), -_c(U+291E, "⤞", e2a49e), -_c(U+291F, "⤟", e2a49f), -_c(U+2920, "⤠", e2a4a0), -_c(U+2921, "⤡", e2a4a1), -_c(U+2922, "⤢", e2a4a2), -_c(U+2923, "⤣", e2a4a3), -_c(U+2924, "⤤", e2a4a4), -_c(U+2925, "⤥", e2a4a5), -_c(U+2926, "⤦", e2a4a6), -_c(U+2927, "⤧", e2a4a7), -_c(U+2928, "⤨", e2a4a8), -_c(U+2929, "⤩", e2a4a9), -_c(U+292A, "⤪", e2a4aa), -_c(U+292B, "⤫", e2a4ab), -_c(U+292C, "⤬", e2a4ac), -_c(U+292D, "⤭", e2a4ad), -_c(U+292E, "⤮", e2a4ae), -_c(U+292F, "⤯", e2a4af), -_c(U+2930, "⤰", e2a4b0), -_c(U+2931, "⤱", e2a4b1), -_c(U+2932, "⤲", e2a4b2), -_c(U+2933, "⤳", e2a4b3), -_c(U+2934, "⤴", e2a4b4), -_c(U+2935, "⤵", e2a4b5), -_c(U+2936, "⤶", e2a4b6), -_c(U+2937, "⤷", e2a4b7), -_c(U+2938, "⤸", e2a4b8), -_c(U+2939, "⤹", e2a4b9), -_c(U+293A, "⤺", e2a4ba), -_c(U+293B, "⤻", e2a4bb), -_c(U+293C, "⤼", e2a4bc), -_c(U+293D, "⤽", e2a4bd), -_c(U+293E, "⤾", e2a4be), -_c(U+293F, "⤿", e2a4bf), -_c(U+2940, "⥀", e2a580), -_c(U+2941, "⥁", e2a581), -_c(U+2942, "⥂", e2a582), -_c(U+2943, "⥃", e2a583), -_c(U+2944, "⥄", e2a584), -_c(U+2945, "⥅", e2a585), -_c(U+2946, "⥆", e2a586), -_c(U+2947, "⥇", e2a587), -_c(U+2948, "⥈", e2a588), -_c(U+2949, "⥉", e2a589), -_c(U+294A, "⥊", e2a58a), -_c(U+294B, "⥋", e2a58b), -_c(U+294C, "⥌", e2a58c), -_c(U+294D, "⥍", e2a58d), -_c(U+294E, "⥎", e2a58e), -_c(U+294F, "⥏", e2a58f), -_c(U+2950, "⥐", e2a590), -_c(U+2951, "⥑", e2a591), -_c(U+2952, "⥒", e2a592), -_c(U+2953, "⥓", e2a593), -_c(U+2954, "⥔", e2a594), -_c(U+2955, "⥕", e2a595), -_c(U+2956, "⥖", e2a596), -_c(U+2957, "⥗", e2a597), -_c(U+2958, "⥘", e2a598), -_c(U+2959, "⥙", e2a599), -_c(U+295A, "⥚", e2a59a), -_c(U+295B, "⥛", e2a59b), -_c(U+295C, "⥜", e2a59c), -_c(U+295D, "⥝", e2a59d), -_c(U+295E, "⥞", e2a59e), -_c(U+295F, "⥟", e2a59f), -_c(U+2960, "⥠", e2a5a0), -_c(U+2961, "⥡", e2a5a1), -_c(U+2962, "⥢", e2a5a2), -_c(U+2963, "⥣", e2a5a3), -_c(U+2964, "⥤", e2a5a4), -_c(U+2965, "⥥", e2a5a5), -_c(U+2966, "⥦", e2a5a6), -_c(U+2967, "⥧", e2a5a7), -_c(U+2968, "⥨", e2a5a8), -_c(U+2969, "⥩", e2a5a9), -_c(U+296A, "⥪", e2a5aa), -_c(U+296B, "⥫", e2a5ab), -_c(U+296C, "⥬", e2a5ac), -_c(U+296D, "⥭", e2a5ad), -_c(U+296E, "⥮", e2a5ae), -_c(U+296F, "⥯", e2a5af), -_c(U+2970, "⥰", e2a5b0), -_c(U+2971, "⥱", e2a5b1), -_c(U+2972, "⥲", e2a5b2), -_c(U+2973, "⥳", e2a5b3), -_c(U+2974, "⥴", e2a5b4), -_c(U+2975, "⥵", e2a5b5), -_c(U+2976, "⥶", e2a5b6), -_c(U+2977, "⥷", e2a5b7), -_c(U+2978, "⥸", e2a5b8), -_c(U+2979, "⥹", e2a5b9), -_c(U+297A, "⥺", e2a5ba), -_c(U+297B, "⥻", e2a5bb), -_c(U+297C, "⥼", e2a5bc), -_c(U+297D, "⥽", e2a5bd), -_c(U+297E, "⥾", e2a5be), -_c(U+297F, "⥿", e2a5bf), -_c(U+2980, "⦀", e2a680), -_c(U+2981, "⦁", e2a681), -_c(U+2982, "⦂", e2a682), -_c(U+2983, "⦃", e2a683), -_c(U+2984, "⦄", e2a684), -_c(U+2985, "⦅", e2a685), -_c(U+2986, "⦆", e2a686), -_c(U+2987, "⦇", e2a687), -_c(U+2988, "⦈", e2a688), -_c(U+2989, "⦉", e2a689), -_c(U+298A, "⦊", e2a68a), -_c(U+298B, "⦋", e2a68b), -_c(U+298C, "⦌", e2a68c), -_c(U+298D, "⦍", e2a68d), -_c(U+298E, "⦎", e2a68e), -_c(U+298F, "⦏", e2a68f), -_c(U+2990, "⦐", e2a690), -_c(U+2991, "⦑", e2a691), -_c(U+2992, "⦒", e2a692), -_c(U+2993, "⦓", e2a693), -_c(U+2994, "⦔", e2a694), -_c(U+2995, "⦕", e2a695), -_c(U+2996, "⦖", e2a696), -_c(U+2997, "⦗", e2a697), -_c(U+2998, "⦘", e2a698), -_c(U+2999, "⦙", e2a699), -_c(U+299A, "⦚", e2a69a), -_c(U+299B, "⦛", e2a69b), -_c(U+299C, "⦜", e2a69c), -_c(U+299D, "⦝", e2a69d), -_c(U+299E, "⦞", e2a69e), -_c(U+299F, "⦟", e2a69f), -_c(U+29A0, "⦠", e2a6a0), -_c(U+29A1, "⦡", e2a6a1), -_c(U+29A2, "⦢", e2a6a2), -_c(U+29A3, "⦣", e2a6a3), -_c(U+29A4, "⦤", e2a6a4), -_c(U+29A5, "⦥", e2a6a5), -_c(U+29A6, "⦦", e2a6a6), -_c(U+29A7, "⦧", e2a6a7), -_c(U+29A8, "⦨", e2a6a8), -_c(U+29A9, "⦩", e2a6a9), -_c(U+29AA, "⦪", e2a6aa), -_c(U+29AB, "⦫", e2a6ab), -_c(U+29AC, "⦬", e2a6ac), -_c(U+29AD, "⦭", e2a6ad), -_c(U+29AE, "⦮", e2a6ae), -_c(U+29AF, "⦯", e2a6af), -_c(U+29B0, "⦰", e2a6b0), -_c(U+29B1, "⦱", e2a6b1), -_c(U+29B2, "⦲", e2a6b2), -_c(U+29B3, "⦳", e2a6b3), -_c(U+29B4, "⦴", e2a6b4), -_c(U+29B5, "⦵", e2a6b5), -_c(U+29B6, "⦶", e2a6b6), -_c(U+29B7, "⦷", e2a6b7), -_c(U+29B8, "⦸", e2a6b8), -_c(U+29B9, "⦹", e2a6b9), -_c(U+29BA, "⦺", e2a6ba), -_c(U+29BB, "⦻", e2a6bb), -_c(U+29BC, "⦼", e2a6bc), -_c(U+29BD, "⦽", e2a6bd), -_c(U+29BE, "⦾", e2a6be), -_c(U+29BF, "⦿", e2a6bf), -_c(U+29C0, "⧀", e2a780), -_c(U+29C1, "⧁", e2a781), -_c(U+29C2, "⧂", e2a782), -_c(U+29C3, "⧃", e2a783), -_c(U+29C4, "⧄", e2a784), -_c(U+29C5, "⧅", e2a785), -_c(U+29C6, "⧆", e2a786), -_c(U+29C7, "⧇", e2a787), -_c(U+29C8, "⧈", e2a788), -_c(U+29C9, "⧉", e2a789), -_c(U+29CA, "⧊", e2a78a), -_c(U+29CB, "⧋", e2a78b), -_c(U+29CC, "⧌", e2a78c), -_c(U+29CD, "⧍", e2a78d), -_c(U+29CE, "⧎", e2a78e), -_c(U+29CF, "⧏", e2a78f), -_c(U+29D0, "⧐", e2a790), -_c(U+29D1, "⧑", e2a791), -_c(U+29D2, "⧒", e2a792), -_c(U+29D3, "⧓", e2a793), -_c(U+29D4, "⧔", e2a794), -_c(U+29D5, "⧕", e2a795), -_c(U+29D6, "⧖", e2a796), -_c(U+29D7, "⧗", e2a797), -_c(U+29D8, "⧘", e2a798), -_c(U+29D9, "⧙", e2a799), -_c(U+29DA, "⧚", e2a79a), -_c(U+29DB, "⧛", e2a79b), -_c(U+29DC, "⧜", e2a79c), -_c(U+29DD, "⧝", e2a79d), -_c(U+29DE, "⧞", e2a79e), -_c(U+29DF, "⧟", e2a79f), -_c(U+29E0, "⧠", e2a7a0), -_c(U+29E1, "⧡", e2a7a1), -_c(U+29E2, "⧢", e2a7a2), -_c(U+29E3, "⧣", e2a7a3), -_c(U+29E4, "⧤", e2a7a4), -_c(U+29E5, "⧥", e2a7a5), -_c(U+29E6, "⧦", e2a7a6), -_c(U+29E7, "⧧", e2a7a7), -_c(U+29E8, "⧨", e2a7a8), -_c(U+29E9, "⧩", e2a7a9), -_c(U+29EA, "⧪", e2a7aa), -_c(U+29EB, "⧫", e2a7ab), -_c(U+29EC, "⧬", e2a7ac), -_c(U+29ED, "⧭", e2a7ad), -_c(U+29EE, "⧮", e2a7ae), -_c(U+29EF, "⧯", e2a7af), -_c(U+29F0, "⧰", e2a7b0), -_c(U+29F1, "⧱", e2a7b1), -_c(U+29F2, "⧲", e2a7b2), -_c(U+29F3, "⧳", e2a7b3), -_c(U+29F4, "⧴", e2a7b4), -_c(U+29F5, "⧵", e2a7b5), -_c(U+29F6, "⧶", e2a7b6), -_c(U+29F7, "⧷", e2a7b7), -_c(U+29F8, "⧸", e2a7b8), -_c(U+29F9, "⧹", e2a7b9), -_c(U+29FA, "⧺", e2a7ba), -_c(U+29FB, "⧻", e2a7bb), -_c(U+29FC, "⧼", e2a7bc), -_c(U+29FD, "⧽", e2a7bd), -_c(U+29FE, "⧾", e2a7be), -_c(U+29FF, "⧿", e2a7bf), -_c(U+1D100, "𝄀", f09d8480), -_c(U+1D101, "𝄁", f09d8481), -_c(U+1D102, "𝄂", f09d8482), -_c(U+1D103, "𝄃", f09d8483), -_c(U+1D104, "𝄄", f09d8484), -_c(U+1D105, "𝄅", f09d8485), -_c(U+1D106, "𝄆", f09d8486), -_c(U+1D107, "𝄇", f09d8487), -_c(U+1D108, "𝄈", f09d8488), -_c(U+1D109, "𝄉", f09d8489), -_c(U+1D10A, "𝄊", f09d848a), -_c(U+1D10B, "𝄋", f09d848b), -_c(U+1D10C, "𝄌", f09d848c), -_c(U+1D10D, "𝄍", f09d848d), -_c(U+1D10E, "𝄎", f09d848e), -_c(U+1D10F, "𝄏", f09d848f), -_c(U+1D110, "𝄐", f09d8490), -_c(U+1D111, "𝄑", f09d8491), -_c(U+1D112, "𝄒", f09d8492), -_c(U+1D113, "𝄓", f09d8493), -_c(U+1D114, "𝄔", f09d8494), -_c(U+1D115, "𝄕", f09d8495), -_c(U+1D116, "𝄖", f09d8496), -_c(U+1D117, "𝄗", f09d8497), -_c(U+1D118, "𝄘", f09d8498), -_c(U+1D119, "𝄙", f09d8499), -_c(U+1D11A, "𝄚", f09d849a), -_c(U+1D11B, "𝄛", f09d849b), -_c(U+1D11C, "𝄜", f09d849c), -_c(U+1D11D, "𝄝", f09d849d), -_c(U+1D11E, "𝄞", f09d849e), -_c(U+1D11F, "𝄟", f09d849f), -_c(U+1D120, "𝄠", f09d84a0), -_c(U+1D121, "𝄡", f09d84a1), -_c(U+1D122, "𝄢", f09d84a2), -_c(U+1D123, "𝄣", f09d84a3), -_c(U+1D124, "𝄤", f09d84a4), -_c(U+1D125, "𝄥", f09d84a5), -_c(U+1D126, "𝄦", f09d84a6), -_c(U+1D127, "𝄧", f09d84a7), -_c(U+1D128, "𝄨", f09d84a8), -_c(U+1D129, "𝄩", f09d84a9), -_c(U+1D12A, "𝄪", f09d84aa), -_c(U+1D12B, "𝄫", f09d84ab), -_c(U+1D12C, "𝄬", f09d84ac), -_c(U+1D12D, "𝄭", f09d84ad), -_c(U+1D12E, "𝄮", f09d84ae), -_c(U+1D12F, "𝄯", f09d84af), -_c(U+1D130, "𝄰", f09d84b0), -_c(U+1D131, "𝄱", f09d84b1), -_c(U+1D132, "𝄲", f09d84b2), -_c(U+1D133, "𝄳", f09d84b3), -_c(U+1D134, "𝄴", f09d84b4), -_c(U+1D135, "𝄵", f09d84b5), -_c(U+1D136, "𝄶", f09d84b6), -_c(U+1D137, "𝄷", f09d84b7), -_c(U+1D138, "𝄸", f09d84b8), -_c(U+1D139, "𝄹", f09d84b9), -_c(U+1D13A, "𝄺", f09d84ba), -_c(U+1D13B, "𝄻", f09d84bb), -_c(U+1D13C, "𝄼", f09d84bc), -_c(U+1D13D, "𝄽", f09d84bd), -_c(U+1D13E, "𝄾", f09d84be), -_c(U+1D13F, "𝄿", f09d84bf), -_c(U+1D140, "𝅀", f09d8580), -_c(U+1D141, "𝅁", f09d8581), -_c(U+1D142, "𝅂", f09d8582), -_c(U+1D143, "𝅃", f09d8583), -_c(U+1D144, "𝅄", f09d8584), -_c(U+1D145, "𝅅", f09d8585), -_c(U+1D146, "𝅆", f09d8586), -_c(U+1D147, "𝅇", f09d8587), -_c(U+1D148, "𝅈", f09d8588), -_c(U+1D149, "𝅉", f09d8589), -_c(U+1D14A, "𝅊", f09d858a), -_c(U+1D14B, "𝅋", f09d858b), -_c(U+1D14C, "𝅌", f09d858c), -_c(U+1D14D, "𝅍", f09d858d), -_c(U+1D14E, "𝅎", f09d858e), -_c(U+1D14F, "𝅏", f09d858f), -_c(U+1D150, "𝅐", f09d8590), -_c(U+1D151, "𝅑", f09d8591), -_c(U+1D152, "𝅒", f09d8592), -_c(U+1D153, "𝅓", f09d8593), -_c(U+1D154, "𝅔", f09d8594), -_c(U+1D155, "𝅕", f09d8595), -_c(U+1D156, "𝅖", f09d8596), -_c(U+1D157, "𝅗", f09d8597), -_c(U+1D158, "𝅘", f09d8598), -_c(U+1D159, "𝅙", f09d8599), -_c(U+1D15A, "𝅚", f09d859a), -_c(U+1D15B, "𝅛", f09d859b), -_c(U+1D15C, "𝅜", f09d859c), -_c(U+1D15D, "𝅝", f09d859d), -_c(U+1D15E, "𝅗𝅥", f09d859e), -_c(U+1D15F, "𝅘𝅥", f09d859f), -_c(U+1D160, "𝅘𝅥𝅮", f09d85a0), -_c(U+1D161, "𝅘𝅥𝅯", f09d85a1), -_c(U+1D162, "𝅘𝅥𝅰", f09d85a2), -_c(U+1D163, "𝅘𝅥𝅱", f09d85a3), -_c(U+1D164, "𝅘𝅥𝅲", f09d85a4), -_c(U+1D165, "𝅥", f09d85a5), -_c(U+1D166, "𝅦", f09d85a6), -_c(U+1D167, "𝅧", f09d85a7), -_c(U+1D168, "𝅨", f09d85a8), -_c(U+1D169, "𝅩", f09d85a9), -_c(U+1D16A, "𝅪", f09d85aa), -_c(U+1D16B, "𝅫", f09d85ab), -_c(U+1D16C, "𝅬", f09d85ac), -_c(U+1D16D, "𝅭", f09d85ad), -_c(U+1D16E, "𝅮", f09d85ae), -_c(U+1D16F, "𝅯", f09d85af), -_c(U+1D170, "𝅰", f09d85b0), -_c(U+1D171, "𝅱", f09d85b1), -_c(U+1D172, "𝅲", f09d85b2), -_c(U+1D173, "𝅳", f09d85b3), -_c(U+1D174, "𝅴", f09d85b4), -_c(U+1D175, "𝅵", f09d85b5), -_c(U+1D176, "𝅶", f09d85b6), -_c(U+1D177, "𝅷", f09d85b7), -_c(U+1D178, "𝅸", f09d85b8), -_c(U+1D179, "𝅹", f09d85b9), -_c(U+1D17A, "𝅺", f09d85ba), -_c(U+1D17B, "𝅻", f09d85bb), -_c(U+1D17C, "𝅼", f09d85bc), -_c(U+1D17D, "𝅽", f09d85bd), -_c(U+1D17E, "𝅾", f09d85be), -_c(U+1D17F, "𝅿", f09d85bf), -_c(U+1D180, "𝆀", f09d8680), -_c(U+1D181, "𝆁", f09d8681), -_c(U+1D182, "𝆂", f09d8682), -_c(U+1D183, "𝆃", f09d8683), -_c(U+1D184, "𝆄", f09d8684), -_c(U+1D185, "𝆅", f09d8685), -_c(U+1D186, "𝆆", f09d8686), -_c(U+1D187, "𝆇", f09d8687), -_c(U+1D188, "𝆈", f09d8688), -_c(U+1D189, "𝆉", f09d8689), -_c(U+1D18A, "𝆊", f09d868a), -_c(U+1D18B, "𝆋", f09d868b), -_c(U+1D18C, "𝆌", f09d868c), -_c(U+1D18D, "𝆍", f09d868d), -_c(U+1D18E, "𝆎", f09d868e), -_c(U+1D18F, "𝆏", f09d868f), -_c(U+1D190, "𝆐", f09d8690), -_c(U+1D191, "𝆑", f09d8691), -_c(U+1D192, "𝆒", f09d8692), -_c(U+1D193, "𝆓", f09d8693), -_c(U+1D194, "𝆔", f09d8694), -_c(U+1D195, "𝆕", f09d8695), -_c(U+1D196, "𝆖", f09d8696), -_c(U+1D197, "𝆗", f09d8697), -_c(U+1D198, "𝆘", f09d8698), -_c(U+1D199, "𝆙", f09d8699), -_c(U+1D19A, "𝆚", f09d869a), -_c(U+1D19B, "𝆛", f09d869b), -_c(U+1D19C, "𝆜", f09d869c), -_c(U+1D19D, "𝆝", f09d869d), -_c(U+1D19E, "𝆞", f09d869e), -_c(U+1D19F, "𝆟", f09d869f), -_c(U+1D1A0, "𝆠", f09d86a0), -_c(U+1D1A1, "𝆡", f09d86a1), -_c(U+1D1A2, "𝆢", f09d86a2), -_c(U+1D1A3, "𝆣", f09d86a3), -_c(U+1D1A4, "𝆤", f09d86a4), -_c(U+1D1A5, "𝆥", f09d86a5), -_c(U+1D1A6, "𝆦", f09d86a6), -_c(U+1D1A7, "𝆧", f09d86a7), -_c(U+1D1A8, "𝆨", f09d86a8), -_c(U+1D1A9, "𝆩", f09d86a9), -_c(U+1D1AA, "𝆪", f09d86aa), -_c(U+1D1AB, "𝆫", f09d86ab), -_c(U+1D1AC, "𝆬", f09d86ac), -_c(U+1D1AD, "𝆭", f09d86ad), -_c(U+1D1AE, "𝆮", f09d86ae), -_c(U+1D1AF, "𝆯", f09d86af), -_c(U+1D1B0, "𝆰", f09d86b0), -_c(U+1D1B1, "𝆱", f09d86b1), -_c(U+1D1B2, "𝆲", f09d86b2), -_c(U+1D1B3, "𝆳", f09d86b3), -_c(U+1D1B4, "𝆴", f09d86b4), -_c(U+1D1B5, "𝆵", f09d86b5), -_c(U+1D1B6, "𝆶", f09d86b6), -_c(U+1D1B7, "𝆷", f09d86b7), -_c(U+1D1B8, "𝆸", f09d86b8), -_c(U+1D1B9, "𝆹", f09d86b9), -_c(U+1D1BA, "𝆺", f09d86ba), -_c(U+1D1BB, "𝆹𝅥", f09d86bb), -_c(U+1D1BC, "𝆺𝅥", f09d86bc), -_c(U+1D1BD, "𝆹𝅥𝅮", f09d86bd), -_c(U+1D1BE, "𝆺𝅥𝅮", f09d86be), -_c(U+1D1BF, "𝆹𝅥𝅯", f09d86bf), -_c(U+1D1C0, "𝆺𝅥𝅯", f09d8780), -_c(U+1D1C1, "𝇁", f09d8781), -_c(U+1D1C2, "𝇂", f09d8782), -_c(U+1D1C3, "𝇃", f09d8783), -_c(U+1D1C4, "𝇄", f09d8784), -_c(U+1D1C5, "𝇅", f09d8785), -_c(U+1D1C6, "𝇆", f09d8786), -_c(U+1D1C7, "𝇇", f09d8787), -_c(U+1D1C8, "𝇈", f09d8788), -_c(U+1D1C9, "𝇉", f09d8789), -_c(U+1D1CA, "𝇊", f09d878a), -_c(U+1D1CB, "𝇋", f09d878b), -_c(U+1D1CC, "𝇌", f09d878c), -_c(U+1D1CD, "𝇍", f09d878d), -_c(U+1D1CE, "𝇎", f09d878e), -_c(U+1D1CF, "𝇏", f09d878f), -_c(U+1D1D0, "𝇐", f09d8790), -_c(U+1D1D1, "𝇑", f09d8791), -_c(U+1D1D2, "𝇒", f09d8792), -_c(U+1D1D3, "𝇓", f09d8793), -_c(U+1D1D4, "𝇔", f09d8794), -_c(U+1D1D5, "𝇕", f09d8795), -_c(U+1D1D6, "𝇖", f09d8796), -_c(U+1D1D7, "𝇗", f09d8797), -_c(U+1D1D8, "𝇘", f09d8798), -_c(U+1D1D9, "𝇙", f09d8799), -_c(U+1D1DA, "𝇚", f09d879a), -_c(U+1D1DB, "𝇛", f09d879b), -_c(U+1D1DC, "𝇜", f09d879c), -_c(U+1D1DD, "𝇝", f09d879d), -_c(U+1D1DE, "𝇞", f09d879e), -_c(U+1D1DF, "𝇟", f09d879f), -_c(U+1D1E0, "𝇠", f09d87a0), -_c(U+1D1E1, "𝇡", f09d87a1), -_c(U+1D1E2, "𝇢", f09d87a2), -_c(U+1D1E3, "𝇣", f09d87a3), -_c(U+1D1E4, "𝇤", f09d87a4), -_c(U+1D1E5, "𝇥", f09d87a5), -_c(U+1D1E6, "𝇦", f09d87a6), -_c(U+1D1E7, "𝇧", f09d87a7), -_c(U+1D1E8, "𝇨", f09d87a8), -_c(U+1D1E9, "𝇩", f09d87a9), -_c(U+1D1EA, "𝇪", f09d87aa), -_c(U+1D1EB, "𝇫", f09d87ab), -_c(U+1D1EC, "𝇬", f09d87ac), -_c(U+1D1ED, "𝇭", f09d87ad), -_c(U+1D1EE, "𝇮", f09d87ae), -_c(U+1D1EF, "𝇯", f09d87af), -_c(U+1D1F0, "𝇰", f09d87b0), -_c(U+1D1F1, "𝇱", f09d87b1), -_c(U+1D1F2, "𝇲", f09d87b2), -_c(U+1D1F3, "𝇳", f09d87b3), -_c(U+1D1F4, "𝇴", f09d87b4), -_c(U+1D1F5, "𝇵", f09d87b5), -_c(U+1D1F6, "𝇶", f09d87b6), -_c(U+1D1F7, "𝇷", f09d87b7), -_c(U+1D1F8, "𝇸", f09d87b8), -_c(U+1D1F9, "𝇹", f09d87b9), -_c(U+1D1FA, "𝇺", f09d87ba), -_c(U+1D1FB, "𝇻", f09d87bb), -_c(U+1D1FC, "𝇼", f09d87bc), -_c(U+1D1FD, "𝇽", f09d87bd), -_c(U+1D1FE, "𝇾", f09d87be), -_c(U+1D1FF, "𝇿", f09d87bf), -_c(U+1D200, "𝈀", f09d8880), -_c(U+1D201, "𝈁", f09d8881), -_c(U+1D202, "𝈂", f09d8882), -_c(U+1D203, "𝈃", f09d8883), -_c(U+1D204, "𝈄", f09d8884), -_c(U+1D205, "𝈅", f09d8885), -_c(U+1D206, "𝈆", f09d8886), -_c(U+1D207, "𝈇", f09d8887), -_c(U+1D208, "𝈈", f09d8888), -_c(U+1D209, "𝈉", f09d8889), -_c(U+1D20A, "𝈊", f09d888a), -_c(U+1D20B, "𝈋", f09d888b), -_c(U+1D20C, "𝈌", f09d888c), -_c(U+1D20D, "𝈍", f09d888d), -_c(U+1D20E, "𝈎", f09d888e), -_c(U+1D20F, "𝈏", f09d888f), -_c(U+1D210, "𝈐", f09d8890), -_c(U+1D211, "𝈑", f09d8891), -_c(U+1D212, "𝈒", f09d8892), -_c(U+1D213, "𝈓", f09d8893), -_c(U+1D214, "𝈔", f09d8894), -_c(U+1D215, "𝈕", f09d8895), -_c(U+1D216, "𝈖", f09d8896), -_c(U+1D217, "𝈗", f09d8897), -_c(U+1D218, "𝈘", f09d8898), -_c(U+1D219, "𝈙", f09d8899), -_c(U+1D21A, "𝈚", f09d889a), -_c(U+1D21B, "𝈛", f09d889b), -_c(U+1D21C, "𝈜", f09d889c), -_c(U+1D21D, "𝈝", f09d889d), -_c(U+1D21E, "𝈞", f09d889e), -_c(U+1D21F, "𝈟", f09d889f), -_c(U+1D220, "𝈠", f09d88a0), -_c(U+1D221, "𝈡", f09d88a1), -_c(U+1D222, "𝈢", f09d88a2), -_c(U+1D223, "𝈣", f09d88a3), -_c(U+1D224, "𝈤", f09d88a4), -_c(U+1D225, "𝈥", f09d88a5), -_c(U+1D226, "𝈦", f09d88a6), -_c(U+1D227, "𝈧", f09d88a7), -_c(U+1D228, "𝈨", f09d88a8), -_c(U+1D229, "𝈩", f09d88a9), -_c(U+1D22A, "𝈪", f09d88aa), -_c(U+1D22B, "𝈫", f09d88ab), -_c(U+1D22C, "𝈬", f09d88ac), -_c(U+1D22D, "𝈭", f09d88ad), -_c(U+1D22E, "𝈮", f09d88ae), -_c(U+1D22F, "𝈯", f09d88af), -_c(U+1D230, "𝈰", f09d88b0), -_c(U+1D231, "𝈱", f09d88b1), -_c(U+1D232, "𝈲", f09d88b2), -_c(U+1D233, "𝈳", f09d88b3), -_c(U+1D234, "𝈴", f09d88b4), -_c(U+1D235, "𝈵", f09d88b5), -_c(U+1D236, "𝈶", f09d88b6), -_c(U+1D237, "𝈷", f09d88b7), -_c(U+1D238, "𝈸", f09d88b8), -_c(U+1D239, "𝈹", f09d88b9), -_c(U+1D23A, "𝈺", f09d88ba), -_c(U+1D23B, "𝈻", f09d88bb), -_c(U+1D23C, "𝈼", f09d88bc), -_c(U+1D23D, "𝈽", f09d88bd), -_c(U+1D23E, "𝈾", f09d88be), -_c(U+1D23F, "𝈿", f09d88bf), -_c(U+1D240, "𝉀", f09d8980), -_c(U+1D241, "𝉁", f09d8981), -_c(U+1D242, "𝉂", f09d8982), -_c(U+1D243, "𝉃", f09d8983), -_c(U+1D244, "𝉄", f09d8984), -_c(U+1D245, "𝉅", f09d8985), -_c(U+1D246, "𝉆", f09d8986), -_c(U+1D247, "𝉇", f09d8987), -_c(U+1D248, "𝉈", f09d8988), -_c(U+1D249, "𝉉", f09d8989), -_c(U+1D24A, "𝉊", f09d898a), -_c(U+1D24B, "𝉋", f09d898b), -_c(U+1D24C, "𝉌", f09d898c), -_c(U+1D24D, "𝉍", f09d898d), -_c(U+1D24E, "𝉎", f09d898e), -_c(U+1D24F, "𝉏", f09d898f), -_c(U+1D250, "𝉐", f09d8990), -_c(U+1D251, "𝉑", f09d8991), -_c(U+1D252, "𝉒", f09d8992), -_c(U+1D253, "𝉓", f09d8993), -_c(U+1D254, "𝉔", f09d8994), -_c(U+1D255, "𝉕", f09d8995), -_c(U+1D256, "𝉖", f09d8996), -_c(U+1D257, "𝉗", f09d8997), -_c(U+1D258, "𝉘", f09d8998), -_c(U+1D259, "𝉙", f09d8999), -_c(U+1D25A, "𝉚", f09d899a), -_c(U+1D25B, "𝉛", f09d899b), -_c(U+1D25C, "𝉜", f09d899c), -_c(U+1D25D, "𝉝", f09d899d), -_c(U+1D25E, "𝉞", f09d899e), -_c(U+1D25F, "𝉟", f09d899f), -_c(U+1D260, "𝉠", f09d89a0), -_c(U+1D261, "𝉡", f09d89a1), -_c(U+1D262, "𝉢", f09d89a2), -_c(U+1D263, "𝉣", f09d89a3), -_c(U+1D264, "𝉤", f09d89a4), -_c(U+1D265, "𝉥", f09d89a5), -_c(U+1D266, "𝉦", f09d89a6), -_c(U+1D267, "𝉧", f09d89a7), -_c(U+1D268, "𝉨", f09d89a8), -_c(U+1D269, "𝉩", f09d89a9), -_c(U+1D26A, "𝉪", f09d89aa), -_c(U+1D26B, "𝉫", f09d89ab), -_c(U+1D26C, "𝉬", f09d89ac), -_c(U+1D26D, "𝉭", f09d89ad), -_c(U+1D26E, "𝉮", f09d89ae), -_c(U+1D26F, "𝉯", f09d89af), -_c(U+1D270, "𝉰", f09d89b0), -_c(U+1D271, "𝉱", f09d89b1), -_c(U+1D272, "𝉲", f09d89b2), -_c(U+1D273, "𝉳", f09d89b3), -_c(U+1D274, "𝉴", f09d89b4), -_c(U+1D275, "𝉵", f09d89b5), -_c(U+1D276, "𝉶", f09d89b6), -_c(U+1D277, "𝉷", f09d89b7), -_c(U+1D278, "𝉸", f09d89b8), -_c(U+1D279, "𝉹", f09d89b9), -_c(U+1D27A, "𝉺", f09d89ba), -_c(U+1D27B, "𝉻", f09d89bb), -_c(U+1D27C, "𝉼", f09d89bc), -_c(U+1D27D, "𝉽", f09d89bd), -_c(U+1D27E, "𝉾", f09d89be), -_c(U+1D27F, "𝉿", f09d89bf), -_c(U+1D280, "𝊀", f09d8a80), -_c(U+1D281, "𝊁", f09d8a81), -_c(U+1D282, "𝊂", f09d8a82), -_c(U+1D283, "𝊃", f09d8a83), -_c(U+1D284, "𝊄", f09d8a84), -_c(U+1D285, "𝊅", f09d8a85), -_c(U+1D286, "𝊆", f09d8a86), -_c(U+1D287, "𝊇", f09d8a87), -_c(U+1D288, "𝊈", f09d8a88), -_c(U+1D289, "𝊉", f09d8a89), -_c(U+1D28A, "𝊊", f09d8a8a), -_c(U+1D28B, "𝊋", f09d8a8b), -_c(U+1D28C, "𝊌", f09d8a8c), -_c(U+1D28D, "𝊍", f09d8a8d), -_c(U+1D28E, "𝊎", f09d8a8e), -_c(U+1D28F, "𝊏", f09d8a8f), -_c(U+1D290, "𝊐", f09d8a90), -_c(U+1D291, "𝊑", f09d8a91), -_c(U+1D292, "𝊒", f09d8a92), -_c(U+1D293, "𝊓", f09d8a93), -_c(U+1D294, "𝊔", f09d8a94), -_c(U+1D295, "𝊕", f09d8a95), -_c(U+1D296, "𝊖", f09d8a96), -_c(U+1D297, "𝊗", f09d8a97), -_c(U+1D298, "𝊘", f09d8a98), -_c(U+1D299, "𝊙", f09d8a99), -_c(U+1D29A, "𝊚", f09d8a9a), -_c(U+1D29B, "𝊛", f09d8a9b), -_c(U+1D29C, "𝊜", f09d8a9c), -_c(U+1D29D, "𝊝", f09d8a9d), -_c(U+1D29E, "𝊞", f09d8a9e), -_c(U+1D29F, "𝊟", f09d8a9f), -_c(U+1D2A0, "𝊠", f09d8aa0), -_c(U+1D2A1, "𝊡", f09d8aa1), -_c(U+1D2A2, "𝊢", f09d8aa2), -_c(U+1D2A3, "𝊣", f09d8aa3), -_c(U+1D2A4, "𝊤", f09d8aa4), -_c(U+1D2A5, "𝊥", f09d8aa5), -_c(U+1D2A6, "𝊦", f09d8aa6), -_c(U+1D2A7, "𝊧", f09d8aa7), -_c(U+1D2A8, "𝊨", f09d8aa8), -_c(U+1D2A9, "𝊩", f09d8aa9), -_c(U+1D2AA, "𝊪", f09d8aaa), -_c(U+1D2AB, "𝊫", f09d8aab), -_c(U+1D2AC, "𝊬", f09d8aac), -_c(U+1D2AD, "𝊭", f09d8aad), -_c(U+1D2AE, "𝊮", f09d8aae), -_c(U+1D2AF, "𝊯", f09d8aaf), -_c(U+1D2B0, "𝊰", f09d8ab0), -_c(U+1D2B1, "𝊱", f09d8ab1), -_c(U+1D2B2, "𝊲", f09d8ab2), -_c(U+1D2B3, "𝊳", f09d8ab3), -_c(U+1D2B4, "𝊴", f09d8ab4), -_c(U+1D2B5, "𝊵", f09d8ab5), -_c(U+1D2B6, "𝊶", f09d8ab6), -_c(U+1D2B7, "𝊷", f09d8ab7), -_c(U+1D2B8, "𝊸", f09d8ab8), -_c(U+1D2B9, "𝊹", f09d8ab9), -_c(U+1D2BA, "𝊺", f09d8aba), -_c(U+1D2BB, "𝊻", f09d8abb), -_c(U+1D2BC, "𝊼", f09d8abc), -_c(U+1D2BD, "𝊽", f09d8abd), -_c(U+1D2BE, "𝊾", f09d8abe), -_c(U+1D2BF, "𝊿", f09d8abf), -_c(U+1D2C0, "𝋀", f09d8b80), -_c(U+1D2C1, "𝋁", f09d8b81), -_c(U+1D2C2, "𝋂", f09d8b82), -_c(U+1D2C3, "𝋃", f09d8b83), -_c(U+1D2C4, "𝋄", f09d8b84), -_c(U+1D2C5, "𝋅", f09d8b85), -_c(U+1D2C6, "𝋆", f09d8b86), -_c(U+1D2C7, "𝋇", f09d8b87), -_c(U+1D2C8, "𝋈", f09d8b88), -_c(U+1D2C9, "𝋉", f09d8b89), -_c(U+1D2CA, "𝋊", f09d8b8a), -_c(U+1D2CB, "𝋋", f09d8b8b), -_c(U+1D2CC, "𝋌", f09d8b8c), -_c(U+1D2CD, "𝋍", f09d8b8d), -_c(U+1D2CE, "𝋎", f09d8b8e), -_c(U+1D2CF, "𝋏", f09d8b8f), -_c(U+1D2D0, "𝋐", f09d8b90), -_c(U+1D2D1, "𝋑", f09d8b91), -_c(U+1D2D2, "𝋒", f09d8b92), -_c(U+1D2D3, "𝋓", f09d8b93), -_c(U+1D2D4, "𝋔", f09d8b94), -_c(U+1D2D5, "𝋕", f09d8b95), -_c(U+1D2D6, "𝋖", f09d8b96), -_c(U+1D2D7, "𝋗", f09d8b97), -_c(U+1D2D8, "𝋘", f09d8b98), -_c(U+1D2D9, "𝋙", f09d8b99), -_c(U+1D2DA, "𝋚", f09d8b9a), -_c(U+1D2DB, "𝋛", f09d8b9b), -_c(U+1D2DC, "𝋜", f09d8b9c), -_c(U+1D2DD, "𝋝", f09d8b9d), -_c(U+1D2DE, "𝋞", f09d8b9e), -_c(U+1D2DF, "𝋟", f09d8b9f), -_c(U+1D2E0, "𝋠", f09d8ba0), -_c(U+1D2E1, "𝋡", f09d8ba1), -_c(U+1D2E2, "𝋢", f09d8ba2), -_c(U+1D2E3, "𝋣", f09d8ba3), -_c(U+1D2E4, "𝋤", f09d8ba4), -_c(U+1D2E5, "𝋥", f09d8ba5), -_c(U+1D2E6, "𝋦", f09d8ba6), -_c(U+1D2E7, "𝋧", f09d8ba7), -_c(U+1D2E8, "𝋨", f09d8ba8), -_c(U+1D2E9, "𝋩", f09d8ba9), -_c(U+1D2EA, "𝋪", f09d8baa), -_c(U+1D2EB, "𝋫", f09d8bab), -_c(U+1D2EC, "𝋬", f09d8bac), -_c(U+1D2ED, "𝋭", f09d8bad), -_c(U+1D2EE, "𝋮", f09d8bae), -_c(U+1D2EF, "𝋯", f09d8baf), -_c(U+1D2F0, "𝋰", f09d8bb0), -_c(U+1D2F1, "𝋱", f09d8bb1), -_c(U+1D2F2, "𝋲", f09d8bb2), -_c(U+1D2F3, "𝋳", f09d8bb3), -_c(U+1D2F4, "𝋴", f09d8bb4), -_c(U+1D2F5, "𝋵", f09d8bb5), -_c(U+1D2F6, "𝋶", f09d8bb6), -_c(U+1D2F7, "𝋷", f09d8bb7), -_c(U+1D2F8, "𝋸", f09d8bb8), -_c(U+1D2F9, "𝋹", f09d8bb9), -_c(U+1D2FA, "𝋺", f09d8bba), -_c(U+1D2FB, "𝋻", f09d8bbb), -_c(U+1D2FC, "𝋼", f09d8bbc), -_c(U+1D2FD, "𝋽", f09d8bbd), -_c(U+1D2FE, "𝋾", f09d8bbe), -_c(U+1D2FF, "𝋿", f09d8bbf), -_c(U+1D300, "𝌀", f09d8c80), -_c(U+1D301, "𝌁", f09d8c81), -_c(U+1D302, "𝌂", f09d8c82), -_c(U+1D303, "𝌃", f09d8c83), -_c(U+1D304, "𝌄", f09d8c84), -_c(U+1D305, "𝌅", f09d8c85), -_c(U+1D306, "𝌆", f09d8c86), -_c(U+1D307, "𝌇", f09d8c87), -_c(U+1D308, "𝌈", f09d8c88), -_c(U+1D309, "𝌉", f09d8c89), -_c(U+1D30A, "𝌊", f09d8c8a), -_c(U+1D30B, "𝌋", f09d8c8b), -_c(U+1D30C, "𝌌", f09d8c8c), -_c(U+1D30D, "𝌍", f09d8c8d), -_c(U+1D30E, "𝌎", f09d8c8e), -_c(U+1D30F, "𝌏", f09d8c8f), -_c(U+1D310, "𝌐", f09d8c90), -_c(U+1D311, "𝌑", f09d8c91), -_c(U+1D312, "𝌒", f09d8c92), -_c(U+1D313, "𝌓", f09d8c93), -_c(U+1D314, "𝌔", f09d8c94), -_c(U+1D315, "𝌕", f09d8c95), -_c(U+1D316, "𝌖", f09d8c96), -_c(U+1D317, "𝌗", f09d8c97), -_c(U+1D318, "𝌘", f09d8c98), -_c(U+1D319, "𝌙", f09d8c99), -_c(U+1D31A, "𝌚", f09d8c9a), -_c(U+1D31B, "𝌛", f09d8c9b), -_c(U+1D31C, "𝌜", f09d8c9c), -_c(U+1D31D, "𝌝", f09d8c9d), -_c(U+1D31E, "𝌞", f09d8c9e), -_c(U+1D31F, "𝌟", f09d8c9f), -_c(U+1D320, "𝌠", f09d8ca0), -_c(U+1D321, "𝌡", f09d8ca1), -_c(U+1D322, "𝌢", f09d8ca2), -_c(U+1D323, "𝌣", f09d8ca3), -_c(U+1D324, "𝌤", f09d8ca4), -_c(U+1D325, "𝌥", f09d8ca5), -_c(U+1D326, "𝌦", f09d8ca6), -_c(U+1D327, "𝌧", f09d8ca7), -_c(U+1D328, "𝌨", f09d8ca8), -_c(U+1D329, "𝌩", f09d8ca9), -_c(U+1D32A, "𝌪", f09d8caa), -_c(U+1D32B, "𝌫", f09d8cab), -_c(U+1D32C, "𝌬", f09d8cac), -_c(U+1D32D, "𝌭", f09d8cad), -_c(U+1D32E, "𝌮", f09d8cae), -_c(U+1D32F, "𝌯", f09d8caf), -_c(U+1D330, "𝌰", f09d8cb0), -_c(U+1D331, "𝌱", f09d8cb1), -_c(U+1D332, "𝌲", f09d8cb2), -_c(U+1D333, "𝌳", f09d8cb3), -_c(U+1D334, "𝌴", f09d8cb4), -_c(U+1D335, "𝌵", f09d8cb5), -_c(U+1D336, "𝌶", f09d8cb6), -_c(U+1D337, "𝌷", f09d8cb7), -_c(U+1D338, "𝌸", f09d8cb8), -_c(U+1D339, "𝌹", f09d8cb9), -_c(U+1D33A, "𝌺", f09d8cba), -_c(U+1D33B, "𝌻", f09d8cbb), -_c(U+1D33C, "𝌼", f09d8cbc), -_c(U+1D33D, "𝌽", f09d8cbd), -_c(U+1D33E, "𝌾", f09d8cbe), -_c(U+1D33F, "𝌿", f09d8cbf), -_c(U+1D340, "𝍀", f09d8d80), -_c(U+1D341, "𝍁", f09d8d81), -_c(U+1D342, "𝍂", f09d8d82), -_c(U+1D343, "𝍃", f09d8d83), -_c(U+1D344, "𝍄", f09d8d84), -_c(U+1D345, "𝍅", f09d8d85), -_c(U+1D346, "𝍆", f09d8d86), -_c(U+1D347, "𝍇", f09d8d87), -_c(U+1D348, "𝍈", f09d8d88), -_c(U+1D349, "𝍉", f09d8d89), -_c(U+1D34A, "𝍊", f09d8d8a), -_c(U+1D34B, "𝍋", f09d8d8b), -_c(U+1D34C, "𝍌", f09d8d8c), -_c(U+1D34D, "𝍍", f09d8d8d), -_c(U+1D34E, "𝍎", f09d8d8e), -_c(U+1D34F, "𝍏", f09d8d8f), -_c(U+1D350, "𝍐", f09d8d90), -_c(U+1D351, "𝍑", f09d8d91), -_c(U+1D352, "𝍒", f09d8d92), -_c(U+1D353, "𝍓", f09d8d93), -_c(U+1D354, "𝍔", f09d8d94), -_c(U+1D355, "𝍕", f09d8d95), -_c(U+1D356, "𝍖", f09d8d96), -_c(U+1D357, "𝍗", f09d8d97), -_c(U+1D358, "𝍘", f09d8d98), -_c(U+1D359, "𝍙", f09d8d99), -_c(U+1D35A, "𝍚", f09d8d9a), -_c(U+1D35B, "𝍛", f09d8d9b), -_c(U+1D35C, "𝍜", f09d8d9c), -_c(U+1D35D, "𝍝", f09d8d9d), -_c(U+1D35E, "𝍞", f09d8d9e), -_c(U+1D35F, "𝍟", f09d8d9f), -_c(U+1D360, "𝍠", f09d8da0), -_c(U+1D361, "𝍡", f09d8da1), -_c(U+1D362, "𝍢", f09d8da2), -_c(U+1D363, "𝍣", f09d8da3), -_c(U+1D364, "𝍤", f09d8da4), -_c(U+1D365, "𝍥", f09d8da5), -_c(U+1D366, "𝍦", f09d8da6), -_c(U+1D367, "𝍧", f09d8da7), -_c(U+1D368, "𝍨", f09d8da8), -_c(U+1D369, "𝍩", f09d8da9), -_c(U+1D36A, "𝍪", f09d8daa), -_c(U+1D36B, "𝍫", f09d8dab), -_c(U+1D36C, "𝍬", f09d8dac), -_c(U+1D36D, "𝍭", f09d8dad), -_c(U+1D36E, "𝍮", f09d8dae), -_c(U+1D36F, "𝍯", f09d8daf), -_c(U+1D370, "𝍰", f09d8db0), -_c(U+1D371, "𝍱", f09d8db1), -_c(U+1D372, "𝍲", f09d8db2), -_c(U+1D373, "𝍳", f09d8db3), -_c(U+1D374, "𝍴", f09d8db4), -_c(U+1D375, "𝍵", f09d8db5), -_c(U+1D376, "𝍶", f09d8db6), -_c(U+1D377, "𝍷", f09d8db7), -_c(U+1D378, "𝍸", f09d8db8), -_c(U+1D379, "𝍹", f09d8db9), -_c(U+1D37A, "𝍺", f09d8dba), -_c(U+1D37B, "𝍻", f09d8dbb), -_c(U+1D37C, "𝍼", f09d8dbc), -_c(U+1D37D, "𝍽", f09d8dbd), -_c(U+1D37E, "𝍾", f09d8dbe), -_c(U+1D37F, "𝍿", f09d8dbf), -_c(U+1D380, "𝎀", f09d8e80), -_c(U+1D381, "𝎁", f09d8e81), -_c(U+1D382, "𝎂", f09d8e82), -_c(U+1D383, "𝎃", f09d8e83), -_c(U+1D384, "𝎄", f09d8e84), -_c(U+1D385, "𝎅", f09d8e85), -_c(U+1D386, "𝎆", f09d8e86), -_c(U+1D387, "𝎇", f09d8e87), -_c(U+1D388, "𝎈", f09d8e88), -_c(U+1D389, "𝎉", f09d8e89), -_c(U+1D38A, "𝎊", f09d8e8a), -_c(U+1D38B, "𝎋", f09d8e8b), -_c(U+1D38C, "𝎌", f09d8e8c), -_c(U+1D38D, "𝎍", f09d8e8d), -_c(U+1D38E, "𝎎", f09d8e8e), -_c(U+1D38F, "𝎏", f09d8e8f), -_c(U+1D390, "𝎐", f09d8e90), -_c(U+1D391, "𝎑", f09d8e91), -_c(U+1D392, "𝎒", f09d8e92), -_c(U+1D393, "𝎓", f09d8e93), -_c(U+1D394, "𝎔", f09d8e94), -_c(U+1D395, "𝎕", f09d8e95), -_c(U+1D396, "𝎖", f09d8e96), -_c(U+1D397, "𝎗", f09d8e97), -_c(U+1D398, "𝎘", f09d8e98), -_c(U+1D399, "𝎙", f09d8e99), -_c(U+1D39A, "𝎚", f09d8e9a), -_c(U+1D39B, "𝎛", f09d8e9b), -_c(U+1D39C, "𝎜", f09d8e9c), -_c(U+1D39D, "𝎝", f09d8e9d), -_c(U+1D39E, "𝎞", f09d8e9e), -_c(U+1D39F, "𝎟", f09d8e9f), -_c(U+1D3A0, "𝎠", f09d8ea0), -_c(U+1D3A1, "𝎡", f09d8ea1), -_c(U+1D3A2, "𝎢", f09d8ea2), -_c(U+1D3A3, "𝎣", f09d8ea3), -_c(U+1D3A4, "𝎤", f09d8ea4), -_c(U+1D3A5, "𝎥", f09d8ea5), -_c(U+1D3A6, "𝎦", f09d8ea6), -_c(U+1D3A7, "𝎧", f09d8ea7), -_c(U+1D3A8, "𝎨", f09d8ea8), -_c(U+1D3A9, "𝎩", f09d8ea9), -_c(U+1D3AA, "𝎪", f09d8eaa), -_c(U+1D3AB, "𝎫", f09d8eab), -_c(U+1D3AC, "𝎬", f09d8eac), -_c(U+1D3AD, "𝎭", f09d8ead), -_c(U+1D3AE, "𝎮", f09d8eae), -_c(U+1D3AF, "𝎯", f09d8eaf), -_c(U+1D3B0, "𝎰", f09d8eb0), -_c(U+1D3B1, "𝎱", f09d8eb1), -_c(U+1D3B2, "𝎲", f09d8eb2), -_c(U+1D3B3, "𝎳", f09d8eb3), -_c(U+1D3B4, "𝎴", f09d8eb4), -_c(U+1D3B5, "𝎵", f09d8eb5), -_c(U+1D3B6, "𝎶", f09d8eb6), -_c(U+1D3B7, "𝎷", f09d8eb7), -_c(U+1D3B8, "𝎸", f09d8eb8), -_c(U+1D3B9, "𝎹", f09d8eb9), -_c(U+1D3BA, "𝎺", f09d8eba), -_c(U+1D3BB, "𝎻", f09d8ebb), -_c(U+1D3BC, "𝎼", f09d8ebc), -_c(U+1D3BD, "𝎽", f09d8ebd), -_c(U+1D3BE, "𝎾", f09d8ebe), -_c(U+1D3BF, "𝎿", f09d8ebf), -_c(U+1D3C0, "𝏀", f09d8f80), -_c(U+1D3C1, "𝏁", f09d8f81), -_c(U+1D3C2, "𝏂", f09d8f82), -_c(U+1D3C3, "𝏃", f09d8f83), -_c(U+1D3C4, "𝏄", f09d8f84), -_c(U+1D3C5, "𝏅", f09d8f85), -_c(U+1D3C6, "𝏆", f09d8f86), -_c(U+1D3C7, "𝏇", f09d8f87), -_c(U+1D3C8, "𝏈", f09d8f88), -_c(U+1D3C9, "𝏉", f09d8f89), -_c(U+1D3CA, "𝏊", f09d8f8a), -_c(U+1D3CB, "𝏋", f09d8f8b), -_c(U+1D3CC, "𝏌", f09d8f8c), -_c(U+1D3CD, "𝏍", f09d8f8d), -_c(U+1D3CE, "𝏎", f09d8f8e), -_c(U+1D3CF, "𝏏", f09d8f8f), -_c(U+1D3D0, "𝏐", f09d8f90), -_c(U+1D3D1, "𝏑", f09d8f91), -_c(U+1D3D2, "𝏒", f09d8f92), -_c(U+1D3D3, "𝏓", f09d8f93), -_c(U+1D3D4, "𝏔", f09d8f94), -_c(U+1D3D5, "𝏕", f09d8f95), -_c(U+1D3D6, "𝏖", f09d8f96), -_c(U+1D3D7, "𝏗", f09d8f97), -_c(U+1D3D8, "𝏘", f09d8f98), -_c(U+1D3D9, "𝏙", f09d8f99), -_c(U+1D3DA, "𝏚", f09d8f9a), -_c(U+1D3DB, "𝏛", f09d8f9b), -_c(U+1D3DC, "𝏜", f09d8f9c), -_c(U+1D3DD, "𝏝", f09d8f9d), -_c(U+1D3DE, "𝏞", f09d8f9e), -_c(U+1D3DF, "𝏟", f09d8f9f), -_c(U+1D3E0, "𝏠", f09d8fa0), -_c(U+1D3E1, "𝏡", f09d8fa1), -_c(U+1D3E2, "𝏢", f09d8fa2), -_c(U+1D3E3, "𝏣", f09d8fa3), -_c(U+1D3E4, "𝏤", f09d8fa4), -_c(U+1D3E5, "𝏥", f09d8fa5), -_c(U+1D3E6, "𝏦", f09d8fa6), -_c(U+1D3E7, "𝏧", f09d8fa7), -_c(U+1D3E8, "𝏨", f09d8fa8), -_c(U+1D3E9, "𝏩", f09d8fa9), -_c(U+1D3EA, "𝏪", f09d8faa), -_c(U+1D3EB, "𝏫", f09d8fab), -_c(U+1D3EC, "𝏬", f09d8fac), -_c(U+1D3ED, "𝏭", f09d8fad), -_c(U+1D3EE, "𝏮", f09d8fae), -_c(U+1D3EF, "𝏯", f09d8faf), -_c(U+1D3F0, "𝏰", f09d8fb0), -_c(U+1D3F1, "𝏱", f09d8fb1), -_c(U+1D3F2, "𝏲", f09d8fb2), -_c(U+1D3F3, "𝏳", f09d8fb3), -_c(U+1D3F4, "𝏴", f09d8fb4), -_c(U+1D3F5, "𝏵", f09d8fb5), -_c(U+1D3F6, "𝏶", f09d8fb6), -_c(U+1D3F7, "𝏷", f09d8fb7), -_c(U+1D3F8, "𝏸", f09d8fb8), -_c(U+1D3F9, "𝏹", f09d8fb9), -_c(U+1D3FA, "𝏺", f09d8fba), -_c(U+1D3FB, "𝏻", f09d8fbb), -_c(U+1D3FC, "𝏼", f09d8fbc), -_c(U+1D3FD, "𝏽", f09d8fbd), -_c(U+1D3FE, "𝏾", f09d8fbe), -_c(U+1D3FF, "𝏿", f09d8fbf), -_c(U+1D400, "𝐀", f09d9080), -_c(U+1D401, "𝐁", f09d9081), -_c(U+1D402, "𝐂", f09d9082), -_c(U+1D403, "𝐃", f09d9083), -_c(U+1D404, "𝐄", f09d9084), -_c(U+1D405, "𝐅", f09d9085), -_c(U+1D406, "𝐆", f09d9086), -_c(U+1D407, "𝐇", f09d9087), -_c(U+1D408, "𝐈", f09d9088), -_c(U+1D409, "𝐉", f09d9089), -_c(U+1D40A, "𝐊", f09d908a), -_c(U+1D40B, "𝐋", f09d908b), -_c(U+1D40C, "𝐌", f09d908c), -_c(U+1D40D, "𝐍", f09d908d), -_c(U+1D40E, "𝐎", f09d908e), -_c(U+1D40F, "𝐏", f09d908f), -_c(U+1D410, "𝐐", f09d9090), -_c(U+1D411, "𝐑", f09d9091), -_c(U+1D412, "𝐒", f09d9092), -_c(U+1D413, "𝐓", f09d9093), -_c(U+1D414, "𝐔", f09d9094), -_c(U+1D415, "𝐕", f09d9095), -_c(U+1D416, "𝐖", f09d9096), -_c(U+1D417, "𝐗", f09d9097), -_c(U+1D418, "𝐘", f09d9098), -_c(U+1D419, "𝐙", f09d9099), -_c(U+1D41A, "𝐚", f09d909a), -_c(U+1D41B, "𝐛", f09d909b), -_c(U+1D41C, "𝐜", f09d909c), -_c(U+1D41D, "𝐝", f09d909d), -_c(U+1D41E, "𝐞", f09d909e), -_c(U+1D41F, "𝐟", f09d909f), -_c(U+1D420, "𝐠", f09d90a0), -_c(U+1D421, "𝐡", f09d90a1), -_c(U+1D422, "𝐢", f09d90a2), -_c(U+1D423, "𝐣", f09d90a3), -_c(U+1D424, "𝐤", f09d90a4), -_c(U+1D425, "𝐥", f09d90a5), -_c(U+1D426, "𝐦", f09d90a6), -_c(U+1D427, "𝐧", f09d90a7), -_c(U+1D428, "𝐨", f09d90a8), -_c(U+1D429, "𝐩", f09d90a9), -_c(U+1D42A, "𝐪", f09d90aa), -_c(U+1D42B, "𝐫", f09d90ab), -_c(U+1D42C, "𝐬", f09d90ac), -_c(U+1D42D, "𝐭", f09d90ad), -_c(U+1D42E, "𝐮", f09d90ae), -_c(U+1D42F, "𝐯", f09d90af), -_c(U+1D430, "𝐰", f09d90b0), -_c(U+1D431, "𝐱", f09d90b1), -_c(U+1D432, "𝐲", f09d90b2), -_c(U+1D433, "𝐳", f09d90b3), -_c(U+1D434, "𝐴", f09d90b4), -_c(U+1D435, "𝐵", f09d90b5), -_c(U+1D436, "𝐶", f09d90b6), -_c(U+1D437, "𝐷", f09d90b7), -_c(U+1D438, "𝐸", f09d90b8), -_c(U+1D439, "𝐹", f09d90b9), -_c(U+1D43A, "𝐺", f09d90ba), -_c(U+1D43B, "𝐻", f09d90bb), -_c(U+1D43C, "𝐼", f09d90bc), -_c(U+1D43D, "𝐽", f09d90bd), -_c(U+1D43E, "𝐾", f09d90be), -_c(U+1D43F, "𝐿", f09d90bf), -_c(U+1D440, "𝑀", f09d9180), -_c(U+1D441, "𝑁", f09d9181), -_c(U+1D442, "𝑂", f09d9182), -_c(U+1D443, "𝑃", f09d9183), -_c(U+1D444, "𝑄", f09d9184), -_c(U+1D445, "𝑅", f09d9185), -_c(U+1D446, "𝑆", f09d9186), -_c(U+1D447, "𝑇", f09d9187), -_c(U+1D448, "𝑈", f09d9188), -_c(U+1D449, "𝑉", f09d9189), -_c(U+1D44A, "𝑊", f09d918a), -_c(U+1D44B, "𝑋", f09d918b), -_c(U+1D44C, "𝑌", f09d918c), -_c(U+1D44D, "𝑍", f09d918d), -_c(U+1D44E, "𝑎", f09d918e), -_c(U+1D44F, "𝑏", f09d918f), -_c(U+1D450, "𝑐", f09d9190), -_c(U+1D451, "𝑑", f09d9191), -_c(U+1D452, "𝑒", f09d9192), -_c(U+1D453, "𝑓", f09d9193), -_c(U+1D454, "𝑔", f09d9194), -_c(U+1D455, "𝑕", f09d9195), -_c(U+1D456, "𝑖", f09d9196), -_c(U+1D457, "𝑗", f09d9197), -_c(U+1D458, "𝑘", f09d9198), -_c(U+1D459, "𝑙", f09d9199), -_c(U+1D45A, "𝑚", f09d919a), -_c(U+1D45B, "𝑛", f09d919b), -_c(U+1D45C, "𝑜", f09d919c), -_c(U+1D45D, "𝑝", f09d919d), -_c(U+1D45E, "𝑞", f09d919e), -_c(U+1D45F, "𝑟", f09d919f), -_c(U+1D460, "𝑠", f09d91a0), -_c(U+1D461, "𝑡", f09d91a1), -_c(U+1D462, "𝑢", f09d91a2), -_c(U+1D463, "𝑣", f09d91a3), -_c(U+1D464, "𝑤", f09d91a4), -_c(U+1D465, "𝑥", f09d91a5), -_c(U+1D466, "𝑦", f09d91a6), -_c(U+1D467, "𝑧", f09d91a7), -_c(U+1D468, "𝑨", f09d91a8), -_c(U+1D469, "𝑩", f09d91a9), -_c(U+1D46A, "𝑪", f09d91aa), -_c(U+1D46B, "𝑫", f09d91ab), -_c(U+1D46C, "𝑬", f09d91ac), -_c(U+1D46D, "𝑭", f09d91ad), -_c(U+1D46E, "𝑮", f09d91ae), -_c(U+1D46F, "𝑯", f09d91af), -_c(U+1D470, "𝑰", f09d91b0), -_c(U+1D471, "𝑱", f09d91b1), -_c(U+1D472, "𝑲", f09d91b2), -_c(U+1D473, "𝑳", f09d91b3), -_c(U+1D474, "𝑴", f09d91b4), -_c(U+1D475, "𝑵", f09d91b5), -_c(U+1D476, "𝑶", f09d91b6), -_c(U+1D477, "𝑷", f09d91b7), -_c(U+1D478, "𝑸", f09d91b8), -_c(U+1D479, "𝑹", f09d91b9), -_c(U+1D47A, "𝑺", f09d91ba), -_c(U+1D47B, "𝑻", f09d91bb), -_c(U+1D47C, "𝑼", f09d91bc), -_c(U+1D47D, "𝑽", f09d91bd), -_c(U+1D47E, "𝑾", f09d91be), -_c(U+1D47F, "𝑿", f09d91bf), -_c(U+1D480, "𝒀", f09d9280), -_c(U+1D481, "𝒁", f09d9281), -_c(U+1D482, "𝒂", f09d9282), -_c(U+1D483, "𝒃", f09d9283), -_c(U+1D484, "𝒄", f09d9284), -_c(U+1D485, "𝒅", f09d9285), -_c(U+1D486, "𝒆", f09d9286), -_c(U+1D487, "𝒇", f09d9287), -_c(U+1D488, "𝒈", f09d9288), -_c(U+1D489, "𝒉", f09d9289), -_c(U+1D48A, "𝒊", f09d928a), -_c(U+1D48B, "𝒋", f09d928b), -_c(U+1D48C, "𝒌", f09d928c), -_c(U+1D48D, "𝒍", f09d928d), -_c(U+1D48E, "𝒎", f09d928e), -_c(U+1D48F, "𝒏", f09d928f), -_c(U+1D490, "𝒐", f09d9290), -_c(U+1D491, "𝒑", f09d9291), -_c(U+1D492, "𝒒", f09d9292), -_c(U+1D493, "𝒓", f09d9293), -_c(U+1D494, "𝒔", f09d9294), -_c(U+1D495, "𝒕", f09d9295), -_c(U+1D496, "𝒖", f09d9296), -_c(U+1D497, "𝒗", f09d9297), -_c(U+1D498, "𝒘", f09d9298), -_c(U+1D499, "𝒙", f09d9299), -_c(U+1D49A, "𝒚", f09d929a), -_c(U+1D49B, "𝒛", f09d929b), -_c(U+1D49C, "𝒜", f09d929c), -_c(U+1D49D, "𝒝", f09d929d), -_c(U+1D49E, "𝒞", f09d929e), -_c(U+1D49F, "𝒟", f09d929f), -_c(U+1D4A0, "𝒠", f09d92a0), -_c(U+1D4A1, "𝒡", f09d92a1), -_c(U+1D4A2, "𝒢", f09d92a2), -_c(U+1D4A3, "𝒣", f09d92a3), -_c(U+1D4A4, "𝒤", f09d92a4), -_c(U+1D4A5, "𝒥", f09d92a5), -_c(U+1D4A6, "𝒦", f09d92a6), -_c(U+1D4A7, "𝒧", f09d92a7), -_c(U+1D4A8, "𝒨", f09d92a8), -_c(U+1D4A9, "𝒩", f09d92a9), -_c(U+1D4AA, "𝒪", f09d92aa), -_c(U+1D4AB, "𝒫", f09d92ab), -_c(U+1D4AC, "𝒬", f09d92ac), -_c(U+1D4AD, "𝒭", f09d92ad), -_c(U+1D4AE, "𝒮", f09d92ae), -_c(U+1D4AF, "𝒯", f09d92af), -_c(U+1D4B0, "𝒰", f09d92b0), -_c(U+1D4B1, "𝒱", f09d92b1), -_c(U+1D4B2, "𝒲", f09d92b2), -_c(U+1D4B3, "𝒳", f09d92b3), -_c(U+1D4B4, "𝒴", f09d92b4), -_c(U+1D4B5, "𝒵", f09d92b5), -_c(U+1D4B6, "𝒶", f09d92b6), -_c(U+1D4B7, "𝒷", f09d92b7), -_c(U+1D4B8, "𝒸", f09d92b8), -_c(U+1D4B9, "𝒹", f09d92b9), -_c(U+1D4BA, "𝒺", f09d92ba), -_c(U+1D4BB, "𝒻", f09d92bb), -_c(U+1D4BC, "𝒼", f09d92bc), -_c(U+1D4BD, "𝒽", f09d92bd), -_c(U+1D4BE, "𝒾", f09d92be), -_c(U+1D4BF, "𝒿", f09d92bf), -_c(U+1D4C0, "𝓀", f09d9380), -_c(U+1D4C1, "𝓁", f09d9381), -_c(U+1D4C2, "𝓂", f09d9382), -_c(U+1D4C3, "𝓃", f09d9383), -_c(U+1D4C4, "𝓄", f09d9384), -_c(U+1D4C5, "𝓅", f09d9385), -_c(U+1D4C6, "𝓆", f09d9386), -_c(U+1D4C7, "𝓇", f09d9387), -_c(U+1D4C8, "𝓈", f09d9388), -_c(U+1D4C9, "𝓉", f09d9389), -_c(U+1D4CA, "𝓊", f09d938a), -_c(U+1D4CB, "𝓋", f09d938b), -_c(U+1D4CC, "𝓌", f09d938c), -_c(U+1D4CD, "𝓍", f09d938d), -_c(U+1D4CE, "𝓎", f09d938e), -_c(U+1D4CF, "𝓏", f09d938f), -_c(U+1D4D0, "𝓐", f09d9390), -_c(U+1D4D1, "𝓑", f09d9391), -_c(U+1D4D2, "𝓒", f09d9392), -_c(U+1D4D3, "𝓓", f09d9393), -_c(U+1D4D4, "𝓔", f09d9394), -_c(U+1D4D5, "𝓕", f09d9395), -_c(U+1D4D6, "𝓖", f09d9396), -_c(U+1D4D7, "𝓗", f09d9397), -_c(U+1D4D8, "𝓘", f09d9398), -_c(U+1D4D9, "𝓙", f09d9399), -_c(U+1D4DA, "𝓚", f09d939a), -_c(U+1D4DB, "𝓛", f09d939b), -_c(U+1D4DC, "𝓜", f09d939c), -_c(U+1D4DD, "𝓝", f09d939d), -_c(U+1D4DE, "𝓞", f09d939e), -_c(U+1D4DF, "𝓟", f09d939f), -_c(U+1D4E0, "𝓠", f09d93a0), -_c(U+1D4E1, "𝓡", f09d93a1), -_c(U+1D4E2, "𝓢", f09d93a2), -_c(U+1D4E3, "𝓣", f09d93a3), -_c(U+1D4E4, "𝓤", f09d93a4), -_c(U+1D4E5, "𝓥", f09d93a5), -_c(U+1D4E6, "𝓦", f09d93a6), -_c(U+1D4E7, "𝓧", f09d93a7), -_c(U+1D4E8, "𝓨", f09d93a8), -_c(U+1D4E9, "𝓩", f09d93a9), -_c(U+1D4EA, "𝓪", f09d93aa), -_c(U+1D4EB, "𝓫", f09d93ab), -_c(U+1D4EC, "𝓬", f09d93ac), -_c(U+1D4ED, "𝓭", f09d93ad), -_c(U+1D4EE, "𝓮", f09d93ae), -_c(U+1D4EF, "𝓯", f09d93af), -_c(U+1D4F0, "𝓰", f09d93b0), -_c(U+1D4F1, "𝓱", f09d93b1), -_c(U+1D4F2, "𝓲", f09d93b2), -_c(U+1D4F3, "𝓳", f09d93b3), -_c(U+1D4F4, "𝓴", f09d93b4), -_c(U+1D4F5, "𝓵", f09d93b5), -_c(U+1D4F6, "𝓶", f09d93b6), -_c(U+1D4F7, "𝓷", f09d93b7), -_c(U+1D4F8, "𝓸", f09d93b8), -_c(U+1D4F9, "𝓹", f09d93b9), -_c(U+1D4FA, "𝓺", f09d93ba), -_c(U+1D4FB, "𝓻", f09d93bb), -_c(U+1D4FC, "𝓼", f09d93bc), -_c(U+1D4FD, "𝓽", f09d93bd), -_c(U+1D4FE, "𝓾", f09d93be), -_c(U+1D4FF, "𝓿", f09d93bf), -_c(U+1F600, "😀", f09f9880), -_c(U+1F601, "😁", f09f9881), -_c(U+1F602, "😂", f09f9882), -_c(U+1F603, "😃", f09f9883), -_c(U+1F604, "😄", f09f9884), -_c(U+1F605, "😅", f09f9885), -_c(U+1F606, "😆", f09f9886), -_c(U+1F607, "😇", f09f9887), -_c(U+1F608, "😈", f09f9888), -_c(U+1F609, "😉", f09f9889), -_c(U+1F60A, "😊", f09f988a), -_c(U+1F60B, "😋", f09f988b), -_c(U+1F60C, "😌", f09f988c), -_c(U+1F60D, "😍", f09f988d), -_c(U+1F60E, "😎", f09f988e), -_c(U+1F60F, "😏", f09f988f), -_c(U+1F610, "😐", f09f9890), -_c(U+1F611, "😑", f09f9891), -_c(U+1F612, "😒", f09f9892), -_c(U+1F613, "😓", f09f9893), -_c(U+1F614, "😔", f09f9894), -_c(U+1F615, "😕", f09f9895), -_c(U+1F616, "😖", f09f9896), -_c(U+1F617, "😗", f09f9897), -_c(U+1F618, "😘", f09f9898), -_c(U+1F619, "😙", f09f9899), -_c(U+1F61A, "😚", f09f989a), -_c(U+1F61B, "😛", f09f989b), -_c(U+1F61C, "😜", f09f989c), -_c(U+1F61D, "😝", f09f989d), -_c(U+1F61E, "😞", f09f989e), -_c(U+1F61F, "😟", f09f989f), -_c(U+1F620, "😠", f09f98a0), -_c(U+1F621, "😡", f09f98a1), -_c(U+1F622, "😢", f09f98a2), -_c(U+1F623, "😣", f09f98a3), -_c(U+1F624, "😤", f09f98a4), -_c(U+1F625, "😥", f09f98a5), -_c(U+1F626, "😦", f09f98a6), -_c(U+1F627, "😧", f09f98a7), -_c(U+1F628, "😨", f09f98a8), -_c(U+1F629, "😩", f09f98a9), -_c(U+1F62A, "😪", f09f98aa), -_c(U+1F62B, "😫", f09f98ab), -_c(U+1F62C, "😬", f09f98ac), -_c(U+1F62D, "😭", f09f98ad), -_c(U+1F62E, "😮", f09f98ae), -_c(U+1F62F, "😯", f09f98af), -_c(U+1F630, "😰", f09f98b0), -_c(U+1F631, "😱", f09f98b1), -_c(U+1F632, "😲", f09f98b2), -_c(U+1F633, "😳", f09f98b3), -_c(U+1F634, "😴", f09f98b4), -_c(U+1F635, "😵", f09f98b5), -_c(U+1F636, "😶", f09f98b6), -_c(U+1F637, "😷", f09f98b7), -_c(U+1F638, "😸", f09f98b8), -_c(U+1F639, "😹", f09f98b9), -_c(U+1F63A, "😺", f09f98ba), -_c(U+1F63B, "😻", f09f98bb), -_c(U+1F63C, "😼", f09f98bc), -_c(U+1F63D, "😽", f09f98bd), -_c(U+1F63E, "😾", f09f98be), -_c(U+1F63F, "😿", f09f98bf), -_c(U+1F640, "🙀", f09f9980), -_c(U+1F641, "🙁", f09f9981), -_c(U+1F642, "🙂", f09f9982), -_c(U+1F643, "🙃", f09f9983), -_c(U+1F644, "🙄", f09f9984), -_c(U+1F645, "🙅", f09f9985), -_c(U+1F646, "🙆", f09f9986), -_c(U+1F647, "🙇", f09f9987), -_c(U+1F648, "🙈", f09f9988), -_c(U+1F649, "🙉", f09f9989), -_c(U+1F64A, "🙊", f09f998a), -_c(U+1F64B, "🙋", f09f998b), -_c(U+1F64C, "🙌", f09f998c), -_c(U+1F64D, "🙍", f09f998d), -_c(U+1F64E, "🙎", f09f998e), -_c(U+1F64F, "🙏", f09f998f), -_c(U+1F650, "🙐", f09f9990), -_c(U+1F651, "🙑", f09f9991), -_c(U+1F652, "🙒", f09f9992), -_c(U+1F653, "🙓", f09f9993), -_c(U+1F654, "🙔", f09f9994), -_c(U+1F655, "🙕", f09f9995), -_c(U+1F656, "🙖", f09f9996), -_c(U+1F657, "🙗", f09f9997), -_c(U+1F658, "🙘", f09f9998), -_c(U+1F659, "🙙", f09f9999), -_c(U+1F65A, "🙚", f09f999a), -_c(U+1F65B, "🙛", f09f999b), -_c(U+1F65C, "🙜", f09f999c), -_c(U+1F65D, "🙝", f09f999d), -_c(U+1F65E, "🙞", f09f999e), -_c(U+1F65F, "🙟", f09f999f), -_c(U+1F660, "🙠", f09f99a0), -_c(U+1F661, "🙡", f09f99a1), -_c(U+1F662, "🙢", f09f99a2), -_c(U+1F663, "🙣", f09f99a3), -_c(U+1F664, "🙤", f09f99a4), -_c(U+1F665, "🙥", f09f99a5), -_c(U+1F666, "🙦", f09f99a6), -_c(U+1F667, "🙧", f09f99a7), -_c(U+1F668, "🙨", f09f99a8), -_c(U+1F669, "🙩", f09f99a9), -_c(U+1F66A, "🙪", f09f99aa), -_c(U+1F66B, "🙫", f09f99ab), -_c(U+1F66C, "🙬", f09f99ac), -_c(U+1F66D, "🙭", f09f99ad), -_c(U+1F66E, "🙮", f09f99ae), -_c(U+1F66F, "🙯", f09f99af), -_c(U+1F670, "🙰", f09f99b0), -_c(U+1F671, "🙱", f09f99b1), -_c(U+1F672, "🙲", f09f99b2), -_c(U+1F673, "🙳", f09f99b3), -_c(U+1F674, "🙴", f09f99b4), -_c(U+1F675, "🙵", f09f99b5), -_c(U+1F676, "🙶", f09f99b6), -_c(U+1F677, "🙷", f09f99b7), -_c(U+1F678, "🙸", f09f99b8), -_c(U+1F679, "🙹", f09f99b9), -_c(U+1F67A, "🙺", f09f99ba), -_c(U+1F67B, "🙻", f09f99bb), -_c(U+1F67C, "🙼", f09f99bc), -_c(U+1F67D, "🙽", f09f99bd), -_c(U+1F67E, "🙾", f09f99be), -_c(U+1F67F, "🙿", f09f99bf), -_c(U+1F680, "🚀", f09f9a80), -_c(U+1F681, "🚁", f09f9a81), -_c(U+1F682, "🚂", f09f9a82), -_c(U+1F683, "🚃", f09f9a83), -_c(U+1F684, "🚄", f09f9a84), -_c(U+1F685, "🚅", f09f9a85), -_c(U+1F686, "🚆", f09f9a86), -_c(U+1F687, "🚇", f09f9a87), -_c(U+1F688, "🚈", f09f9a88), -_c(U+1F689, "🚉", f09f9a89), -_c(U+1F68A, "🚊", f09f9a8a), -_c(U+1F68B, "🚋", f09f9a8b), -_c(U+1F68C, "🚌", f09f9a8c), -_c(U+1F68D, "🚍", f09f9a8d), -_c(U+1F68E, "🚎", f09f9a8e), -_c(U+1F68F, "🚏", f09f9a8f), -_c(U+1F690, "🚐", f09f9a90), -_c(U+1F691, "🚑", f09f9a91), -_c(U+1F692, "🚒", f09f9a92), -_c(U+1F693, "🚓", f09f9a93), -_c(U+1F694, "🚔", f09f9a94), -_c(U+1F695, "🚕", f09f9a95), -_c(U+1F696, "🚖", f09f9a96), -_c(U+1F697, "🚗", f09f9a97), -_c(U+1F698, "🚘", f09f9a98), -_c(U+1F699, "🚙", f09f9a99), -_c(U+1F69A, "🚚", f09f9a9a), -_c(U+1F69B, "🚛", f09f9a9b), -_c(U+1F69C, "🚜", f09f9a9c), -_c(U+1F69D, "🚝", f09f9a9d), -_c(U+1F69E, "🚞", f09f9a9e), -_c(U+1F69F, "🚟", f09f9a9f), -_c(U+1F6A0, "🚠", f09f9aa0), -_c(U+1F6A1, "🚡", f09f9aa1), -_c(U+1F6A2, "🚢", f09f9aa2), -_c(U+1F6A3, "🚣", f09f9aa3), -_c(U+1F6A4, "🚤", f09f9aa4), -_c(U+1F6A5, "🚥", f09f9aa5), -_c(U+1F6A6, "🚦", f09f9aa6), -_c(U+1F6A7, "🚧", f09f9aa7), -_c(U+1F6A8, "🚨", f09f9aa8), -_c(U+1F6A9, "🚩", f09f9aa9), -_c(U+1F6AA, "🚪", f09f9aaa), -_c(U+1F6AB, "🚫", f09f9aab), -_c(U+1F6AC, "🚬", f09f9aac), -_c(U+1F6AD, "🚭", f09f9aad), -_c(U+1F6AE, "🚮", f09f9aae), -_c(U+1F6AF, "🚯", f09f9aaf), -_c(U+1F6B0, "🚰", f09f9ab0), -_c(U+1F6B1, "🚱", f09f9ab1), -_c(U+1F6B2, "🚲", f09f9ab2), -_c(U+1F6B3, "🚳", f09f9ab3), -_c(U+1F6B4, "🚴", f09f9ab4), -_c(U+1F6B5, "🚵", f09f9ab5), -_c(U+1F6B6, "🚶", f09f9ab6), -_c(U+1F6B7, "🚷", f09f9ab7), -_c(U+1F6B8, "🚸", f09f9ab8), -_c(U+1F6B9, "🚹", f09f9ab9), -_c(U+1F6BA, "🚺", f09f9aba), -_c(U+1F6BB, "🚻", f09f9abb), -_c(U+1F6BC, "🚼", f09f9abc), -_c(U+1F6BD, "🚽", f09f9abd), -_c(U+1F6BE, "🚾", f09f9abe), -_c(U+1F6BF, "🚿", f09f9abf), -_c(U+1F6C0, "🛀", f09f9b80), -_c(U+1F6C1, "🛁", f09f9b81), -_c(U+1F6C2, "🛂", f09f9b82), -_c(U+1F6C3, "🛃", f09f9b83), -_c(U+1F6C4, "🛄", f09f9b84), -_c(U+1F6C5, "🛅", f09f9b85), -_c(U+1F6C6, "🛆", f09f9b86), -_c(U+1F6C7, "🛇", f09f9b87), -_c(U+1F6C8, "🛈", f09f9b88), -_c(U+1F6C9, "🛉", f09f9b89), -_c(U+1F6CA, "🛊", f09f9b8a), -_c(U+1F6CB, "🛋", f09f9b8b), -_c(U+1F6CC, "🛌", f09f9b8c), -_c(U+1F6CD, "🛍", f09f9b8d), -_c(U+1F6CE, "🛎", f09f9b8e), -_c(U+1F6CF, "🛏", f09f9b8f), -_c(U+1F6D0, "🛐", f09f9b90), -_c(U+1F6D1, "🛑", f09f9b91), -_c(U+1F6D2, "🛒", f09f9b92), -_c(U+1F6D3, "🛓", f09f9b93), -_c(U+1F6D4, "🛔", f09f9b94), -_c(U+1F6D5, "🛕", f09f9b95), -_c(U+1F6D6, "🛖", f09f9b96), -_c(U+1F6D7, "🛗", f09f9b97), -_c(U+1F6D8, "🛘", f09f9b98), -_c(U+1F6D9, "🛙", f09f9b99), -_c(U+1F6DA, "🛚", f09f9b9a), -_c(U+1F6DB, "🛛", f09f9b9b), -_c(U+1F6DC, "🛜", f09f9b9c), -_c(U+1F6DD, "🛝", f09f9b9d), -_c(U+1F6DE, "🛞", f09f9b9e), -_c(U+1F6DF, "🛟", f09f9b9f), -_c(U+1F6E0, "🛠", f09f9ba0), -_c(U+1F6E1, "🛡", f09f9ba1), -_c(U+1F6E2, "🛢", f09f9ba2), -_c(U+1F6E3, "🛣", f09f9ba3), -_c(U+1F6E4, "🛤", f09f9ba4), -_c(U+1F6E5, "🛥", f09f9ba5), -_c(U+1F6E6, "🛦", f09f9ba6), -_c(U+1F6E7, "🛧", f09f9ba7), -_c(U+1F6E8, "🛨", f09f9ba8), -_c(U+1F6E9, "🛩", f09f9ba9), -_c(U+1F6EA, "🛪", f09f9baa), -_c(U+1F6EB, "🛫", f09f9bab), -_c(U+1F6EC, "🛬", f09f9bac), -_c(U+1F6ED, "🛭", f09f9bad), -_c(U+1F6EE, "🛮", f09f9bae), -_c(U+1F6EF, "🛯", f09f9baf), -_c(U+1F6F0, "🛰", f09f9bb0), -_c(U+1F6F1, "🛱", f09f9bb1), -_c(U+1F6F2, "🛲", f09f9bb2), -_c(U+1F6F3, "🛳", f09f9bb3), -_c(U+1F6F4, "🛴", f09f9bb4), -_c(U+1F6F5, "🛵", f09f9bb5), -_c(U+1F6F6, "🛶", f09f9bb6), -_c(U+1F6F7, "🛷", f09f9bb7), -_c(U+1F6F8, "🛸", f09f9bb8), -_c(U+1F6F9, "🛹", f09f9bb9), -_c(U+1F6FA, "🛺", f09f9bba), -_c(U+1F6FB, "🛻", f09f9bbb), -_c(U+1F6FC, "🛼", f09f9bbc), -_c(U+1F6FD, "🛽", f09f9bbd), -_c(U+1F6FE, "🛾", f09f9bbe), -_c(U+1F6FF, "🛿", f09f9bbf), -_c(U+1F700, "🜀", f09f9c80), -_c(U+1F701, "🜁", f09f9c81), -_c(U+1F702, "🜂", f09f9c82), -_c(U+1F703, "🜃", f09f9c83), -_c(U+1F704, "🜄", f09f9c84), -_c(U+1F705, "🜅", f09f9c85), -_c(U+1F706, "🜆", f09f9c86), -_c(U+1F707, "🜇", f09f9c87), -_c(U+1F708, "🜈", f09f9c88), -_c(U+1F709, "🜉", f09f9c89), -_c(U+1F70A, "🜊", f09f9c8a), -_c(U+1F70B, "🜋", f09f9c8b), -_c(U+1F70C, "🜌", f09f9c8c), -_c(U+1F70D, "🜍", f09f9c8d), -_c(U+1F70E, "🜎", f09f9c8e), -_c(U+1F70F, "🜏", f09f9c8f), -_c(U+1F710, "🜐", f09f9c90), -_c(U+1F711, "🜑", f09f9c91), -_c(U+1F712, "🜒", f09f9c92), -_c(U+1F713, "🜓", f09f9c93), -_c(U+1F714, "🜔", f09f9c94), -_c(U+1F715, "🜕", f09f9c95), -_c(U+1F716, "🜖", f09f9c96), -_c(U+1F717, "🜗", f09f9c97), -_c(U+1F718, "🜘", f09f9c98), -_c(U+1F719, "🜙", f09f9c99), -_c(U+1F71A, "🜚", f09f9c9a), -_c(U+1F71B, "🜛", f09f9c9b), -_c(U+1F71C, "🜜", f09f9c9c), -_c(U+1F71D, "🜝", f09f9c9d), -_c(U+1F71E, "🜞", f09f9c9e), -_c(U+1F71F, "🜟", f09f9c9f), -_c(U+1F720, "🜠", f09f9ca0), -_c(U+1F721, "🜡", f09f9ca1), -_c(U+1F722, "🜢", f09f9ca2), -_c(U+1F723, "🜣", f09f9ca3), -_c(U+1F724, "🜤", f09f9ca4), -_c(U+1F725, "🜥", f09f9ca5), -_c(U+1F726, "🜦", f09f9ca6), -_c(U+1F727, "🜧", f09f9ca7), -_c(U+1F728, "🜨", f09f9ca8), -_c(U+1F729, "🜩", f09f9ca9), -_c(U+1F72A, "🜪", f09f9caa), -_c(U+1F72B, "🜫", f09f9cab), -_c(U+1F72C, "🜬", f09f9cac), -_c(U+1F72D, "🜭", f09f9cad), -_c(U+1F72E, "🜮", f09f9cae), -_c(U+1F72F, "🜯", f09f9caf), -_c(U+1F730, "🜰", f09f9cb0), -_c(U+1F731, "🜱", f09f9cb1), -_c(U+1F732, "🜲", f09f9cb2), -_c(U+1F733, "🜳", f09f9cb3), -_c(U+1F734, "🜴", f09f9cb4), -_c(U+1F735, "🜵", f09f9cb5), -_c(U+1F736, "🜶", f09f9cb6), -_c(U+1F737, "🜷", f09f9cb7), -_c(U+1F738, "🜸", f09f9cb8), -_c(U+1F739, "🜹", f09f9cb9), -_c(U+1F73A, "🜺", f09f9cba), -_c(U+1F73B, "🜻", f09f9cbb), -_c(U+1F73C, "🜼", f09f9cbc), -_c(U+1F73D, "🜽", f09f9cbd), -_c(U+1F73E, "🜾", f09f9cbe), -_c(U+1F73F, "🜿", f09f9cbf), -_c(U+1F740, "🝀", f09f9d80), -_c(U+1F741, "🝁", f09f9d81), -_c(U+1F742, "🝂", f09f9d82), -_c(U+1F743, "🝃", f09f9d83), -_c(U+1F744, "🝄", f09f9d84), -_c(U+1F745, "🝅", f09f9d85), -_c(U+1F746, "🝆", f09f9d86), -_c(U+1F747, "🝇", f09f9d87), -_c(U+1F748, "🝈", f09f9d88), -_c(U+1F749, "🝉", f09f9d89), -_c(U+1F74A, "🝊", f09f9d8a), -_c(U+1F74B, "🝋", f09f9d8b), -_c(U+1F74C, "🝌", f09f9d8c), -_c(U+1F74D, "🝍", f09f9d8d), -_c(U+1F74E, "🝎", f09f9d8e), -_c(U+1F74F, "🝏", f09f9d8f), -_c(U+1F750, "🝐", f09f9d90), -_c(U+1F751, "🝑", f09f9d91), -_c(U+1F752, "🝒", f09f9d92), -_c(U+1F753, "🝓", f09f9d93), -_c(U+1F754, "🝔", f09f9d94), -_c(U+1F755, "🝕", f09f9d95), -_c(U+1F756, "🝖", f09f9d96), -_c(U+1F757, "🝗", f09f9d97), -_c(U+1F758, "🝘", f09f9d98), -_c(U+1F759, "🝙", f09f9d99), -_c(U+1F75A, "🝚", f09f9d9a), -_c(U+1F75B, "🝛", f09f9d9b), -_c(U+1F75C, "🝜", f09f9d9c), -_c(U+1F75D, "🝝", f09f9d9d), -_c(U+1F75E, "🝞", f09f9d9e), -_c(U+1F75F, "🝟", f09f9d9f), -_c(U+1F760, "🝠", f09f9da0), -_c(U+1F761, "🝡", f09f9da1), -_c(U+1F762, "🝢", f09f9da2), -_c(U+1F763, "🝣", f09f9da3), -_c(U+1F764, "🝤", f09f9da4), -_c(U+1F765, "🝥", f09f9da5), -_c(U+1F766, "🝦", f09f9da6), -_c(U+1F767, "🝧", f09f9da7), -_c(U+1F768, "🝨", f09f9da8), -_c(U+1F769, "🝩", f09f9da9), -_c(U+1F76A, "🝪", f09f9daa), -_c(U+1F76B, "🝫", f09f9dab), -_c(U+1F76C, "🝬", f09f9dac), -_c(U+1F76D, "🝭", f09f9dad), -_c(U+1F76E, "🝮", f09f9dae), -_c(U+1F76F, "🝯", f09f9daf), -_c(U+1F770, "🝰", f09f9db0), -_c(U+1F771, "🝱", f09f9db1), -_c(U+1F772, "🝲", f09f9db2), -_c(U+1F773, "🝳", f09f9db3), -_c(U+1F774, "🝴", f09f9db4), -_c(U+1F775, "🝵", f09f9db5), -_c(U+1F776, "🝶", f09f9db6), -_c(U+1F777, "🝷", f09f9db7), -_c(U+1F778, "🝸", f09f9db8), -_c(U+1F779, "🝹", f09f9db9), -_c(U+1F77A, "🝺", f09f9dba), -_c(U+1F77B, "🝻", f09f9dbb), -_c(U+1F77C, "🝼", f09f9dbc), -_c(U+1F77D, "🝽", f09f9dbd), -_c(U+1F77E, "🝾", f09f9dbe), -_c(U+1F77F, "🝿", f09f9dbf), -_c(U+1F780, "🞀", f09f9e80), -_c(U+1F781, "🞁", f09f9e81), -_c(U+1F782, "🞂", f09f9e82), -_c(U+1F783, "🞃", f09f9e83), -_c(U+1F784, "🞄", f09f9e84), -_c(U+1F785, "🞅", f09f9e85), -_c(U+1F786, "🞆", f09f9e86), -_c(U+1F787, "🞇", f09f9e87), -_c(U+1F788, "🞈", f09f9e88), -_c(U+1F789, "🞉", f09f9e89), -_c(U+1F78A, "🞊", f09f9e8a), -_c(U+1F78B, "🞋", f09f9e8b), -_c(U+1F78C, "🞌", f09f9e8c), -_c(U+1F78D, "🞍", f09f9e8d), -_c(U+1F78E, "🞎", f09f9e8e), -_c(U+1F78F, "🞏", f09f9e8f), -_c(U+1F790, "🞐", f09f9e90), -_c(U+1F791, "🞑", f09f9e91), -_c(U+1F792, "🞒", f09f9e92), -_c(U+1F793, "🞓", f09f9e93), -_c(U+1F794, "🞔", f09f9e94), -_c(U+1F795, "🞕", f09f9e95), -_c(U+1F796, "🞖", f09f9e96), -_c(U+1F797, "🞗", f09f9e97), -_c(U+1F798, "🞘", f09f9e98), -_c(U+1F799, "🞙", f09f9e99), -_c(U+1F79A, "🞚", f09f9e9a), -_c(U+1F79B, "🞛", f09f9e9b), -_c(U+1F79C, "🞜", f09f9e9c), -_c(U+1F79D, "🞝", f09f9e9d), -_c(U+1F79E, "🞞", f09f9e9e), -_c(U+1F79F, "🞟", f09f9e9f), -_c(U+1F7A0, "🞠", f09f9ea0), -_c(U+1F7A1, "🞡", f09f9ea1), -_c(U+1F7A2, "🞢", f09f9ea2), -_c(U+1F7A3, "🞣", f09f9ea3), -_c(U+1F7A4, "🞤", f09f9ea4), -_c(U+1F7A5, "🞥", f09f9ea5), -_c(U+1F7A6, "🞦", f09f9ea6), -_c(U+1F7A7, "🞧", f09f9ea7), -_c(U+1F7A8, "🞨", f09f9ea8), -_c(U+1F7A9, "🞩", f09f9ea9), -_c(U+1F7AA, "🞪", f09f9eaa), -_c(U+1F7AB, "🞫", f09f9eab), -_c(U+1F7AC, "🞬", f09f9eac), -_c(U+1F7AD, "🞭", f09f9ead), -_c(U+1F7AE, "🞮", f09f9eae), -_c(U+1F7AF, "🞯", f09f9eaf), -_c(U+1F7B0, "🞰", f09f9eb0), -_c(U+1F7B1, "🞱", f09f9eb1), -_c(U+1F7B2, "🞲", f09f9eb2), -_c(U+1F7B3, "🞳", f09f9eb3), -_c(U+1F7B4, "🞴", f09f9eb4), -_c(U+1F7B5, "🞵", f09f9eb5), -_c(U+1F7B6, "🞶", f09f9eb6), -_c(U+1F7B7, "🞷", f09f9eb7), -_c(U+1F7B8, "🞸", f09f9eb8), -_c(U+1F7B9, "🞹", f09f9eb9), -_c(U+1F7BA, "🞺", f09f9eba), -_c(U+1F7BB, "🞻", f09f9ebb), -_c(U+1F7BC, "🞼", f09f9ebc), -_c(U+1F7BD, "🞽", f09f9ebd), -_c(U+1F7BE, "🞾", f09f9ebe), -_c(U+1F7BF, "🞿", f09f9ebf), -_c(U+1F7C0, "🟀", f09f9f80), -_c(U+1F7C1, "🟁", f09f9f81), -_c(U+1F7C2, "🟂", f09f9f82), -_c(U+1F7C3, "🟃", f09f9f83), -_c(U+1F7C4, "🟄", f09f9f84), -_c(U+1F7C5, "🟅", f09f9f85), -_c(U+1F7C6, "🟆", f09f9f86), -_c(U+1F7C7, "🟇", f09f9f87), -_c(U+1F7C8, "🟈", f09f9f88), -_c(U+1F7C9, "🟉", f09f9f89), -_c(U+1F7CA, "🟊", f09f9f8a), -_c(U+1F7CB, "🟋", f09f9f8b), -_c(U+1F7CC, "🟌", f09f9f8c), -_c(U+1F7CD, "🟍", f09f9f8d), -_c(U+1F7CE, "🟎", f09f9f8e), -_c(U+1F7CF, "🟏", f09f9f8f), -_c(U+1F7D0, "🟐", f09f9f90), -_c(U+1F7D1, "🟑", f09f9f91), -_c(U+1F7D2, "🟒", f09f9f92), -_c(U+1F7D3, "🟓", f09f9f93), -_c(U+1F7D4, "🟔", f09f9f94), -_c(U+1F7D5, "🟕", f09f9f95), -_c(U+1F7D6, "🟖", f09f9f96), -_c(U+1F7D7, "🟗", f09f9f97), -_c(U+1F7D8, "🟘", f09f9f98), -_c(U+1F7D9, "🟙", f09f9f99), -_c(U+1F7DA, "🟚", f09f9f9a), -_c(U+1F7DB, "🟛", f09f9f9b), -_c(U+1F7DC, "🟜", f09f9f9c), -_c(U+1F7DD, "🟝", f09f9f9d), -_c(U+1F7DE, "🟞", f09f9f9e), -_c(U+1F7DF, "🟟", f09f9f9f), -_c(U+1F7E0, "🟠", f09f9fa0), -_c(U+1F7E1, "🟡", f09f9fa1), -_c(U+1F7E2, "🟢", f09f9fa2), -_c(U+1F7E3, "🟣", f09f9fa3), -_c(U+1F7E4, "🟤", f09f9fa4), -_c(U+1F7E5, "🟥", f09f9fa5), -_c(U+1F7E6, "🟦", f09f9fa6), -_c(U+1F7E7, "🟧", f09f9fa7), -_c(U+1F7E8, "🟨", f09f9fa8), -_c(U+1F7E9, "🟩", f09f9fa9), -_c(U+1F7EA, "🟪", f09f9faa), -_c(U+1F7EB, "🟫", f09f9fab), -_c(U+1F7EC, "🟬", f09f9fac), -_c(U+1F7ED, "🟭", f09f9fad), -_c(U+1F7EE, "🟮", f09f9fae), -_c(U+1F7EF, "🟯", f09f9faf), -_c(U+1F7F0, "🟰", f09f9fb0), -_c(U+1F7F1, "🟱", f09f9fb1), -_c(U+1F7F2, "🟲", f09f9fb2), -_c(U+1F7F3, "🟳", f09f9fb3), -_c(U+1F7F4, "🟴", f09f9fb4), -_c(U+1F7F5, "🟵", f09f9fb5), -_c(U+1F7F6, "🟶", f09f9fb6), -_c(U+1F7F7, "🟷", f09f9fb7), -_c(U+1F7F8, "🟸", f09f9fb8), -_c(U+1F7F9, "🟹", f09f9fb9), -_c(U+1F7FA, "🟺", f09f9fba), -_c(U+1F7FB, "🟻", f09f9fbb), -_c(U+1F7FC, "🟼", f09f9fbc), -_c(U+1F7FD, "🟽", f09f9fbd), -_c(U+1F7FE, "🟾", f09f9fbe), -_c(U+1F7FF, "🟿", f09f9fbf), -_c(U+1F800, "🠀", f09fa080), -_c(U+1F801, "🠁", f09fa081), -_c(U+1F802, "🠂", f09fa082), -_c(U+1F803, "🠃", f09fa083), -_c(U+1F804, "🠄", f09fa084), -_c(U+1F805, "🠅", f09fa085), -_c(U+1F806, "🠆", f09fa086), -_c(U+1F807, "🠇", f09fa087), -_c(U+1F808, "🠈", f09fa088), -_c(U+1F809, "🠉", f09fa089), -_c(U+1F80A, "🠊", f09fa08a), -_c(U+1F80B, "🠋", f09fa08b), -_c(U+1F80C, "🠌", f09fa08c), -_c(U+1F80D, "🠍", f09fa08d), -_c(U+1F80E, "🠎", f09fa08e), -_c(U+1F80F, "🠏", f09fa08f), -_c(U+1F810, "🠐", f09fa090), -_c(U+1F811, "🠑", f09fa091), -_c(U+1F812, "🠒", f09fa092), -_c(U+1F813, "🠓", f09fa093), -_c(U+1F814, "🠔", f09fa094), -_c(U+1F815, "🠕", f09fa095), -_c(U+1F816, "🠖", f09fa096), -_c(U+1F817, "🠗", f09fa097), -_c(U+1F818, "🠘", f09fa098), -_c(U+1F819, "🠙", f09fa099), -_c(U+1F81A, "🠚", f09fa09a), -_c(U+1F81B, "🠛", f09fa09b), -_c(U+1F81C, "🠜", f09fa09c), -_c(U+1F81D, "🠝", f09fa09d), -_c(U+1F81E, "🠞", f09fa09e), -_c(U+1F81F, "🠟", f09fa09f), -_c(U+1F820, "🠠", f09fa0a0), -_c(U+1F821, "🠡", f09fa0a1), -_c(U+1F822, "🠢", f09fa0a2), -_c(U+1F823, "🠣", f09fa0a3), -_c(U+1F824, "🠤", f09fa0a4), -_c(U+1F825, "🠥", f09fa0a5), -_c(U+1F826, "🠦", f09fa0a6), -_c(U+1F827, "🠧", f09fa0a7), -_c(U+1F828, "🠨", f09fa0a8), -_c(U+1F829, "🠩", f09fa0a9), -_c(U+1F82A, "🠪", f09fa0aa), -_c(U+1F82B, "🠫", f09fa0ab), -_c(U+1F82C, "🠬", f09fa0ac), -_c(U+1F82D, "🠭", f09fa0ad), -_c(U+1F82E, "🠮", f09fa0ae), -_c(U+1F82F, "🠯", f09fa0af), -_c(U+1F830, "🠰", f09fa0b0), -_c(U+1F831, "🠱", f09fa0b1), -_c(U+1F832, "🠲", f09fa0b2), -_c(U+1F833, "🠳", f09fa0b3), -_c(U+1F834, "🠴", f09fa0b4), -_c(U+1F835, "🠵", f09fa0b5), -_c(U+1F836, "🠶", f09fa0b6), -_c(U+1F837, "🠷", f09fa0b7), -_c(U+1F838, "🠸", f09fa0b8), -_c(U+1F839, "🠹", f09fa0b9), -_c(U+1F83A, "🠺", f09fa0ba), -_c(U+1F83B, "🠻", f09fa0bb), -_c(U+1F83C, "🠼", f09fa0bc), -_c(U+1F83D, "🠽", f09fa0bd), -_c(U+1F83E, "🠾", f09fa0be), -_c(U+1F83F, "🠿", f09fa0bf), -_c(U+1F840, "🡀", f09fa180), -_c(U+1F841, "🡁", f09fa181), -_c(U+1F842, "🡂", f09fa182), -_c(U+1F843, "🡃", f09fa183), -_c(U+1F844, "🡄", f09fa184), -_c(U+1F845, "🡅", f09fa185), -_c(U+1F846, "🡆", f09fa186), -_c(U+1F847, "🡇", f09fa187), -_c(U+1F848, "🡈", f09fa188), -_c(U+1F849, "🡉", f09fa189), -_c(U+1F84A, "🡊", f09fa18a), -_c(U+1F84B, "🡋", f09fa18b), -_c(U+1F84C, "🡌", f09fa18c), -_c(U+1F84D, "🡍", f09fa18d), -_c(U+1F84E, "🡎", f09fa18e), -_c(U+1F84F, "🡏", f09fa18f), -_c(U+1F850, "🡐", f09fa190), -_c(U+1F851, "🡑", f09fa191), -_c(U+1F852, "🡒", f09fa192), -_c(U+1F853, "🡓", f09fa193), -_c(U+1F854, "🡔", f09fa194), -_c(U+1F855, "🡕", f09fa195), -_c(U+1F856, "🡖", f09fa196), -_c(U+1F857, "🡗", f09fa197), -_c(U+1F858, "🡘", f09fa198), -_c(U+1F859, "🡙", f09fa199), -_c(U+1F85A, "🡚", f09fa19a), -_c(U+1F85B, "🡛", f09fa19b), -_c(U+1F85C, "🡜", f09fa19c), -_c(U+1F85D, "🡝", f09fa19d), -_c(U+1F85E, "🡞", f09fa19e), -_c(U+1F85F, "🡟", f09fa19f), -_c(U+1F860, "🡠", f09fa1a0), -_c(U+1F861, "🡡", f09fa1a1), -_c(U+1F862, "🡢", f09fa1a2), -_c(U+1F863, "🡣", f09fa1a3), -_c(U+1F864, "🡤", f09fa1a4), -_c(U+1F865, "🡥", f09fa1a5), -_c(U+1F866, "🡦", f09fa1a6), -_c(U+1F867, "🡧", f09fa1a7), -_c(U+1F868, "🡨", f09fa1a8), -_c(U+1F869, "🡩", f09fa1a9), -_c(U+1F86A, "🡪", f09fa1aa), -_c(U+1F86B, "🡫", f09fa1ab), -_c(U+1F86C, "🡬", f09fa1ac), -_c(U+1F86D, "🡭", f09fa1ad), -_c(U+1F86E, "🡮", f09fa1ae), -_c(U+1F86F, "🡯", f09fa1af), -_c(U+1F870, "🡰", f09fa1b0), -_c(U+1F871, "🡱", f09fa1b1), -_c(U+1F872, "🡲", f09fa1b2), -_c(U+1F873, "🡳", f09fa1b3), -_c(U+1F874, "🡴", f09fa1b4), -_c(U+1F875, "🡵", f09fa1b5), -_c(U+1F876, "🡶", f09fa1b6), -_c(U+1F877, "🡷", f09fa1b7), -_c(U+1F878, "🡸", f09fa1b8), -_c(U+1F879, "🡹", f09fa1b9), -_c(U+1F87A, "🡺", f09fa1ba), -_c(U+1F87B, "🡻", f09fa1bb), -_c(U+1F87C, "🡼", f09fa1bc), -_c(U+1F87D, "🡽", f09fa1bd), -_c(U+1F87E, "🡾", f09fa1be), -_c(U+1F87F, "🡿", f09fa1bf), -_c(U+1F880, "🢀", f09fa280), -_c(U+1F881, "🢁", f09fa281), -_c(U+1F882, "🢂", f09fa282), -_c(U+1F883, "🢃", f09fa283), -_c(U+1F884, "🢄", f09fa284), -_c(U+1F885, "🢅", f09fa285), -_c(U+1F886, "🢆", f09fa286), -_c(U+1F887, "🢇", f09fa287), -_c(U+1F888, "🢈", f09fa288), -_c(U+1F889, "🢉", f09fa289), -_c(U+1F88A, "🢊", f09fa28a), -_c(U+1F88B, "🢋", f09fa28b), -_c(U+1F88C, "🢌", f09fa28c), -_c(U+1F88D, "🢍", f09fa28d), -_c(U+1F88E, "🢎", f09fa28e), -_c(U+1F88F, "🢏", f09fa28f), -_c(U+1F890, "🢐", f09fa290), -_c(U+1F891, "🢑", f09fa291), -_c(U+1F892, "🢒", f09fa292), -_c(U+1F893, "🢓", f09fa293), -_c(U+1F894, "🢔", f09fa294), -_c(U+1F895, "🢕", f09fa295), -_c(U+1F896, "🢖", f09fa296), -_c(U+1F897, "🢗", f09fa297), -_c(U+1F898, "🢘", f09fa298), -_c(U+1F899, "🢙", f09fa299), -_c(U+1F89A, "🢚", f09fa29a), -_c(U+1F89B, "🢛", f09fa29b), -_c(U+1F89C, "🢜", f09fa29c), -_c(U+1F89D, "🢝", f09fa29d), -_c(U+1F89E, "🢞", f09fa29e), -_c(U+1F89F, "🢟", f09fa29f), -_c(U+1F8A0, "🢠", f09fa2a0), -_c(U+1F8A1, "🢡", f09fa2a1), -_c(U+1F8A2, "🢢", f09fa2a2), -_c(U+1F8A3, "🢣", f09fa2a3), -_c(U+1F8A4, "🢤", f09fa2a4), -_c(U+1F8A5, "🢥", f09fa2a5), -_c(U+1F8A6, "🢦", f09fa2a6), -_c(U+1F8A7, "🢧", f09fa2a7), -_c(U+1F8A8, "🢨", f09fa2a8), -_c(U+1F8A9, "🢩", f09fa2a9), -_c(U+1F8AA, "🢪", f09fa2aa), -_c(U+1F8AB, "🢫", f09fa2ab), -_c(U+1F8AC, "🢬", f09fa2ac), -_c(U+1F8AD, "🢭", f09fa2ad), -_c(U+1F8AE, "🢮", f09fa2ae), -_c(U+1F8AF, "🢯", f09fa2af), -_c(U+1F8B0, "🢰", f09fa2b0), -_c(U+1F8B1, "🢱", f09fa2b1), -_c(U+1F8B2, "🢲", f09fa2b2), -_c(U+1F8B3, "🢳", f09fa2b3), -_c(U+1F8B4, "🢴", f09fa2b4), -_c(U+1F8B5, "🢵", f09fa2b5), -_c(U+1F8B6, "🢶", f09fa2b6), -_c(U+1F8B7, "🢷", f09fa2b7), -_c(U+1F8B8, "🢸", f09fa2b8), -_c(U+1F8B9, "🢹", f09fa2b9), -_c(U+1F8BA, "🢺", f09fa2ba), -_c(U+1F8BB, "🢻", f09fa2bb), -_c(U+1F8BC, "🢼", f09fa2bc), -_c(U+1F8BD, "🢽", f09fa2bd), -_c(U+1F8BE, "🢾", f09fa2be), -_c(U+1F8BF, "🢿", f09fa2bf), -_c(U+1F8C0, "🣀", f09fa380), -_c(U+1F8C1, "🣁", f09fa381), -_c(U+1F8C2, "🣂", f09fa382), -_c(U+1F8C3, "🣃", f09fa383), -_c(U+1F8C4, "🣄", f09fa384), -_c(U+1F8C5, "🣅", f09fa385), -_c(U+1F8C6, "🣆", f09fa386), -_c(U+1F8C7, "🣇", f09fa387), -_c(U+1F8C8, "🣈", f09fa388), -_c(U+1F8C9, "🣉", f09fa389), -_c(U+1F8CA, "🣊", f09fa38a), -_c(U+1F8CB, "🣋", f09fa38b), -_c(U+1F8CC, "🣌", f09fa38c), -_c(U+1F8CD, "🣍", f09fa38d), -_c(U+1F8CE, "🣎", f09fa38e), -_c(U+1F8CF, "🣏", f09fa38f), -_c(U+1F8D0, "🣐", f09fa390), -_c(U+1F8D1, "🣑", f09fa391), -_c(U+1F8D2, "🣒", f09fa392), -_c(U+1F8D3, "🣓", f09fa393), -_c(U+1F8D4, "🣔", f09fa394), -_c(U+1F8D5, "🣕", f09fa395), -_c(U+1F8D6, "🣖", f09fa396), -_c(U+1F8D7, "🣗", f09fa397), -_c(U+1F8D8, "🣘", f09fa398), -_c(U+1F8D9, "🣙", f09fa399), -_c(U+1F8DA, "🣚", f09fa39a), -_c(U+1F8DB, "🣛", f09fa39b), -_c(U+1F8DC, "🣜", f09fa39c), -_c(U+1F8DD, "🣝", f09fa39d), -_c(U+1F8DE, "🣞", f09fa39e), -_c(U+1F8DF, "🣟", f09fa39f), -_c(U+1F8E0, "🣠", f09fa3a0), -_c(U+1F8E1, "🣡", f09fa3a1), -_c(U+1F8E2, "🣢", f09fa3a2), -_c(U+1F8E3, "🣣", f09fa3a3), -_c(U+1F8E4, "🣤", f09fa3a4), -_c(U+1F8E5, "🣥", f09fa3a5), -_c(U+1F8E6, "🣦", f09fa3a6), -_c(U+1F8E7, "🣧", f09fa3a7), -_c(U+1F8E8, "🣨", f09fa3a8), -_c(U+1F8E9, "🣩", f09fa3a9), -_c(U+1F8EA, "🣪", f09fa3aa), -_c(U+1F8EB, "🣫", f09fa3ab), -_c(U+1F8EC, "🣬", f09fa3ac), -_c(U+1F8ED, "🣭", f09fa3ad), -_c(U+1F8EE, "🣮", f09fa3ae), -_c(U+1F8EF, "🣯", f09fa3af), -_c(U+1F8F0, "🣰", f09fa3b0), -_c(U+1F8F1, "🣱", f09fa3b1), -_c(U+1F8F2, "🣲", f09fa3b2), -_c(U+1F8F3, "🣳", f09fa3b3), -_c(U+1F8F4, "🣴", f09fa3b4), -_c(U+1F8F5, "🣵", f09fa3b5), -_c(U+1F8F6, "🣶", f09fa3b6), -_c(U+1F8F7, "🣷", f09fa3b7), -_c(U+1F8F8, "🣸", f09fa3b8), -_c(U+1F8F9, "🣹", f09fa3b9), -_c(U+1F8FA, "🣺", f09fa3ba), -_c(U+1F8FB, "🣻", f09fa3bb), -_c(U+1F8FC, "🣼", f09fa3bc), -_c(U+1F8FD, "🣽", f09fa3bd), -_c(U+1F8FE, "🣾", f09fa3be), -_c(U+1F8FF, "🣿", f09fa3bf), -_c(U+1F900, "🤀", f09fa480), -_c(U+1F901, "🤁", f09fa481), -_c(U+1F902, "🤂", f09fa482), -_c(U+1F903, "🤃", f09fa483), -_c(U+1F904, "🤄", f09fa484), -_c(U+1F905, "🤅", f09fa485), -_c(U+1F906, "🤆", f09fa486), -_c(U+1F907, "🤇", f09fa487), -_c(U+1F908, "🤈", f09fa488), -_c(U+1F909, "🤉", f09fa489), -_c(U+1F90A, "🤊", f09fa48a), -_c(U+1F90B, "🤋", f09fa48b), -_c(U+1F90C, "🤌", f09fa48c), -_c(U+1F90D, "🤍", f09fa48d), -_c(U+1F90E, "🤎", f09fa48e), -_c(U+1F90F, "🤏", f09fa48f), -_c(U+1F910, "🤐", f09fa490), -_c(U+1F911, "🤑", f09fa491), -_c(U+1F912, "🤒", f09fa492), -_c(U+1F913, "🤓", f09fa493), -_c(U+1F914, "🤔", f09fa494), -_c(U+1F915, "🤕", f09fa495), -_c(U+1F916, "🤖", f09fa496), -_c(U+1F917, "🤗", f09fa497), -_c(U+1F918, "🤘", f09fa498), -_c(U+1F919, "🤙", f09fa499), -_c(U+1F91A, "🤚", f09fa49a), -_c(U+1F91B, "🤛", f09fa49b), -_c(U+1F91C, "🤜", f09fa49c), -_c(U+1F91D, "🤝", f09fa49d), -_c(U+1F91E, "🤞", f09fa49e), -_c(U+1F91F, "🤟", f09fa49f), -_c(U+1F920, "🤠", f09fa4a0), -_c(U+1F921, "🤡", f09fa4a1), -_c(U+1F922, "🤢", f09fa4a2), -_c(U+1F923, "🤣", f09fa4a3), -_c(U+1F924, "🤤", f09fa4a4), -_c(U+1F925, "🤥", f09fa4a5), -_c(U+1F926, "🤦", f09fa4a6), -_c(U+1F927, "🤧", f09fa4a7), -_c(U+1F928, "🤨", f09fa4a8), -_c(U+1F929, "🤩", f09fa4a9), -_c(U+1F92A, "🤪", f09fa4aa), -_c(U+1F92B, "🤫", f09fa4ab), -_c(U+1F92C, "🤬", f09fa4ac), -_c(U+1F92D, "🤭", f09fa4ad), -_c(U+1F92E, "🤮", f09fa4ae), -_c(U+1F92F, "🤯", f09fa4af), -_c(U+1F930, "🤰", f09fa4b0), -_c(U+1F931, "🤱", f09fa4b1), -_c(U+1F932, "🤲", f09fa4b2), -_c(U+1F933, "🤳", f09fa4b3), -_c(U+1F934, "🤴", f09fa4b4), -_c(U+1F935, "🤵", f09fa4b5), -_c(U+1F936, "🤶", f09fa4b6), -_c(U+1F937, "🤷", f09fa4b7), -_c(U+1F938, "🤸", f09fa4b8), -_c(U+1F939, "🤹", f09fa4b9), -_c(U+1F93A, "🤺", f09fa4ba), -_c(U+1F93B, "🤻", f09fa4bb), -_c(U+1F93C, "🤼", f09fa4bc), -_c(U+1F93D, "🤽", f09fa4bd), -_c(U+1F93E, "🤾", f09fa4be), -_c(U+1F93F, "🤿", f09fa4bf), -_c(U+1F940, "🥀", f09fa580), -_c(U+1F941, "🥁", f09fa581), -_c(U+1F942, "🥂", f09fa582), -_c(U+1F943, "🥃", f09fa583), -_c(U+1F944, "🥄", f09fa584), -_c(U+1F945, "🥅", f09fa585), -_c(U+1F946, "🥆", f09fa586), -_c(U+1F947, "🥇", f09fa587), -_c(U+1F948, "🥈", f09fa588), -_c(U+1F949, "🥉", f09fa589), -_c(U+1F94A, "🥊", f09fa58a), -_c(U+1F94B, "🥋", f09fa58b), -_c(U+1F94C, "🥌", f09fa58c), -_c(U+1F94D, "🥍", f09fa58d), -_c(U+1F94E, "🥎", f09fa58e), -_c(U+1F94F, "🥏", f09fa58f), -_c(U+1F950, "🥐", f09fa590), -_c(U+1F951, "🥑", f09fa591), -_c(U+1F952, "🥒", f09fa592), -_c(U+1F953, "🥓", f09fa593), -_c(U+1F954, "🥔", f09fa594), -_c(U+1F955, "🥕", f09fa595), -_c(U+1F956, "🥖", f09fa596), -_c(U+1F957, "🥗", f09fa597), -_c(U+1F958, "🥘", f09fa598), -_c(U+1F959, "🥙", f09fa599), -_c(U+1F95A, "🥚", f09fa59a), -_c(U+1F95B, "🥛", f09fa59b), -_c(U+1F95C, "🥜", f09fa59c), -_c(U+1F95D, "🥝", f09fa59d), -_c(U+1F95E, "🥞", f09fa59e), -_c(U+1F95F, "🥟", f09fa59f), -_c(U+1F960, "🥠", f09fa5a0), -_c(U+1F961, "🥡", f09fa5a1), -_c(U+1F962, "🥢", f09fa5a2), -_c(U+1F963, "🥣", f09fa5a3), -_c(U+1F964, "🥤", f09fa5a4), -_c(U+1F965, "🥥", f09fa5a5), -_c(U+1F966, "🥦", f09fa5a6), -_c(U+1F967, "🥧", f09fa5a7), -_c(U+1F968, "🥨", f09fa5a8), -_c(U+1F969, "🥩", f09fa5a9), -_c(U+1F96A, "🥪", f09fa5aa), -_c(U+1F96B, "🥫", f09fa5ab), -_c(U+1F96C, "🥬", f09fa5ac), -_c(U+1F96D, "🥭", f09fa5ad), -_c(U+1F96E, "🥮", f09fa5ae), -_c(U+1F96F, "🥯", f09fa5af), -_c(U+1F970, "🥰", f09fa5b0), -_c(U+1F971, "🥱", f09fa5b1), -_c(U+1F972, "🥲", f09fa5b2), -_c(U+1F973, "🥳", f09fa5b3), -_c(U+1F974, "🥴", f09fa5b4), -_c(U+1F975, "🥵", f09fa5b5), -_c(U+1F976, "🥶", f09fa5b6), -_c(U+1F977, "🥷", f09fa5b7), -_c(U+1F978, "🥸", f09fa5b8), -_c(U+1F979, "🥹", f09fa5b9), -_c(U+1F97A, "🥺", f09fa5ba), -_c(U+1F97B, "🥻", f09fa5bb), -_c(U+1F97C, "🥼", f09fa5bc), -_c(U+1F97D, "🥽", f09fa5bd), -_c(U+1F97E, "🥾", f09fa5be), -_c(U+1F97F, "🥿", f09fa5bf), -_c(U+1F980, "🦀", f09fa680), -_c(U+1F981, "🦁", f09fa681), -_c(U+1F982, "🦂", f09fa682), -_c(U+1F983, "🦃", f09fa683), -_c(U+1F984, "🦄", f09fa684), -_c(U+1F985, "🦅", f09fa685), -_c(U+1F986, "🦆", f09fa686), -_c(U+1F987, "🦇", f09fa687), -_c(U+1F988, "🦈", f09fa688), -_c(U+1F989, "🦉", f09fa689), -_c(U+1F98A, "🦊", f09fa68a), -_c(U+1F98B, "🦋", f09fa68b), -_c(U+1F98C, "🦌", f09fa68c), -_c(U+1F98D, "🦍", f09fa68d), -_c(U+1F98E, "🦎", f09fa68e), -_c(U+1F98F, "🦏", f09fa68f), -_c(U+1F990, "🦐", f09fa690), -_c(U+1F991, "🦑", f09fa691), -_c(U+1F992, "🦒", f09fa692), -_c(U+1F993, "🦓", f09fa693), -_c(U+1F994, "🦔", f09fa694), -_c(U+1F995, "🦕", f09fa695), -_c(U+1F996, "🦖", f09fa696), -_c(U+1F997, "🦗", f09fa697), -_c(U+1F998, "🦘", f09fa698), -_c(U+1F999, "🦙", f09fa699), -_c(U+1F99A, "🦚", f09fa69a), -_c(U+1F99B, "🦛", f09fa69b), -_c(U+1F99C, "🦜", f09fa69c), -_c(U+1F99D, "🦝", f09fa69d), -_c(U+1F99E, "🦞", f09fa69e), -_c(U+1F99F, "🦟", f09fa69f), -_c(U+1F9A0, "🦠", f09fa6a0), -_c(U+1F9A1, "🦡", f09fa6a1), -_c(U+1F9A2, "🦢", f09fa6a2), -_c(U+1F9A3, "🦣", f09fa6a3), -_c(U+1F9A4, "🦤", f09fa6a4), -_c(U+1F9A5, "🦥", f09fa6a5), -_c(U+1F9A6, "🦦", f09fa6a6), -_c(U+1F9A7, "🦧", f09fa6a7), -_c(U+1F9A8, "🦨", f09fa6a8), -_c(U+1F9A9, "🦩", f09fa6a9), -_c(U+1F9AA, "🦪", f09fa6aa), -_c(U+1F9AB, "🦫", f09fa6ab), -_c(U+1F9AC, "🦬", f09fa6ac), -_c(U+1F9AD, "🦭", f09fa6ad), -_c(U+1F9AE, "🦮", f09fa6ae), -_c(U+1F9AF, "🦯", f09fa6af), -_c(U+1F9B0, "🦰", f09fa6b0), -_c(U+1F9B1, "🦱", f09fa6b1), -_c(U+1F9B2, "🦲", f09fa6b2), -_c(U+1F9B3, "🦳", f09fa6b3), -_c(U+1F9B4, "🦴", f09fa6b4), -_c(U+1F9B5, "🦵", f09fa6b5), -_c(U+1F9B6, "🦶", f09fa6b6), -_c(U+1F9B7, "🦷", f09fa6b7), -_c(U+1F9B8, "🦸", f09fa6b8), -_c(U+1F9B9, "🦹", f09fa6b9), -_c(U+1F9BA, "🦺", f09fa6ba), -_c(U+1F9BB, "🦻", f09fa6bb), -_c(U+1F9BC, "🦼", f09fa6bc), -_c(U+1F9BD, "🦽", f09fa6bd), -_c(U+1F9BE, "🦾", f09fa6be), -_c(U+1F9BF, "🦿", f09fa6bf), -_c(U+1F9C0, "🧀", f09fa780), -_c(U+1F9C1, "🧁", f09fa781), -_c(U+1F9C2, "🧂", f09fa782), -_c(U+1F9C3, "🧃", f09fa783), -_c(U+1F9C4, "🧄", f09fa784), -_c(U+1F9C5, "🧅", f09fa785), -_c(U+1F9C6, "🧆", f09fa786), -_c(U+1F9C7, "🧇", f09fa787), -_c(U+1F9C8, "🧈", f09fa788), -_c(U+1F9C9, "🧉", f09fa789), -_c(U+1F9CA, "🧊", f09fa78a), -_c(U+1F9CB, "🧋", f09fa78b), -_c(U+1F9CC, "🧌", f09fa78c), -_c(U+1F9CD, "🧍", f09fa78d), -_c(U+1F9CE, "🧎", f09fa78e), -_c(U+1F9CF, "🧏", f09fa78f), -_c(U+1F9D0, "🧐", f09fa790), -_c(U+1F9D1, "🧑", f09fa791), -_c(U+1F9D2, "🧒", f09fa792), -_c(U+1F9D3, "🧓", f09fa793), -_c(U+1F9D4, "🧔", f09fa794), -_c(U+1F9D5, "🧕", f09fa795), -_c(U+1F9D6, "🧖", f09fa796), -_c(U+1F9D7, "🧗", f09fa797), -_c(U+1F9D8, "🧘", f09fa798), -_c(U+1F9D9, "🧙", f09fa799), -_c(U+1F9DA, "🧚", f09fa79a), -_c(U+1F9DB, "🧛", f09fa79b), -_c(U+1F9DC, "🧜", f09fa79c), -_c(U+1F9DD, "🧝", f09fa79d), -_c(U+1F9DE, "🧞", f09fa79e), -_c(U+1F9DF, "🧟", f09fa79f), -_c(U+1F9E0, "🧠", f09fa7a0), -_c(U+1F9E1, "🧡", f09fa7a1), -_c(U+1F9E2, "🧢", f09fa7a2), -_c(U+1F9E3, "🧣", f09fa7a3), -_c(U+1F9E4, "🧤", f09fa7a4), -_c(U+1F9E5, "🧥", f09fa7a5), -_c(U+1F9E6, "🧦", f09fa7a6), -_c(U+1F9E7, "🧧", f09fa7a7), -_c(U+1F9E8, "🧨", f09fa7a8), -_c(U+1F9E9, "🧩", f09fa7a9), -_c(U+1F9EA, "🧪", f09fa7aa), -_c(U+1F9EB, "🧫", f09fa7ab), -_c(U+1F9EC, "🧬", f09fa7ac), -_c(U+1F9ED, "🧭", f09fa7ad), -_c(U+1F9EE, "🧮", f09fa7ae), -_c(U+1F9EF, "🧯", f09fa7af), -_c(U+1F9F0, "🧰", f09fa7b0), -_c(U+1F9F1, "🧱", f09fa7b1), -_c(U+1F9F2, "🧲", f09fa7b2), -_c(U+1F9F3, "🧳", f09fa7b3), -_c(U+1F9F4, "🧴", f09fa7b4), -_c(U+1F9F5, "🧵", f09fa7b5), -_c(U+1F9F6, "🧶", f09fa7b6), -_c(U+1F9F7, "🧷", f09fa7b7), -_c(U+1F9F8, "🧸", f09fa7b8), -_c(U+1F9F9, "🧹", f09fa7b9), -_c(U+1F9FA, "🧺", f09fa7ba), -_c(U+1F9FB, "🧻", f09fa7bb), -_c(U+1F9FC, "🧼", f09fa7bc), -_c(U+1F9FD, "🧽", f09fa7bd), -_c(U+1F9FE, "🧾", f09fa7be), -_c(U+1F9FF, "🧿", f09fa7bf), -_c(U+100000, "􀀀", f4808080), -_c(U+100001, "􀀁", f4808081), -_c(U+100002, "􀀂", f4808082), -_c(U+100003, "􀀃", f4808083), -_c(U+100004, "􀀄", f4808084), -_c(U+100005, "􀀅", f4808085), -_c(U+100006, "􀀆", f4808086), -_c(U+100007, "􀀇", f4808087), -_c(U+100008, "􀀈", f4808088), -_c(U+100009, "􀀉", f4808089), -_c(U+10000A, "􀀊", f480808a), -_c(U+10000B, "􀀋", f480808b), -_c(U+10000C, "􀀌", f480808c), -_c(U+10000D, "􀀍", f480808d), -_c(U+10000E, "􀀎", f480808e), -_c(U+10000F, "􀀏", f480808f), -_c(U+100010, "􀀐", f4808090), -_c(U+100011, "􀀑", f4808091), -_c(U+100012, "􀀒", f4808092), -_c(U+100013, "􀀓", f4808093), -_c(U+100014, "􀀔", f4808094), -_c(U+100015, "􀀕", f4808095), -_c(U+100016, "􀀖", f4808096), -_c(U+100017, "􀀗", f4808097), -_c(U+100018, "􀀘", f4808098), -_c(U+100019, "􀀙", f4808099), -_c(U+10001A, "􀀚", f480809a), -_c(U+10001B, "􀀛", f480809b), -_c(U+10001C, "􀀜", f480809c), -_c(U+10001D, "􀀝", f480809d), -_c(U+10001E, "􀀞", f480809e), -_c(U+10001F, "􀀟", f480809f), -_c(U+100020, "􀀠", f48080a0), -_c(U+100021, "􀀡", f48080a1), -_c(U+100022, "􀀢", f48080a2), -_c(U+100023, "􀀣", f48080a3), -_c(U+100024, "􀀤", f48080a4), -_c(U+100025, "􀀥", f48080a5), -_c(U+100026, "􀀦", f48080a6), -_c(U+100027, "􀀧", f48080a7), -_c(U+100028, "􀀨", f48080a8), -_c(U+100029, "􀀩", f48080a9), -_c(U+10002A, "􀀪", f48080aa), -_c(U+10002B, "􀀫", f48080ab), -_c(U+10002C, "􀀬", f48080ac), -_c(U+10002D, "􀀭", f48080ad), -_c(U+10002E, "􀀮", f48080ae), -_c(U+10002F, "􀀯", f48080af), -_c(U+100030, "􀀰", f48080b0), -_c(U+100031, "􀀱", f48080b1), -_c(U+100032, "􀀲", f48080b2), -_c(U+100033, "􀀳", f48080b3), -_c(U+100034, "􀀴", f48080b4), -_c(U+100035, "􀀵", f48080b5), -_c(U+100036, "􀀶", f48080b6), -_c(U+100037, "􀀷", f48080b7), -_c(U+100038, "􀀸", f48080b8), -_c(U+100039, "􀀹", f48080b9), -_c(U+10003A, "􀀺", f48080ba), -_c(U+10003B, "􀀻", f48080bb), -_c(U+10003C, "􀀼", f48080bc), -_c(U+10003D, "􀀽", f48080bd), -_c(U+10003E, "􀀾", f48080be), -_c(U+10003F, "􀀿", f48080bf), -_c(U+100040, "􀁀", f4808180), -_c(U+100041, "􀁁", f4808181), -_c(U+100042, "􀁂", f4808182), -_c(U+100043, "􀁃", f4808183), -_c(U+100044, "􀁄", f4808184), -_c(U+100045, "􀁅", f4808185), -_c(U+100046, "􀁆", f4808186), -_c(U+100047, "􀁇", f4808187), -_c(U+100048, "􀁈", f4808188), -_c(U+100049, "􀁉", f4808189), -_c(U+10004A, "􀁊", f480818a), -_c(U+10004B, "􀁋", f480818b), -_c(U+10004C, "􀁌", f480818c), -_c(U+10004D, "􀁍", f480818d), -_c(U+10004E, "􀁎", f480818e), -_c(U+10004F, "􀁏", f480818f), -_c(U+100050, "􀁐", f4808190), -_c(U+100051, "􀁑", f4808191), -_c(U+100052, "􀁒", f4808192), -_c(U+100053, "􀁓", f4808193), -_c(U+100054, "􀁔", f4808194), -_c(U+100055, "􀁕", f4808195), -_c(U+100056, "􀁖", f4808196), -_c(U+100057, "􀁗", f4808197), -_c(U+100058, "􀁘", f4808198), -_c(U+100059, "􀁙", f4808199), -_c(U+10005A, "􀁚", f480819a), -_c(U+10005B, "􀁛", f480819b), -_c(U+10005C, "􀁜", f480819c), -_c(U+10005D, "􀁝", f480819d), -_c(U+10005E, "􀁞", f480819e), -_c(U+10005F, "􀁟", f480819f), -_c(U+100060, "􀁠", f48081a0), -_c(U+100061, "􀁡", f48081a1), -_c(U+100062, "􀁢", f48081a2), -_c(U+100063, "􀁣", f48081a3), -_c(U+100064, "􀁤", f48081a4), -_c(U+100065, "􀁥", f48081a5), -_c(U+100066, "􀁦", f48081a6), -_c(U+100067, "􀁧", f48081a7), -_c(U+100068, "􀁨", f48081a8), -_c(U+100069, "􀁩", f48081a9), -_c(U+10006A, "􀁪", f48081aa), -_c(U+10006B, "􀁫", f48081ab), -_c(U+10006C, "􀁬", f48081ac), -_c(U+10006D, "􀁭", f48081ad), -_c(U+10006E, "􀁮", f48081ae), -_c(U+10006F, "􀁯", f48081af), -_c(U+100070, "􀁰", f48081b0), -_c(U+100071, "􀁱", f48081b1), -_c(U+100072, "􀁲", f48081b2), -_c(U+100073, "􀁳", f48081b3), -_c(U+100074, "􀁴", f48081b4), -_c(U+100075, "􀁵", f48081b5), -_c(U+100076, "􀁶", f48081b6), -_c(U+100077, "􀁷", f48081b7), -_c(U+100078, "􀁸", f48081b8), -_c(U+100079, "􀁹", f48081b9), -_c(U+10007A, "􀁺", f48081ba), -_c(U+10007B, "􀁻", f48081bb), -_c(U+10007C, "􀁼", f48081bc), -_c(U+10007D, "􀁽", f48081bd), -_c(U+10007E, "􀁾", f48081be), -_c(U+10007F, "􀁿", f48081bf), -_c(U+100080, "􀂀", f4808280), -_c(U+100081, "􀂁", f4808281), -_c(U+100082, "􀂂", f4808282), -_c(U+100083, "􀂃", f4808283), -_c(U+100084, "􀂄", f4808284), -_c(U+100085, "􀂅", f4808285), -_c(U+100086, "􀂆", f4808286), -_c(U+100087, "􀂇", f4808287), -_c(U+100088, "􀂈", f4808288), -_c(U+100089, "􀂉", f4808289), -_c(U+10008A, "􀂊", f480828a), -_c(U+10008B, "􀂋", f480828b), -_c(U+10008C, "􀂌", f480828c), -_c(U+10008D, "􀂍", f480828d), -_c(U+10008E, "􀂎", f480828e), -_c(U+10008F, "􀂏", f480828f), -_c(U+100090, "􀂐", f4808290), -_c(U+100091, "􀂑", f4808291), -_c(U+100092, "􀂒", f4808292), -_c(U+100093, "􀂓", f4808293), -_c(U+100094, "􀂔", f4808294), -_c(U+100095, "􀂕", f4808295), -_c(U+100096, "􀂖", f4808296), -_c(U+100097, "􀂗", f4808297), -_c(U+100098, "􀂘", f4808298), -_c(U+100099, "􀂙", f4808299), -_c(U+10009A, "􀂚", f480829a), -_c(U+10009B, "􀂛", f480829b), -_c(U+10009C, "􀂜", f480829c), -_c(U+10009D, "􀂝", f480829d), -_c(U+10009E, "􀂞", f480829e), -_c(U+10009F, "􀂟", f480829f), -_c(U+1000A0, "􀂠", f48082a0), -_c(U+1000A1, "􀂡", f48082a1), -_c(U+1000A2, "􀂢", f48082a2), -_c(U+1000A3, "􀂣", f48082a3), -_c(U+1000A4, "􀂤", f48082a4), -_c(U+1000A5, "􀂥", f48082a5), -_c(U+1000A6, "􀂦", f48082a6), -_c(U+1000A7, "􀂧", f48082a7), -_c(U+1000A8, "􀂨", f48082a8), -_c(U+1000A9, "􀂩", f48082a9), -_c(U+1000AA, "􀂪", f48082aa), -_c(U+1000AB, "􀂫", f48082ab), -_c(U+1000AC, "􀂬", f48082ac), -_c(U+1000AD, "􀂭", f48082ad), -_c(U+1000AE, "􀂮", f48082ae), -_c(U+1000AF, "􀂯", f48082af), -_c(U+1000B0, "􀂰", f48082b0), -_c(U+1000B1, "􀂱", f48082b1), -_c(U+1000B2, "􀂲", f48082b2), -_c(U+1000B3, "􀂳", f48082b3), -_c(U+1000B4, "􀂴", f48082b4), -_c(U+1000B5, "􀂵", f48082b5), -_c(U+1000B6, "􀂶", f48082b6), -_c(U+1000B7, "􀂷", f48082b7), -_c(U+1000B8, "􀂸", f48082b8), -_c(U+1000B9, "􀂹", f48082b9), -_c(U+1000BA, "􀂺", f48082ba), -_c(U+1000BB, "􀂻", f48082bb), -_c(U+1000BC, "􀂼", f48082bc), -_c(U+1000BD, "􀂽", f48082bd), -_c(U+1000BE, "􀂾", f48082be), -_c(U+1000BF, "􀂿", f48082bf), -_c(U+1000C0, "􀃀", f4808380), -_c(U+1000C1, "􀃁", f4808381), -_c(U+1000C2, "􀃂", f4808382), -_c(U+1000C3, "􀃃", f4808383), -_c(U+1000C4, "􀃄", f4808384), -_c(U+1000C5, "􀃅", f4808385), -_c(U+1000C6, "􀃆", f4808386), -_c(U+1000C7, "􀃇", f4808387), -_c(U+1000C8, "􀃈", f4808388), -_c(U+1000C9, "􀃉", f4808389), -_c(U+1000CA, "􀃊", f480838a), -_c(U+1000CB, "􀃋", f480838b), -_c(U+1000CC, "􀃌", f480838c), -_c(U+1000CD, "􀃍", f480838d), -_c(U+1000CE, "􀃎", f480838e), -_c(U+1000CF, "􀃏", f480838f), -_c(U+1000D0, "􀃐", f4808390), -_c(U+1000D1, "􀃑", f4808391), -_c(U+1000D2, "􀃒", f4808392), -_c(U+1000D3, "􀃓", f4808393), -_c(U+1000D4, "􀃔", f4808394), -_c(U+1000D5, "􀃕", f4808395), -_c(U+1000D6, "􀃖", f4808396), -_c(U+1000D7, "􀃗", f4808397), -_c(U+1000D8, "􀃘", f4808398), -_c(U+1000D9, "􀃙", f4808399), -_c(U+1000DA, "􀃚", f480839a), -_c(U+1000DB, "􀃛", f480839b), -_c(U+1000DC, "􀃜", f480839c), -_c(U+1000DD, "􀃝", f480839d), -_c(U+1000DE, "􀃞", f480839e), -_c(U+1000DF, "􀃟", f480839f), -_c(U+1000E0, "􀃠", f48083a0), -_c(U+1000E1, "􀃡", f48083a1), -_c(U+1000E2, "􀃢", f48083a2), -_c(U+1000E3, "􀃣", f48083a3), -_c(U+1000E4, "􀃤", f48083a4), -_c(U+1000E5, "􀃥", f48083a5), -_c(U+1000E6, "􀃦", f48083a6), -_c(U+1000E7, "􀃧", f48083a7), -_c(U+1000E8, "􀃨", f48083a8), -_c(U+1000E9, "􀃩", f48083a9), -_c(U+1000EA, "􀃪", f48083aa), -_c(U+1000EB, "􀃫", f48083ab), -_c(U+1000EC, "􀃬", f48083ac), -_c(U+1000ED, "􀃭", f48083ad), -_c(U+1000EE, "􀃮", f48083ae), -_c(U+1000EF, "􀃯", f48083af), -_c(U+1000F0, "􀃰", f48083b0), -_c(U+1000F1, "􀃱", f48083b1), -_c(U+1000F2, "􀃲", f48083b2), -_c(U+1000F3, "􀃳", f48083b3), -_c(U+1000F4, "􀃴", f48083b4), -_c(U+1000F5, "􀃵", f48083b5), -_c(U+1000F6, "􀃶", f48083b6), -_c(U+1000F7, "􀃷", f48083b7), -_c(U+1000F8, "􀃸", f48083b8), -_c(U+1000F9, "􀃹", f48083b9), -_c(U+1000FA, "􀃺", f48083ba), -_c(U+1000FB, "􀃻", f48083bb), -_c(U+1000FC, "􀃼", f48083bc), -_c(U+1000FD, "􀃽", f48083bd), -_c(U+1000FE, "􀃾", f48083be), -_c(U+1000FF, "􀃿", f48083bf), -_c(U+100100, "􀄀", f4808480), -_c(U+100101, "􀄁", f4808481), -_c(U+100102, "􀄂", f4808482), -_c(U+100103, "􀄃", f4808483), -_c(U+100104, "􀄄", f4808484), -_c(U+100105, "􀄅", f4808485), -_c(U+100106, "􀄆", f4808486), -_c(U+100107, "􀄇", f4808487), -_c(U+100108, "􀄈", f4808488), -_c(U+100109, "􀄉", f4808489), -_c(U+10010A, "􀄊", f480848a), -_c(U+10010B, "􀄋", f480848b), -_c(U+10010C, "􀄌", f480848c), -_c(U+10010D, "􀄍", f480848d), -_c(U+10010E, "􀄎", f480848e), -_c(U+10010F, "􀄏", f480848f), -_c(U+100110, "􀄐", f4808490), -_c(U+100111, "􀄑", f4808491), -_c(U+100112, "􀄒", f4808492), -_c(U+100113, "􀄓", f4808493), -_c(U+100114, "􀄔", f4808494), -_c(U+100115, "􀄕", f4808495), -_c(U+100116, "􀄖", f4808496), -_c(U+100117, "􀄗", f4808497), -_c(U+100118, "􀄘", f4808498), -_c(U+100119, "􀄙", f4808499), -_c(U+10011A, "􀄚", f480849a), -_c(U+10011B, "􀄛", f480849b), -_c(U+10011C, "􀄜", f480849c), -_c(U+10011D, "􀄝", f480849d), -_c(U+10011E, "􀄞", f480849e), -_c(U+10011F, "􀄟", f480849f), -_c(U+100120, "􀄠", f48084a0), -_c(U+100121, "􀄡", f48084a1), -_c(U+100122, "􀄢", f48084a2), -_c(U+100123, "􀄣", f48084a3), -_c(U+100124, "􀄤", f48084a4), -_c(U+100125, "􀄥", f48084a5), -_c(U+100126, "􀄦", f48084a6), -_c(U+100127, "􀄧", f48084a7), -_c(U+100128, "􀄨", f48084a8), -_c(U+100129, "􀄩", f48084a9), -_c(U+10012A, "􀄪", f48084aa), -_c(U+10012B, "􀄫", f48084ab), -_c(U+10012C, "􀄬", f48084ac), -_c(U+10012D, "􀄭", f48084ad), -_c(U+10012E, "􀄮", f48084ae), -_c(U+10012F, "􀄯", f48084af), -_c(U+100130, "􀄰", f48084b0), -_c(U+100131, "􀄱", f48084b1), -_c(U+100132, "􀄲", f48084b2), -_c(U+100133, "􀄳", f48084b3), -_c(U+100134, "􀄴", f48084b4), -_c(U+100135, "􀄵", f48084b5), -_c(U+100136, "􀄶", f48084b6), -_c(U+100137, "􀄷", f48084b7), -_c(U+100138, "􀄸", f48084b8), -_c(U+100139, "􀄹", f48084b9), -_c(U+10013A, "􀄺", f48084ba), -_c(U+10013B, "􀄻", f48084bb), -_c(U+10013C, "􀄼", f48084bc), -_c(U+10013D, "􀄽", f48084bd), -_c(U+10013E, "􀄾", f48084be), -_c(U+10013F, "􀄿", f48084bf), -_c(U+100140, "􀅀", f4808580), -_c(U+100141, "􀅁", f4808581), -_c(U+100142, "􀅂", f4808582), -_c(U+100143, "􀅃", f4808583), -_c(U+100144, "􀅄", f4808584), -_c(U+100145, "􀅅", f4808585), -_c(U+100146, "􀅆", f4808586), -_c(U+100147, "􀅇", f4808587), -_c(U+100148, "􀅈", f4808588), -_c(U+100149, "􀅉", f4808589), -_c(U+10014A, "􀅊", f480858a), -_c(U+10014B, "􀅋", f480858b), -_c(U+10014C, "􀅌", f480858c), -_c(U+10014D, "􀅍", f480858d), -_c(U+10014E, "􀅎", f480858e), -_c(U+10014F, "􀅏", f480858f), -_c(U+100150, "􀅐", f4808590), -_c(U+100151, "􀅑", f4808591), -_c(U+100152, "􀅒", f4808592), -_c(U+100153, "􀅓", f4808593), -_c(U+100154, "􀅔", f4808594), -_c(U+100155, "􀅕", f4808595), -_c(U+100156, "􀅖", f4808596), -_c(U+100157, "􀅗", f4808597), -_c(U+100158, "􀅘", f4808598), -_c(U+100159, "􀅙", f4808599), -_c(U+10015A, "􀅚", f480859a), -_c(U+10015B, "􀅛", f480859b), -_c(U+10015C, "􀅜", f480859c), -_c(U+10015D, "􀅝", f480859d), -_c(U+10015E, "􀅞", f480859e), -_c(U+10015F, "􀅟", f480859f), -_c(U+100160, "􀅠", f48085a0), -_c(U+100161, "􀅡", f48085a1), -_c(U+100162, "􀅢", f48085a2), -_c(U+100163, "􀅣", f48085a3), -_c(U+100164, "􀅤", f48085a4), -_c(U+100165, "􀅥", f48085a5), -_c(U+100166, "􀅦", f48085a6), -_c(U+100167, "􀅧", f48085a7), -_c(U+100168, "􀅨", f48085a8), -_c(U+100169, "􀅩", f48085a9), -_c(U+10016A, "􀅪", f48085aa), -_c(U+10016B, "􀅫", f48085ab), -_c(U+10016C, "􀅬", f48085ac), -_c(U+10016D, "􀅭", f48085ad), -_c(U+10016E, "􀅮", f48085ae), -_c(U+10016F, "􀅯", f48085af), -_c(U+100170, "􀅰", f48085b0), -_c(U+100171, "􀅱", f48085b1), -_c(U+100172, "􀅲", f48085b2), -_c(U+100173, "􀅳", f48085b3), -_c(U+100174, "􀅴", f48085b4), -_c(U+100175, "􀅵", f48085b5), -_c(U+100176, "􀅶", f48085b6), -_c(U+100177, "􀅷", f48085b7), -_c(U+100178, "􀅸", f48085b8), -_c(U+100179, "􀅹", f48085b9), -_c(U+10017A, "􀅺", f48085ba), -_c(U+10017B, "􀅻", f48085bb), -_c(U+10017C, "􀅼", f48085bc), -_c(U+10017D, "􀅽", f48085bd), -_c(U+10017E, "􀅾", f48085be), -_c(U+10017F, "􀅿", f48085bf), -_c(U+100180, "􀆀", f4808680), -_c(U+100181, "􀆁", f4808681), -_c(U+100182, "􀆂", f4808682), -_c(U+100183, "􀆃", f4808683), -_c(U+100184, "􀆄", f4808684), -_c(U+100185, "􀆅", f4808685), -_c(U+100186, "􀆆", f4808686), -_c(U+100187, "􀆇", f4808687), -_c(U+100188, "􀆈", f4808688), -_c(U+100189, "􀆉", f4808689), -_c(U+10018A, "􀆊", f480868a), -_c(U+10018B, "􀆋", f480868b), -_c(U+10018C, "􀆌", f480868c), -_c(U+10018D, "􀆍", f480868d), -_c(U+10018E, "􀆎", f480868e), -_c(U+10018F, "􀆏", f480868f), -_c(U+100190, "􀆐", f4808690), -_c(U+100191, "􀆑", f4808691), -_c(U+100192, "􀆒", f4808692), -_c(U+100193, "􀆓", f4808693), -_c(U+100194, "􀆔", f4808694), -_c(U+100195, "􀆕", f4808695), -_c(U+100196, "􀆖", f4808696), -_c(U+100197, "􀆗", f4808697), -_c(U+100198, "􀆘", f4808698), -_c(U+100199, "􀆙", f4808699), -_c(U+10019A, "􀆚", f480869a), -_c(U+10019B, "􀆛", f480869b), -_c(U+10019C, "􀆜", f480869c), -_c(U+10019D, "􀆝", f480869d), -_c(U+10019E, "􀆞", f480869e), -_c(U+10019F, "􀆟", f480869f), -_c(U+1001A0, "􀆠", f48086a0), -_c(U+1001A1, "􀆡", f48086a1), -_c(U+1001A2, "􀆢", f48086a2), -_c(U+1001A3, "􀆣", f48086a3), -_c(U+1001A4, "􀆤", f48086a4), -_c(U+1001A5, "􀆥", f48086a5), -_c(U+1001A6, "􀆦", f48086a6), -_c(U+1001A7, "􀆧", f48086a7), -_c(U+1001A8, "􀆨", f48086a8), -_c(U+1001A9, "􀆩", f48086a9), -_c(U+1001AA, "􀆪", f48086aa), -_c(U+1001AB, "􀆫", f48086ab), -_c(U+1001AC, "􀆬", f48086ac), -_c(U+1001AD, "􀆭", f48086ad), -_c(U+1001AE, "􀆮", f48086ae), -_c(U+1001AF, "􀆯", f48086af), -_c(U+1001B0, "􀆰", f48086b0), -_c(U+1001B1, "􀆱", f48086b1), -_c(U+1001B2, "􀆲", f48086b2), -_c(U+1001B3, "􀆳", f48086b3), -_c(U+1001B4, "􀆴", f48086b4), -_c(U+1001B5, "􀆵", f48086b5), -_c(U+1001B6, "􀆶", f48086b6), -_c(U+1001B7, "􀆷", f48086b7), -_c(U+1001B8, "􀆸", f48086b8), -_c(U+1001B9, "􀆹", f48086b9), -_c(U+1001BA, "􀆺", f48086ba), -_c(U+1001BB, "􀆻", f48086bb), -_c(U+1001BC, "􀆼", f48086bc), -_c(U+1001BD, "􀆽", f48086bd), -_c(U+1001BE, "􀆾", f48086be), -_c(U+1001BF, "􀆿", f48086bf), -_c(U+1001C0, "􀇀", f4808780), -_c(U+1001C1, "􀇁", f4808781), -_c(U+1001C2, "􀇂", f4808782), -_c(U+1001C3, "􀇃", f4808783), -_c(U+1001C4, "􀇄", f4808784), -_c(U+1001C5, "􀇅", f4808785), -_c(U+1001C6, "􀇆", f4808786), -_c(U+1001C7, "􀇇", f4808787), -_c(U+1001C8, "􀇈", f4808788), -_c(U+1001C9, "􀇉", f4808789), -_c(U+1001CA, "􀇊", f480878a), -_c(U+1001CB, "􀇋", f480878b), -_c(U+1001CC, "􀇌", f480878c), -_c(U+1001CD, "􀇍", f480878d), -_c(U+1001CE, "􀇎", f480878e), -_c(U+1001CF, "􀇏", f480878f), -_c(U+1001D0, "􀇐", f4808790), -_c(U+1001D1, "􀇑", f4808791), -_c(U+1001D2, "􀇒", f4808792), -_c(U+1001D3, "􀇓", f4808793), -_c(U+1001D4, "􀇔", f4808794), -_c(U+1001D5, "􀇕", f4808795), -_c(U+1001D6, "􀇖", f4808796), -_c(U+1001D7, "􀇗", f4808797), -_c(U+1001D8, "􀇘", f4808798), -_c(U+1001D9, "􀇙", f4808799), -_c(U+1001DA, "􀇚", f480879a), -_c(U+1001DB, "􀇛", f480879b), -_c(U+1001DC, "􀇜", f480879c), -_c(U+1001DD, "􀇝", f480879d), -_c(U+1001DE, "􀇞", f480879e), -_c(U+1001DF, "􀇟", f480879f), -_c(U+1001E0, "􀇠", f48087a0), -_c(U+1001E1, "􀇡", f48087a1), -_c(U+1001E2, "􀇢", f48087a2), -_c(U+1001E3, "􀇣", f48087a3), -_c(U+1001E4, "􀇤", f48087a4), -_c(U+1001E5, "􀇥", f48087a5), -_c(U+1001E6, "􀇦", f48087a6), -_c(U+1001E7, "􀇧", f48087a7), -_c(U+1001E8, "􀇨", f48087a8), -_c(U+1001E9, "􀇩", f48087a9), -_c(U+1001EA, "􀇪", f48087aa), -_c(U+1001EB, "􀇫", f48087ab), -_c(U+1001EC, "􀇬", f48087ac), -_c(U+1001ED, "􀇭", f48087ad), -_c(U+1001EE, "􀇮", f48087ae), -_c(U+1001EF, "􀇯", f48087af), -_c(U+1001F0, "􀇰", f48087b0), -_c(U+1001F1, "􀇱", f48087b1), -_c(U+1001F2, "􀇲", f48087b2), -_c(U+1001F3, "􀇳", f48087b3), -_c(U+1001F4, "􀇴", f48087b4), -_c(U+1001F5, "􀇵", f48087b5), -_c(U+1001F6, "􀇶", f48087b6), -_c(U+1001F7, "􀇷", f48087b7), -_c(U+1001F8, "􀇸", f48087b8), -_c(U+1001F9, "􀇹", f48087b9), -_c(U+1001FA, "􀇺", f48087ba), -_c(U+1001FB, "􀇻", f48087bb), -_c(U+1001FC, "􀇼", f48087bc), -_c(U+1001FD, "􀇽", f48087bd), -_c(U+1001FE, "􀇾", f48087be), -_c(U+1001FF, "􀇿", f48087bf), -_c(U+100200, "􀈀", f4808880), -_c(U+100201, "􀈁", f4808881), -_c(U+100202, "􀈂", f4808882), -_c(U+100203, "􀈃", f4808883), -_c(U+100204, "􀈄", f4808884), -_c(U+100205, "􀈅", f4808885), -_c(U+100206, "􀈆", f4808886), -_c(U+100207, "􀈇", f4808887), -_c(U+100208, "􀈈", f4808888), -_c(U+100209, "􀈉", f4808889), -_c(U+10020A, "􀈊", f480888a), -_c(U+10020B, "􀈋", f480888b), -_c(U+10020C, "􀈌", f480888c), -_c(U+10020D, "􀈍", f480888d), -_c(U+10020E, "􀈎", f480888e), -_c(U+10020F, "􀈏", f480888f), -_c(U+100210, "􀈐", f4808890), -_c(U+100211, "􀈑", f4808891), -_c(U+100212, "􀈒", f4808892), -_c(U+100213, "􀈓", f4808893), -_c(U+100214, "􀈔", f4808894), -_c(U+100215, "􀈕", f4808895), -_c(U+100216, "􀈖", f4808896), -_c(U+100217, "􀈗", f4808897), -_c(U+100218, "􀈘", f4808898), -_c(U+100219, "􀈙", f4808899), -_c(U+10021A, "􀈚", f480889a), -_c(U+10021B, "􀈛", f480889b), -_c(U+10021C, "􀈜", f480889c), -_c(U+10021D, "􀈝", f480889d), -_c(U+10021E, "􀈞", f480889e), -_c(U+10021F, "􀈟", f480889f), -_c(U+100220, "􀈠", f48088a0), -_c(U+100221, "􀈡", f48088a1), -_c(U+100222, "􀈢", f48088a2), -_c(U+100223, "􀈣", f48088a3), -_c(U+100224, "􀈤", f48088a4), -_c(U+100225, "􀈥", f48088a5), -_c(U+100226, "􀈦", f48088a6), -_c(U+100227, "􀈧", f48088a7), -_c(U+100228, "􀈨", f48088a8), -_c(U+100229, "􀈩", f48088a9), -_c(U+10022A, "􀈪", f48088aa), -_c(U+10022B, "􀈫", f48088ab), -_c(U+10022C, "􀈬", f48088ac), -_c(U+10022D, "􀈭", f48088ad), -_c(U+10022E, "􀈮", f48088ae), -_c(U+10022F, "􀈯", f48088af), -_c(U+100230, "􀈰", f48088b0), -_c(U+100231, "􀈱", f48088b1), -_c(U+100232, "􀈲", f48088b2), -_c(U+100233, "􀈳", f48088b3), -_c(U+100234, "􀈴", f48088b4), -_c(U+100235, "􀈵", f48088b5), -_c(U+100236, "􀈶", f48088b6), -_c(U+100237, "􀈷", f48088b7), -_c(U+100238, "􀈸", f48088b8), -_c(U+100239, "􀈹", f48088b9), -_c(U+10023A, "􀈺", f48088ba), -_c(U+10023B, "􀈻", f48088bb), -_c(U+10023C, "􀈼", f48088bc), -_c(U+10023D, "􀈽", f48088bd), -_c(U+10023E, "􀈾", f48088be), -_c(U+10023F, "􀈿", f48088bf), -_c(U+100240, "􀉀", f4808980), -_c(U+100241, "􀉁", f4808981), -_c(U+100242, "􀉂", f4808982), -_c(U+100243, "􀉃", f4808983), -_c(U+100244, "􀉄", f4808984), -_c(U+100245, "􀉅", f4808985), -_c(U+100246, "􀉆", f4808986), -_c(U+100247, "􀉇", f4808987), -_c(U+100248, "􀉈", f4808988), -_c(U+100249, "􀉉", f4808989), -_c(U+10024A, "􀉊", f480898a), -_c(U+10024B, "􀉋", f480898b), -_c(U+10024C, "􀉌", f480898c), -_c(U+10024D, "􀉍", f480898d), -_c(U+10024E, "􀉎", f480898e), -_c(U+10024F, "􀉏", f480898f), -_c(U+100250, "􀉐", f4808990), -_c(U+100251, "􀉑", f4808991), -_c(U+100252, "􀉒", f4808992), -_c(U+100253, "􀉓", f4808993), -_c(U+100254, "􀉔", f4808994), -_c(U+100255, "􀉕", f4808995), -_c(U+100256, "􀉖", f4808996), -_c(U+100257, "􀉗", f4808997), -_c(U+100258, "􀉘", f4808998), -_c(U+100259, "􀉙", f4808999), -_c(U+10025A, "􀉚", f480899a), -_c(U+10025B, "􀉛", f480899b), -_c(U+10025C, "􀉜", f480899c), -_c(U+10025D, "􀉝", f480899d), -_c(U+10025E, "􀉞", f480899e), -_c(U+10025F, "􀉟", f480899f), -_c(U+100260, "􀉠", f48089a0), -_c(U+100261, "􀉡", f48089a1), -_c(U+100262, "􀉢", f48089a2), -_c(U+100263, "􀉣", f48089a3), -_c(U+100264, "􀉤", f48089a4), -_c(U+100265, "􀉥", f48089a5), -_c(U+100266, "􀉦", f48089a6), -_c(U+100267, "􀉧", f48089a7), -_c(U+100268, "􀉨", f48089a8), -_c(U+100269, "􀉩", f48089a9), -_c(U+10026A, "􀉪", f48089aa), -_c(U+10026B, "􀉫", f48089ab), -_c(U+10026C, "􀉬", f48089ac), -_c(U+10026D, "􀉭", f48089ad), -_c(U+10026E, "􀉮", f48089ae), -_c(U+10026F, "􀉯", f48089af), -_c(U+100270, "􀉰", f48089b0), -_c(U+100271, "􀉱", f48089b1), -_c(U+100272, "􀉲", f48089b2), -_c(U+100273, "􀉳", f48089b3), -_c(U+100274, "􀉴", f48089b4), -_c(U+100275, "􀉵", f48089b5), -_c(U+100276, "􀉶", f48089b6), -_c(U+100277, "􀉷", f48089b7), -_c(U+100278, "􀉸", f48089b8), -_c(U+100279, "􀉹", f48089b9), -_c(U+10027A, "􀉺", f48089ba), -_c(U+10027B, "􀉻", f48089bb), -_c(U+10027C, "􀉼", f48089bc), -_c(U+10027D, "􀉽", f48089bd), -_c(U+10027E, "􀉾", f48089be), -_c(U+10027F, "􀉿", f48089bf), -_c(U+100280, "􀊀", f4808a80), -_c(U+100281, "􀊁", f4808a81), -_c(U+100282, "􀊂", f4808a82), -_c(U+100283, "􀊃", f4808a83), -_c(U+100284, "􀊄", f4808a84), -_c(U+100285, "􀊅", f4808a85), -_c(U+100286, "􀊆", f4808a86), -_c(U+100287, "􀊇", f4808a87), -_c(U+100288, "􀊈", f4808a88), -_c(U+100289, "􀊉", f4808a89), -_c(U+10028A, "􀊊", f4808a8a), -_c(U+10028B, "􀊋", f4808a8b), -_c(U+10028C, "􀊌", f4808a8c), -_c(U+10028D, "􀊍", f4808a8d), -_c(U+10028E, "􀊎", f4808a8e), -_c(U+10028F, "􀊏", f4808a8f), -_c(U+100290, "􀊐", f4808a90), -_c(U+100291, "􀊑", f4808a91), -_c(U+100292, "􀊒", f4808a92), -_c(U+100293, "􀊓", f4808a93), -_c(U+100294, "􀊔", f4808a94), -_c(U+100295, "􀊕", f4808a95), -_c(U+100296, "􀊖", f4808a96), -_c(U+100297, "􀊗", f4808a97), -_c(U+100298, "􀊘", f4808a98), -_c(U+100299, "􀊙", f4808a99), -_c(U+10029A, "􀊚", f4808a9a), -_c(U+10029B, "􀊛", f4808a9b), -_c(U+10029C, "􀊜", f4808a9c), -_c(U+10029D, "􀊝", f4808a9d), -_c(U+10029E, "􀊞", f4808a9e), -_c(U+10029F, "􀊟", f4808a9f), -_c(U+1002A0, "􀊠", f4808aa0), -_c(U+1002A1, "􀊡", f4808aa1), -_c(U+1002A2, "􀊢", f4808aa2), -_c(U+1002A3, "􀊣", f4808aa3), -_c(U+1002A4, "􀊤", f4808aa4), -_c(U+1002A5, "􀊥", f4808aa5), -_c(U+1002A6, "􀊦", f4808aa6), -_c(U+1002A7, "􀊧", f4808aa7), -_c(U+1002A8, "􀊨", f4808aa8), -_c(U+1002A9, "􀊩", f4808aa9), -_c(U+1002AA, "􀊪", f4808aaa), -_c(U+1002AB, "􀊫", f4808aab), -_c(U+1002AC, "􀊬", f4808aac), -_c(U+1002AD, "􀊭", f4808aad), -_c(U+1002AE, "􀊮", f4808aae), -_c(U+1002AF, "􀊯", f4808aaf), -_c(U+1002B0, "􀊰", f4808ab0), -_c(U+1002B1, "􀊱", f4808ab1), -_c(U+1002B2, "􀊲", f4808ab2), -_c(U+1002B3, "􀊳", f4808ab3), -_c(U+1002B4, "􀊴", f4808ab4), -_c(U+1002B5, "􀊵", f4808ab5), -_c(U+1002B6, "􀊶", f4808ab6), -_c(U+1002B7, "􀊷", f4808ab7), -_c(U+1002B8, "􀊸", f4808ab8), -_c(U+1002B9, "􀊹", f4808ab9), -_c(U+1002BA, "􀊺", f4808aba), -_c(U+1002BB, "􀊻", f4808abb), -_c(U+1002BC, "􀊼", f4808abc), -_c(U+1002BD, "􀊽", f4808abd), -_c(U+1002BE, "􀊾", f4808abe), -_c(U+1002BF, "􀊿", f4808abf), -_c(U+1002C0, "􀋀", f4808b80), -_c(U+1002C1, "􀋁", f4808b81), -_c(U+1002C2, "􀋂", f4808b82), -_c(U+1002C3, "􀋃", f4808b83), -_c(U+1002C4, "􀋄", f4808b84), -_c(U+1002C5, "􀋅", f4808b85), -_c(U+1002C6, "􀋆", f4808b86), -_c(U+1002C7, "􀋇", f4808b87), -_c(U+1002C8, "􀋈", f4808b88), -_c(U+1002C9, "􀋉", f4808b89), -_c(U+1002CA, "􀋊", f4808b8a), -_c(U+1002CB, "􀋋", f4808b8b), -_c(U+1002CC, "􀋌", f4808b8c), -_c(U+1002CD, "􀋍", f4808b8d), -_c(U+1002CE, "􀋎", f4808b8e), -_c(U+1002CF, "􀋏", f4808b8f), -_c(U+1002D0, "􀋐", f4808b90), -_c(U+1002D1, "􀋑", f4808b91), -_c(U+1002D2, "􀋒", f4808b92), -_c(U+1002D3, "􀋓", f4808b93), -_c(U+1002D4, "􀋔", f4808b94), -_c(U+1002D5, "􀋕", f4808b95), -_c(U+1002D6, "􀋖", f4808b96), -_c(U+1002D7, "􀋗", f4808b97), -_c(U+1002D8, "􀋘", f4808b98), -_c(U+1002D9, "􀋙", f4808b99), -_c(U+1002DA, "􀋚", f4808b9a), -_c(U+1002DB, "􀋛", f4808b9b), -_c(U+1002DC, "􀋜", f4808b9c), -_c(U+1002DD, "􀋝", f4808b9d), -_c(U+1002DE, "􀋞", f4808b9e), -_c(U+1002DF, "􀋟", f4808b9f), -_c(U+1002E0, "􀋠", f4808ba0), -_c(U+1002E1, "􀋡", f4808ba1), -_c(U+1002E2, "􀋢", f4808ba2), -_c(U+1002E3, "􀋣", f4808ba3), -_c(U+1002E4, "􀋤", f4808ba4), -_c(U+1002E5, "􀋥", f4808ba5), -_c(U+1002E6, "􀋦", f4808ba6), -_c(U+1002E7, "􀋧", f4808ba7), -_c(U+1002E8, "􀋨", f4808ba8), -_c(U+1002E9, "􀋩", f4808ba9), -_c(U+1002EA, "􀋪", f4808baa), -_c(U+1002EB, "􀋫", f4808bab), -_c(U+1002EC, "􀋬", f4808bac), -_c(U+1002ED, "􀋭", f4808bad), -_c(U+1002EE, "􀋮", f4808bae), -_c(U+1002EF, "􀋯", f4808baf), -_c(U+1002F0, "􀋰", f4808bb0), -_c(U+1002F1, "􀋱", f4808bb1), -_c(U+1002F2, "􀋲", f4808bb2), -_c(U+1002F3, "􀋳", f4808bb3), -_c(U+1002F4, "􀋴", f4808bb4), -_c(U+1002F5, "􀋵", f4808bb5), -_c(U+1002F6, "􀋶", f4808bb6), -_c(U+1002F7, "􀋷", f4808bb7), -_c(U+1002F8, "􀋸", f4808bb8), -_c(U+1002F9, "􀋹", f4808bb9), -_c(U+1002FA, "􀋺", f4808bba), -_c(U+1002FB, "􀋻", f4808bbb), -_c(U+1002FC, "􀋼", f4808bbc), -_c(U+1002FD, "􀋽", f4808bbd), -_c(U+1002FE, "􀋾", f4808bbe), -_c(U+1002FF, "􀋿", f4808bbf), -_c(U+100300, "􀌀", f4808c80), -_c(U+100301, "􀌁", f4808c81), -_c(U+100302, "􀌂", f4808c82), -_c(U+100303, "􀌃", f4808c83), -_c(U+100304, "􀌄", f4808c84), -_c(U+100305, "􀌅", f4808c85), -_c(U+100306, "􀌆", f4808c86), -_c(U+100307, "􀌇", f4808c87), -_c(U+100308, "􀌈", f4808c88), -_c(U+100309, "􀌉", f4808c89), -_c(U+10030A, "􀌊", f4808c8a), -_c(U+10030B, "􀌋", f4808c8b), -_c(U+10030C, "􀌌", f4808c8c), -_c(U+10030D, "􀌍", f4808c8d), -_c(U+10030E, "􀌎", f4808c8e), -_c(U+10030F, "􀌏", f4808c8f), -_c(U+100310, "􀌐", f4808c90), -_c(U+100311, "􀌑", f4808c91), -_c(U+100312, "􀌒", f4808c92), -_c(U+100313, "􀌓", f4808c93), -_c(U+100314, "􀌔", f4808c94), -_c(U+100315, "􀌕", f4808c95), -_c(U+100316, "􀌖", f4808c96), -_c(U+100317, "􀌗", f4808c97), -_c(U+100318, "􀌘", f4808c98), -_c(U+100319, "􀌙", f4808c99), -_c(U+10031A, "􀌚", f4808c9a), -_c(U+10031B, "􀌛", f4808c9b), -_c(U+10031C, "􀌜", f4808c9c), -_c(U+10031D, "􀌝", f4808c9d), -_c(U+10031E, "􀌞", f4808c9e), -_c(U+10031F, "􀌟", f4808c9f), -_c(U+100320, "􀌠", f4808ca0), -_c(U+100321, "􀌡", f4808ca1), -_c(U+100322, "􀌢", f4808ca2), -_c(U+100323, "􀌣", f4808ca3), -_c(U+100324, "􀌤", f4808ca4), -_c(U+100325, "􀌥", f4808ca5), -_c(U+100326, "􀌦", f4808ca6), -_c(U+100327, "􀌧", f4808ca7), -_c(U+100328, "􀌨", f4808ca8), -_c(U+100329, "􀌩", f4808ca9), -_c(U+10032A, "􀌪", f4808caa), -_c(U+10032B, "􀌫", f4808cab), -_c(U+10032C, "􀌬", f4808cac), -_c(U+10032D, "􀌭", f4808cad), -_c(U+10032E, "􀌮", f4808cae), -_c(U+10032F, "􀌯", f4808caf), -_c(U+100330, "􀌰", f4808cb0), -_c(U+100331, "􀌱", f4808cb1), -_c(U+100332, "􀌲", f4808cb2), -_c(U+100333, "􀌳", f4808cb3), -_c(U+100334, "􀌴", f4808cb4), -_c(U+100335, "􀌵", f4808cb5), -_c(U+100336, "􀌶", f4808cb6), -_c(U+100337, "􀌷", f4808cb7), -_c(U+100338, "􀌸", f4808cb8), -_c(U+100339, "􀌹", f4808cb9), -_c(U+10033A, "􀌺", f4808cba), -_c(U+10033B, "􀌻", f4808cbb), -_c(U+10033C, "􀌼", f4808cbc), -_c(U+10033D, "􀌽", f4808cbd), -_c(U+10033E, "􀌾", f4808cbe), -_c(U+10033F, "􀌿", f4808cbf), -_c(U+100340, "􀍀", f4808d80), -_c(U+100341, "􀍁", f4808d81), -_c(U+100342, "􀍂", f4808d82), -_c(U+100343, "􀍃", f4808d83), -_c(U+100344, "􀍄", f4808d84), -_c(U+100345, "􀍅", f4808d85), -_c(U+100346, "􀍆", f4808d86), -_c(U+100347, "􀍇", f4808d87), -_c(U+100348, "􀍈", f4808d88), -_c(U+100349, "􀍉", f4808d89), -_c(U+10034A, "􀍊", f4808d8a), -_c(U+10034B, "􀍋", f4808d8b), -_c(U+10034C, "􀍌", f4808d8c), -_c(U+10034D, "􀍍", f4808d8d), -_c(U+10034E, "􀍎", f4808d8e), -_c(U+10034F, "􀍏", f4808d8f), -_c(U+100350, "􀍐", f4808d90), -_c(U+100351, "􀍑", f4808d91), -_c(U+100352, "􀍒", f4808d92), -_c(U+100353, "􀍓", f4808d93), -_c(U+100354, "􀍔", f4808d94), -_c(U+100355, "􀍕", f4808d95), -_c(U+100356, "􀍖", f4808d96), -_c(U+100357, "􀍗", f4808d97), -_c(U+100358, "􀍘", f4808d98), -_c(U+100359, "􀍙", f4808d99), -_c(U+10035A, "􀍚", f4808d9a), -_c(U+10035B, "􀍛", f4808d9b), -_c(U+10035C, "􀍜", f4808d9c), -_c(U+10035D, "􀍝", f4808d9d), -_c(U+10035E, "􀍞", f4808d9e), -_c(U+10035F, "􀍟", f4808d9f), -_c(U+100360, "􀍠", f4808da0), -_c(U+100361, "􀍡", f4808da1), -_c(U+100362, "􀍢", f4808da2), -_c(U+100363, "􀍣", f4808da3), -_c(U+100364, "􀍤", f4808da4), -_c(U+100365, "􀍥", f4808da5), -_c(U+100366, "􀍦", f4808da6), -_c(U+100367, "􀍧", f4808da7), -_c(U+100368, "􀍨", f4808da8), -_c(U+100369, "􀍩", f4808da9), -_c(U+10036A, "􀍪", f4808daa), -_c(U+10036B, "􀍫", f4808dab), -_c(U+10036C, "􀍬", f4808dac), -_c(U+10036D, "􀍭", f4808dad), -_c(U+10036E, "􀍮", f4808dae), -_c(U+10036F, "􀍯", f4808daf), -_c(U+100370, "􀍰", f4808db0), -_c(U+100371, "􀍱", f4808db1), -_c(U+100372, "􀍲", f4808db2), -_c(U+100373, "􀍳", f4808db3), -_c(U+100374, "􀍴", f4808db4), -_c(U+100375, "􀍵", f4808db5), -_c(U+100376, "􀍶", f4808db6), -_c(U+100377, "􀍷", f4808db7), -_c(U+100378, "􀍸", f4808db8), -_c(U+100379, "􀍹", f4808db9), -_c(U+10037A, "􀍺", f4808dba), -_c(U+10037B, "􀍻", f4808dbb), -_c(U+10037C, "􀍼", f4808dbc), -_c(U+10037D, "􀍽", f4808dbd), -_c(U+10037E, "􀍾", f4808dbe), -_c(U+10037F, "􀍿", f4808dbf), -_c(U+100380, "􀎀", f4808e80), -_c(U+100381, "􀎁", f4808e81), -_c(U+100382, "􀎂", f4808e82), -_c(U+100383, "􀎃", f4808e83), -_c(U+100384, "􀎄", f4808e84), -_c(U+100385, "􀎅", f4808e85), -_c(U+100386, "􀎆", f4808e86), -_c(U+100387, "􀎇", f4808e87), -_c(U+100388, "􀎈", f4808e88), -_c(U+100389, "􀎉", f4808e89), -_c(U+10038A, "􀎊", f4808e8a), -_c(U+10038B, "􀎋", f4808e8b), -_c(U+10038C, "􀎌", f4808e8c), -_c(U+10038D, "􀎍", f4808e8d), -_c(U+10038E, "􀎎", f4808e8e), -_c(U+10038F, "􀎏", f4808e8f), -_c(U+100390, "􀎐", f4808e90), -_c(U+100391, "􀎑", f4808e91), -_c(U+100392, "􀎒", f4808e92), -_c(U+100393, "􀎓", f4808e93), -_c(U+100394, "􀎔", f4808e94), -_c(U+100395, "􀎕", f4808e95), -_c(U+100396, "􀎖", f4808e96), -_c(U+100397, "􀎗", f4808e97), -_c(U+100398, "􀎘", f4808e98), -_c(U+100399, "􀎙", f4808e99), -_c(U+10039A, "􀎚", f4808e9a), -_c(U+10039B, "􀎛", f4808e9b), -_c(U+10039C, "􀎜", f4808e9c), -_c(U+10039D, "􀎝", f4808e9d), -_c(U+10039E, "􀎞", f4808e9e), -_c(U+10039F, "􀎟", f4808e9f), -_c(U+1003A0, "􀎠", f4808ea0), -_c(U+1003A1, "􀎡", f4808ea1), -_c(U+1003A2, "􀎢", f4808ea2), -_c(U+1003A3, "􀎣", f4808ea3), -_c(U+1003A4, "􀎤", f4808ea4), -_c(U+1003A5, "􀎥", f4808ea5), -_c(U+1003A6, "􀎦", f4808ea6), -_c(U+1003A7, "􀎧", f4808ea7), -_c(U+1003A8, "􀎨", f4808ea8), -_c(U+1003A9, "􀎩", f4808ea9), -_c(U+1003AA, "􀎪", f4808eaa), -_c(U+1003AB, "􀎫", f4808eab), -_c(U+1003AC, "􀎬", f4808eac), -_c(U+1003AD, "􀎭", f4808ead), -_c(U+1003AE, "􀎮", f4808eae), -_c(U+1003AF, "􀎯", f4808eaf), -_c(U+1003B0, "􀎰", f4808eb0), -_c(U+1003B1, "􀎱", f4808eb1), -_c(U+1003B2, "􀎲", f4808eb2), -_c(U+1003B3, "􀎳", f4808eb3), -_c(U+1003B4, "􀎴", f4808eb4), -_c(U+1003B5, "􀎵", f4808eb5), -_c(U+1003B6, "􀎶", f4808eb6), -_c(U+1003B7, "􀎷", f4808eb7), -_c(U+1003B8, "􀎸", f4808eb8), -_c(U+1003B9, "􀎹", f4808eb9), -_c(U+1003BA, "􀎺", f4808eba), -_c(U+1003BB, "􀎻", f4808ebb), -_c(U+1003BC, "􀎼", f4808ebc), -_c(U+1003BD, "􀎽", f4808ebd), -_c(U+1003BE, "􀎾", f4808ebe), -_c(U+1003BF, "􀎿", f4808ebf), -_c(U+1003C0, "􀏀", f4808f80), -_c(U+1003C1, "􀏁", f4808f81), -_c(U+1003C2, "􀏂", f4808f82), -_c(U+1003C3, "􀏃", f4808f83), -_c(U+1003C4, "􀏄", f4808f84), -_c(U+1003C5, "􀏅", f4808f85), -_c(U+1003C6, "􀏆", f4808f86), -_c(U+1003C7, "􀏇", f4808f87), -_c(U+1003C8, "􀏈", f4808f88), -_c(U+1003C9, "􀏉", f4808f89), -_c(U+1003CA, "􀏊", f4808f8a), -_c(U+1003CB, "􀏋", f4808f8b), -_c(U+1003CC, "􀏌", f4808f8c), -_c(U+1003CD, "􀏍", f4808f8d), -_c(U+1003CE, "􀏎", f4808f8e), -_c(U+1003CF, "􀏏", f4808f8f), -_c(U+1003D0, "􀏐", f4808f90), -_c(U+1003D1, "􀏑", f4808f91), -_c(U+1003D2, "􀏒", f4808f92), -_c(U+1003D3, "􀏓", f4808f93), -_c(U+1003D4, "􀏔", f4808f94), -_c(U+1003D5, "􀏕", f4808f95), -_c(U+1003D6, "􀏖", f4808f96), -_c(U+1003D7, "􀏗", f4808f97), -_c(U+1003D8, "􀏘", f4808f98), -_c(U+1003D9, "􀏙", f4808f99), -_c(U+1003DA, "􀏚", f4808f9a), -_c(U+1003DB, "􀏛", f4808f9b), -_c(U+1003DC, "􀏜", f4808f9c), -_c(U+1003DD, "􀏝", f4808f9d), -_c(U+1003DE, "􀏞", f4808f9e), -_c(U+1003DF, "􀏟", f4808f9f), -_c(U+1003E0, "􀏠", f4808fa0), -_c(U+1003E1, "􀏡", f4808fa1), -_c(U+1003E2, "􀏢", f4808fa2), -_c(U+1003E3, "􀏣", f4808fa3), -_c(U+1003E4, "􀏤", f4808fa4), -_c(U+1003E5, "􀏥", f4808fa5), -_c(U+1003E6, "􀏦", f4808fa6), -_c(U+1003E7, "􀏧", f4808fa7), -_c(U+1003E8, "􀏨", f4808fa8), -_c(U+1003E9, "􀏩", f4808fa9), -_c(U+1003EA, "􀏪", f4808faa), -_c(U+1003EB, "􀏫", f4808fab), -_c(U+1003EC, "􀏬", f4808fac), -_c(U+1003ED, "􀏭", f4808fad), -_c(U+1003EE, "􀏮", f4808fae), -_c(U+1003EF, "􀏯", f4808faf), -_c(U+1003F0, "􀏰", f4808fb0), -_c(U+1003F1, "􀏱", f4808fb1), -_c(U+1003F2, "􀏲", f4808fb2), -_c(U+1003F3, "􀏳", f4808fb3), -_c(U+1003F4, "􀏴", f4808fb4), -_c(U+1003F5, "􀏵", f4808fb5), -_c(U+1003F6, "􀏶", f4808fb6), -_c(U+1003F7, "􀏷", f4808fb7), -_c(U+1003F8, "􀏸", f4808fb8), -_c(U+1003F9, "􀏹", f4808fb9), -_c(U+1003FA, "􀏺", f4808fba), -_c(U+1003FB, "􀏻", f4808fbb), -_c(U+1003FC, "􀏼", f4808fbc), -_c(U+1003FD, "􀏽", f4808fbd), -_c(U+1003FE, "􀏾", f4808fbe), -_c(U+1003FF, "􀏿", f4808fbf), -#undef _c diff --git a/thirdparty/ryml/ext/c4core/tools/amalgamate.py b/thirdparty/ryml/ext/c4core/tools/amalgamate.py deleted file mode 100644 index eabcb4891..000000000 --- a/thirdparty/ryml/ext/c4core/tools/amalgamate.py +++ /dev/null @@ -1,143 +0,0 @@ -import re -from os.path import abspath, dirname -import sys -import subprocess - -projdir = abspath(dirname(dirname(__file__))) -sys.path.insert(0, f"{projdir}/cmake") -import amalgamate_utils as am - - -def amalgamate_fastfloat(): - fastfloatdir = f"{projdir}/src/c4/ext/fast_float" - subprocess.run([ - sys.executable, - f"{fastfloatdir}/script/amalgamate.py", - "--license", "MIT", - "--output", f"{fastfloatdir}/../fast_float_all.h" - ], cwd=fastfloatdir).check_returncode() - - -def amalgamate_c4core(filename: str, - with_stl: bool=True, - with_fastfloat: bool=True): - if with_fastfloat: - amalgamate_fastfloat() - repo = "https://github.com/biojppm/c4core" - defmacro = "C4CORE_SINGLE_HDR_DEFINE_NOW" - exports_def_code = f"""// shared library: export when defining -#if defined(C4CORE_SHARED) && defined({defmacro}) && !defined(C4CORE_EXPORTS) -#define C4CORE_EXPORTS -#endif -""" - required_gcc4_8_include = """// these includes are needed to work around conditional -// includes in the gcc4.8 shim -#include -#include -#include -""" - srcblocks = [ - am.cmttext(f""" -c4core - C++ utilities - -{repo} - -DO NOT EDIT. This file is generated automatically. -This is an amalgamated single-header version of the library. - -INSTRUCTIONS: - - Include at will in any header of your project - - In one (and only one) of your project source files, - #define {defmacro} and then include this header. - This will enable the function and class definitions in - the header file. - - To compile into a shared library, just define the - preprocessor symbol C4CORE_SHARED . This will take - care of symbol export/import. -"""), - am.cmtfile("LICENSE.txt"), - am.injcode(exports_def_code), - "src/c4/export.hpp", - "src/c4/preprocessor.hpp", - "src/c4/platform.hpp", - "src/c4/cpu.hpp", - "src/c4/compiler.hpp", - am.injcode(required_gcc4_8_include), - "cmake/compat/c4/gcc-4.8.hpp", - "src/c4/language.hpp", - "src/c4/types.hpp", - "src/c4/config.hpp", - am.hdrfile("src/c4/ext/debugbreak/debugbreak.h", "c4/ext/debugbreak/debugbreak.h", "DEBUG_BREAK_H"), - "src/c4/error.hpp", - "src/c4/memory_util.hpp", - "src/c4/memory_resource.hpp", - "src/c4/ctor_dtor.hpp", - "src/c4/allocator.hpp", - "src/c4/char_traits.hpp", - "src/c4/hash.hpp", - "src/c4/szconv.hpp", - "src/c4/blob.hpp", - "src/c4/substr_fwd.hpp", - "src/c4/substr.hpp", - am.onlyif(with_fastfloat, am.injfile("src/c4/ext/fast_float_all.h", "c4/ext/fast_float_all.h")), - am.onlyif(with_fastfloat, "src/c4/ext/fast_float.hpp"), - "src/c4/std/vector_fwd.hpp", - "src/c4/std/string_fwd.hpp", - "src/c4/std/std_fwd.hpp", - "src/c4/charconv.hpp", - "src/c4/utf.hpp", - "src/c4/format.hpp", - "src/c4/dump.hpp", - "src/c4/enum.hpp", - "src/c4/bitmask.hpp", - "src/c4/span.hpp", - "src/c4/type_name.hpp", - "src/c4/base64.hpp", - am.onlyif(with_stl, am.ignfile("src/c4/std/std.hpp")), # this is an umbrella include - am.onlyif(with_stl, "src/c4/std/string.hpp"), - am.onlyif(with_stl, "src/c4/std/vector.hpp"), - am.onlyif(with_stl, "src/c4/std/tuple.hpp"), - "src/c4/ext/rng/rng.hpp", - "src/c4/ext/sg14/inplace_function.h", - am.ignfile("src/c4/common.hpp"), - am.ignfile("src/c4/c4_push.hpp"), - am.ignfile("src/c4/c4_pop.hpp"), - am.ignfile("src/c4/restrict.hpp"), - am.ignfile("src/c4/unrestrict.hpp"), - "src/c4/language.cpp", - "src/c4/format.cpp", - "src/c4/memory_util.cpp", - "src/c4/char_traits.cpp", - "src/c4/memory_resource.cpp", - "src/c4/utf.cpp", - "src/c4/base64.cpp", - am.injcode("#define C4_WINDOWS_POP_HPP_"), - "src/c4/windows_push.hpp", - "src/c4/windows.hpp", - "src/c4/windows_pop.hpp", # do NOT include this before windows.hpp - "src/c4/error.cpp", - ] - result = am.catfiles(srcblocks, - projdir, - # comment out lines with these patterns: - include_regexes=[ - re.compile(r'^\s*#\s*include "(c4/.*)".*$'), - re.compile(r'^\s*#\s*include <(c4/.*)>.*$'), - ], - definition_macro=defmacro, - repo=repo, - result_incguard="_C4CORE_SINGLE_HEADER_AMALGAMATED_HPP_") - result_with_only_first_includes = am.include_only_first(result) - am.file_put_contents(filename, result_with_only_first_includes) - - -def mkparser(): - return am.mkparser(fastfloat=(True, "enable fastfloat library"), - stl=(True, "enable stl interop")) - - -if __name__ == "__main__": - args = mkparser().parse_args() - amalgamate_c4core(filename=args.output, - with_fastfloat=args.fastfloat, - with_stl=args.stl) diff --git a/thirdparty/ryml/ext/testbm.cmake b/thirdparty/ryml/ext/testbm.cmake deleted file mode 100644 index 7a1a4366e..000000000 --- a/thirdparty/ryml/ext/testbm.cmake +++ /dev/null @@ -1,4 +0,0 @@ -# these are used both for testing and benchmarking -c4_require_subproject(c4fs REMOTE - GIT_REPOSITORY https://github.com/biojppm/c4fs - GIT_TAG master) diff --git a/thirdparty/ryml/img/first_comparison_yaml_cpp.png b/thirdparty/ryml/img/first_comparison_yaml_cpp.png deleted file mode 100644 index d8f749def..000000000 Binary files a/thirdparty/ryml/img/first_comparison_yaml_cpp.png and /dev/null differ diff --git a/thirdparty/ryml/pyproject.toml b/thirdparty/ryml/pyproject.toml deleted file mode 100644 index 2a5bc9029..000000000 --- a/thirdparty/ryml/pyproject.toml +++ /dev/null @@ -1,8 +0,0 @@ -[build-system] -requires = [ - "setuptools>=42", - "setuptools_scm[toml]>=3.4", - "setuptools-git", - "wheel", - "ninja", - "cmake_build_extension"] diff --git a/thirdparty/ryml/requirements.txt b/thirdparty/ryml/requirements.txt deleted file mode 100644 index 815f8c397..000000000 --- a/thirdparty/ryml/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -tbump -wheel -cmake-build-extension -build -twine -setuptools_scm -setuptools-git -deprecation diff --git a/thirdparty/ryml/samples/add_subdirectory/run.sh b/thirdparty/ryml/samples/add_subdirectory/run.sh deleted file mode 100755 index fc2f7d858..000000000 --- a/thirdparty/ryml/samples/add_subdirectory/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -x - -# take the build type from the command line, or default to release -cfg=${1:-Release} -# make sure to run from where this file is -cd $(dirname $0) -# configure the sample -cmake -S . -B ./build/$cfg -DCMAKE_BUILD_TYPE=$cfg -# build and run the sample -cmake --build ./build/$cfg --config $cfg --target run diff --git a/thirdparty/ryml/samples/custom_c4core/run.sh b/thirdparty/ryml/samples/custom_c4core/run.sh deleted file mode 100755 index fc2f7d858..000000000 --- a/thirdparty/ryml/samples/custom_c4core/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -x - -# take the build type from the command line, or default to release -cfg=${1:-Release} -# make sure to run from where this file is -cd $(dirname $0) -# configure the sample -cmake -S . -B ./build/$cfg -DCMAKE_BUILD_TYPE=$cfg -# build and run the sample -cmake --build ./build/$cfg --config $cfg --target run diff --git a/thirdparty/ryml/samples/fetch_content/run.sh b/thirdparty/ryml/samples/fetch_content/run.sh deleted file mode 100755 index cb1d7a008..000000000 --- a/thirdparty/ryml/samples/fetch_content/run.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -x - -# take the build type from the command line, or default to release -cfg=${1:-Release} -# force cmake's FetchContent to choose a specific branch, or default to nothing -branch=${2:-} -# make sure to run from where this file is -cd $(dirname $0) -# configure the sample -cmake -S . -B ./build/$cfg -DCMAKE_BUILD_TYPE=$cfg -DRYML_BRANCH_NAME="$branch" -# build and run the sample -cmake --build ./build/$cfg --config $cfg --target run diff --git a/thirdparty/ryml/samples/find_package/run.sh b/thirdparty/ryml/samples/find_package/run.sh deleted file mode 100755 index 930783b9c..000000000 --- a/thirdparty/ryml/samples/find_package/run.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -x - -# take the build type from the command line, or default to release -cfg=${1:-Release} -# make sure to run from where this file is -cd $(dirname $0) - -#------------------------------ -# first, build and install ryml -#------------------------------ -RYML_SRC=$(cd ../.. ; pwd) -RYML_DIR=./build/$cfg/ryml-install # install ryml to this directory -# configure ryml -cmake -S "$RYML_SRC" -B ./build/$cfg/ryml-build "-DCMAKE_INSTALL_PREFIX=$RYML_DIR" -DCMAKE_BUILD_TYPE=$cfg -# build ryml -cmake --build ./build/$cfg/ryml-build --parallel --config $cfg -# install ryml -cmake --build ./build/$cfg/ryml-build --config $cfg --target install - -#----------------------------- -# now build and run the sample -#----------------------------- -# configure the sample -cmake -S . -B ./build/$cfg "-DCMAKE_PREFIX_PATH=$RYML_DIR" -DCMAKE_BUILD_TYPE=$cfg -# build and run the sample -cmake --build ./build/$cfg --config $cfg --target run diff --git a/thirdparty/ryml/samples/quickstart.cpp b/thirdparty/ryml/samples/quickstart.cpp deleted file mode 100644 index 0652f9fd5..000000000 --- a/thirdparty/ryml/samples/quickstart.cpp +++ /dev/null @@ -1,4161 +0,0 @@ -// ryml: quickstart -// -// This file does a quick tour of ryml. It has multiple self-contained -// and commented samples that illustrate how to use ryml, and how it -// works. Although this is not a unit test, the samples are written as -// a sequence of actions and predicate checks to better convey what is -// the expected result at any stage. And to ensure the code here is -// correct and up to date, it's also run as part of the CI tests. -// -// The directories that exist side-by-side with this file contain -// several examples on how to build this with cmake, such that you can -// hit the ground running. I suggest starting first with the -// add_subdirectory example, treating it just like any other -// self-contained cmake project. -// -// If something is unclear, please open an issue or send a pull -// request at https://github.com/biojppm/rapidyaml . If you have an -// issue while using ryml, it is also encouraged to try to reproduce -// the issue here, or look first through the relevant section. -// -// Happy ryml'ing! - - -//----------------------------------------------------------------------------- - -// ryml can be used as a single header, or as a simple library: -#if defined(RYML_SINGLE_HEADER) // using the single header directly in the executable - #define RYML_SINGLE_HDR_DEFINE_NOW - #include -#elif defined(RYML_SINGLE_HEADER_LIB) // using the single header from a library - #include -#else - #include - // is needed if interop with std containers is - // desired; ryml itself does not use any STL container. - // For this sample, we will be using std interop, so... - #include // optional header, provided for std:: interop - #include // needed for the examples below -#endif - -// these are needed for the examples below -#include -#include -#include -#include -#include - - -//----------------------------------------------------------------------------- - -// CONTENTS: -// -// (Each function addresses a topic and is fully self-contained. Jump -// to the function to find out about its topic.) -namespace sample { -void sample_quick_overview(); ///< briefly skim over most of the features -void sample_substr(); ///< about ryml's string views (from c4core) -void sample_parse_file(); ///< ready-to-go example of parsing a file from disk -void sample_parse_in_place(); ///< parse a mutable YAML source buffer -void sample_parse_in_arena(); ///< parse a read-only YAML source buffer -void sample_parse_reuse_tree(); ///< parse into an existing tree, maybe into a node -void sample_parse_reuse_parser(); ///< reuse an existing parser -void sample_parse_reuse_tree_and_parser(); ///< how to reuse existing trees and parsers -void sample_iterate_trees(); ///< visit individual nodes and iterate through trees -void sample_create_trees(); ///< programatically create trees -void sample_tree_arena(); ///< interact with the tree's serialization arena -void sample_fundamental_types(); ///< serialize/deserialize fundamental types -void sample_formatting(); ///< control formatting when serializing/deserializing -void sample_base64(); ///< encode/decode base64 -void sample_user_scalar_types(); ///< serialize/deserialize scalar (leaf/string) types -void sample_user_container_types(); ///< serialize/deserialize container (map or seq) types -void sample_std_types(); ///< serialize/deserialize STL containers -void sample_emit_to_container(); ///< emit to memory, eg a string or vector-like container -void sample_emit_to_stream(); ///< emit to a stream, eg std::ostream -void sample_emit_to_file(); ///< emit to a FILE* -void sample_emit_nested_node(); ///< pick a nested node as the root when emitting -void sample_json(); ///< JSON parsing and emitting -void sample_anchors_and_aliases(); ///< deal with YAML anchors and aliases -void sample_tags(); ///< deal with YAML type tags -void sample_docs(); ///< deal with YAML docs -void sample_error_handler(); ///< set a custom error handler -void sample_global_allocator(); ///< set a global allocator for ryml -void sample_per_tree_allocator(); ///< set per-tree allocators -void sample_static_trees(); ///< how to use static trees in ryml -void sample_location_tracking(); ///< track node locations in the parsed source tree -int report_checks(); -} /* namespace sample */ - -int main() -{ - sample::sample_quick_overview(); - sample::sample_substr(); - sample::sample_parse_file(); - sample::sample_parse_in_place(); - sample::sample_parse_in_arena(); - sample::sample_parse_reuse_tree(); - sample::sample_parse_reuse_parser(); - sample::sample_parse_reuse_tree_and_parser(); - sample::sample_iterate_trees(); - sample::sample_create_trees(); - sample::sample_tree_arena(); - sample::sample_fundamental_types(); - sample::sample_formatting(); - sample::sample_base64(); - sample::sample_user_scalar_types(); - sample::sample_user_container_types(); - sample::sample_std_types(); - sample::sample_emit_to_container(); - sample::sample_emit_to_stream(); - sample::sample_emit_to_file(); - sample::sample_emit_nested_node(); - sample::sample_json(); - sample::sample_anchors_and_aliases(); - sample::sample_tags(); - sample::sample_docs(); - sample::sample_error_handler(); - sample::sample_global_allocator(); - sample::sample_per_tree_allocator(); - sample::sample_static_trees(); - sample::sample_location_tracking(); - return sample::report_checks(); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace sample { - -bool report_check(int line, const char *predicate, bool result); -#ifdef __GNUC__ -#if __GNUC__ == 4 && __GNUC_MINOR__ >= 8 -struct CheckPredicate { - const char *file; - const int line; - - void operator() (bool predicate) const - { - if (!report_check(line, nullptr, predicate)) - { - RYML_DEBUG_BREAK(); - } - } -}; -#define CHECK CheckPredicate{__FILE__, __LINE__} -#endif -#endif - -#if !defined(CHECK) -/// a quick'n'dirty assertion to verify a predicate -#define CHECK(predicate) do { if(!report_check(__LINE__, #predicate, (predicate))) { RYML_DEBUG_BREAK(); } } while(0) -#endif - -//----------------------------------------------------------------------------- - -/** a brief tour over most features */ -void sample_quick_overview() -{ - // Parse YAML code in place, potentially mutating the buffer. - // It is also possible to: - // - parse a read-only buffer using parse_in_arena() - // - reuse an existing tree (advised) - // - reuse an existing parser (advised) - char yml_buf[] = "{foo: 1, bar: [2, 3], john: doe}"; - ryml::Tree tree = ryml::parse_in_place(yml_buf); - - // Note: it will always be significantly faster to use mutable - // buffers and reuse tree+parser. - // - // Below you will find samples that show how to achieve reuse; but - // please note that for brevity and clarity, many of the examples - // here are parsing immutable buffers, and not reusing tree or - // parser. - - - //------------------------------------------------------------------ - // API overview - - // ryml has a two-level API: - // - // The lower level index API is based on the indices of nodes, - // where the node's id is the node's position in the tree's data - // array. This API is very efficient, but somewhat difficult to use: - size_t root_id = tree.root_id(); - size_t bar_id = tree.find_child(root_id, "bar"); // need to get the index right - CHECK(tree.is_map(root_id)); // all of the index methods are in the tree - CHECK(tree.is_seq(bar_id)); // ... and receive the subject index - - // The node API is a lightweight abstraction sitting on top of the - // index API, but offering a much more convenient interaction: - ryml::ConstNodeRef root = tree.rootref(); - ryml::ConstNodeRef bar = tree["bar"]; - CHECK(root.is_map()); - CHECK(bar.is_seq()); - // A node ref is a lightweight handle to the tree and associated id: - CHECK(root.tree() == &tree); // a node ref points at its tree, WITHOUT refcount - CHECK(root.id() == root_id); // a node ref's id is the index of the node - CHECK(bar.id() == bar_id); // a node ref's id is the index of the node - - // The node API translates very cleanly to the index API, so most - // of the code examples below are using the node API. - - // One significant point of the node API is that it holds a raw - // pointer to the tree. Care must be taken to ensure the lifetimes - // match, so that a node will never access the tree after the tree - // went out of scope. - - - //------------------------------------------------------------------ - // To read the parsed tree - - // ConstNodeRef::operator[] does a lookup, is O(num_children[node]). - CHECK(tree["foo"].is_keyval()); - CHECK(tree["foo"].key() == "foo"); - CHECK(tree["foo"].val() == "1"); - CHECK(tree["bar"].is_seq()); - CHECK(tree["bar"].has_key()); - CHECK(tree["bar"].key() == "bar"); - // maps use string keys, seqs use integral keys: - CHECK(tree["bar"][0].val() == "2"); - CHECK(tree["bar"][1].val() == "3"); - CHECK(tree["john"].val() == "doe"); - // An integral key is the position of the child within its parent, - // so even maps can also use int keys, if the key position is - // known. - CHECK(tree[0].id() == tree["foo"].id()); - CHECK(tree[1].id() == tree["bar"].id()); - CHECK(tree[2].id() == tree["john"].id()); - // Tree::operator[](int) searches a ***root*** child by its position. - CHECK(tree[0].id() == tree["foo"].id()); // 0: first child of root - CHECK(tree[1].id() == tree["bar"].id()); // 1: first child of root - CHECK(tree[2].id() == tree["john"].id()); // 2: first child of root - // NodeRef::operator[](int) searches a ***node*** child by its position: - CHECK(bar[0].val() == "2"); // 0 means first child of bar - CHECK(bar[1].val() == "3"); // 1 means second child of bar - // NodeRef::operator[](string): - // A string key is the key of the node: lookup is by name. So it - // is only available for maps, and it is NOT available for seqs, - // since seq members do not have keys. - CHECK(tree["foo"].key() == "foo"); - CHECK(tree["bar"].key() == "bar"); - CHECK(tree["john"].key() == "john"); - CHECK(bar.is_seq()); - // CHECK(bar["BOOM!"].is_seed()); // error, seqs do not have key lookup - - // Note that maps can also use index keys as well as string keys: - CHECK(root["foo"].id() == root[0].id()); - CHECK(root["bar"].id() == root[1].id()); - CHECK(root["john"].id() == root[2].id()); - - // IMPORTANT. The ryml tree uses indexed linked lists for storing - // children, so the complexity of `Tree::operator[csubstr]` and - // `Tree::operator[size_t]` is linear on the number of root - // children. If you use `Tree::operator[]` with a large tree where - // the root has many children, you will see a performance hit. - // - // To avoid this hit, you can create your own accelerator - // structure. For example, before doing a lookup, do a single - // traverse at the root level to fill an `map` - // mapping key names to node indices; with a node index, a lookup - // (via `Tree::get()`) is O(1), so this way you can get O(log n) - // lookup from a key. (But please do not use `std::map` if you - // care about performance; use something else like a flat map or - // sorted vector). - // - // As for node refs, the difference from `NodeRef::operator[]` and - // `ConstNodeRef::operator[]` to `Tree::operator[]` is that the - // latter refers to the root node, whereas the former are invoked - // on their target node. But the lookup process works the same for - // both and their algorithmic complexity is the same: they are - // both linear in the number of direct children. But of course, - // depending on the data, that number may be very different from - // one to another. - - //------------------------------------------------------------------ - // Hierarchy: - - { - ryml::ConstNodeRef foo = root.first_child(); - ryml::ConstNodeRef john = root.last_child(); - CHECK(tree.size() == 6); // O(1) number of nodes in the tree - CHECK(root.num_children() == 3); // O(num_children[root]) - CHECK(foo.num_siblings() == 3); // O(num_children[parent(foo)]) - CHECK(foo.parent().id() == root.id()); // parent() is O(1) - CHECK(root.first_child().id() == root["foo"].id()); // first_child() is O(1) - CHECK(root.last_child().id() == root["john"].id()); // last_child() is O(1) - CHECK(john.first_sibling().id() == foo.id()); - CHECK(foo.last_sibling().id() == john.id()); - // prev_sibling(), next_sibling(): (both are O(1)) - CHECK(foo.num_siblings() == root.num_children()); - CHECK(foo.prev_sibling().id() == ryml::NONE); // foo is the first_child() - CHECK(foo.next_sibling().key() == "bar"); - CHECK(foo.next_sibling().next_sibling().key() == "john"); - CHECK(foo.next_sibling().next_sibling().next_sibling().id() == ryml::NONE); // john is the last_child() - } - - - //------------------------------------------------------------------ - // Iterating: - { - ryml::csubstr expected_keys[] = {"foo", "bar", "john"}; - // iterate children using the high-level node API: - { - size_t count = 0; - for(ryml::ConstNodeRef const& child : root.children()) - CHECK(child.key() == expected_keys[count++]); - } - // iterate siblings using the high-level node API: - { - size_t count = 0; - for(ryml::ConstNodeRef const& child : root["foo"].siblings()) - CHECK(child.key() == expected_keys[count++]); - } - // iterate children using the lower-level tree index API: - { - size_t count = 0; - for(size_t child_id = tree.first_child(root_id); child_id != ryml::NONE; child_id = tree.next_sibling(child_id)) - CHECK(tree.key(child_id) == expected_keys[count++]); - } - // iterate siblings using the lower-level tree index API: - // (notice the only difference from above is in the loop - // preamble, which calls tree.first_sibling(bar_id) instead of - // tree.first_child(root_id)) - { - size_t count = 0; - for(size_t child_id = tree.first_sibling(bar_id); child_id != ryml::NONE; child_id = tree.next_sibling(child_id)) - CHECK(tree.key(child_id) == expected_keys[count++]); - } - } - - - //------------------------------------------------------------------ - // Gotchas: - CHECK(!tree["bar"].has_val()); // seq is a container, so no val - CHECK(!tree["bar"][0].has_key()); // belongs to a seq, so no key - CHECK(!tree["bar"][1].has_key()); // belongs to a seq, so no key - //CHECK(tree["bar"].val() == BOOM!); // ... so attempting to get a val is undefined behavior - //CHECK(tree["bar"][0].key() == BOOM!); // ... so attempting to get a key is undefined behavior - //CHECK(tree["bar"][1].key() == BOOM!); // ... so attempting to get a key is undefined behavior - - - //------------------------------------------------------------------ - // Deserializing: use operator>> - { - int foo = 0, bar0 = 0, bar1 = 0; - std::string john; - root["foo"] >> foo; - root["bar"][0] >> bar0; - root["bar"][1] >> bar1; - root["john"] >> john; // requires from_chars(std::string). see serialization samples below. - CHECK(foo == 1); - CHECK(bar0 == 2); - CHECK(bar1 == 3); - CHECK(john == "doe"); - } - - - //------------------------------------------------------------------ - // Modifying existing nodes: operator<< vs operator= - - // As implied by its name, ConstNodeRef is a reference to a const - // node. It can be used to read from the node, but not write to it - // or modify the hierarchy of the node. If any modification is - // desired then a NodeRef must be used instead: - ryml::NodeRef wroot = tree.rootref(); - - // operator= assigns an existing string to the receiving node. - // This pointer will be in effect until the tree goes out of scope - // so beware to only assign from strings outliving the tree. - wroot["foo"] = "says you"; - wroot["bar"][0] = "-2"; - wroot["bar"][1] = "-3"; - wroot["john"] = "ron"; - // Now the tree is _pointing_ at the memory of the strings above. - // That is OK because those are static strings and will outlive - // the tree. - CHECK(root["foo"].val() == "says you"); - CHECK(root["bar"][0].val() == "-2"); - CHECK(root["bar"][1].val() == "-3"); - CHECK(root["john"].val() == "ron"); - // WATCHOUT: do not assign from temporary objects: - // { - // std::string crash("will dangle"); - // root["john"] = ryml::to_csubstr(crash); - // } - // CHECK(root["john"] == "dangling"); // CRASH! the string was deallocated - - // operator<< first serializes the input to the tree's arena, then - // assigns the serialized string to the receiving node. This avoids - // constraints with the lifetime, since the arena lives with the tree. - CHECK(tree.arena().empty()); - wroot["foo"] << "says who"; // requires to_chars(). see serialization samples below. - wroot["bar"][0] << 20; - wroot["bar"][1] << 30; - wroot["john"] << "deere"; - CHECK(root["foo"].val() == "says who"); - CHECK(root["bar"][0].val() == "20"); - CHECK(root["bar"][1].val() == "30"); - CHECK(root["john"].val() == "deere"); - CHECK(tree.arena() == "says who2030deere"); // the result of serializations to the tree arena - // using operator<< instead of operator=, the crash above is avoided: - { - std::string ok("in_scope"); - // root["john"] = ryml::to_csubstr(ok); // don't, will dangle - wroot["john"] << ryml::to_csubstr(ok); // OK, copy to the tree's arena - } - CHECK(root["john"] == "in_scope"); // OK! - CHECK(tree.arena() == "says who2030deerein_scope"); // the result of serializations to the tree arena - - - //------------------------------------------------------------------ - // Adding new nodes: - - // adding a keyval node to a map: - CHECK(root.num_children() == 3); - wroot["newkeyval"] = "shiny and new"; // using these strings - wroot.append_child() << ryml::key("newkeyval (serialized)") << "shiny and new (serialized)"; // serializes and assigns the serialization - CHECK(root.num_children() == 5); - CHECK(root["newkeyval"].key() == "newkeyval"); - CHECK(root["newkeyval"].val() == "shiny and new"); - CHECK(root["newkeyval (serialized)"].key() == "newkeyval (serialized)"); - CHECK(root["newkeyval (serialized)"].val() == "shiny and new (serialized)"); - CHECK( ! tree.in_arena(root["newkeyval"].key())); // it's using directly the static string above - CHECK( ! tree.in_arena(root["newkeyval"].val())); // it's using directly the static string above - CHECK( tree.in_arena(root["newkeyval (serialized)"].key())); // it's using a serialization of the string above - CHECK( tree.in_arena(root["newkeyval (serialized)"].val())); // it's using a serialization of the string above - // adding a val node to a seq: - CHECK(root["bar"].num_children() == 2); - wroot["bar"][2] = "oh so nice"; - wroot["bar"][3] << "oh so nice (serialized)"; - CHECK(root["bar"].num_children() == 4); - CHECK(root["bar"][2].val() == "oh so nice"); - CHECK(root["bar"][3].val() == "oh so nice (serialized)"); - // adding a seq node: - CHECK(root.num_children() == 5); - wroot["newseq"] |= ryml::SEQ; - wroot.append_child() << ryml::key("newseq (serialized)") |= ryml::SEQ; - CHECK(root.num_children() == 7); - CHECK(root["newseq"].num_children() == 0); - CHECK(root["newseq"].is_seq()); - CHECK(root["newseq (serialized)"].num_children() == 0); - CHECK(root["newseq (serialized)"].is_seq()); - // adding a map node: - CHECK(root.num_children() == 7); - wroot["newmap"] |= ryml::MAP; - wroot.append_child() << ryml::key("newmap (serialized)") |= ryml::MAP; - CHECK(root.num_children() == 9); - CHECK(root["newmap"].num_children() == 0); - CHECK(root["newmap"].is_map()); - CHECK(root["newmap (serialized)"].num_children() == 0); - CHECK(root["newmap (serialized)"].is_map()); - // - // When the tree is mutable, operator[] does not mutate the tree - // until the returned node is written to. - // - // Until such time, the NodeRef object keeps in itself the required - // information to write to the proper place in the tree. This is - // called being in a "seed" state. - // - // This means that passing a key/index which does not exist will - // not mutate the tree, but will instead store (in the node) the - // proper place of the tree to be able to do so, if and when it is - // required. - // - // This is a significant difference from eg, the behavior of - // std::map, which mutates the map immediately within the call to - // operator[]. - // - // All of the points above apply only if the tree is mutable. If - // the tree is const, then a NodeRef cannot be obtained from it; - // only a ConstNodeRef, which can never be used to mutate the - // tree. - CHECK(!root.has_child("I am not nothing")); - ryml::NodeRef nothing = wroot["I am nothing"]; - CHECK(nothing.valid()); // points at the tree, and a specific place in the tree - CHECK(nothing.is_seed()); // ... but nothing is there yet. - CHECK(!root.has_child("I am nothing")); // same as above - ryml::NodeRef something = wroot["I am something"]; - ryml::ConstNodeRef constsomething = wroot["I am something"]; - CHECK(!root.has_child("I am something")); // same as above - CHECK(something.valid()); - CHECK(something.is_seed()); // same as above - CHECK(!constsomething.valid()); // NOTE: because a ConstNodeRef - // cannot be used to mutate a - // tree, it is only valid() if it - // is pointing at an existing - // node. - something = "indeed"; // this will commit to the tree, mutating at the proper place - CHECK(root.has_child("I am something")); - CHECK(root["I am something"].val() == "indeed"); - CHECK(something.valid()); - CHECK(!something.is_seed()); // now the tree has this node, so the - // ref is no longer a seed - // now the constref is also valid (but it needs to be reassigned): - ryml::ConstNodeRef constsomethingnew = wroot["I am something"]; - CHECK(constsomethingnew.valid()); - // note that the old constref is now stale, because it only keeps - // the state at creation: - CHECK(!constsomething.valid()); - - - //------------------------------------------------------------------ - // Emitting: - - // emit to a FILE* - ryml::emit_yaml(tree, stdout); - // emit to a stream - std::stringstream ss; - ss << tree; - std::string stream_result = ss.str(); - // emit to a buffer: - std::string str_result = ryml::emitrs_yaml(tree); - // can emit to any given buffer: - char buf[1024]; - ryml::csubstr buf_result = ryml::emit_yaml(tree, buf); - // now check - ryml::csubstr expected_result = R"(foo: says who -bar: - - 20 - - 30 - - oh so nice - - oh so nice (serialized) -john: in_scope -newkeyval: shiny and new -newkeyval (serialized): shiny and new (serialized) -newseq: [] -newseq (serialized): [] -newmap: {} -newmap (serialized): {} -I am something: indeed -)"; - CHECK(buf_result == expected_result); - CHECK(str_result == expected_result); - CHECK(stream_result == expected_result); - // There are many possibilities to emit to buffer; - // please look at the emit sample functions below. - - //------------------------------------------------------------------ - // ConstNodeRef vs NodeRef - - ryml::NodeRef noderef = tree["bar"][0]; - ryml::ConstNodeRef constnoderef = tree["bar"][0]; - - // ConstNodeRef cannot be used to mutate the tree, but a NodeRef can: - //constnoderef = "21"; // compile error - //constnoderef << "22"; // compile error - noderef = "21"; // ok, can assign because it's not const - CHECK(tree["bar"][0].val() == "21"); - noderef << "22"; // ok, can serialize and assign because it's not const - CHECK(tree["bar"][0].val() == "22"); - - // it is not possible to obtain a NodeRef from a ConstNodeRef: - // noderef = constnoderef; // compile error - - // it is always possible to obtain a ConstNodeRef from a NodeRef: - constnoderef = noderef; // ok can assign const <- nonconst - - // If a tree is const, then only ConstNodeRef's can be - // obtained from that tree: - ryml::Tree const& consttree = tree; - //noderef = consttree["bar"][0]; // compile error - noderef = tree["bar"][0]; // ok - constnoderef = consttree["bar"][0]; // ok - - // ConstNodeRef and NodeRef can be compared for equality. - // Equality means they point at the same node. - CHECK(constnoderef == noderef); - CHECK(!(constnoderef != noderef)); - - //------------------------------------------------------------------ - // Dealing with UTF8 - ryml::Tree langs = ryml::parse_in_arena(R"( -en: Planet (Gas) -fr: Planète (Gazeuse) -ru: Планета (Газ) -ja: 惑星(ガス) -zh: 行星(气体) -# UTF8 decoding only happens in double-quoted strings,\ -# as per the YAML standard -decode this: "\u263A \xE2\x98\xBA" -and this as well: "\u2705 \U0001D11E" -)"); - // in-place UTF8 just works: - CHECK(langs["en"].val() == "Planet (Gas)"); - CHECK(langs["fr"].val() == "Planète (Gazeuse)"); - CHECK(langs["ru"].val() == "Планета (Газ)"); - CHECK(langs["ja"].val() == "惑星(ガス)"); - CHECK(langs["zh"].val() == "行星(气体)"); - // and \x \u \U codepoints are decoded (but only when they appear - // inside double-quoted strings, as dictated by the YAML - // standard): - CHECK(langs["decode this"].val() == "☺ ☺"); - CHECK(langs["and this as well"].val() == "✅ 𝄞"); - - //------------------------------------------------------------------ - // Getting the location of nodes in the source: - // - // Location tracking is opt-in: - ryml::Parser parser(ryml::ParserOptions().locations(true)); - // Now the parser will start by building the accelerator structure: - ryml::Tree tree2 = parser.parse_in_arena("expected.yml", expected_result); - // ... and use it when querying - ryml::Location loc = parser.location(tree2["bar"][1]); - CHECK(parser.location_contents(loc).begins_with("30")); - CHECK(loc.line == 3u); - CHECK(loc.col == 4u); - // For further details in location tracking, - // refer to the sample function below. -} - - -//----------------------------------------------------------------------------- - -/** demonstrate usage of ryml::substr/ryml::csubstr - * - * These types are imported from the c4core library into the ryml - * namespace You may have noticed above the use of a `csubstr` - * class. This class is defined in another library, - * [c4core](https://github.com/biojppm/c4core), which is imported by - * ryml. This is a library I use with my projects consisting of - * multiplatform low-level utilities. One of these is `c4::csubstr` - * (the name comes from "constant substring") which is a non-owning - * read-only string view, with many methods that make it practical to - * use (I would certainly argue more practical than `std::string`). In - * fact, `c4::csubstr` and its writeable counterpart `c4::substr` are - * the workhorses of the ryml parsing and serialization code. - * - * @see https://c4core.docsforge.com/master/api/c4/basic_substring/ - * @see https://c4core.docsforge.com/master/api/c4/#substr - * @see https://c4core.docsforge.com/master/api/c4/#csubstr - */ -void sample_substr() -{ - // substr is a mutable view: pointer and length to a string in memory. - // csubstr is a const-substr (immutable). - - // construct from explicit args - { - const char foobar_str[] = "foobar"; - auto s = ryml::csubstr(foobar_str, strlen(foobar_str)); - CHECK(s == "foobar"); - CHECK(s.size() == 6); - CHECK(s.data() == foobar_str); - CHECK(s.size() == s.len); - CHECK(s.data() == s.str); - } - - // construct from a string array - { - const char foobar_str[] = "foobar"; - ryml::csubstr s = foobar_str; - CHECK(s == "foobar"); - CHECK(s != "foobar0"); - CHECK(s.size() == 6); - CHECK(s.data() == foobar_str); - CHECK(s.size() == s.len); - CHECK(s.data() == s.str); - } - // you can also declare directly in-place from an array: - { - ryml::csubstr s = "foobar"; - CHECK(s == "foobar"); - CHECK(s != "foobar0"); - CHECK(s.size() == 6); - CHECK(s.size() == s.len); - CHECK(s.data() == s.str); - } - - // construct from a C-string: - // - // Since the input is only a pointer, the string length can only - // be found with a call to strlen(). To make this cost evident, we - // require a call to to_csubstr(): - { - const char *foobar_str = "foobar"; - ryml::csubstr s = ryml::to_csubstr(foobar_str); - CHECK(s == "foobar"); - CHECK(s != "foobar0"); - CHECK(s.size() == 6); - CHECK(s.size() == s.len); - CHECK(s.data() == s.str); - } - - // construct from a std::string: same approach as above. - // requires inclusion of the header - // or of the umbrella header . - // this was a deliberate design choice to avoid requiring - // the heavy std:: allocation machinery - { - std::string foobar_str = "foobar"; - ryml::csubstr s = ryml::to_csubstr(foobar_str); // defined in - CHECK(s == "foobar"); - CHECK(s != "foobar0"); - CHECK(s.size() == 6); - CHECK(s.size() == s.len); - CHECK(s.data() == s.str); - } - - // convert substr -> csubstr - { - char buf[] = "foo"; - ryml::substr foo = buf; - CHECK(foo.len == 3); - CHECK(foo.data() == buf); - ryml::csubstr cfoo = foo; - CHECK(cfoo.data() == buf); - } - // cannot convert csubstr -> substr: - { - // ryml::substr foo2 = cfoo; // compile error: cannot write to csubstr - } - - // construct from char[]/const char[]: mutable vs immutable memory - { - char const foobar_str_ro[] = "foobar"; // ro := read-only - char foobar_str_rw[] = "foobar"; // rw := read-write - static_assert(std::is_array::value, "this is an array"); - static_assert(std::is_array::value, "this is an array"); - // csubstr <- read-only memory - { - ryml::csubstr foobar = foobar_str_ro; - CHECK(foobar.data() == foobar_str_ro); - CHECK(foobar.size() == strlen(foobar_str_ro)); - CHECK(foobar == "foobar"); // AKA strcmp - } - // csubstr <- read-write memory: you can create an immutable csubstr from mutable memory - { - ryml::csubstr foobar = foobar_str_rw; - CHECK(foobar.data() == foobar_str_rw); - CHECK(foobar.size() == strlen(foobar_str_rw)); - CHECK(foobar == "foobar"); // AKA strcmp - } - // substr <- read-write memory. - { - ryml::substr foobar = foobar_str_rw; - CHECK(foobar.data() == foobar_str_rw); - CHECK(foobar.size() == strlen(foobar_str_rw)); - CHECK(foobar == "foobar"); // AKA strcmp - } - // substr <- ro is impossible. - { - //ryml::substr foobar = foobar_str_ro; // compile error! - } - } - - // construct from char*/const char*: mutable vs immutable memory. - // use to_substr()/to_csubstr() - { - char const* foobar_str_ro = "foobar"; // ro := read-only - char foobar_str_rw_[] = "foobar"; // rw := read-write - char * foobar_str_rw = foobar_str_rw_; // rw := read-write - static_assert(!std::is_array::value, "this is a decayed pointer"); - static_assert(!std::is_array::value, "this is a decayed pointer"); - // csubstr <- read-only memory - { - //ryml::csubstr foobar = foobar_str_ro; // compile error: length is not known - ryml::csubstr foobar = ryml::to_csubstr(foobar_str_ro); - CHECK(foobar.data() == foobar_str_ro); - CHECK(foobar.size() == strlen(foobar_str_ro)); - CHECK(foobar == "foobar"); // AKA strcmp - } - // csubstr <- read-write memory: you can create an immutable csubstr from mutable memory - { - ryml::csubstr foobar = ryml::to_csubstr(foobar_str_rw); - CHECK(foobar.data() == foobar_str_rw); - CHECK(foobar.size() == strlen(foobar_str_rw)); - CHECK(foobar == "foobar"); // AKA strcmp - } - // substr <- read-write memory. - { - ryml::substr foobar = ryml::to_substr(foobar_str_rw); - CHECK(foobar.data() == foobar_str_rw); - CHECK(foobar.size() == strlen(foobar_str_rw)); - CHECK(foobar == "foobar"); // AKA strcmp - } - // substr <- read-only is impossible. - { - //ryml::substr foobar = ryml::to_substr(foobar_str_ro); // compile error! - } - } - - // substr is mutable, without changing the size: - { - char buf[] = "foobar"; - ryml::substr foobar = buf; - CHECK(foobar == "foobar"); - foobar[0] = 'F'; CHECK(foobar == "Foobar"); - foobar.back() = 'R'; CHECK(foobar == "FoobaR"); - foobar.reverse(); CHECK(foobar == "RabooF"); - foobar.reverse(); CHECK(foobar == "FoobaR"); - foobar.reverse_sub(1, 4); CHECK(foobar == "FabooR"); - foobar.reverse_sub(1, 4); CHECK(foobar == "FoobaR"); - foobar.reverse_range(2, 5); CHECK(foobar == "FoaboR"); - foobar.reverse_range(2, 5); CHECK(foobar == "FoobaR"); - foobar.replace('o', '0'); CHECK(foobar == "F00baR"); - foobar.replace('a', '_'); CHECK(foobar == "F00b_R"); - foobar.replace("_0b", 'a'); CHECK(foobar == "FaaaaR"); - foobar.toupper(); CHECK(foobar == "FAAAAR"); - foobar.tolower(); CHECK(foobar == "faaaar"); - foobar.fill('.'); CHECK(foobar == "......"); - // see also: - // - .erase() - // - .replace_all() - } - - // sub-views - { - ryml::csubstr s = "fooFOObarBAR"; - CHECK(s.len == 12u); - // sub(): <- first,[num] - CHECK(s.sub(0) == "fooFOObarBAR"); - CHECK(s.sub(0, 12) == "fooFOObarBAR"); - CHECK(s.sub(0, 3) == "foo" ); - CHECK(s.sub(3) == "FOObarBAR"); - CHECK(s.sub(3, 3) == "FOO" ); - CHECK(s.sub(6) == "barBAR"); - CHECK(s.sub(6, 3) == "bar" ); - CHECK(s.sub(9) == "BAR"); - CHECK(s.sub(9, 3) == "BAR"); - // first(): <- length - CHECK(s.first(0) == "" ); - CHECK(s.first(1) == "f" ); - CHECK(s.first(2) != "f" ); - CHECK(s.first(2) == "fo" ); - CHECK(s.first(3) == "foo"); - // last(): <- length - CHECK(s.last(0) == ""); - CHECK(s.last(1) == "R"); - CHECK(s.last(2) == "AR"); - CHECK(s.last(3) == "BAR"); - // range(): <- first, last - CHECK(s.range(0, 12) == "fooFOObarBAR"); - CHECK(s.range(1, 12) == "ooFOObarBAR"); - CHECK(s.range(1, 11) == "ooFOObarBA" ); - CHECK(s.range(2, 10) == "oFOObarB" ); - CHECK(s.range(3, 9) == "FOObar" ); - // offs(): offset from beginning, end - CHECK(s.offs(0, 0) == "fooFOObarBAR"); - CHECK(s.offs(1, 0) == "ooFOObarBAR"); - CHECK(s.offs(1, 1) == "ooFOObarBA" ); - CHECK(s.offs(2, 1) == "oFOObarBA" ); - CHECK(s.offs(2, 2) == "oFOObarB" ); - CHECK(s.offs(3, 3) == "FOObar" ); - // right_of(): <- pos, include_pos - CHECK(s.right_of(0, true) == "fooFOObarBAR"); - CHECK(s.right_of(0, false) == "ooFOObarBAR"); - CHECK(s.right_of(1, true) == "ooFOObarBAR"); - CHECK(s.right_of(1, false) == "oFOObarBAR"); - CHECK(s.right_of(2, true) == "oFOObarBAR"); - CHECK(s.right_of(2, false) == "FOObarBAR"); - CHECK(s.right_of(3, true) == "FOObarBAR"); - CHECK(s.right_of(3, false) == "OObarBAR"); - // left_of() <- pos, include_pos - CHECK(s.left_of(12, false) == "fooFOObarBAR"); - CHECK(s.left_of(11, true) == "fooFOObarBAR"); - CHECK(s.left_of(11, false) == "fooFOObarBA" ); - CHECK(s.left_of(10, true) == "fooFOObarBA" ); - CHECK(s.left_of(10, false) == "fooFOObarB" ); - CHECK(s.left_of( 9, true) == "fooFOObarB" ); - CHECK(s.left_of( 9, false) == "fooFOObar" ); - // left_of(),right_of() <- substr - ryml::csubstr FOO = s.sub(3, 3); - CHECK(s.is_super(FOO)); // required for the following - CHECK(s.left_of(FOO) == "foo"); - CHECK(s.right_of(FOO) == "barBAR"); - } - - // is_sub(),is_super() - { - ryml::csubstr foobar = "foobar"; - ryml::csubstr foo = foobar.first(3); - CHECK(foo.is_sub(foobar)); - CHECK(foo.is_sub(foo)); - CHECK(!foo.is_super(foobar)); - CHECK(!foobar.is_sub(foo)); - // identity comparison is true: - CHECK(foo.is_super(foo)); - CHECK(foo.is_sub(foo)); - CHECK(foobar.is_sub(foobar)); - CHECK(foobar.is_super(foobar)); - } - - // overlaps() - { - ryml::csubstr foobar = "foobar"; - ryml::csubstr foo = foobar.first(3); - ryml::csubstr oba = foobar.offs(2, 1); - ryml::csubstr abc = "abc"; - CHECK(foobar.overlaps(foo)); - CHECK(foobar.overlaps(oba)); - CHECK(foo.overlaps(foobar)); - CHECK(foo.overlaps(oba)); - CHECK(!foo.overlaps(abc)); - CHECK(!abc.overlaps(foo)); - } - - // triml(): trim characters from the left - // trimr(): trim characters from the right - // trim(): trim characters from left AND right - { - CHECK(ryml::csubstr(" \t\n\rcontents without whitespace\t \n\r").trim("\t \n\r") == "contents without whitespace"); - ryml::csubstr aaabbb = "aaabbb"; - ryml::csubstr aaa___bbb = "aaa___bbb"; - // trim a character: - CHECK(aaabbb.triml('a') == aaabbb.last(3)); // bbb - CHECK(aaabbb.trimr('a') == aaabbb); - CHECK(aaabbb.trim ('a') == aaabbb.last(3)); // bbb - CHECK(aaabbb.triml('b') == aaabbb); - CHECK(aaabbb.trimr('b') == aaabbb.first(3)); // aaa - CHECK(aaabbb.trim ('b') == aaabbb.first(3)); // aaa - CHECK(aaabbb.triml('c') == aaabbb); - CHECK(aaabbb.trimr('c') == aaabbb); - CHECK(aaabbb.trim ('c') == aaabbb); - CHECK(aaa___bbb.triml('a') == aaa___bbb.last(6)); // ___bbb - CHECK(aaa___bbb.trimr('a') == aaa___bbb); - CHECK(aaa___bbb.trim ('a') == aaa___bbb.last(6)); // ___bbb - CHECK(aaa___bbb.triml('b') == aaa___bbb); - CHECK(aaa___bbb.trimr('b') == aaa___bbb.first(6)); // aaa___ - CHECK(aaa___bbb.trim ('b') == aaa___bbb.first(6)); // aaa___ - CHECK(aaa___bbb.triml('c') == aaa___bbb); - CHECK(aaa___bbb.trimr('c') == aaa___bbb); - CHECK(aaa___bbb.trim ('c') == aaa___bbb); - // trim ANY of the characters: - CHECK(aaabbb.triml("ab") == ""); - CHECK(aaabbb.trimr("ab") == ""); - CHECK(aaabbb.trim ("ab") == ""); - CHECK(aaabbb.triml("ba") == ""); - CHECK(aaabbb.trimr("ba") == ""); - CHECK(aaabbb.trim ("ba") == ""); - CHECK(aaabbb.triml("cd") == aaabbb); - CHECK(aaabbb.trimr("cd") == aaabbb); - CHECK(aaabbb.trim ("cd") == aaabbb); - CHECK(aaa___bbb.triml("ab") == aaa___bbb.last(6)); // ___bbb - CHECK(aaa___bbb.triml("ba") == aaa___bbb.last(6)); // ___bbb - CHECK(aaa___bbb.triml("cd") == aaa___bbb); - CHECK(aaa___bbb.trimr("ab") == aaa___bbb.first(6)); // aaa___ - CHECK(aaa___bbb.trimr("ba") == aaa___bbb.first(6)); // aaa___ - CHECK(aaa___bbb.trimr("cd") == aaa___bbb); - CHECK(aaa___bbb.trim ("ab") == aaa___bbb.range(3, 6)); // ___ - CHECK(aaa___bbb.trim ("ba") == aaa___bbb.range(3, 6)); // ___ - CHECK(aaa___bbb.trim ("cd") == aaa___bbb); - } - - // unquoted(): - { - CHECK(ryml::csubstr(R"('this is is single quoted')").unquoted() == "this is is single quoted"); - CHECK(ryml::csubstr(R"("this is is double quoted")").unquoted() == "this is is double quoted"); - } - - // stripl(): remove pattern from the left - // stripr(): remove pattern from the right - { - ryml::csubstr abc___cba = "abc___cba"; - ryml::csubstr abc___abc = "abc___abc"; - CHECK(abc___cba.stripl("abc") == abc___cba.last(6)); // ___cba - CHECK(abc___cba.stripr("abc") == abc___cba); - CHECK(abc___cba.stripl("ab") == abc___cba.last(7)); // c___cba - CHECK(abc___cba.stripr("ab") == abc___cba); - CHECK(abc___cba.stripl("a") == abc___cba.last(8)); // bc___cba, same as triml('a') - CHECK(abc___cba.stripr("a") == abc___cba.first(8)); - CHECK(abc___abc.stripl("abc") == abc___abc.last(6)); // ___abc - CHECK(abc___abc.stripr("abc") == abc___abc.first(6)); // abc___ - CHECK(abc___abc.stripl("ab") == abc___abc.last(7)); // c___cba - CHECK(abc___abc.stripr("ab") == abc___abc); - CHECK(abc___abc.stripl("a") == abc___abc.last(8)); // bc___cba, same as triml('a') - CHECK(abc___abc.stripr("a") == abc___abc); - } - - // begins_with()/ends_with() - // begins_with_any()/ends_with_any() - { - ryml::csubstr s = "foobar123"; - // char overloads - CHECK(s.begins_with('f')); - CHECK(s.ends_with('3')); - CHECK(!s.ends_with('2')); - CHECK(!s.ends_with('o')); - // char[] overloads - CHECK(s.begins_with("foobar")); - CHECK(s.begins_with("foo")); - CHECK(s.begins_with_any("foo")); - CHECK(!s.begins_with("oof")); - CHECK(s.begins_with_any("oof")); - CHECK(s.ends_with("23")); - CHECK(s.ends_with("123")); - CHECK(s.ends_with_any("123")); - CHECK(!s.ends_with("321")); - CHECK(s.ends_with_any("231")); - } - - // select() - { - ryml::csubstr s = "0123456789"; - CHECK(s.select('0') == s.sub(0, 1)); - CHECK(s.select('1') == s.sub(1, 1)); - CHECK(s.select('2') == s.sub(2, 1)); - CHECK(s.select('8') == s.sub(8, 1)); - CHECK(s.select('9') == s.sub(9, 1)); - CHECK(s.select("0123") == s.range(0, 4)); - CHECK(s.select("012" ) == s.range(0, 3)); - CHECK(s.select("01" ) == s.range(0, 2)); - CHECK(s.select("0" ) == s.range(0, 1)); - CHECK(s.select( "123") == s.range(1, 4)); - CHECK(s.select( "23") == s.range(2, 4)); - CHECK(s.select( "3") == s.range(3, 4)); - } - - // find() - { - ryml::csubstr s012345 = "012345"; - // find single characters: - CHECK(s012345.find('a') == ryml::npos); - CHECK(s012345.find('0' ) == 0u); - CHECK(s012345.find('0', 1u) == ryml::npos); - CHECK(s012345.find('1' ) == 1u); - CHECK(s012345.find('1', 2u) == ryml::npos); - CHECK(s012345.find('2' ) == 2u); - CHECK(s012345.find('2', 3u) == ryml::npos); - CHECK(s012345.find('3' ) == 3u); - CHECK(s012345.find('3', 4u) == ryml::npos); - // find patterns - CHECK(s012345.find("ab" ) == ryml::npos); - CHECK(s012345.find("01" ) == 0u); - CHECK(s012345.find("01", 1u) == ryml::npos); - CHECK(s012345.find("12" ) == 1u); - CHECK(s012345.find("12", 2u) == ryml::npos); - CHECK(s012345.find("23" ) == 2u); - CHECK(s012345.find("23", 3u) == ryml::npos); - } - - // count(): count the number of occurrences of a character - { - ryml::csubstr buf = "00110022003300440055"; - CHECK(buf.count('1' ) == 2u); - CHECK(buf.count('1', 0u) == 2u); - CHECK(buf.count('1', 1u) == 2u); - CHECK(buf.count('1', 2u) == 2u); - CHECK(buf.count('1', 3u) == 1u); - CHECK(buf.count('1', 4u) == 0u); - CHECK(buf.count('1', 5u) == 0u); - CHECK(buf.count('0' ) == 10u); - CHECK(buf.count('0', 0u) == 10u); - CHECK(buf.count('0', 1u) == 9u); - CHECK(buf.count('0', 2u) == 8u); - CHECK(buf.count('0', 3u) == 8u); - CHECK(buf.count('0', 4u) == 8u); - CHECK(buf.count('0', 5u) == 7u); - CHECK(buf.count('0', 6u) == 6u); - CHECK(buf.count('0', 7u) == 6u); - CHECK(buf.count('0', 8u) == 6u); - CHECK(buf.count('0', 9u) == 5u); - CHECK(buf.count('0', 10u) == 4u); - CHECK(buf.count('0', 11u) == 4u); - CHECK(buf.count('0', 12u) == 4u); - CHECK(buf.count('0', 13u) == 3u); - CHECK(buf.count('0', 14u) == 2u); - CHECK(buf.count('0', 15u) == 2u); - CHECK(buf.count('0', 16u) == 2u); - CHECK(buf.count('0', 17u) == 1u); - CHECK(buf.count('0', 18u) == 0u); - CHECK(buf.count('0', 19u) == 0u); - CHECK(buf.count('0', 20u) == 0u); - } - - // first_of(),last_of() - { - ryml::csubstr s012345 = "012345"; - CHECK(s012345.first_of('a') == ryml::npos); - CHECK(s012345.first_of("ab") == ryml::npos); - CHECK(s012345.first_of('0') == 0u); - CHECK(s012345.first_of("0") == 0u); - CHECK(s012345.first_of("01") == 0u); - CHECK(s012345.first_of("10") == 0u); - CHECK(s012345.first_of("012") == 0u); - CHECK(s012345.first_of("210") == 0u); - CHECK(s012345.first_of("0123") == 0u); - CHECK(s012345.first_of("3210") == 0u); - CHECK(s012345.first_of("01234") == 0u); - CHECK(s012345.first_of("43210") == 0u); - CHECK(s012345.first_of("012345") == 0u); - CHECK(s012345.first_of("543210") == 0u); - CHECK(s012345.first_of('5') == 5u); - CHECK(s012345.first_of("5") == 5u); - CHECK(s012345.first_of("45") == 4u); - CHECK(s012345.first_of("54") == 4u); - CHECK(s012345.first_of("345") == 3u); - CHECK(s012345.first_of("543") == 3u); - CHECK(s012345.first_of("2345") == 2u); - CHECK(s012345.first_of("5432") == 2u); - CHECK(s012345.first_of("12345") == 1u); - CHECK(s012345.first_of("54321") == 1u); - CHECK(s012345.first_of("012345") == 0u); - CHECK(s012345.first_of("543210") == 0u); - CHECK(s012345.first_of('0', 6u) == ryml::npos); - CHECK(s012345.first_of('5', 6u) == ryml::npos); - CHECK(s012345.first_of("012345", 6u) == ryml::npos); - // - CHECK(s012345.last_of('a') == ryml::npos); - CHECK(s012345.last_of("ab") == ryml::npos); - CHECK(s012345.last_of('0') == 0u); - CHECK(s012345.last_of("0") == 0u); - CHECK(s012345.last_of("01") == 1u); - CHECK(s012345.last_of("10") == 1u); - CHECK(s012345.last_of("012") == 2u); - CHECK(s012345.last_of("210") == 2u); - CHECK(s012345.last_of("0123") == 3u); - CHECK(s012345.last_of("3210") == 3u); - CHECK(s012345.last_of("01234") == 4u); - CHECK(s012345.last_of("43210") == 4u); - CHECK(s012345.last_of("012345") == 5u); - CHECK(s012345.last_of("543210") == 5u); - CHECK(s012345.last_of('5') == 5u); - CHECK(s012345.last_of("5") == 5u); - CHECK(s012345.last_of("45") == 5u); - CHECK(s012345.last_of("54") == 5u); - CHECK(s012345.last_of("345") == 5u); - CHECK(s012345.last_of("543") == 5u); - CHECK(s012345.last_of("2345") == 5u); - CHECK(s012345.last_of("5432") == 5u); - CHECK(s012345.last_of("12345") == 5u); - CHECK(s012345.last_of("54321") == 5u); - CHECK(s012345.last_of("012345") == 5u); - CHECK(s012345.last_of("543210") == 5u); - CHECK(s012345.last_of('0', 6u) == 0u); - CHECK(s012345.last_of('5', 6u) == 5u); - CHECK(s012345.last_of("012345", 6u) == 5u); - } - - // first_not_of(), last_not_of() - { - ryml::csubstr s012345 = "012345"; - CHECK(s012345.first_not_of('a') == 0u); - CHECK(s012345.first_not_of("ab") == 0u); - CHECK(s012345.first_not_of('0') == 1u); - CHECK(s012345.first_not_of("0") == 1u); - CHECK(s012345.first_not_of("01") == 2u); - CHECK(s012345.first_not_of("10") == 2u); - CHECK(s012345.first_not_of("012") == 3u); - CHECK(s012345.first_not_of("210") == 3u); - CHECK(s012345.first_not_of("0123") == 4u); - CHECK(s012345.first_not_of("3210") == 4u); - CHECK(s012345.first_not_of("01234") == 5u); - CHECK(s012345.first_not_of("43210") == 5u); - CHECK(s012345.first_not_of("012345") == ryml::npos); - CHECK(s012345.first_not_of("543210") == ryml::npos); - CHECK(s012345.first_not_of('5') == 0u); - CHECK(s012345.first_not_of("5") == 0u); - CHECK(s012345.first_not_of("45") == 0u); - CHECK(s012345.first_not_of("54") == 0u); - CHECK(s012345.first_not_of("345") == 0u); - CHECK(s012345.first_not_of("543") == 0u); - CHECK(s012345.first_not_of("2345") == 0u); - CHECK(s012345.first_not_of("5432") == 0u); - CHECK(s012345.first_not_of("12345") == 0u); - CHECK(s012345.first_not_of("54321") == 0u); - CHECK(s012345.first_not_of("012345") == ryml::npos); - CHECK(s012345.first_not_of("543210") == ryml::npos); - CHECK(s012345.last_not_of('a') == 5u); - CHECK(s012345.last_not_of("ab") == 5u); - CHECK(s012345.last_not_of('5') == 4u); - CHECK(s012345.last_not_of("5") == 4u); - CHECK(s012345.last_not_of("45") == 3u); - CHECK(s012345.last_not_of("54") == 3u); - CHECK(s012345.last_not_of("345") == 2u); - CHECK(s012345.last_not_of("543") == 2u); - CHECK(s012345.last_not_of("2345") == 1u); - CHECK(s012345.last_not_of("5432") == 1u); - CHECK(s012345.last_not_of("12345") == 0u); - CHECK(s012345.last_not_of("54321") == 0u); - CHECK(s012345.last_not_of("012345") == ryml::npos); - CHECK(s012345.last_not_of("543210") == ryml::npos); - CHECK(s012345.last_not_of('0') == 5u); - CHECK(s012345.last_not_of("0") == 5u); - CHECK(s012345.last_not_of("01") == 5u); - CHECK(s012345.last_not_of("10") == 5u); - CHECK(s012345.last_not_of("012") == 5u); - CHECK(s012345.last_not_of("210") == 5u); - CHECK(s012345.last_not_of("0123") == 5u); - CHECK(s012345.last_not_of("3210") == 5u); - CHECK(s012345.last_not_of("01234") == 5u); - CHECK(s012345.last_not_of("43210") == 5u); - CHECK(s012345.last_not_of("012345") == ryml::npos); - CHECK(s012345.last_not_of("543210") == ryml::npos); - } - - // first_non_empty_span() - { - CHECK(ryml::csubstr("foo bar").first_non_empty_span() == "foo"); - CHECK(ryml::csubstr(" foo bar").first_non_empty_span() == "foo"); - CHECK(ryml::csubstr("\n \r \t foo bar").first_non_empty_span() == "foo"); - CHECK(ryml::csubstr("\n \r \t foo\n\r\t bar").first_non_empty_span() == "foo"); - CHECK(ryml::csubstr("\n \r \t foo\n\r\t bar").first_non_empty_span() == "foo"); - CHECK(ryml::csubstr(",\n \r \t foo\n\r\t bar").first_non_empty_span() == ","); - } - // first_uint_span() - { - CHECK(ryml::csubstr("1234 asdkjh").first_uint_span() == "1234"); - CHECK(ryml::csubstr("1234\rasdkjh").first_uint_span() == "1234"); - CHECK(ryml::csubstr("1234\tasdkjh").first_uint_span() == "1234"); - CHECK(ryml::csubstr("1234\nasdkjh").first_uint_span() == "1234"); - CHECK(ryml::csubstr("1234]asdkjh").first_uint_span() == "1234"); - CHECK(ryml::csubstr("1234)asdkjh").first_uint_span() == "1234"); - CHECK(ryml::csubstr("1234gasdkjh").first_uint_span() == ""); - } - // first_int_span() - { - CHECK(ryml::csubstr("-1234 asdkjh").first_int_span() == "-1234"); - CHECK(ryml::csubstr("-1234\rasdkjh").first_int_span() == "-1234"); - CHECK(ryml::csubstr("-1234\tasdkjh").first_int_span() == "-1234"); - CHECK(ryml::csubstr("-1234\nasdkjh").first_int_span() == "-1234"); - CHECK(ryml::csubstr("-1234]asdkjh").first_int_span() == "-1234"); - CHECK(ryml::csubstr("-1234)asdkjh").first_int_span() == "-1234"); - CHECK(ryml::csubstr("-1234gasdkjh").first_int_span() == ""); - } - // first_real_span() - { - CHECK(ryml::csubstr("-1234 asdkjh").first_real_span() == "-1234"); - CHECK(ryml::csubstr("-1234\rasdkjh").first_real_span() == "-1234"); - CHECK(ryml::csubstr("-1234\tasdkjh").first_real_span() == "-1234"); - CHECK(ryml::csubstr("-1234\nasdkjh").first_real_span() == "-1234"); - CHECK(ryml::csubstr("-1234]asdkjh").first_real_span() == "-1234"); - CHECK(ryml::csubstr("-1234)asdkjh").first_real_span() == "-1234"); - CHECK(ryml::csubstr("-1234gasdkjh").first_real_span() == ""); - CHECK(ryml::csubstr("1.234 asdkjh").first_real_span() == "1.234"); - CHECK(ryml::csubstr("1.234e+5 asdkjh").first_real_span() == "1.234e+5"); - CHECK(ryml::csubstr("1.234e-5 asdkjh").first_real_span() == "1.234e-5"); - CHECK(ryml::csubstr("1.234 asdkjh").first_real_span() == "1.234"); - CHECK(ryml::csubstr("1.234e+5 asdkjh").first_real_span() == "1.234e+5"); - CHECK(ryml::csubstr("1.234e-5 asdkjh").first_real_span() == "1.234e-5"); - CHECK(ryml::csubstr("-1.234 asdkjh").first_real_span() == "-1.234"); - CHECK(ryml::csubstr("-1.234e+5 asdkjh").first_real_span() == "-1.234e+5"); - CHECK(ryml::csubstr("-1.234e-5 asdkjh").first_real_span() == "-1.234e-5"); - // hexadecimal real numbers - CHECK(ryml::csubstr("0x1.e8480p+19 asdkjh").first_real_span() == "0x1.e8480p+19"); - CHECK(ryml::csubstr("0x1.e8480p-19 asdkjh").first_real_span() == "0x1.e8480p-19"); - CHECK(ryml::csubstr("-0x1.e8480p+19 asdkjh").first_real_span() == "-0x1.e8480p+19"); - CHECK(ryml::csubstr("-0x1.e8480p-19 asdkjh").first_real_span() == "-0x1.e8480p-19"); - CHECK(ryml::csubstr("+0x1.e8480p+19 asdkjh").first_real_span() == "+0x1.e8480p+19"); - CHECK(ryml::csubstr("+0x1.e8480p-19 asdkjh").first_real_span() == "+0x1.e8480p-19"); - // binary real numbers - CHECK(ryml::csubstr("0b101.011p+19 asdkjh").first_real_span() == "0b101.011p+19"); - CHECK(ryml::csubstr("0b101.011p-19 asdkjh").first_real_span() == "0b101.011p-19"); - CHECK(ryml::csubstr("-0b101.011p+19 asdkjh").first_real_span() == "-0b101.011p+19"); - CHECK(ryml::csubstr("-0b101.011p-19 asdkjh").first_real_span() == "-0b101.011p-19"); - CHECK(ryml::csubstr("+0b101.011p+19 asdkjh").first_real_span() == "+0b101.011p+19"); - CHECK(ryml::csubstr("+0b101.011p-19 asdkjh").first_real_span() == "+0b101.011p-19"); - // octal real numbers - CHECK(ryml::csubstr("0o173.045p+19 asdkjh").first_real_span() == "0o173.045p+19"); - CHECK(ryml::csubstr("0o173.045p-19 asdkjh").first_real_span() == "0o173.045p-19"); - CHECK(ryml::csubstr("-0o173.045p+19 asdkjh").first_real_span() == "-0o173.045p+19"); - CHECK(ryml::csubstr("-0o173.045p-19 asdkjh").first_real_span() == "-0o173.045p-19"); - CHECK(ryml::csubstr("+0o173.045p+19 asdkjh").first_real_span() == "+0o173.045p+19"); - CHECK(ryml::csubstr("+0o173.045p-19 asdkjh").first_real_span() == "+0o173.045p-19"); - } - // see also is_number() - - // basename(), dirname(), extshort(), extlong() - { - CHECK(ryml::csubstr("/path/to/file.tar.gz").basename() == "file.tar.gz"); - CHECK(ryml::csubstr("/path/to/file.tar.gz").dirname() == "/path/to/"); - CHECK(ryml::csubstr("C:\\path\\to\\file.tar.gz").basename('\\') == "file.tar.gz"); - CHECK(ryml::csubstr("C:\\path\\to\\file.tar.gz").dirname('\\') == "C:\\path\\to\\"); - CHECK(ryml::csubstr("/path/to/file.tar.gz").extshort() == "gz"); - CHECK(ryml::csubstr("/path/to/file.tar.gz").extlong() == "tar.gz"); - CHECK(ryml::csubstr("/path/to/file.tar.gz").name_wo_extshort() == "/path/to/file.tar"); - CHECK(ryml::csubstr("/path/to/file.tar.gz").name_wo_extlong() == "/path/to/file"); - } - - // split() - { - using namespace ryml; - csubstr parts[] = {"aa", "bb", "cc", "dd", "ee", "ff"}; - { - size_t count = 0; - for(csubstr part : csubstr("aa/bb/cc/dd/ee/ff").split('/')) - CHECK(part == parts[count++]); - CHECK(count == 6u); - } - { - size_t count = 0; - for(csubstr part : csubstr("aa.bb.cc.dd.ee.ff").split('.')) - CHECK(part == parts[count++]); - CHECK(count == 6u); - } - { - size_t count = 0; - for(csubstr part : csubstr("aa-bb-cc-dd-ee-ff").split('-')) - CHECK(part == parts[count++]); - CHECK(count == 6u); - } - // see also next_split() - } - - // pop_left(), pop_right() --- non-greedy version - // gpop_left(), gpop_right() --- greedy version - { - const bool skip_empty = true; - // pop_left(): pop the last element from the left - CHECK(ryml::csubstr( "0/1/2" ). pop_left('/' ) == "0" ); - CHECK(ryml::csubstr( "/0/1/2" ). pop_left('/' ) == "" ); - CHECK(ryml::csubstr("//0/1/2" ). pop_left('/' ) == "" ); - CHECK(ryml::csubstr( "0/1/2" ). pop_left('/', skip_empty) == "0" ); - CHECK(ryml::csubstr( "/0/1/2" ). pop_left('/', skip_empty) == "/0" ); - CHECK(ryml::csubstr("//0/1/2" ). pop_left('/', skip_empty) == "//0" ); - // gpop_left(): pop all but the first element (greedy pop) - CHECK(ryml::csubstr( "0/1/2" ).gpop_left('/' ) == "0/1" ); - CHECK(ryml::csubstr( "/0/1/2" ).gpop_left('/' ) == "/0/1" ); - CHECK(ryml::csubstr("//0/1/2" ).gpop_left('/' ) == "//0/1" ); - CHECK(ryml::csubstr( "0/1/2/" ).gpop_left('/' ) == "0/1/2"); - CHECK(ryml::csubstr( "/0/1/2/" ).gpop_left('/' ) == "/0/1/2"); - CHECK(ryml::csubstr("//0/1/2/" ).gpop_left('/' ) == "//0/1/2"); - CHECK(ryml::csubstr( "0/1/2//" ).gpop_left('/' ) == "0/1/2/"); - CHECK(ryml::csubstr( "/0/1/2//" ).gpop_left('/' ) == "/0/1/2/"); - CHECK(ryml::csubstr("//0/1/2//" ).gpop_left('/' ) == "//0/1/2/"); - CHECK(ryml::csubstr( "0/1/2" ).gpop_left('/', skip_empty) == "0/1" ); - CHECK(ryml::csubstr( "/0/1/2" ).gpop_left('/', skip_empty) == "/0/1" ); - CHECK(ryml::csubstr("//0/1/2" ).gpop_left('/', skip_empty) == "//0/1" ); - CHECK(ryml::csubstr( "0/1/2/" ).gpop_left('/', skip_empty) == "0/1" ); - CHECK(ryml::csubstr( "/0/1/2/" ).gpop_left('/', skip_empty) == "/0/1" ); - CHECK(ryml::csubstr("//0/1/2/" ).gpop_left('/', skip_empty) == "//0/1" ); - CHECK(ryml::csubstr( "0/1/2//" ).gpop_left('/', skip_empty) == "0/1" ); - CHECK(ryml::csubstr( "/0/1/2//" ).gpop_left('/', skip_empty) == "/0/1" ); - CHECK(ryml::csubstr("//0/1/2//" ).gpop_left('/', skip_empty) == "//0/1" ); - // pop_right(): pop the last element from the right - CHECK(ryml::csubstr( "0/1/2" ). pop_right('/' ) == "2" ); - CHECK(ryml::csubstr( "0/1/2/" ). pop_right('/' ) == "" ); - CHECK(ryml::csubstr( "0/1/2//" ). pop_right('/' ) == "" ); - CHECK(ryml::csubstr( "0/1/2" ). pop_right('/', skip_empty) == "2" ); - CHECK(ryml::csubstr( "0/1/2/" ). pop_right('/', skip_empty) == "2/" ); - CHECK(ryml::csubstr( "0/1/2//" ). pop_right('/', skip_empty) == "2//" ); - // gpop_right(): pop all but the first element (greedy pop) - CHECK(ryml::csubstr( "0/1/2" ).gpop_right('/' ) == "1/2"); - CHECK(ryml::csubstr( "0/1/2/" ).gpop_right('/' ) == "1/2/" ); - CHECK(ryml::csubstr( "0/1/2//" ).gpop_right('/' ) == "1/2//" ); - CHECK(ryml::csubstr( "/0/1/2" ).gpop_right('/' ) == "0/1/2"); - CHECK(ryml::csubstr( "/0/1/2/" ).gpop_right('/' ) == "0/1/2/" ); - CHECK(ryml::csubstr( "/0/1/2//" ).gpop_right('/' ) == "0/1/2//" ); - CHECK(ryml::csubstr("//0/1/2" ).gpop_right('/' ) == "/0/1/2"); - CHECK(ryml::csubstr("//0/1/2/" ).gpop_right('/' ) == "/0/1/2/" ); - CHECK(ryml::csubstr("//0/1/2//" ).gpop_right('/' ) == "/0/1/2//" ); - CHECK(ryml::csubstr( "0/1/2" ).gpop_right('/', skip_empty) == "1/2"); - CHECK(ryml::csubstr( "0/1/2/" ).gpop_right('/', skip_empty) == "1/2/" ); - CHECK(ryml::csubstr( "0/1/2//" ).gpop_right('/', skip_empty) == "1/2//" ); - CHECK(ryml::csubstr( "/0/1/2" ).gpop_right('/', skip_empty) == "1/2"); - CHECK(ryml::csubstr( "/0/1/2/" ).gpop_right('/', skip_empty) == "1/2/" ); - CHECK(ryml::csubstr( "/0/1/2//" ).gpop_right('/', skip_empty) == "1/2//" ); - CHECK(ryml::csubstr("//0/1/2" ).gpop_right('/', skip_empty) == "1/2"); - CHECK(ryml::csubstr("//0/1/2/" ).gpop_right('/', skip_empty) == "1/2/" ); - CHECK(ryml::csubstr("//0/1/2//" ).gpop_right('/', skip_empty) == "1/2//" ); - } - - // see the docs: - // https://c4core.docsforge.com/master/api/c4/basic_substring/ -} - - -//----------------------------------------------------------------------------- - -// helper functions for sample_parse_file() -template CharContainer file_get_contents(const char *filename); -template size_t file_get_contents(const char *filename, CharContainer *v); -template void file_put_contents(const char *filename, CharContainer const& v, const char* access="wb"); -void file_put_contents(const char *filename, const char *buf, size_t sz, const char* access); - - -/** demonstrate how to load a YAML file from disk to parse with ryml. - * - * ryml offers no overload to directly parse files from disk; it only - * parses source buffers (which may be mutable or immutable). It is - * up to the caller to load the file contents into a buffer before - * parsing with ryml. - * - * But that does not mean that loading a file is unimportant. There - * are many ways to achieve this in C++, but for convenience and to - * enable you to quickly get up to speed, here is an example - * implementation loading a file from disk and then parsing the - * resulting buffer with ryml. */ -void sample_parse_file() -{ - const char filename[] = "ryml_example.yml"; - - // because this is a minimal sample, it assumes nothing on the - // environment/OS (other than that it can read/write files). So we - // create the file on the fly: - file_put_contents(filename, ryml::csubstr("foo: 1\nbar:\n - 2\n - 3\n")); - - // now we can load it into a std::string (for example): - { - std::string contents = file_get_contents(filename); - ryml::Tree tree = ryml::parse_in_arena(ryml::to_csubstr(contents)); // immutable (csubstr) overload - CHECK(tree["foo"].val() == "1"); - CHECK(tree["bar"][0].val() == "2"); - CHECK(tree["bar"][1].val() == "3"); - } - - // or we can use a vector instead: - { - std::vector contents = file_get_contents>(filename); - ryml::Tree tree = ryml::parse_in_place(ryml::to_substr(contents)); // mutable (csubstr) overload - CHECK(tree["foo"].val() == "1"); - CHECK(tree["bar"][0].val() == "2"); - CHECK(tree["bar"][1].val() == "3"); - } - - // generally, any contiguous char container can be used with ryml, - // provided that the ryml::substr/ryml::csubstr view can be - // created out of it. - // - // ryml provides the overloads above for these two containers, but - // if you are using another container it should be very easy (only - // requires pointer and length). -} - - -//----------------------------------------------------------------------------- - -/** demonstrate in-place parsing of a mutable YAML source buffer. */ -void sample_parse_in_place() -{ - // Like the name suggests, parse_in_place() directly mutates the - // source buffer in place - char src[] = "{foo: 1, bar: [2, 3]}"; // ryml can parse in situ - ryml::substr srcview = src; // a mutable view to the source buffer - ryml::Tree tree = ryml::parse_in_place(srcview); // you can also reuse the tree and/or parser - ryml::ConstNodeRef root = tree.crootref(); // get a reference to the root - - CHECK(root.is_map()); - CHECK(root["foo"].is_keyval()); - CHECK(root["foo"].key() == "foo"); - CHECK(root["foo"].val() == "1"); - CHECK(root["bar"].is_seq()); - CHECK(root["bar"].has_key()); - CHECK(root["bar"].key() == "bar"); - CHECK(root["bar"][0].val() == "2"); - CHECK(root["bar"][1].val() == "3"); - - // deserializing: - int foo = 0, bar0 = 0, bar1 = 0; - root["foo"] >> foo; - root["bar"][0] >> bar0; - root["bar"][1] >> bar1; - CHECK(foo == 1); - CHECK(bar0 == 2); - CHECK(bar1 == 3); - - // after parsing, the tree holds views to the source buffer: - CHECK(root["foo"].val().data() == src + strlen("{foo: ")); - CHECK(root["foo"].val().begin() == src + strlen("{foo: ")); - CHECK(root["foo"].val().end() == src + strlen("{foo: 1")); - CHECK(root["foo"].val().is_sub(srcview)); // equivalent to the previous three assertions - CHECK(root["bar"][0].val().data() == src + strlen("{foo: 1, bar: [")); - CHECK(root["bar"][0].val().begin() == src + strlen("{foo: 1, bar: [")); - CHECK(root["bar"][0].val().end() == src + strlen("{foo: 1, bar: [2")); - CHECK(root["bar"][0].val().is_sub(srcview)); // equivalent to the previous three assertions - CHECK(root["bar"][1].val().data() == src + strlen("{foo: 1, bar: [2, ")); - CHECK(root["bar"][1].val().begin() == src + strlen("{foo: 1, bar: [2, ")); - CHECK(root["bar"][1].val().end() == src + strlen("{foo: 1, bar: [2, 3")); - CHECK(root["bar"][1].val().is_sub(srcview)); // equivalent to the previous three assertions - - // NOTE. parse_in_place() cannot accept ryml::csubstr - // so this will cause a /compile/ error: - ryml::csubstr csrcview = srcview; // ok, can assign from mutable to immutable - //tree = ryml::parse_in_place(csrcview); // compile error, cannot mutate an immutable view - (void)csrcview; -} - - -//----------------------------------------------------------------------------- - -/** demonstrate parsing of a read-only YAML source buffer */ -void sample_parse_in_arena() -{ - // to parse read-only memory, ryml will copy first to the tree's - // arena, and then parse the copied buffer: - ryml::Tree tree = ryml::parse_in_arena("{foo: 1, bar: [2, 3]}"); - ryml::ConstNodeRef root = tree.crootref(); // get a reference to the root - - CHECK(root.is_map()); - CHECK(root["foo"].is_keyval()); - CHECK(root["foo"].key() == "foo"); - CHECK(root["foo"].val() == "1"); - CHECK(root["bar"].is_seq()); - CHECK(root["bar"].has_key()); - CHECK(root["bar"].key() == "bar"); - CHECK(root["bar"][0].val() == "2"); - CHECK(root["bar"][1].val() == "3"); - - // deserializing: - int foo = 0, bar0 = 0, bar1 = 0; - root["foo"] >> foo; - root["bar"][0] >> bar0; - root["bar"][1] >> bar1; - CHECK(foo == 1); - CHECK(bar0 == 2); - CHECK(bar1 == 3); - - // NOTE. parse_in_arena() cannot accept ryml::substr. Overloads - // receiving substr buffers are declared, but intentionally left - // undefined, so this will cause a /linker/ error - char src[] = "{foo: is it really true}"; - ryml::substr srcview = src; - //tree = ryml::parse_in_place(srcview); // linker error, overload intentionally undefined - - // If you really intend to parse a mutable buffer in the arena, - // then simply convert it to immutable prior to calling - // parse_in_arena(): - ryml::csubstr csrcview = srcview; // assigning from src also works - tree = ryml::parse_in_arena(csrcview); // OK! csrcview is immutable - CHECK(tree["foo"].val() == "is it really true"); -} - - -//----------------------------------------------------------------------------- - -/** demonstrate reuse/modification of tree when parsing */ -void sample_parse_reuse_tree() -{ - ryml::Tree tree; - - // it will always be faster if the tree's size is conveniently reserved: - tree.reserve(30); // reserve 30 nodes (good enough for this sample) - // if you are using the tree's arena to serialize data, - // then reserve also the arena's size: - tree.reserve_arena(256); // reserve 256 characters (good enough for this sample) - - // now parse into the tree: - ryml::parse_in_arena("{foo: 1, bar: [2, 3]}", &tree); - - ryml::ConstNodeRef root = tree.crootref(); - CHECK(root.num_children() == 2); - CHECK(root.is_map()); - CHECK(root["foo"].is_keyval()); - CHECK(root["foo"].key() == "foo"); - CHECK(root["foo"].val() == "1"); - CHECK(root["bar"].is_seq()); - CHECK(root["bar"].has_key()); - CHECK(root["bar"].key() == "bar"); - CHECK(root["bar"][0].val() == "2"); - CHECK(root["bar"][1].val() == "3"); - CHECK(ryml::emitrs_yaml(tree) == R"(foo: 1 -bar: - - 2 - - 3 -)"); - - // WATCHOUT: parsing into an existing tree will APPEND to it: - ryml::parse_in_arena("{foo2: 12, bar2: [22, 32]}", &tree); - CHECK(ryml::emitrs_yaml(tree) == R"(foo: 1 -bar: - - 2 - - 3 -foo2: 12 -bar2: - - 22 - - 32 -)"); - CHECK(root.num_children() == 4); - CHECK(root["foo2"].is_keyval()); - CHECK(root["foo2"].key() == "foo2"); - CHECK(root["foo2"].val() == "12"); - CHECK(root["bar2"].is_seq()); - CHECK(root["bar2"].has_key()); - CHECK(root["bar2"].key() == "bar2"); - CHECK(root["bar2"][0].val() == "22"); - CHECK(root["bar2"][1].val() == "32"); - - // clear first before parsing into an existing tree. - tree.clear(); - tree.clear_arena(); // you may or may not want to clear the arena - ryml::parse_in_arena("[a, b, {x0: 1, x1: 2}]", &tree); - CHECK(ryml::emitrs_yaml(tree) == R"(- a -- b -- x0: 1 - x1: 2 -)"); - CHECK(root.is_seq()); - CHECK(root[0].val() == "a"); - CHECK(root[1].val() == "b"); - CHECK(root[2].is_map()); - CHECK(root[2]["x0"].val() == "1"); - CHECK(root[2]["x1"].val() == "2"); - - // we can parse directly into a node nested deep in an existing tree: - ryml::NodeRef mroot = tree.rootref(); // modifiable root - ryml::parse_in_arena("{champagne: Dom Perignon, coffee: Arabica}", mroot.append_child()); - CHECK(ryml::emitrs_yaml(tree) == R"(- a -- b -- x0: 1 - x1: 2 -- champagne: Dom Perignon - coffee: Arabica -)"); - CHECK(root.is_seq()); - CHECK(root[0].val() == "a"); - CHECK(root[1].val() == "b"); - CHECK(root[2].is_map()); - CHECK(root[2]["x0"].val() == "1"); - CHECK(root[2]["x1"].val() == "2"); - CHECK(root[3].is_map()); - CHECK(root[3]["champagne"].val() == "Dom Perignon"); - CHECK(root[3]["coffee"].val() == "Arabica"); - - // watchout: to add to an existing node within a map, the node's key must first be set: - ryml::NodeRef more = mroot[3].append_child({ryml::KEYMAP, "more"}); - ryml::NodeRef beer = mroot[3].append_child({ryml::KEYSEQ, "beer"}); - ryml::parse_in_arena("{vinho verde: Soalheiro, vinho tinto: Redoma 2017}", more); - ryml::parse_in_arena("[Rochefort 10, Busch, Leffe Rituel]", beer); - CHECK(ryml::emitrs_yaml(tree) == R"(- a -- b -- x0: 1 - x1: 2 -- champagne: Dom Perignon - coffee: Arabica - more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 - beer: - - Rochefort 10 - - Busch - - Leffe Rituel -)"); - - ryml::parse_in_arena("[foo, bar, baz, bat]", mroot); - CHECK(ryml::emitrs_yaml(tree) == R"(- a -- b -- x0: 1 - x1: 2 -- champagne: Dom Perignon - coffee: Arabica - more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 - beer: - - Rochefort 10 - - Busch - - Leffe Rituel -- foo -- bar -- baz -- bat -)"); - - ryml::parse_in_arena("[Kasteel Donker]", beer); - CHECK(ryml::emitrs_yaml(tree) == R"(- a -- b -- x0: 1 - x1: 2 -- champagne: Dom Perignon - coffee: Arabica - more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 - beer: - - Rochefort 10 - - Busch - - Leffe Rituel - - Kasteel Donker -- foo -- bar -- baz -- bat -)"); -} - - -//----------------------------------------------------------------------------- - -/** Demonstrates reuse of an existing parser. Doing this is - recommended when multiple files are parsed. */ -void sample_parse_reuse_parser() -{ - ryml::Parser parser; - - // it is also advised to reserve the parser depth - // to the expected depth of the data tree: - parser.reserve_stack(10); // uses small storage optimization - // defaulting to 16 depth, so this - // instruction is a no-op, and the stack - // will located in the parser object. - parser.reserve_stack(20); // But this will cause an allocation - // because it is above 16. - - auto champagnes = parser.parse_in_arena("champagnes.yml", "[Dom Perignon, Gosset Grande Reserve, Ruinart Blanc de Blancs, Jacquesson 742]"); - CHECK(ryml::emitrs_yaml(champagnes) == R"(- Dom Perignon -- Gosset Grande Reserve -- Ruinart Blanc de Blancs -- Jacquesson 742 -)"); - - auto beers = parser.parse_in_arena("beers.yml", "[Rochefort 10, Busch, Leffe Rituel, Kasteel Donker]"); - CHECK(ryml::emitrs_yaml(beers) == R"(- Rochefort 10 -- Busch -- Leffe Rituel -- Kasteel Donker -)"); - -} - - -//----------------------------------------------------------------------------- - -/** for ultimate speed when parsing multiple times, reuse both the - tree and parser */ -void sample_parse_reuse_tree_and_parser() -{ - ryml::Tree tree; - ryml::Parser parser; - - // it will always be faster if the tree's size is conveniently reserved: - tree.reserve(30); // reserve 30 nodes (good enough for this sample) - // if you are using the tree's arena to serialize data, - // then reserve also the arena's size: - tree.reserve(256); // reserve 256 characters (good enough for this sample) - // it is also advised to reserve the parser depth - // to the expected depth of the data tree: - parser.reserve_stack(10); // the parser uses small storage - // optimization defaulting to 16 depth, - // so this instruction is a no-op, and - // the stack will be located in the - // parser object. - parser.reserve_stack(20); // But this will cause an allocation - // because it is above 16. - - ryml::csubstr champagnes = "[Dom Perignon, Gosset Grande Reserve, Ruinart Blanc de Blancs, Jacquesson 742]"; - ryml::csubstr beers = "[Rochefort 10, Busch, Leffe Rituel, Kasteel Donker]"; - ryml::csubstr wines = "[Soalheiro, Niepoort Redoma 2017, Vina Esmeralda]"; - - parser.parse_in_arena("champagnes.yml", champagnes, &tree); - CHECK(ryml::emitrs_yaml(tree) == R"(- Dom Perignon -- Gosset Grande Reserve -- Ruinart Blanc de Blancs -- Jacquesson 742 -)"); - - // watchout: this will APPEND to the given tree: - parser.parse_in_arena("beers.yml", beers, &tree); - CHECK(ryml::emitrs_yaml(tree) == R"(- Dom Perignon -- Gosset Grande Reserve -- Ruinart Blanc de Blancs -- Jacquesson 742 -- Rochefort 10 -- Busch -- Leffe Rituel -- Kasteel Donker -)"); - - // if you don't wish to append, clear the tree first: - tree.clear(); - parser.parse_in_arena("wines.yml", wines, &tree); - CHECK(ryml::emitrs_yaml(tree) == R"(- Soalheiro -- Niepoort Redoma 2017 -- Vina Esmeralda -)"); -} - - -//----------------------------------------------------------------------------- - -/** shows how to programatically iterate through trees */ -void sample_iterate_trees() -{ - const ryml::Tree tree = ryml::parse_in_arena(R"(doe: "a deer, a female deer" -ray: "a drop of golden sun" -pi: 3.14159 -xmas: true -french-hens: 3 -calling-birds: - - huey - - dewey - - louie - - fred -xmas-fifth-day: - calling-birds: four - french-hens: 3 - golden-rings: 5 - partridges: - count: 1 - location: a pear tree - turtle-doves: two -cars: GTO -)"); - ryml::ConstNodeRef root = tree.crootref(); - - // iterate children - { - std::vector keys, vals; // to store all the root-level keys, vals - for(ryml::ConstNodeRef n : root.children()) - { - keys.emplace_back(n.key()); - vals.emplace_back(n.has_val() ? n.val() : ryml::csubstr{}); - } - CHECK(keys[0] == "doe"); - CHECK(vals[0] == "a deer, a female deer"); - CHECK(keys[1] == "ray"); - CHECK(vals[1] == "a drop of golden sun"); - CHECK(keys[2] == "pi"); - CHECK(vals[2] == "3.14159"); - CHECK(keys[3] == "xmas"); - CHECK(vals[3] == "true"); - CHECK(root[5].has_key()); - CHECK(root[5].is_seq()); - CHECK(root[5].key() == "calling-birds"); - CHECK(!root[5].has_val()); // it is a map, so not a val - //CHECK(root[5].val() == ""); // ERROR! node does not have a val. - CHECK(keys[5] == "calling-birds"); - CHECK(vals[5] == ""); - } - - // iterate siblings - { - size_t count = 0; - ryml::csubstr calling_birds[] = {"huey", "dewey", "louie", "fred"}; - for(ryml::ConstNodeRef n : root["calling-birds"][2].siblings()) - CHECK(n.val() == calling_birds[count++]); - CHECK(count == 4u); - } -} - - -//----------------------------------------------------------------------------- - -/** shows how to programatically create trees */ -void sample_create_trees() -{ - ryml::NodeRef doe; - CHECK(!doe.valid()); // it's pointing at nowhere - - ryml::Tree tree; - ryml::NodeRef root = tree.rootref(); - root |= ryml::MAP; // mark root as a map - doe = root["doe"]; - CHECK(doe.valid()); // it's now pointing at the tree - CHECK(doe.is_seed()); // but the tree has nothing there, so this is only a seed - - // set the value of the node - const char a_deer[] = "a deer, a female deer"; - doe = a_deer; - // now the node really exists in the tree, and this ref is no - // longer a seed: - CHECK(!doe.is_seed()); - // WATCHOUT for lifetimes: - CHECK(doe.val().str == a_deer); // it is pointing at the initial string - // If you need to avoid lifetime dependency, serialize the data: - { - std::string a_drop = "a drop of golden sun"; - // this will copy the string to the tree's arena: - // (see the serialization samples below) - root["ray"] << a_drop; - // and now you can modify the original string without changing - // the tree: - a_drop[0] = 'Z'; - a_drop[1] = 'Z'; - } - CHECK(root["ray"].val() == "a drop of golden sun"); - - // etc. - root["pi"] << ryml::fmt::real(3.141592654, 5); - root["xmas"] << ryml::fmt::boolalpha(true); - root["french-hens"] << 3; - ryml::NodeRef calling_birds = root["calling-birds"]; - calling_birds |= ryml::SEQ; - calling_birds.append_child() = "huey"; - calling_birds.append_child() = "dewey"; - calling_birds.append_child() = "louie"; - calling_birds.append_child() = "fred"; - ryml::NodeRef xmas5 = root["xmas-fifth-day"]; - xmas5 |= ryml::MAP; - xmas5["calling-birds"] = "four"; - xmas5["french-hens"] << 3; - xmas5["golden-rings"] << 5; - xmas5["partridges"] |= ryml::MAP; - xmas5["partridges"]["count"] << 1; - xmas5["partridges"]["location"] = "a pear tree"; - xmas5["turtle-doves"] = "two"; - root["cars"] = "GTO"; - - std::cout << tree; - CHECK(ryml::emitrs_yaml(tree) == R"(doe: 'a deer, a female deer' -ray: a drop of golden sun -pi: 3.14159 -xmas: true -'french-hens': 3 -'calling-birds': - - huey - - dewey - - louie - - fred -'xmas-fifth-day': - 'calling-birds': four - 'french-hens': 3 - 'golden-rings': 5 - partridges: - count: 1 - location: a pear tree - 'turtle-doves': two -cars: GTO -)"); -} - - -//----------------------------------------------------------------------------- - -/** demonstrates explicit and implicit interaction with the tree's string arena. - * Notice that ryml only holds strings in the tree's nodes. */ -void sample_tree_arena() -{ - // mutable buffers are parsed in situ: - { - char buf[] = "[a, b, c, d]"; - ryml::substr yml = buf; - ryml::Tree tree = ryml::parse_in_place(yml); - // notice the arena is empty: - CHECK(tree.arena().empty()); - // and the tree is pointing at the original buffer: - ryml::NodeRef root = tree.rootref(); - CHECK(root[0].val().is_sub(yml)); - CHECK(root[1].val().is_sub(yml)); - CHECK(root[2].val().is_sub(yml)); - CHECK(root[3].val().is_sub(yml)); - CHECK(yml.is_super(root[0].val())); - CHECK(yml.is_super(root[1].val())); - CHECK(yml.is_super(root[2].val())); - CHECK(yml.is_super(root[3].val())); - } - - // when parsing immutable buffers, the buffer is first copied to the - // tree's arena; the copy in the arena is then the buffer which is - // actually parsed - { - ryml::csubstr yml = "[a, b, c, d]"; - ryml::Tree tree = ryml::parse_in_arena(yml); - // notice the buffer was copied to the arena: - CHECK(tree.arena().data() != yml.data()); - CHECK(tree.arena() == yml); - // and the tree is pointing at the arena instead of to the - // original buffer: - ryml::NodeRef root = tree.rootref(); - ryml::csubstr arena = tree.arena(); - CHECK(root[0].val().is_sub(arena)); - CHECK(root[1].val().is_sub(arena)); - CHECK(root[2].val().is_sub(arena)); - CHECK(root[3].val().is_sub(arena)); - CHECK(arena.is_super(root[0].val())); - CHECK(arena.is_super(root[1].val())); - CHECK(arena.is_super(root[2].val())); - CHECK(arena.is_super(root[3].val())); - } - - // the arena is also used when the data is serialized to string - // with NodeRef::operator<<(): mutable buffer - { - char buf[] = "[a, b, c, d]"; // mutable - ryml::substr yml = buf; - ryml::Tree tree = ryml::parse_in_place(yml); - // notice the arena is empty: - CHECK(tree.arena().empty()); - ryml::NodeRef root = tree.rootref(); - - // serialize an integer, and mutate the tree - CHECK(root[2].val() == "c"); - CHECK(root[2].val().is_sub(yml)); // val is first pointing at the buffer - root[2] << 12345; - CHECK(root[2].val() == "12345"); - CHECK(root[2].val().is_sub(tree.arena())); // now val is pointing at the arena - // notice the serialized string was appended to the tree's arena: - CHECK(tree.arena() == "12345"); - - // serialize an integer, and mutate the tree - CHECK(root[3].val() == "d"); - CHECK(root[3].val().is_sub(yml)); // val is first pointing at the buffer - root[3] << 67890; - CHECK(root[3].val() == "67890"); - CHECK(root[3].val().is_sub(tree.arena())); // now val is pointing at the arena - // notice the serialized string was appended to the tree's arena: - CHECK(tree.arena() == "1234567890"); - } - // the arena is also used when the data is serialized to string - // with NodeRef::operator<<(): immutable buffer - { - ryml::csubstr yml = "[a, b, c, d]"; // immutable - ryml::Tree tree = ryml::parse_in_arena(yml); - // notice the buffer was copied to the arena: - CHECK(tree.arena().data() != yml.data()); - CHECK(tree.arena() == yml); - ryml::NodeRef root = tree.rootref(); - - // serialize an integer, and mutate the tree - CHECK(root[2].val() == "c"); - root[2] << 12345; // serialize an integer - CHECK(root[2].val() == "12345"); - // notice the serialized string was appended to the tree's arena: - // notice also the previous values remain there. - // RYML DOES NOT KEEP TRACK OF REFERENCES TO THE ARENA. - CHECK(tree.arena() == "[a, b, c, d]12345"); - // old values: --------------^ - - // serialize an integer, and mutate the tree - root[3] << 67890; - CHECK(root[3].val() == "67890"); - // notice the serialized string was appended to the tree's arena: - // notice also the previous values remain there. - // RYML DOES NOT KEEP TRACK OF REFERENCES TO THE ARENA. - CHECK(tree.arena() == "[a, b, c, d]1234567890"); - // old values: --------------^ ---^^^^^ - } - - // to_arena(): directly serialize values to the arena: - { - ryml::Tree tree = ryml::parse_in_arena("{a: b}"); - ryml::csubstr c10 = tree.to_arena(10101010); - CHECK(c10 == "10101010"); - CHECK(c10.is_sub(tree.arena())); - CHECK(tree.arena() == "{a: b}10101010"); - CHECK(tree.key(1) == "a"); - CHECK(tree.val(1) == "b"); - tree.set_val(1, c10); - CHECK(tree.val(1) == c10); - // and you can also do it through a node: - ryml::NodeRef root = tree.rootref(); - root["a"].set_val_serialized(2222); - CHECK(root["a"].val() == "2222"); - CHECK(tree.arena() == "{a: b}101010102222"); - } - - // copy_to_arena(): manually copy a string to the arena: - { - ryml::Tree tree = ryml::parse_in_arena("{a: b}"); - ryml::csubstr mystr = "Gosset Grande Reserve"; - ryml::csubstr copied = tree.copy_to_arena(mystr); - CHECK(!copied.overlaps(mystr)); - CHECK(copied == mystr); - CHECK(tree.arena() == "{a: b}Gosset Grande Reserve"); - } - - // alloc_arena(): allocate a buffer from the arena: - { - ryml::Tree tree = ryml::parse_in_arena("{a: b}"); - ryml::csubstr mystr = "Gosset Grande Reserve"; - ryml::substr copied = tree.alloc_arena(mystr.size()); - CHECK(!copied.overlaps(mystr)); - memcpy(copied.str, mystr.str, mystr.len); - CHECK(copied == mystr); - CHECK(tree.arena() == "{a: b}Gosset Grande Reserve"); - } - - // reserve_arena(): ensure the arena has a certain size to avoid reallocations - { - ryml::Tree tree = ryml::parse_in_arena("{a: b}"); - CHECK(tree.arena().size() == strlen("{a: b}")); - tree.reserve_arena(100); - CHECK(tree.arena_capacity() >= 100); - CHECK(tree.arena().size() == strlen("{a: b}")); - tree.to_arena(123456); - CHECK(tree.arena().first(12) == "{a: b}123456"); - } -} - - -//----------------------------------------------------------------------------- - -/** ryml provides facilities for serializing the C++ fundamental - types. This is achieved through to_chars()/from_chars(). - See an example below for user scalar types. */ -void sample_fundamental_types() -{ - ryml::Tree tree; - CHECK(tree.arena().empty()); - CHECK(tree.to_arena('a') == "a"); CHECK(tree.arena() == "a"); - CHECK(tree.to_arena("bcde") == "bcde"); CHECK(tree.arena() == "abcde"); - CHECK(tree.to_arena(unsigned(0)) == "0"); CHECK(tree.arena() == "abcde0"); - CHECK(tree.to_arena(int(1)) == "1"); CHECK(tree.arena() == "abcde01"); - CHECK(tree.to_arena(uint8_t(0)) == "0"); CHECK(tree.arena() == "abcde010"); - CHECK(tree.to_arena(uint16_t(1)) == "1"); CHECK(tree.arena() == "abcde0101"); - CHECK(tree.to_arena(uint32_t(2)) == "2"); CHECK(tree.arena() == "abcde01012"); - CHECK(tree.to_arena(uint64_t(3)) == "3"); CHECK(tree.arena() == "abcde010123"); - CHECK(tree.to_arena(int8_t( 4)) == "4"); CHECK(tree.arena() == "abcde0101234"); - CHECK(tree.to_arena(int8_t(-4)) == "-4"); CHECK(tree.arena() == "abcde0101234-4"); - CHECK(tree.to_arena(int16_t( 5)) == "5"); CHECK(tree.arena() == "abcde0101234-45"); - CHECK(tree.to_arena(int16_t(-5)) == "-5"); CHECK(tree.arena() == "abcde0101234-45-5"); - CHECK(tree.to_arena(int32_t( 6)) == "6"); CHECK(tree.arena() == "abcde0101234-45-56"); - CHECK(tree.to_arena(int32_t(-6)) == "-6"); CHECK(tree.arena() == "abcde0101234-45-56-6"); - CHECK(tree.to_arena(int64_t( 7)) == "7"); CHECK(tree.arena() == "abcde0101234-45-56-67"); - CHECK(tree.to_arena(int64_t(-7)) == "-7"); CHECK(tree.arena() == "abcde0101234-45-56-67-7"); - CHECK(tree.to_arena((void*)1) == "0x1"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x1"); - CHECK(tree.to_arena(float(0.124)) == "0.124"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.124"); - CHECK(tree.to_arena(double(0.234)) == "0.234"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.1240.234"); - CHECK(tree.to_arena(bool(true)) == "1"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.1240.2341"); - CHECK(tree.to_arena(bool(false)) == "0"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.1240.23410"); - - // write special float values - const float fnan = std::numeric_limits::quiet_NaN(); - const double dnan = std::numeric_limits::quiet_NaN(); - const float finf = std::numeric_limits::infinity(); - const double dinf = std::numeric_limits::infinity(); - CHECK(tree.to_arena( finf) == ".inf"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.1240.23410.inf"); - CHECK(tree.to_arena( dinf) == ".inf"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.1240.23410.inf.inf"); - CHECK(tree.to_arena(-finf) == "-.inf"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.1240.23410.inf.inf-.inf"); - CHECK(tree.to_arena(-dinf) == "-.inf"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.1240.23410.inf.inf-.inf-.inf"); - CHECK(tree.to_arena( fnan) == ".nan"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.1240.23410.inf.inf-.inf-.inf.nan"); - CHECK(tree.to_arena( dnan) == ".nan"); CHECK(tree.arena() == "abcde0101234-45-56-67-70x10.1240.23410.inf.inf-.inf-.inf.nan.nan"); - // read special float values - tree = ryml::parse_in_arena(R"({ninf: -.inf, pinf: .inf, nan: .nan})"); - C4_SUPPRESS_WARNING_GCC_CLANG_WITH_PUSH("-Wfloat-equal"); - float f = 0.f; - double d = 0.; - CHECK(f == 0.f); - CHECK(d == 0.); - tree["ninf"] >> f; CHECK(f == -finf); - tree["ninf"] >> d; CHECK(d == -dinf); - tree["pinf"] >> f; CHECK(f == finf); - tree["pinf"] >> d; CHECK(d == dinf); - tree["nan" ] >> f; CHECK(std::isnan(f)); - tree["nan" ] >> d; CHECK(std::isnan(d)); - C4_SUPPRESS_WARNING_GCC_CLANG_POP -} - - -//----------------------------------------------------------------------------- - -/** ryml provides facilities for formatting (imported from c4core into - * the ryml namespace) - * - * @see https://c4core.docsforge.com/master/formatting-arguments/ - * @see https://c4core.docsforge.com/master/formatting-strings/ - */ -void sample_formatting() -{ - // format(), format_sub(), formatrs(): format arguments - { - char buf_[256] = {}; - ryml::substr buf = buf_; - size_t size = ryml::format(buf, "a={} foo {} {} bar {}", 0.1, 10, 11, 12); - CHECK(size == strlen("a=0.1 foo 10 11 bar 12")); - CHECK(buf.first(size) == "a=0.1 foo 10 11 bar 12"); - // it is safe to call on an empty buffer: - // returns the size needed for the result, and no overflow occurs: - size = ryml::format({} , "a={} foo {} {} bar {}", "this_is_a", 10, 11, 12); - CHECK(size == ryml::format(buf, "a={} foo {} {} bar {}", "this_is_a", 10, 11, 12)); - CHECK(size == strlen("a=this_is_a foo 10 11 bar 12")); - // it is also safe to call on an insufficient buffer: - char smallbuf[8] = {}; - size = ryml::format(smallbuf, "{} is too large {}", "this", "for the buffer"); - CHECK(size == strlen("this is too large for the buffer")); - // ... and the result is truncated at the buffer size: - CHECK(ryml::substr(smallbuf, sizeof(smallbuf)) == "this is\0"); - - // format_sub() directly returns the written string: - ryml::csubstr result = ryml::format_sub(buf, "b={}, damn it.", 1); - CHECK(result == "b=1, damn it."); - CHECK(result.is_sub(buf)); - - // formatrs() means FORMAT & ReSize: - // - // Instead of a substr, it receives any owning linear char container - // for which to_substr() is defined (using ADL). - // has to_substr() definitions for std::string and - // std::vector. - // - // formatrs() starts by calling format(), and if needed, resizes the container - // and calls format() again. - // - // Note that unless the container is previously sized, this - // may cause an allocation, which will make your code slower. - // Make sure to call .reserve() on the container for real - // production code. - std::string sbuf; - ryml::formatrs(&sbuf, "and c={} seems about right", 2); - CHECK(sbuf == "and c=2 seems about right"); - std::vector vbuf; // works with any linear char container - ryml::formatrs(&vbuf, "and c={} seems about right", 2); - CHECK(sbuf == "and c=2 seems about right"); - // with formatrs() it is also possible to append: - ryml::formatrs(ryml::append, &sbuf, ", and finally d={} - done", 3); - CHECK(sbuf == "and c=2 seems about right, and finally d=3 - done"); - } - - // unformat(): read arguments - opposite of format() - { - char buf_[256]; - - int a = 0, b = 1, c = 2; - ryml::csubstr result = ryml::format_sub(buf_, "{} and {} and {}", a, b, c); - CHECK(result == "0 and 1 and 2"); - int aa = -1, bb = -2, cc = -3; - size_t num_characters = ryml::unformat(result, "{} and {} and {}", aa, bb, cc); - CHECK(num_characters != ryml::csubstr::npos); // if a conversion fails, returns ryml::csubstr::npos - CHECK(num_characters == result.size()); - CHECK(aa == a); - CHECK(bb == b); - CHECK(cc == c); - - result = ryml::format_sub(buf_, "{} and {} and {}", 10, 20, 30); - CHECK(result == "10 and 20 and 30"); - num_characters = ryml::unformat(result, "{} and {} and {}", aa, bb, cc); - CHECK(num_characters != ryml::csubstr::npos); // if a conversion fails, returns ryml::csubstr::npos - CHECK(num_characters == result.size()); - CHECK(aa == 10); - CHECK(bb == 20); - CHECK(cc == 30); - } - - // cat(), cat_sub(), catrs(): concatenate arguments - { - char buf_[256] = {}; - ryml::substr buf = buf_; - size_t size = ryml::cat(buf, "a=", 0.1, "foo", 10, 11, "bar", 12); - CHECK(size == strlen("a=0.1foo1011bar12")); - CHECK(buf.first(size) == "a=0.1foo1011bar12"); - // it is safe to call on an empty buffer: - // returns the size needed for the result, and no overflow occurs: - CHECK(ryml::cat({}, "a=", 0) == 3); - // it is also safe to call on an insufficient buffer: - char smallbuf[8] = {}; - size = ryml::cat(smallbuf, "this", " is too large ", "for the buffer"); - CHECK(size == strlen("this is too large for the buffer")); - // ... and the result is truncated at the buffer size: - CHECK(ryml::substr(smallbuf, sizeof(smallbuf)) == "this is\0"); - - // cat_sub() directly returns the written string: - ryml::csubstr result = ryml::cat_sub(buf, "b=", 1, ", damn it."); - CHECK(result == "b=1, damn it."); - CHECK(result.is_sub(buf)); - - // catrs() means CAT & ReSize: - // - // Instead of a substr, it receives any owning linear char container - // for which to_substr() is defined (using ADL). - // has to_substr() definitions for std::string and - // std::vector. - // - // catrs() starts by calling cat(), and if needed, resizes the container - // and calls cat() again. - // - // Note that unless the container is previously sized, this - // may cause an allocation, which will make your code slower. - // Make sure to call .reserve() on the container for real - // production code. - std::string sbuf; - ryml::catrs(&sbuf, "and c=", 2, " seems about right"); - CHECK(sbuf == "and c=2 seems about right"); - std::vector vbuf; // works with any linear char container - ryml::catrs(&vbuf, "and c=", 2, " seems about right"); - CHECK(sbuf == "and c=2 seems about right"); - // with catrs() it is also possible to append: - ryml::catrs(ryml::append, &sbuf, ", and finally d=", 3, " - done"); - CHECK(sbuf == "and c=2 seems about right, and finally d=3 - done"); - } - - // uncat(): read arguments - opposite of cat() - { - char buf_[256]; - - int a = 0, b = 1, c = 2; - ryml::csubstr result = ryml::cat_sub(buf_, a, ' ', b, ' ', c); - CHECK(result == "0 1 2"); - int aa = -1, bb = -2, cc = -3; - char sep1 = 'a', sep2 = 'b'; - size_t num_characters = ryml::uncat(result, aa, sep1, bb, sep2, cc); - CHECK(num_characters == result.size()); - CHECK(aa == a); - CHECK(bb == b); - CHECK(cc == c); - CHECK(sep1 == ' '); - CHECK(sep2 == ' '); - - result = ryml::cat_sub(buf_, 10, ' ', 20, ' ', 30); - CHECK(result == "10 20 30"); - num_characters = ryml::uncat(result, aa, sep1, bb, sep2, cc); - CHECK(num_characters == result.size()); - CHECK(aa == 10); - CHECK(bb == 20); - CHECK(cc == 30); - CHECK(sep1 == ' '); - CHECK(sep2 == ' '); - } - - // catsep(), catsep_sub(), catseprs(): concatenate arguments, with a separator - { - char buf_[256] = {}; - ryml::substr buf = buf_; - // use ' ' as a separator - size_t size = ryml::catsep(buf, ' ', "a=", 0, "b=", 1, "c=", 2, 45, 67); - CHECK(buf.first(size) == "a= 0 b= 1 c= 2 45 67"); - // any separator may be used - // use " and " as a separator - size = ryml::catsep(buf, " and ", "a=0", "b=1", "c=2", 45, 67); - CHECK(buf.first(size) == "a=0 and b=1 and c=2 and 45 and 67"); - // use " ... " as a separator - size = ryml::catsep(buf, " ... ", "a=0", "b=1", "c=2", 45, 67); - CHECK(buf.first(size) == "a=0 ... b=1 ... c=2 ... 45 ... 67"); - // use '/' as a separator - size = ryml::catsep(buf, '/', "a=", 0, "b=", 1, "c=", 2, 45, 67); - CHECK(buf.first(size) == "a=/0/b=/1/c=/2/45/67"); - // use 888 as a separator - size = ryml::catsep(buf, 888, "a=0", "b=1", "c=2", 45, 67); - CHECK(buf.first(size) == "a=0888b=1888c=28884588867"); - - // it is safe to call on an empty buffer: - // returns the size needed for the result, and no overflow occurs: - CHECK(size == ryml::catsep({}, 888, "a=0", "b=1", "c=2", 45, 67)); - // it is also safe to call on an insufficient buffer: - char smallbuf[8] = {}; - CHECK(size == ryml::catsep(smallbuf, 888, "a=0", "b=1", "c=2", 45, 67)); - CHECK(size == strlen("a=0888b=1888c=28884588867")); - // ... and the result is truncated: - CHECK(ryml::substr(smallbuf, sizeof(smallbuf)) == "a=0888b\0"); - - // catsep_sub() directly returns the written substr: - ryml::csubstr result = ryml::catsep_sub(buf, " and ", "a=0", "b=1", "c=2", 45, 67); - CHECK(result == "a=0 and b=1 and c=2 and 45 and 67"); - CHECK(result.is_sub(buf)); - - // catseprs() means CATSEP & ReSize: - // - // Instead of a substr, it receives any owning linear char container - // for which to_substr() is defined (using ADL). - // has to_substr() definitions for std::string and - // std::vector. - // - // catseprs() starts by calling catsep(), and if needed, resizes the container - // and calls catsep() again. - // - // Note that unless the container is previously sized, this - // may cause an allocation, which will make your code slower. - // Make sure to call .reserve() on the container for real - // production code. - std::string sbuf; - ryml::catseprs(&sbuf, " and ", "a=0", "b=1", "c=2", 45, 67); - CHECK(sbuf == "a=0 and b=1 and c=2 and 45 and 67"); - std::vector vbuf; // works with any linear char container - ryml::catseprs(&vbuf, " and ", "a=0", "b=1", "c=2", 45, 67); - CHECK(ryml::to_csubstr(vbuf) == "a=0 and b=1 and c=2 and 45 and 67"); - - // with catseprs() it is also possible to append: - ryml::catseprs(ryml::append, &sbuf, " well ", " --- a=0", "b=11", "c=12", 145, 167); - CHECK(sbuf == "a=0 and b=1 and c=2 and 45 and 67 --- a=0 well b=11 well c=12 well 145 well 167"); - } - - // uncatsep(): read arguments with a separator - opposite of catsep() - { - char buf_[256] = {}; - - int a = 0, b = 1, c = 2; - ryml::csubstr result = ryml::catsep_sub(buf_, ' ', a, b, c); - CHECK(result == "0 1 2"); - int aa = -1, bb = -2, cc = -3; - char sep = 'b'; - size_t num_characters = ryml::uncatsep(result, sep, aa, bb, cc); - CHECK(num_characters == result.size()); - CHECK(aa == a); - CHECK(bb == b); - CHECK(cc == c); - CHECK(sep == ' '); - - sep = '_'; - result = ryml::catsep_sub(buf_, ' ', 10, 20, 30); - CHECK(result == "10 20 30"); - num_characters = ryml::uncatsep(result, sep, aa, bb, cc); - CHECK(num_characters == result.size()); - CHECK(aa == 10); - CHECK(bb == 20); - CHECK(cc == 30); - CHECK(sep == ' '); - } - - // formatting individual arguments - { - using namespace ryml; // all the symbols below are in the ryml namespace. - char buf_[256] = {}; // all the results below are written in this buffer - substr buf = buf_; - // -------------------------------------- - // fmt::boolalpha(): format as true/false - // -------------------------------------- - // just as with std streams, printing a bool will output the integer value: - CHECK("0" == cat_sub(buf, false)); - CHECK("1" == cat_sub(buf, true)); - // to force a "true"/"false", use fmt::boolalpha: - CHECK("false" == cat_sub(buf, fmt::boolalpha(false))); - CHECK("true" == cat_sub(buf, fmt::boolalpha(true))); - - // --------------------------------- - // fmt::hex(): format as hexadecimal - // --------------------------------- - CHECK("0xff" == cat_sub(buf, fmt::hex(255))); - CHECK("0x100" == cat_sub(buf, fmt::hex(256))); - CHECK("-0xff" == cat_sub(buf, fmt::hex(-255))); - CHECK("-0x100" == cat_sub(buf, fmt::hex(-256))); - CHECK("3735928559" == cat_sub(buf, UINT32_C(0xdeadbeef))); - CHECK("0xdeadbeef" == cat_sub(buf, fmt::hex(UINT32_C(0xdeadbeef)))); - // ---------------------------- - // fmt::bin(): format as binary - // ---------------------------- - CHECK("0b1000" == cat_sub(buf, fmt::bin(8))); - CHECK("0b1001" == cat_sub(buf, fmt::bin(9))); - CHECK("0b10001" == cat_sub(buf, fmt::bin(17))); - CHECK("0b11001" == cat_sub(buf, fmt::bin(25))); - CHECK("-0b1000" == cat_sub(buf, fmt::bin(-8))); - CHECK("-0b1001" == cat_sub(buf, fmt::bin(-9))); - CHECK("-0b10001" == cat_sub(buf, fmt::bin(-17))); - CHECK("-0b11001" == cat_sub(buf, fmt::bin(-25))); - // --------------------------- - // fmt::bin(): format as octal - // --------------------------- - CHECK("0o77" == cat_sub(buf, fmt::oct(63))); - CHECK("0o100" == cat_sub(buf, fmt::oct(64))); - CHECK("0o377" == cat_sub(buf, fmt::oct(255))); - CHECK("0o400" == cat_sub(buf, fmt::oct(256))); - CHECK("0o1000" == cat_sub(buf, fmt::oct(512))); - CHECK("-0o77" == cat_sub(buf, fmt::oct(-63))); - CHECK("-0o100" == cat_sub(buf, fmt::oct(-64))); - CHECK("-0o377" == cat_sub(buf, fmt::oct(-255))); - CHECK("-0o400" == cat_sub(buf, fmt::oct(-256))); - CHECK("-0o1000" == cat_sub(buf, fmt::oct(-512))); - // --------------------------- - // fmt::zpad(): pad with zeros - // --------------------------- - CHECK("000063" == cat_sub(buf, fmt::zpad(63, 6))); - CHECK( "00063" == cat_sub(buf, fmt::zpad(63, 5))); - CHECK( "0063" == cat_sub(buf, fmt::zpad(63, 4))); - CHECK( "063" == cat_sub(buf, fmt::zpad(63, 3))); - CHECK( "63" == cat_sub(buf, fmt::zpad(63, 2))); - CHECK( "63" == cat_sub(buf, fmt::zpad(63, 1))); // will never trim the result - CHECK( "63" == cat_sub(buf, fmt::zpad(63, 0))); // will never trim the result - CHECK("0x00003f" == cat_sub(buf, fmt::zpad(fmt::hex(63), 6))); - CHECK("0o000077" == cat_sub(buf, fmt::zpad(fmt::oct(63), 6))); - CHECK("0b00011001" == cat_sub(buf, fmt::zpad(fmt::bin(25), 8))); - // ------------------------------------------------ - // fmt::left(): align left with a given field width - // ------------------------------------------------ - CHECK("63 " == cat_sub(buf, fmt::left(63, 6))); - CHECK("63 " == cat_sub(buf, fmt::left(63, 5))); - CHECK("63 " == cat_sub(buf, fmt::left(63, 4))); - CHECK("63 " == cat_sub(buf, fmt::left(63, 3))); - CHECK("63" == cat_sub(buf, fmt::left(63, 2))); - CHECK("63" == cat_sub(buf, fmt::left(63, 1))); // will never trim the result - CHECK("63" == cat_sub(buf, fmt::left(63, 0))); // will never trim the result - // the fill character can be specified (defaults to ' '): - CHECK("63----" == cat_sub(buf, fmt::left(63, 6, '-'))); - CHECK("63++++" == cat_sub(buf, fmt::left(63, 6, '+'))); - CHECK("63////" == cat_sub(buf, fmt::left(63, 6, '/'))); - CHECK("630000" == cat_sub(buf, fmt::left(63, 6, '0'))); - CHECK("63@@@@" == cat_sub(buf, fmt::left(63, 6, '@'))); - CHECK("0x003f " == cat_sub(buf, fmt::left(fmt::zpad(fmt::hex(63), 4), 10))); - // -------------------------------------------------- - // fmt::right(): align right with a given field width - // -------------------------------------------------- - CHECK(" 63" == cat_sub(buf, fmt::right(63, 6))); - CHECK(" 63" == cat_sub(buf, fmt::right(63, 5))); - CHECK(" 63" == cat_sub(buf, fmt::right(63, 4))); - CHECK(" 63" == cat_sub(buf, fmt::right(63, 3))); - CHECK("63" == cat_sub(buf, fmt::right(63, 2))); - CHECK("63" == cat_sub(buf, fmt::right(63, 1))); // will never trim the result - CHECK("63" == cat_sub(buf, fmt::right(63, 0))); // will never trim the result - // the fill character can be specified (defaults to ' '): - CHECK("----63" == cat_sub(buf, fmt::right(63, 6, '-'))); - CHECK("++++63" == cat_sub(buf, fmt::right(63, 6, '+'))); - CHECK("////63" == cat_sub(buf, fmt::right(63, 6, '/'))); - CHECK("000063" == cat_sub(buf, fmt::right(63, 6, '0'))); - CHECK("@@@@63" == cat_sub(buf, fmt::right(63, 6, '@'))); - CHECK(" 0x003f" == cat_sub(buf, fmt::right(fmt::zpad(fmt::hex(63), 4), 10))); - - // ------------------------------------------ - // fmt::real(): format floating point numbers - // ------------------------------------------ - CHECK("0" == cat_sub(buf, fmt::real(0.01f, 0))); - CHECK("0.0" == cat_sub(buf, fmt::real(0.01f, 1))); - CHECK("0.01" == cat_sub(buf, fmt::real(0.01f, 2))); - CHECK("0.010" == cat_sub(buf, fmt::real(0.01f, 3))); - CHECK("0.0100" == cat_sub(buf, fmt::real(0.01f, 4))); - CHECK("0.01000" == cat_sub(buf, fmt::real(0.01f, 5))); - CHECK("1" == cat_sub(buf, fmt::real(1.01f, 0))); - CHECK("1.0" == cat_sub(buf, fmt::real(1.01f, 1))); - CHECK("1.01" == cat_sub(buf, fmt::real(1.01f, 2))); - CHECK("1.010" == cat_sub(buf, fmt::real(1.01f, 3))); - CHECK("1.0100" == cat_sub(buf, fmt::real(1.01f, 4))); - CHECK("1.01000" == cat_sub(buf, fmt::real(1.01f, 5))); - CHECK("1" == cat_sub(buf, fmt::real(1.234234234, 0))); - CHECK("1.2" == cat_sub(buf, fmt::real(1.234234234, 1))); - CHECK("1.23" == cat_sub(buf, fmt::real(1.234234234, 2))); - CHECK("1.234" == cat_sub(buf, fmt::real(1.234234234, 3))); - CHECK("1.2342" == cat_sub(buf, fmt::real(1.234234234, 4))); - CHECK("1.23423" == cat_sub(buf, fmt::real(1.234234234, 5))); - CHECK("1000000.00000" == cat_sub(buf, fmt::real(1000000.000000000, 5))); - CHECK("1234234.23423" == cat_sub(buf, fmt::real(1234234.234234234, 5))); - // AKA %f - CHECK("1000000.00000" == cat_sub(buf, fmt::real(1000000.000000000, 5, FTOA_FLOAT))); // AKA %f, same as above - CHECK("1234234.23423" == cat_sub(buf, fmt::real(1234234.234234234, 5, FTOA_FLOAT))); // AKA %f - CHECK("1234234.2342" == cat_sub(buf, fmt::real(1234234.234234234, 4, FTOA_FLOAT))); // AKA %f - CHECK("1234234.234" == cat_sub(buf, fmt::real(1234234.234234234, 3, FTOA_FLOAT))); // AKA %f - CHECK("1234234.23" == cat_sub(buf, fmt::real(1234234.234234234, 2, FTOA_FLOAT))); // AKA %f - // AKA %e - CHECK("1.00000e+06" == cat_sub(buf, fmt::real(1000000.000000000, 5, FTOA_SCIENT))); // AKA %e - CHECK("1.23423e+06" == cat_sub(buf, fmt::real(1234234.234234234, 5, FTOA_SCIENT))); // AKA %e - CHECK("1.2342e+06" == cat_sub(buf, fmt::real(1234234.234234234, 4, FTOA_SCIENT))); // AKA %e - CHECK("1.234e+06" == cat_sub(buf, fmt::real(1234234.234234234, 3, FTOA_SCIENT))); // AKA %e - CHECK("1.23e+06" == cat_sub(buf, fmt::real(1234234.234234234, 2, FTOA_SCIENT))); // AKA %e - // AKA %g - CHECK("1e+06" == cat_sub(buf, fmt::real(1000000.000000000, 5, FTOA_FLEX))); // AKA %g - CHECK("1.2342e+06" == cat_sub(buf, fmt::real(1234234.234234234, 5, FTOA_FLEX))); // AKA %g - CHECK("1.234e+06" == cat_sub(buf, fmt::real(1234234.234234234, 4, FTOA_FLEX))); // AKA %g - CHECK("1.23e+06" == cat_sub(buf, fmt::real(1234234.234234234, 3, FTOA_FLEX))); // AKA %g - CHECK("1.2e+06" == cat_sub(buf, fmt::real(1234234.234234234, 2, FTOA_FLEX))); // AKA %g - // AKA %a (hexadecimal formatting of floats) - // Earlier versions of emscripten's sprintf() (from MUSL) do not - // respect some precision values when printing in hexadecimal - // format. - // - // @see https://github.com/biojppm/c4core/pull/52 - #if defined(__EMSCRIPTEN__) && __EMSCRIPTEN_major__ < 3 - #define _c4emscripten_alt(alt1, alt2) alt2 - #else - #define _c4emscripten_alt(alt1, alt2) alt1 - #endif - CHECK("0x1.e8480p+19" == cat_sub(buf, fmt::real(1000000.000000000, 5, FTOA_HEXA))); // AKA %a - CHECK("0x1.2d53ap+20" == cat_sub(buf, fmt::real(1234234.234234234, 5, FTOA_HEXA))); // AKA %a - CHECK(_c4emscripten_alt("0x1.2d54p+20", "0x1.2d538p+20") - == cat_sub(buf, fmt::real(1234234.234234234, 4, FTOA_HEXA))); // AKA %a - CHECK("0x1.2d5p+20" == cat_sub(buf, fmt::real(1234234.234234234, 3, FTOA_HEXA))); // AKA %a - CHECK(_c4emscripten_alt("0x1.2dp+20", "0x1.2d8p+20") - == cat_sub(buf, fmt::real(1234234.234234234, 2, FTOA_HEXA))); // AKA %a - #undef _c4emscripten_alt - - // -------------------------------------------------------------- - // fmt::raw(): dump data in machine format (respecting alignment) - // -------------------------------------------------------------- - { - C4_SUPPRESS_WARNING_CLANG_WITH_PUSH("-Wcast-align") // we're casting the values directly, so alignment is strictly respected. - const uint32_t payload[] = {10, 20, 30, 40, UINT32_C(0xdeadbeef)}; - // (package payload as a substr, for comparison only) - csubstr expected = csubstr((const char *)payload, sizeof(payload)); - csubstr actual = cat_sub(buf, fmt::raw(payload)); - CHECK(!actual.overlaps(expected)); - CHECK(0 == memcmp(expected.str, actual.str, expected.len)); - // also possible with variables: - for(const uint32_t value : payload) - { - // (package payload as a substr, for comparison only) - expected = csubstr((const char *)&value, sizeof(value)); - actual = cat_sub(buf, fmt::raw(value)); - CHECK(actual.size() == sizeof(uint32_t)); - CHECK(!actual.overlaps(expected)); - CHECK(0 == memcmp(expected.str, actual.str, expected.len)); - // with non-const data, fmt::craw() may be needed for disambiguation: - actual = cat_sub(buf, fmt::craw(value)); - CHECK(actual.size() == sizeof(uint32_t)); - CHECK(!actual.overlaps(expected)); - CHECK(0 == memcmp(expected.str, actual.str, expected.len)); - // - // read back: - uint32_t result; - auto reader = fmt::raw(result); // keeps a reference to result - CHECK(&result == (uint32_t*)reader.buf); - CHECK(reader.len == sizeof(uint32_t)); - uncat(actual, reader); - // and compare: - // (vs2017/release/32bit does not reload result from cache, so force it) - result = *(uint32_t*)reader.buf; - CHECK(result == value); // roundtrip completed successfully - } - C4_SUPPRESS_WARNING_CLANG_POP - } - - // ------------------------- - // fmt::base64(): see below! - // ------------------------- - } -} - - -//----------------------------------------------------------------------------- - -/** demonstrates how to read and write base64-encoded blobs. - @see https://c4core.docsforge.com/master/base64/ - */ -void sample_base64() -{ - ryml::Tree tree; - tree.rootref() |= ryml::MAP; - struct text_and_base64 { ryml::csubstr text, base64; }; - text_and_base64 cases[] = { - {{"Love all, trust a few, do wrong to none."}, {"TG92ZSBhbGwsIHRydXN0IGEgZmV3LCBkbyB3cm9uZyB0byBub25lLg=="}}, - {{"The fool doth think he is wise, but the wise man knows himself to be a fool."}, {"VGhlIGZvb2wgZG90aCB0aGluayBoZSBpcyB3aXNlLCBidXQgdGhlIHdpc2UgbWFuIGtub3dzIGhpbXNlbGYgdG8gYmUgYSBmb29sLg=="}}, - {{"Brevity is the soul of wit."}, {"QnJldml0eSBpcyB0aGUgc291bCBvZiB3aXQu"}}, - {{"All that glitters is not gold."}, {"QWxsIHRoYXQgZ2xpdHRlcnMgaXMgbm90IGdvbGQu"}}, - {{"These violent delights have violent ends..."}, {"VGhlc2UgdmlvbGVudCBkZWxpZ2h0cyBoYXZlIHZpb2xlbnQgZW5kcy4uLg=="}}, - {{"How now, my love?"}, {"SG93IG5vdywgbXkgbG92ZT8="}}, - {{"Why is your cheek so pale?"}, {"V2h5IGlzIHlvdXIgY2hlZWsgc28gcGFsZT8="}}, - {{"How chance the roses there do fade so fast?"}, {"SG93IGNoYW5jZSB0aGUgcm9zZXMgdGhlcmUgZG8gZmFkZSBzbyBmYXN0Pw=="}}, - {{"Belike for want of rain, which I could well beteem them from the tempest of my eyes."}, {"QmVsaWtlIGZvciB3YW50IG9mIHJhaW4sIHdoaWNoIEkgY291bGQgd2VsbCBiZXRlZW0gdGhlbSBmcm9tIHRoZSB0ZW1wZXN0IG9mIG15IGV5ZXMu"}}, - }; - // to encode base64 and write the result to val: - for(text_and_base64 c : cases) - { - tree[c.text] << ryml::fmt::base64(c.text); - CHECK(tree[c.text].val() == c.base64); - } - // to encode base64 and write the result to key: - for(text_and_base64 c : cases) - { - tree.rootref().append_child() << ryml::key(ryml::fmt::base64(c.text)) << c.text; - CHECK(tree[c.base64].val() == c.text); - } - CHECK(ryml::emitrs_yaml(tree) == R"('Love all, trust a few, do wrong to none.': TG92ZSBhbGwsIHRydXN0IGEgZmV3LCBkbyB3cm9uZyB0byBub25lLg== -'The fool doth think he is wise, but the wise man knows himself to be a fool.': VGhlIGZvb2wgZG90aCB0aGluayBoZSBpcyB3aXNlLCBidXQgdGhlIHdpc2UgbWFuIGtub3dzIGhpbXNlbGYgdG8gYmUgYSBmb29sLg== -Brevity is the soul of wit.: QnJldml0eSBpcyB0aGUgc291bCBvZiB3aXQu -All that glitters is not gold.: QWxsIHRoYXQgZ2xpdHRlcnMgaXMgbm90IGdvbGQu -These violent delights have violent ends...: VGhlc2UgdmlvbGVudCBkZWxpZ2h0cyBoYXZlIHZpb2xlbnQgZW5kcy4uLg== -'How now, my love?': SG93IG5vdywgbXkgbG92ZT8= -'Why is your cheek so pale?': V2h5IGlzIHlvdXIgY2hlZWsgc28gcGFsZT8= -'How chance the roses there do fade so fast?': SG93IGNoYW5jZSB0aGUgcm9zZXMgdGhlcmUgZG8gZmFkZSBzbyBmYXN0Pw== -'Belike for want of rain, which I could well beteem them from the tempest of my eyes.': QmVsaWtlIGZvciB3YW50IG9mIHJhaW4sIHdoaWNoIEkgY291bGQgd2VsbCBiZXRlZW0gdGhlbSBmcm9tIHRoZSB0ZW1wZXN0IG9mIG15IGV5ZXMu -TG92ZSBhbGwsIHRydXN0IGEgZmV3LCBkbyB3cm9uZyB0byBub25lLg==: 'Love all, trust a few, do wrong to none.' -VGhlIGZvb2wgZG90aCB0aGluayBoZSBpcyB3aXNlLCBidXQgdGhlIHdpc2UgbWFuIGtub3dzIGhpbXNlbGYgdG8gYmUgYSBmb29sLg==: 'The fool doth think he is wise, but the wise man knows himself to be a fool.' -QnJldml0eSBpcyB0aGUgc291bCBvZiB3aXQu: Brevity is the soul of wit. -QWxsIHRoYXQgZ2xpdHRlcnMgaXMgbm90IGdvbGQu: All that glitters is not gold. -VGhlc2UgdmlvbGVudCBkZWxpZ2h0cyBoYXZlIHZpb2xlbnQgZW5kcy4uLg==: These violent delights have violent ends... -SG93IG5vdywgbXkgbG92ZT8=: 'How now, my love?' -V2h5IGlzIHlvdXIgY2hlZWsgc28gcGFsZT8=: 'Why is your cheek so pale?' -SG93IGNoYW5jZSB0aGUgcm9zZXMgdGhlcmUgZG8gZmFkZSBzbyBmYXN0Pw==: 'How chance the roses there do fade so fast?' -QmVsaWtlIGZvciB3YW50IG9mIHJhaW4sIHdoaWNoIEkgY291bGQgd2VsbCBiZXRlZW0gdGhlbSBmcm9tIHRoZSB0ZW1wZXN0IG9mIG15IGV5ZXMu: 'Belike for want of rain, which I could well beteem them from the tempest of my eyes.' -)"); - // to decode the val base64 and write the result to buf: - char buf1_[128], buf2_[128]; - ryml::substr buf1 = buf1_; // this is where we will write the result - ryml::substr buf2 = buf2_; // this is where we will write the result - for(auto c : cases) - { - // write the decoded result into the given buffer - tree[c.text] >> ryml::fmt::base64(buf1); // cannot know the needed size - size_t len = tree[c.text].deserialize_val(ryml::fmt::base64(buf2)); // returns the needed size - CHECK(len <= buf1.len); - CHECK(len <= buf2.len); - CHECK(c.text.len == len); - CHECK(buf1.first(len) == c.text); - CHECK(buf2.first(len) == c.text); - } - // to decode the val base64 and write the result to buf: - for(text_and_base64 c : cases) - { - // write the decoded result into the given buffer - tree[c.base64] >> ryml::key(ryml::fmt::base64(buf1)); // cannot know the needed size - size_t len = tree[c.base64].deserialize_key(ryml::fmt::base64(buf2)); // returns the needed size - CHECK(len <= buf1.len); - CHECK(len <= buf2.len); - CHECK(c.text.len == len); - CHECK(buf1.first(len) == c.text); - CHECK(buf2.first(len) == c.text); - } - // directly encode variables - { - const uint64_t valin = UINT64_C(0xdeadbeef); - uint64_t valout = 0; - tree["deadbeef"] << c4::fmt::base64(valin); // sometimes cbase64() is needed to avoid ambiguity - size_t len = tree["deadbeef"].deserialize_val(ryml::fmt::base64(valout)); - CHECK(len <= sizeof(valout)); - CHECK(valout == UINT64_C(0xdeadbeef)); // base64 roundtrip is bit-accurate - } - // directly encode memory ranges - { - const uint32_t data_in[11] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0xdeadbeef}; - uint32_t data_out[11] = {}; - CHECK(memcmp(data_in, data_out, sizeof(data_in)) != 0); // before the roundtrip - tree["int_data"] << c4::fmt::base64(data_in); - size_t len = tree["int_data"].deserialize_val(ryml::fmt::base64(data_out)); - CHECK(len <= sizeof(data_out)); - CHECK(memcmp(data_in, data_out, sizeof(data_in)) == 0); // after the roundtrip - } -} - - -//----------------------------------------------------------------------------- - -// user scalar types: implemented in ryml through to_chars() + from_chars() - -// To harness [C++'s ADL rules](http://en.cppreference.com/w/cpp/language/adl), -// it is important to overload these functions in the namespace of the type -// you're serializing (or in the c4::yml namespace). -// -// Please take note of the following pitfall when using serialization -// functions: you have to include the header with the serialization -// before any other headers that use functions from it. see the -// include order at the top of this file. -// -// This constraint also applies to the conversion functions for your -// types; just like with the STL's headers, they should be included -// prior to ryml's headers. - -template struct vec2 { T x, y; }; -template struct vec3 { T x, y, z; }; -template struct vec4 { T x, y, z, w; }; - -template struct parse_only_vec2 { T x, y; }; -template struct parse_only_vec3 { T x, y, z; }; -template struct parse_only_vec4 { T x, y, z, w; }; - -template struct emit_only_vec2 { T x, y; }; -template struct emit_only_vec3 { T x, y, z; }; -template struct emit_only_vec4 { T x, y, z, w; }; - -// to_chars(): only needed for emitting -// -// format v to the given string view + return the number of -// characters written into it. The view size (buf.len) must -// be strictly respected. Return the number of characters -// that need to be written. So if the return value -// is larger than buf.len, ryml will resize the buffer and -// call this again with a larger buffer of the correct size. - -template size_t to_chars(ryml::substr buf, vec2 v) { return ryml::format(buf, "({},{})", v.x, v.y); } -template size_t to_chars(ryml::substr buf, vec3 v) { return ryml::format(buf, "({},{},{})", v.x, v.y, v.z); } -template size_t to_chars(ryml::substr buf, vec4 v) { return ryml::format(buf, "({},{},{},{})", v.x, v.y, v.z, v.w); } - -template size_t to_chars(ryml::substr buf, emit_only_vec2 v) { return ryml::format(buf, "({},{})", v.x, v.y); } -template size_t to_chars(ryml::substr buf, emit_only_vec3 v) { return ryml::format(buf, "({},{},{})", v.x, v.y, v.z); } -template size_t to_chars(ryml::substr buf, emit_only_vec4 v) { return ryml::format(buf, "({},{},{},{})", v.x, v.y, v.z, v.w); } - - -// from_chars(): only needed for parsing - -template bool from_chars(ryml::csubstr buf, vec2 *v) { size_t ret = ryml::unformat(buf, "({},{})", v->x, v->y); return ret != ryml::yml::npos; } -template bool from_chars(ryml::csubstr buf, vec3 *v) { size_t ret = ryml::unformat(buf, "({},{},{})", v->x, v->y, v->z); return ret != ryml::yml::npos; } -template bool from_chars(ryml::csubstr buf, vec4 *v) { size_t ret = ryml::unformat(buf, "({},{},{},{})", v->x, v->y, v->z, v->w); return ret != ryml::yml::npos; } - -template bool from_chars(ryml::csubstr buf, parse_only_vec2 *v) { size_t ret = ryml::unformat(buf, "({},{})", v->x, v->y); return ret != ryml::yml::npos; } -template bool from_chars(ryml::csubstr buf, parse_only_vec3 *v) { size_t ret = ryml::unformat(buf, "({},{},{})", v->x, v->y, v->z); return ret != ryml::yml::npos; } -template bool from_chars(ryml::csubstr buf, parse_only_vec4 *v) { size_t ret = ryml::unformat(buf, "({},{},{},{})", v->x, v->y, v->z, v->w); return ret != ryml::yml::npos; } - - -/** to add scalar types (ie leaf types converting to/from string), - define the functions above for those types. */ -void sample_user_scalar_types() -{ - ryml::Tree t; - - auto r = t.rootref(); - r |= ryml::MAP; - - vec2 v2in{10, 11}; - vec2 v2out{1, 2}; - r["v2"] << v2in; // serializes to the tree's arena, and then sets the keyval - r["v2"] >> v2out; - CHECK(v2in.x == v2out.x); - CHECK(v2in.y == v2out.y); - vec3 v3in{100, 101, 102}; - vec3 v3out{1, 2, 3}; - r["v3"] << v3in; // serializes to the tree's arena, and then sets the keyval - r["v3"] >> v3out; - CHECK(v3in.x == v3out.x); - CHECK(v3in.y == v3out.y); - CHECK(v3in.z == v3out.z); - vec4 v4in{1000, 1001, 1002, 1003}; - vec4 v4out{1, 2, 3, 4}; - r["v4"] << v4in; // serializes to the tree's arena, and then sets the keyval - r["v4"] >> v4out; - CHECK(v4in.x == v4out.x); - CHECK(v4in.y == v4out.y); - CHECK(v4in.z == v4out.z); - CHECK(v4in.w == v4out.w); - CHECK(ryml::emitrs_yaml(t) == R"(v2: '(10,11)' -v3: '(100,101,102)' -v4: '(1000,1001,1002,1003)' -)"); - - // note that only the used functions are needed: - // - if a type is only parsed, then only from_chars() is needed - // - if a type is only emitted, then only to_chars() is needed - emit_only_vec2 eov2in{20, 21}; // only has to_chars() - parse_only_vec2 pov2out{1, 2}; // only has from_chars() - r["v2"] << eov2in; // serializes to the tree's arena, and then sets the keyval - r["v2"] >> pov2out; - CHECK(eov2in.x == pov2out.x); - CHECK(eov2in.y == pov2out.y); - emit_only_vec3 eov3in{30, 31, 32}; // only has to_chars() - parse_only_vec3 pov3out{1, 2, 3}; // only has from_chars() - r["v3"] << eov3in; // serializes to the tree's arena, and then sets the keyval - r["v3"] >> pov3out; - CHECK(eov3in.x == pov3out.x); - CHECK(eov3in.y == pov3out.y); - CHECK(eov3in.z == pov3out.z); - emit_only_vec4 eov4in{40, 41, 42, 43}; // only has to_chars() - parse_only_vec4 pov4out{1, 2, 3, 4}; // only has from_chars() - r["v4"] << eov4in; // serializes to the tree's arena, and then sets the keyval - r["v4"] >> pov4out; - CHECK(eov4in.x == pov4out.x); - CHECK(eov4in.y == pov4out.y); - CHECK(eov4in.z == pov4out.z); - CHECK(ryml::emitrs_yaml(t) == R"(v2: '(20,21)' -v3: '(30,31,32)' -v4: '(40,41,42,43)' -)"); -} - - -//----------------------------------------------------------------------------- - -// user container types: implemented in ryml through write() + read() -// -// Please take note of the following pitfall when using serialization -// functions: you have to include the header with the serialization -// before any other headers that use functions from it. see the -// include order at the top of this file. -// -// This constraint also applies to the conversion functions for your -// types; just like with the STL's headers, they should be included -// prior to ryml's headers. - -// user container type: seq-like -template struct my_seq_type { std::vector seq_member; }; -template -void write(ryml::NodeRef *n, my_seq_type const& seq) -{ - *n |= ryml::SEQ; - for(auto const& v : seq.seq_member) - n->append_child() << v; -} -template -bool read(ryml::ConstNodeRef const& n, my_seq_type *seq) -{ - seq->seq_member.resize(n.num_children()); // num_children() is O(N) - size_t pos = 0; - for(auto const ch : n.children()) - ch >> seq->seq_member[pos++]; - return true; -} - - -// user container type: map-like -template struct my_map_type { std::map map_member; }; -template -void write(ryml::NodeRef *n, my_map_type const& map) -{ - *n |= ryml::MAP; - for(auto const& v : map.map_member) - n->append_child() << ryml::key(v.first) << v.second; -} -template -bool read(ryml::ConstNodeRef const& n, my_map_type *map) -{ - K k{}; - V v{}; - for(auto const ch : n) - { - ch >> c4::yml::key(k) >> v; - map->map_member.emplace(std::make_pair(std::move(k), std::move(v))); - } - return true; -} - - -// user container type: notice all the members are complex types -// defined above -struct my_type -{ - vec2 v2; - vec3 v3; - vec4 v4; - my_seq_type seq; - my_map_type map; -}; -void write(ryml::NodeRef *n, my_type const& val) -{ - *n |= ryml::MAP; - n->append_child() << ryml::key("v2") << val.v2; - n->append_child() << ryml::key("v3") << val.v3; - n->append_child() << ryml::key("v4") << val.v4; - n->append_child() << ryml::key("seq") << val.seq; - n->append_child() << ryml::key("map") << val.map; -} -bool read(ryml::ConstNodeRef const& n, my_type *val) -{ - n["v2"] >> val->v2; - n["v3"] >> val->v3; - n["v4"] >> val->v4; - n["seq"] >> val->seq; - n["map"] >> val->map; - return true; -} - - -void sample_user_container_types() -{ - my_type mt_in{ - {20, 21}, - {30, 31, 32}, - {40, 41, 42, 43}, - {{101, 102, 103, 104, 105, 106, 107}}, - {{{1001, 2001}, {1002, 2002}, {1003, 2003}}}, - }; - my_type mt_out; - - ryml::Tree t; - t.rootref() << mt_in; // read from this - t.crootref() >> mt_out; // assign here - CHECK(mt_out.v2.x == mt_in.v2.x); - CHECK(mt_out.v2.y == mt_in.v2.y); - CHECK(mt_out.v3.x == mt_in.v3.x); - CHECK(mt_out.v3.y == mt_in.v3.y); - CHECK(mt_out.v3.z == mt_in.v3.z); - CHECK(mt_out.v4.x == mt_in.v4.x); - CHECK(mt_out.v4.y == mt_in.v4.y); - CHECK(mt_out.v4.z == mt_in.v4.z); - CHECK(mt_out.v4.w == mt_in.v4.w); - CHECK(mt_in.seq.seq_member.size() > 0); - CHECK(mt_out.seq.seq_member.size() == mt_in.seq.seq_member.size()); - for(size_t i = 0; i < mt_in.seq.seq_member.size(); ++i) - { - CHECK(mt_out.seq.seq_member[i] == mt_in.seq.seq_member[i]); - } - CHECK(mt_in.map.map_member.size() > 0); - CHECK(mt_out.map.map_member.size() == mt_in.map.map_member.size()); - for(auto const& kv : mt_in.map.map_member) - { - CHECK(mt_out.map.map_member.find(kv.first) != mt_out.map.map_member.end()); - CHECK(mt_out.map.map_member[kv.first] == kv.second); - } - CHECK(ryml::emitrs_yaml(t) == R"(v2: '(20,21)' -v3: '(30,31,32)' -v4: '(40,41,42,43)' -seq: - - 101 - - 102 - - 103 - - 104 - - 105 - - 106 - - 107 -map: - 1001: 2001 - 1002: 2002 - 1003: 2003 -)"); -} - - -//----------------------------------------------------------------------------- -// -// Please take note of the following pitfall when using serialization -// functions: you have to include the header with the serialization -// before any other headers that use functions from it. see the -// include order at the top of this file. -// -// This constraint also applies to the conversion functions for your -// types; just like with the STL's headers, they should be included -// prior to ryml's headers. - -/** demonstrates usage with the std implementations provided by ryml in the ryml_std.hpp header */ -void sample_std_types() -{ - std::string yml_std_string = R"(- v2: '(20,21)' - v3: '(30,31,32)' - v4: '(40,41,42,43)' - seq: - - 101 - - 102 - - 103 - - 104 - - 105 - - 106 - - 107 - map: - 1001: 2001 - 1002: 2002 - 1003: 2003 -- v2: '(120,121)' - v3: '(130,131,132)' - v4: '(140,141,142,143)' - seq: - - 1101 - - 1102 - - 1103 - - 1104 - - 1105 - - 1106 - - 1107 - map: - 11001: 12001 - 11002: 12002 - 11003: 12003 -- v2: '(220,221)' - v3: '(230,231,232)' - v4: '(240,241,242,243)' - seq: - - 2101 - - 2102 - - 2103 - - 2104 - - 2105 - - 2106 - - 2107 - map: - 21001: 22001 - 21002: 22002 - 21003: 22003 -)"; - // parse in-place using the std::string above - auto tree = ryml::parse_in_place(ryml::to_substr(yml_std_string)); - // my_type is a container-of-containers type. see above its - // definition implementation for ryml. - std::vector vmt; - tree.rootref() >> vmt; - CHECK(vmt.size() == 3); - ryml::Tree tree_out; - tree_out.rootref() << vmt; - CHECK(ryml::emitrs_yaml(tree_out) == yml_std_string); -} - - -//----------------------------------------------------------------------------- - -/** demonstrates how to emit to a linear container of char */ -void sample_emit_to_container() -{ - // it is possible to emit to any linear container of char. - - ryml::csubstr ymla = "- 1\n- 2\n"; - ryml::csubstr ymlb = R"(- a -- b -- x0: 1 - x1: 2 -- champagne: Dom Perignon - coffee: Arabica - more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 - beer: - - Rochefort 10 - - Busch - - Leffe Rituel -- foo -- bar -- baz -- bat -)"; - auto treea = ryml::parse_in_arena(ymla); - auto treeb = ryml::parse_in_arena(ymlb); - - // eg, std::vector - { - // do a blank call on an empty buffer to find the required size. - // no overflow will occur, and returns a substr with the size - // required to output - ryml::csubstr output = ryml::emit_yaml(treea, treea.root_id(), ryml::substr{}, /*error_on_excess*/false); - CHECK(output.str == nullptr); - CHECK(output.len > 0); - size_t num_needed_chars = output.len; - std::vector buf(num_needed_chars); - // now try again with the proper buffer - output = ryml::emit_yaml(treea, treea.root_id(), ryml::to_substr(buf), /*error_on_excess*/true); - CHECK(output == ymla); - - // it is possible to reuse the buffer and grow it as needed. - // first do a blank run to find the size: - output = ryml::emit_yaml(treeb, treeb.root_id(), ryml::substr{}, /*error_on_excess*/false); - CHECK(output.str == nullptr); - CHECK(output.len > 0); - CHECK(output.len == ymlb.len); - num_needed_chars = output.len; - buf.resize(num_needed_chars); - // now try again with the proper buffer - output = ryml::emit_yaml(treeb, treeb.root_id(), ryml::to_substr(buf), /*error_on_excess*/true); - CHECK(output == ymlb); - - // there is a convenience wrapper performing the same as above: - // provided to_substr() is defined for that container. - output = ryml::emitrs_yaml(treeb, &buf); - CHECK(output == ymlb); - - // or you can just output a new container: - // provided to_substr() is defined for that container. - std::vector another = ryml::emitrs_yaml>(treeb); - CHECK(ryml::to_csubstr(another) == ymlb); - - // you can also emit nested nodes: - another = ryml::emitrs_yaml>(treeb[3][2]); - CHECK(ryml::to_csubstr(another) == R"(more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 -)"); - } - - - // eg, std::string. notice this is the same code as above - { - // do a blank call on an empty buffer to find the required size. - // no overflow will occur, and returns a substr with the size - // required to output - ryml::csubstr output = ryml::emit_yaml(treea, treea.root_id(), ryml::substr{}, /*error_on_excess*/false); - CHECK(output.str == nullptr); - CHECK(output.len > 0); - size_t num_needed_chars = output.len; - std::string buf; - buf.resize(num_needed_chars); - // now try again with the proper buffer - output = ryml::emit_yaml(treea, treea.root_id(), ryml::to_substr(buf), /*error_on_excess*/true); - CHECK(output == ymla); - - // it is possible to reuse the buffer and grow it as needed. - // first do a blank run to find the size: - output = ryml::emit_yaml(treeb, treeb.root_id(), ryml::substr{}, /*error_on_excess*/false); - CHECK(output.str == nullptr); - CHECK(output.len > 0); - CHECK(output.len == ymlb.len); - num_needed_chars = output.len; - buf.resize(num_needed_chars); - // now try again with the proper buffer - output = ryml::emit_yaml(treeb, treeb.root_id(), ryml::to_substr(buf), /*error_on_excess*/true); - CHECK(output == ymlb); - - // there is a convenience wrapper performing the above instructions: - // provided to_substr() is defined for that container - output = ryml::emitrs_yaml(treeb, &buf); - CHECK(output == ymlb); - - // or you can just output a new container: - // provided to_substr() is defined for that container. - std::string another = ryml::emitrs_yaml(treeb); - CHECK(ryml::to_csubstr(another) == ymlb); - - // you can also emit nested nodes: - another = ryml::emitrs_yaml(treeb[3][2]); - CHECK(ryml::to_csubstr(another) == R"(more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 -)"); - } -} - - -//----------------------------------------------------------------------------- - -/** demonstrates how to emit to a stream-like structure */ -void sample_emit_to_stream() -{ - ryml::csubstr ymlb = R"(- a -- b -- x0: 1 - x1: 2 -- champagne: Dom Perignon - coffee: Arabica - more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 - beer: - - Rochefort 10 - - Busch - - Leffe Rituel -- foo -- bar -- baz -- bat -)"; - auto tree = ryml::parse_in_arena(ymlb); - - std::string s; - - // emit a full tree - { - std::stringstream ss; - ss << tree; // works with any stream having .operator<<() and .write() - s = ss.str(); - CHECK(ryml::to_csubstr(s) == ymlb); - } - - // emit a full tree as json - { - std::stringstream ss; - ss << ryml::as_json(tree); // works with any stream having .operator<<() and .write() - s = ss.str(); - CHECK(ryml::to_csubstr(s) == R"(["a","b",{"x0": 1,"x1": 2},{"champagne": "Dom Perignon","coffee": "Arabica","more": {"vinho verde": "Soalheiro","vinho tinto": "Redoma 2017"},"beer": ["Rochefort 10","Busch","Leffe Rituel"]},"foo","bar","baz","bat"])"); - } - - // emit a nested node - { - std::stringstream ss; - ss << tree[3][2]; // works with any stream having .operator<<() and .write() - s = ss.str(); - CHECK(ryml::to_csubstr(s) == R"(more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 -)"); - } - - // emit a nested node as json - { - std::stringstream ss; - ss << ryml::as_json(tree[3][2]); // works with any stream having .operator<<() and .write() - s = ss.str(); - CHECK(ryml::to_csubstr(s) == R"("more": {"vinho verde": "Soalheiro","vinho tinto": "Redoma 2017"})"); - } -} - - -//----------------------------------------------------------------------------- - -/** demonstrates how to emit to a FILE* */ -void sample_emit_to_file() -{ - ryml::csubstr yml = R"(- a -- b -- x0: 1 - x1: 2 -- champagne: Dom Perignon - coffee: Arabica - more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 - beer: - - Rochefort 10 - - Busch - - Leffe Rituel -- foo -- bar -- baz -- bat -)"; - auto tree = ryml::parse_in_arena(yml); - // this is emitting to stdout, but of course you can pass in any - // FILE* obtained from fopen() - size_t len = ryml::emit_yaml(tree, tree.root_id(), stdout); - // the return value is the number of characters that were written - // to the file - CHECK(len == yml.len); -} - - -//----------------------------------------------------------------------------- - -/** just like parsing into a nested node, you can also emit from a nested node. */ -void sample_emit_nested_node() -{ - const ryml::Tree tree = ryml::parse_in_arena(R"(- a -- b -- x0: 1 - x1: 2 -- champagne: Dom Perignon - coffee: Arabica - more: - vinho verde: Soalheiro - vinho tinto: Redoma 2017 - beer: - - Rochefort 10 - - Busch - - Leffe Rituel - - - and so - - many other - - wonderful beers -- more -- seq -- members -- here -)"); - CHECK(ryml::emitrs_yaml(tree[3]["beer"]) == R"(beer: - - Rochefort 10 - - Busch - - Leffe Rituel - - - and so - - many other - - wonderful beers -)"); - CHECK(ryml::emitrs_yaml(tree[3]["beer"][0]) == "Rochefort 10\n"); - CHECK(ryml::emitrs_yaml(tree[3]["beer"][3]) == R"(- and so -- many other -- wonderful beers -)"); -} - - -//----------------------------------------------------------------------------- - - -/** shows how to parse and emit JSON. */ -void sample_json() -{ - // Since JSON is a subset of YAML, parsing JSON is just the - // same as YAML: - ryml::Tree tree = ryml::parse_in_arena(R"({ -"doe":"a deer, a female deer", -"ray":"a drop of golden sun", -"me":"a name, I call myself", -"far":"a long long way to go" -})"); - // However, emitting still defaults to YAML - CHECK(ryml::emitrs_yaml(tree) == R"('doe': 'a deer, a female deer' -'ray': 'a drop of golden sun' -'me': 'a name, I call myself' -'far': 'a long long way to go' -)"); - // to emit JSON, use the proper overload: - CHECK(ryml::emitrs_json(tree) == R"({"doe": "a deer, a female deer","ray": "a drop of golden sun","me": "a name, I call myself","far": "a long long way to go"})"); - // to emit JSON to a stream: - std::stringstream ss; - ss << ryml::as_json(tree); // <- mark it like this - CHECK(ss.str() == R"({"doe": "a deer, a female deer","ray": "a drop of golden sun","me": "a name, I call myself","far": "a long long way to go"})"); - // Note the following limitations: - // - // - YAML streams cannot be emitted as json, and are not - // allowed. But you can work around this by emitting the - // individual documents separately; see the sample_docs() - // below for such an example. - // - // - tags cannot be emitted as json, and are not allowed. - // - // - anchors and aliases cannot be emitted as json and are not allowed. -} - - -//----------------------------------------------------------------------------- - -/** demonstrates usage with anchors and alias references. - -Note that dereferencing is opt-in; after parsing, you have to call -`Tree::resolve()` explicitly if you want resolved references in the -tree. This method will resolve all references and substitute the anchored -values in place of the reference. - -The `Tree::resolve()` method first does a full traversal of the tree to -gather all anchors and references in a separate collection, then it goes -through that collection to locate the names, which it does by obeying the -YAML standard diktat that - - an alias node refers to the most recent node in - the serialization having the specified anchor - -So, depending on the number of anchor/alias nodes, this is a potentially -expensive operation, with a best-case linear complexity (from the initial -traversal) and a worst-case quadratic complexity (if every node has an -alias/anchor). This potential cost is the reason for requiring an explicit -call to `Tree::resolve()`. */ -void sample_anchors_and_aliases() -{ - std::string unresolved = R"(base: &base - name: Everyone has same name -foo: &foo - <<: *base - age: 10 -bar: &bar - <<: *base - age: 20 -bill_to: &id001 - street: |- - 123 Tornado Alley - Suite 16 - city: East Centerville - state: KS -ship_to: *id001 -&keyref key: &valref val -*valref: *keyref -)"; - - ryml::Tree tree = ryml::parse_in_arena(ryml::to_csubstr(unresolved)); - // by default, references are not resolved when parsing: - CHECK( ! tree["base"].has_key_anchor()); - CHECK( tree["base"].has_val_anchor()); - CHECK( tree["base"].val_anchor() == "base"); - CHECK( tree["key"].key_anchor() == "keyref"); - CHECK( tree["key"].val_anchor() == "valref"); - CHECK( tree["*valref"].is_key_ref()); - CHECK( tree["*valref"].is_val_ref()); - CHECK( tree["*valref"].key_ref() == "valref"); - CHECK( tree["*valref"].val_ref() == "keyref"); - - // to resolve references, simply call tree.resolve(), - // which will perform the reference instantiations: - tree.resolve(); - - // all the anchors and references are substistuted and then removed: - CHECK( ! tree["base"].has_key_anchor()); - CHECK( ! tree["base"].has_val_anchor()); - CHECK( ! tree["base"].has_val_anchor()); - CHECK( ! tree["key"].has_key_anchor()); - CHECK( ! tree["key"].has_val_anchor()); - CHECK( ! tree["val"].is_key_ref()); // notice *valref is now turned to val - CHECK( ! tree["val"].is_val_ref()); // notice *valref is now turned to val - - CHECK(tree["ship_to"]["city"] == "East Centerville"); - CHECK(tree["ship_to"]["state"] == "KS"); -} - - -//----------------------------------------------------------------------------- - -void sample_tags() -{ - std::string unresolved = R"(--- !!map -a: 0 -b: 1 ---- !map -? a -: b ---- !!seq -- a -- b ---- !!str -a - b -... ---- !!str a b -... ---- !!str a b ---- !!str -a: b ---- !!str a: b ---- -!!str a: b ---- -!!str a - b ---- -!!set -? a -? b ---- !!set -? a ---- -[!!int 0, !!str 0] -)"; - std::string resolved = R"(--- !!map -a: 0 -b: 1 ---- !map -a: b ---- !!seq -- a -- b ---- !!str a b ---- !!str a b ---- !!str a b ---- !!str 'a: b' ---- !!str 'a: b' ---- -!!str a: b ---- !!str a b ---- !!set -a: -b: ---- !!set -a: ---- -- !!int 0 -- !!str 0 -)"; - ryml::Tree tree = ryml::parse_in_arena(ryml::to_csubstr(unresolved)); - CHECK(ryml::emitrs_yaml(tree) == resolved); - const ryml::ConstNodeRef stream = tree.rootref(); - CHECK(stream.is_stream()); - CHECK(stream.num_children() == 13); - for(auto node : stream.children()) - CHECK(node.is_doc()); - CHECK(stream[11].val_tag() == "!!set"); -} - - -//----------------------------------------------------------------------------- - -void sample_docs() -{ - std::string yml = R"(--- -a: 0 -b: 1 ---- -c: 2 -d: 3 ---- -- 4 -- 5 -- 6 -- 7 -)"; - ryml::Tree tree = ryml::parse_in_place(ryml::to_substr(yml)); - CHECK(ryml::emitrs_yaml(tree) == yml); - - // iteration through docs - { - // using the node API - const ryml::ConstNodeRef stream = tree.rootref(); - CHECK(stream.is_root()); - CHECK(stream.is_stream()); - CHECK(!stream.is_doc()); - CHECK(stream.num_children() == 3); - for(const ryml::ConstNodeRef doc : stream.children()) - CHECK(doc.is_doc()); - CHECK(tree.docref(0).id() == stream.child(0).id()); - CHECK(tree.docref(1).id() == stream.child(1).id()); - CHECK(tree.docref(2).id() == stream.child(2).id()); - // equivalent: using the lower level index API - const size_t stream_id = tree.root_id(); - CHECK(tree.is_root(stream_id)); - CHECK(tree.is_stream(stream_id)); - CHECK(!tree.is_doc(stream_id)); - CHECK(tree.num_children(stream_id) == 3); - for(size_t doc_id = tree.first_child(stream_id); doc_id != ryml::NONE; doc_id = tree.next_sibling(stream_id)) - CHECK(tree.is_doc(doc_id)); - CHECK(tree.doc(0) == tree.child(stream_id, 0)); - CHECK(tree.doc(1) == tree.child(stream_id, 1)); - CHECK(tree.doc(2) == tree.child(stream_id, 2)); - - // using the node API - CHECK(stream[0].is_doc()); - CHECK(stream[0].is_map()); - CHECK(stream[0]["a"].val() == "0"); - CHECK(stream[0]["b"].val() == "1"); - // equivalent: using the index API - const size_t doc0_id = tree.first_child(stream_id); - CHECK(tree.is_doc(doc0_id)); - CHECK(tree.is_map(doc0_id)); - CHECK(tree.val(tree.find_child(doc0_id, "a")) == "0"); - CHECK(tree.val(tree.find_child(doc0_id, "b")) == "1"); - - // using the node API - CHECK(stream[1].is_doc()); - CHECK(stream[1].is_map()); - CHECK(stream[1]["c"].val() == "2"); - CHECK(stream[1]["d"].val() == "3"); - // equivalent: using the index API - const size_t doc1_id = tree.next_sibling(doc0_id); - CHECK(tree.is_doc(doc1_id)); - CHECK(tree.is_map(doc1_id)); - CHECK(tree.val(tree.find_child(doc1_id, "c")) == "2"); - CHECK(tree.val(tree.find_child(doc1_id, "d")) == "3"); - - // using the node API - CHECK(stream[2].is_doc()); - CHECK(stream[2].is_seq()); - CHECK(stream[2][0].val() == "4"); - CHECK(stream[2][1].val() == "5"); - CHECK(stream[2][2].val() == "6"); - CHECK(stream[2][3].val() == "7"); - // equivalent: using the index API - const size_t doc2_id = tree.next_sibling(doc1_id); - CHECK(tree.is_doc(doc2_id)); - CHECK(tree.is_seq(doc2_id)); - CHECK(tree.val(tree.child(doc2_id, 0)) == "4"); - CHECK(tree.val(tree.child(doc2_id, 1)) == "5"); - CHECK(tree.val(tree.child(doc2_id, 2)) == "6"); - CHECK(tree.val(tree.child(doc2_id, 3)) == "7"); - } - - // Note: since json does not have streams, you cannot emit the above - // tree as json when you start from the root: - //CHECK(ryml::emitrs_json(tree) == yml); // RUNTIME ERROR! - - // emitting streams as json is not possible, but - // you can iterate through individual documents and emit - // them separately: - { - const std::string expected_json[] = { - R"({"a": 0,"b": 1})", - R"({"c": 2,"d": 3})", - R"([4,5,6,7])", - }; - // using the node API - { - size_t count = 0; - const ryml::ConstNodeRef stream = tree.rootref(); - CHECK(stream.num_children() == C4_COUNTOF(expected_json)); - for(ryml::ConstNodeRef doc : stream.children()) - CHECK(ryml::emitrs_json(doc) == expected_json[count++]); - } - // equivalent: using the index API - { - size_t count = 0; - const size_t stream_id = tree.root_id(); - CHECK(tree.num_children(stream_id) == C4_COUNTOF(expected_json)); - for(size_t doc_id = tree.first_child(stream_id); doc_id != ryml::NONE; doc_id = tree.next_sibling(doc_id)) - CHECK(ryml::emitrs_json(tree, doc_id) == expected_json[count++]); - } - } -} - - -//----------------------------------------------------------------------------- - -// To avoid imposing a particular type of error handling, ryml accepts -// custom error handlers. This enables users to use exceptions, or -// plain calls to abort(), as they see fit. -// -// However, it is important to note that the error callback must never -// return to the caller! Otherwise, an infinite loop or program crash -// may occur. - - -struct ErrorHandlerExample -{ - // this will be called on error - void on_error(const char* msg, size_t len, ryml::Location loc) - { - throw std::runtime_error(ryml::formatrs("{}:{}:{} ({}B): ERROR: {}", - loc.name, loc.line, loc.col, loc.offset, ryml::csubstr(msg, len))); - } - - // bridge - ryml::Callbacks callbacks() - { - return ryml::Callbacks(this, nullptr, nullptr, ErrorHandlerExample::s_error); - } - static void s_error(const char* msg, size_t len, ryml::Location loc, void *this_) - { - return ((ErrorHandlerExample*)this_)->on_error(msg, len, loc); - } - - // checking - template - void check_error_occurs(Fn &&fn) const - { - bool expected_error_occurred = false; - try { fn(); } - catch(std::runtime_error const&) { expected_error_occurred = true; } - CHECK(expected_error_occurred); - } - void check_effect(bool committed) const - { - ryml::Callbacks const& current = ryml::get_callbacks(); - if(committed) - { - CHECK(current.m_error == &s_error); - } - else - { - CHECK(current.m_error != &s_error); - } - CHECK(current.m_allocate == defaults.m_allocate); - CHECK(current.m_free == defaults.m_free); - } - // save the default callbacks for checking - ErrorHandlerExample() : defaults(ryml::get_callbacks()) {} - ryml::Callbacks defaults; -}; - -/** demonstrates how to set a custom error handler for ryml */ -void sample_error_handler() -{ - ErrorHandlerExample errh; - - // set a global error handler. Note the error callback must never - // return: it must either abort or throw an exception. Otherwise, - // the parser will enter into an infinite loop, or the program may - // crash. - ryml::set_callbacks(errh.callbacks()); - errh.check_effect(/*committed*/true); - errh.check_error_occurs([]{ - ryml::Tree tree = ryml::parse_in_arena("errorhandler.yml", "[a: b\n}"); - std::cout << tree; - }); - - ryml::set_callbacks(errh.defaults); // restore defaults. - errh.check_effect(/*committed*/false); -} - - -//----------------------------------------------------------------------------- - -// Please note the following about the use of custom allocators with -// ryml. Due to [the static initialization order -// fiasco](https://en.cppreference.com/w/cpp/language/siof), if you -// use static ryml trees or parsers, you need to make sure that their -// callbacks have the same lifetime. So you can't use ryml's default -// callbacks structure, as it is declared in a ryml file, and the standard -// provides no guarantee on the relative initialization order, such -// that it is constructed before and destroyed after your -// variables (in fact you are pretty much guaranteed to see this -// fail). So please carefully consider your choices, and ponder -// whether you really need to use ryml static trees and parsers. If -// you do need this, then you will need to declare and use a ryml -// callbacks structure that outlives the tree and/or parser. - -struct GlobalAllocatorExample -{ - std::vector memory_pool = std::vector(10u * 1024u); // 10KB - size_t num_allocs = 0, alloc_size = 0; - size_t num_deallocs = 0, dealloc_size = 0; - - void *allocate(size_t len) - { - void *ptr = &memory_pool[alloc_size]; - alloc_size += len; - ++num_allocs; - if(C4_UNLIKELY(alloc_size > memory_pool.size())) - { - std::cerr << "out of memory! requested=" << alloc_size << " vs " << memory_pool.size() << " available" << std::endl; - std::abort(); - } - return ptr; - } - - void free(void *mem, size_t len) - { - CHECK((char*)mem >= &memory_pool.front() && (char*)mem < &memory_pool.back()); - CHECK((char*)mem+len >= &memory_pool.front() && (char*)mem+len <= &memory_pool.back()); - dealloc_size += len; - ++num_deallocs; - // no need to free here - } - - // bridge - ryml::Callbacks callbacks() - { - return ryml::Callbacks(this, &GlobalAllocatorExample::s_allocate, &GlobalAllocatorExample::s_free, nullptr); - } - static void* s_allocate(size_t len, void* /*hint*/, void *this_) - { - return ((GlobalAllocatorExample*)this_)->allocate(len); - } - static void s_free(void *mem, size_t len, void *this_) - { - return ((GlobalAllocatorExample*)this_)->free(mem, len); - } - - // checking - ~GlobalAllocatorExample() - { - check_and_reset(); - } - void check_and_reset() - { - std::cout << "size: alloc=" << alloc_size << " dealloc=" << dealloc_size << std::endl; - std::cout << "count: #allocs=" << num_allocs << " #deallocs=" << num_deallocs << std::endl; - CHECK(num_allocs == num_deallocs); - CHECK(alloc_size >= dealloc_size); // failure here means a double free - CHECK(alloc_size == dealloc_size); // failure here means a leak - num_allocs = 0; - num_deallocs = 0; - alloc_size = 0; - dealloc_size = 0; - } -}; - - -/** demonstrates how to set the global allocator for ryml */ -void sample_global_allocator() -{ - GlobalAllocatorExample mem; - - // save the existing callbacks for restoring - ryml::Callbacks defaults = ryml::get_callbacks(); - - // set to our callbacks - ryml::set_callbacks(mem.callbacks()); - - // verify that the allocator is in effect - ryml::Callbacks const& current = ryml::get_callbacks(); - CHECK(current.m_allocate == &mem.s_allocate); - CHECK(current.m_free == &mem.s_free); - - // so far nothing was allocated - CHECK(mem.alloc_size == 0); - - // parse one tree and check - (void)ryml::parse_in_arena(R"({foo: bar})"); - mem.check_and_reset(); - - // parse another tree and check - (void)ryml::parse_in_arena(R"([a, b, c, d, {foo: bar, money: pennys}])"); - mem.check_and_reset(); - - // verify that by reserving we save allocations - { - ryml::Parser parser; // reuse a parser - ryml::Tree tree; // reuse a tree - - tree.reserve(10); // reserve the number of nodes - tree.reserve_arena(100); // reserve the arena size - parser.reserve_stack(10); // reserve the parser depth. - - // since the parser stack uses Small Storage Optimization, - // allocations will only happen with capacities higher than 16. - CHECK(mem.num_allocs == 2); // tree, tree_arena and NOT the parser - - parser.reserve_stack(20); // reserve the parser depth. - CHECK(mem.num_allocs == 3); // tree, tree_arena and now the parser as well - - // verify that no other allocations occur when parsing - size_t size_before = mem.alloc_size; - parser.parse_in_arena("", R"([a, b, c, d, {foo: bar, money: pennys}])", &tree); - CHECK(mem.alloc_size == size_before); - CHECK(mem.num_allocs == 3); - } - mem.check_and_reset(); - - // restore defaults. - ryml::set_callbacks(defaults); -} - - -//----------------------------------------------------------------------------- - -/** an example for a per-tree memory allocator */ -struct PerTreeMemoryExample -{ - std::vector memory_pool = std::vector(10u * 1024u); // 10KB - size_t num_allocs = 0, alloc_size = 0; - size_t num_deallocs = 0, dealloc_size = 0; - - ryml::Callbacks callbacks() const - { - // Above we used static functions to bridge to our methods. - // To show a different approach, we employ lambdas here. - // Note that there can be no captures in the lambdas - // because these are C-style function pointers. - ryml::Callbacks cb; - cb.m_user_data = (void*) this; - cb.m_allocate = [](size_t len, void *, void *data){ return ((PerTreeMemoryExample*) data)->allocate(len); }; - cb.m_free = [](void *mem, size_t len, void *data){ return ((PerTreeMemoryExample*) data)->free(mem, len); }; - return cb; - } - - void *allocate(size_t len) - { - void *ptr = &memory_pool[alloc_size]; - alloc_size += len; - ++num_allocs; - if(C4_UNLIKELY(alloc_size > memory_pool.size())) - { - std::cerr << "out of memory! requested=" << alloc_size << " vs " << memory_pool.size() << " available" << std::endl; - std::abort(); - } - return ptr; - } - - void free(void *mem, size_t len) - { - CHECK((char*)mem >= &memory_pool.front() && (char*)mem < &memory_pool.back()); - CHECK((char*)mem+len >= &memory_pool.front() && (char*)mem+len <= &memory_pool.back()); - dealloc_size += len; - ++num_deallocs; - // no need to free here - } - - // checking - ~PerTreeMemoryExample() - { - check_and_reset(); - } - void check_and_reset() - { - std::cout << "size: alloc=" << alloc_size << " dealloc=" << dealloc_size << std::endl; - std::cout << "count: #allocs=" << num_allocs << " #deallocs=" << num_deallocs << std::endl; - CHECK(num_allocs == num_deallocs); - CHECK(alloc_size >= dealloc_size); // failure here means a double free - CHECK(alloc_size == dealloc_size); // failure here means a leak - num_allocs = 0; - num_deallocs = 0; - alloc_size = 0; - dealloc_size = 0; - } -}; - -void sample_per_tree_allocator() -{ - PerTreeMemoryExample mrp; - PerTreeMemoryExample mr1; - PerTreeMemoryExample mr2; - - // the trees will use the memory in the resources above, - // with each tree using a separate resource - { - // Watchout: ensure that the lifetime of the callbacks target - // exceeds the lifetime of the tree. - auto parser = ryml::Parser(mrp.callbacks()); - auto tree1 = ryml::Tree(mr1.callbacks()); - auto tree2 = ryml::Tree(mr2.callbacks()); - - ryml::csubstr yml1 = "{a: b}"; - ryml::csubstr yml2 = "{c: d, e: f, g: [h, i, 0, 1, 2, 3]}"; - - parser.parse_in_arena("file1.yml", yml1, &tree1); - parser.parse_in_arena("file2.yml", yml2, &tree2); - } - - CHECK(mrp.num_allocs == 0); // YAML depth not large enough to warrant a parser allocation - CHECK(mr1.alloc_size <= mr2.alloc_size); // because yml2 has more nodes -} - - -//----------------------------------------------------------------------------- - - -/** shows how to work around the static initialization order fiasco - * when using a static-duration ryml tree - * @see https://en.cppreference.com/w/cpp/language/siof */ -void sample_static_trees() -{ - // Using static trees incurs may incur a static initialization - // order problem. This happens because a default-constructed tree will - // obtain the callbacks from the current global setting, which may - // not have been initialized due to undefined static initialization - // order: - // - //static ryml::Tree tree; // ERROR! depends on ryml::get_callbacks() which may not have been initialized. - // - // To work around the issue, declare static callbacks - // to explicitly initialize the static tree: - static ryml::Callbacks callbacks = {}; // use default callback members - static ryml::Tree tree(callbacks); // OK - // now you can use the tree as normal: - ryml::parse_in_arena(R"(doe: "a deer, a female deer")", &tree); - CHECK(tree["doe"].val() == "a deer, a female deer"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** demonstrates how to obtain the (zero-based) location of a node - * from a recently parsed tree */ -void sample_location_tracking() -{ - // NOTE: locations are zero-based. If you intend to show the - // location to a human user, you may want to pre-increment the line - // and column by 1. - ryml::csubstr yaml = R"({ -aa: contents, -foo: [one, [two, three]] -})"; - // A parser is needed to track locations, and it has to be - // explicitly set to do it. Location tracking is disabled by - // default. - ryml::ParserOptions opts = {}; - opts.locations(true); // enable locations, default is false - ryml::Parser parser(opts); - CHECK(parser.options().locations()); - // When locations are enabled, the first task while parsing will - // consist of building and caching (in the parser) a - // source-to-node lookup structure to accelerate location lookups. - // - // The cost of building the location accelerator is linear in the - // size of the source buffer. This increased cost is the reason - // for the opt-in requirement. When locations are disabled there - // is no cost. - // - // Building the location accelerator may trigger an allocation, - // but this can and should be avoided by reserving prior to - // parsing: - parser.reserve_locations(50u); // reserve for 50 lines - // Now the structure will be built during parsing: - ryml::Tree tree = parser.parse_in_arena("source.yml", yaml); - // Now we are ready to query the location from the parser: - ryml::Location loc = parser.location(tree.rootref()); - // As for the complexity of the query: for large buffers it is - // O(log(numlines)). For short source buffers (30 lines and less), - // it is O(numlines), as a plain linear search is faster in this - // case. - CHECK(parser.location_contents(loc).begins_with("{")); - CHECK(loc.offset == 0u); - CHECK(loc.line == 0u); - CHECK(loc.col == 0u); - // on the next call, we only pay O(log(numlines)) because the - // rebuild is already available: - loc = parser.location(tree["aa"]); - CHECK(parser.location_contents(loc).begins_with("aa")); - CHECK(loc.offset == 2u); - CHECK(loc.line == 1u); - CHECK(loc.col == 0u); - // KEYSEQ in flow style: points at the key - loc = parser.location(tree["foo"]); - CHECK(parser.location_contents(loc).begins_with("foo")); - CHECK(loc.offset == 16u); - CHECK(loc.line == 2u); - CHECK(loc.col == 0u); - loc = parser.location(tree["foo"][0]); - CHECK(parser.location_contents(loc).begins_with("one")); - CHECK(loc.line == 2u); - CHECK(loc.col == 6u); - // SEQ in flow style: location points at the opening '[' (there's no key) - loc = parser.location(tree["foo"][1]); - CHECK(parser.location_contents(loc).begins_with("[")); - CHECK(loc.line == 2u); - CHECK(loc.col == 11u); - loc = parser.location(tree["foo"][1][0]); - CHECK(parser.location_contents(loc).begins_with("two")); - CHECK(loc.line == 2u); - CHECK(loc.col == 12u); - loc = parser.location(tree["foo"][1][1]); - CHECK(parser.location_contents(loc).begins_with("three")); - CHECK(loc.line == 2u); - CHECK(loc.col == 17u); - // NOTE. The parser locations always point at the latest buffer to - // be parsed with the parser object, so they must be queried using - // the corresponding latest tree to be parsed. This means that if - // the parser is reused, earlier trees will loose the possibility - // of querying for location. It is undefined behavior to query the - // parser for the location of a node from an earlier tree: - ryml::Tree docval = parser.parse_in_arena("docval.yaml", "this is a docval"); - // From now on, none of the locations from the previous tree can - // be queried: - //loc = parser.location(tree.rootref()); // ERROR, undefined behavior - loc = parser.location(docval.rootref()); // OK. this is the latest tree from this parser - CHECK(parser.location_contents(loc).begins_with("this is a docval")); - CHECK(loc.line == 0u); - CHECK(loc.col == 0u); - - // NOTES ABOUT CONTAINER LOCATIONS - ryml::Tree tree2 = parser.parse_in_arena("containers.yaml", R"( -a new: buffer -to: be parsed -map with key: - first: value - second: value -seq with key: - - first value - - second value - - - - nested first value - - nested second value - - - nested first: value - nested second: value -)"); - // (Likewise, the docval tree can no longer be used to query.) - // - // For key-less block-style maps, the location of the container - // points at the first child's key. For example, in this case - // the root does not have a key, so its location is taken - // to be at the first child: - loc = parser.location(tree2.rootref()); - CHECK(parser.location_contents(loc).begins_with("a new")); - CHECK(loc.offset == 1u); - CHECK(loc.line == 1u); - CHECK(loc.col == 0u); - // note the first child points exactly at the same place: - loc = parser.location(tree2["a new"]); - CHECK(parser.location_contents(loc).begins_with("a new")); - CHECK(loc.offset == 1u); - CHECK(loc.line == 1u); - CHECK(loc.col == 0u); - loc = parser.location(tree2["to"]); - CHECK(parser.location_contents(loc).begins_with("to")); - CHECK(loc.line == 2u); - CHECK(loc.col == 0u); - // but of course, if the block-style map is a KEYMAP, then the - // location is the map's key, and not the first child's key: - loc = parser.location(tree2["map with key"]); - CHECK(parser.location_contents(loc).begins_with("map with key")); - CHECK(loc.line == 3u); - CHECK(loc.col == 0u); - loc = parser.location(tree2["map with key"]["first"]); - CHECK(parser.location_contents(loc).begins_with("first")); - CHECK(loc.line == 4u); - CHECK(loc.col == 2u); - loc = parser.location(tree2["map with key"]["second"]); - CHECK(parser.location_contents(loc).begins_with("second")); - CHECK(loc.line == 5u); - CHECK(loc.col == 2u); - // same thing for KEYSEQ: - loc = parser.location(tree2["seq with key"]); - CHECK(parser.location_contents(loc).begins_with("seq with key")); - CHECK(loc.line == 6u); - CHECK(loc.col == 0u); - loc = parser.location(tree2["seq with key"][0]); - CHECK(parser.location_contents(loc).begins_with("first value")); - CHECK(loc.line == 7u); - CHECK(loc.col == 4u); - loc = parser.location(tree2["seq with key"][1]); - CHECK(parser.location_contents(loc).begins_with("second value")); - CHECK(loc.line == 8u); - CHECK(loc.col == 4u); - // SEQ nested in SEQ: container location points at the first child's "- " dash - loc = parser.location(tree2["seq with key"][2]); - CHECK(parser.location_contents(loc).begins_with("- nested first value")); - CHECK(loc.line == 10u); - CHECK(loc.col == 4u); - loc = parser.location(tree2["seq with key"][2][0]); - CHECK(parser.location_contents(loc).begins_with("nested first value")); - CHECK(loc.line == 10u); - CHECK(loc.col == 6u); - // MAP nested in SEQ: same as above: point to key - loc = parser.location(tree2["seq with key"][3]); - CHECK(parser.location_contents(loc).begins_with("nested first: ")); - CHECK(loc.line == 13u); - CHECK(loc.col == 4u); - loc = parser.location(tree2["seq with key"][3][0]); - CHECK(parser.location_contents(loc).begins_with("nested first: ")); - CHECK(loc.line == 13u); - CHECK(loc.col == 4u); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace /*anon*/ { -static int num_checks = 0; -static int num_failed_checks = 0; -} // namespace /*anon*/ - - -bool report_check(int line, const char *predicate, bool result) -{ - ++num_checks; - const char *msg = predicate ? "OK! " : "OK!"; - if(!result) - { - ++num_failed_checks; - msg = predicate ? "ERROR: " : "ERROR"; - } - std::cout << __FILE__ << ':' << line << ": " << msg << (predicate ? predicate : "") << std::endl; - return result; -} - - -int report_checks() -{ - std::cout << "Completed " << num_checks << " checks." << std::endl; - if(num_failed_checks) - std::cout << "ERROR: " << num_failed_checks << '/' << num_checks << " checks failed." << std::endl; - else - std::cout << "SUCCESS!" << std::endl; - return num_failed_checks; -} - - -// helper functions for sample_parse_file() - -C4_SUPPRESS_WARNING_MSVC_WITH_PUSH(4996) // fopen: this function or variable may be unsafe -/** load a file from disk and return a newly created CharContainer */ -template -size_t file_get_contents(const char *filename, CharContainer *v) -{ - ::FILE *fp = ::fopen(filename, "rb"); - C4_CHECK_MSG(fp != nullptr, "could not open file"); - ::fseek(fp, 0, SEEK_END); - long sz = ::ftell(fp); - v->resize(static_cast(sz)); - if(sz) - { - ::rewind(fp); - size_t ret = ::fread(&(*v)[0], 1, v->size(), fp); - C4_CHECK(ret == (size_t)sz); - } - ::fclose(fp); - return v->size(); -} - -/** load a file from disk into an existing CharContainer */ -template -CharContainer file_get_contents(const char *filename) -{ - CharContainer cc; - file_get_contents(filename, &cc); - return cc; -} - -/** save a buffer into a file */ -template -void file_put_contents(const char *filename, CharContainer const& v, const char* access) -{ - file_put_contents(filename, v.empty() ? "" : &v[0], v.size(), access); -} - -/** save a buffer into a file */ -void file_put_contents(const char *filename, const char *buf, size_t sz, const char* access) -{ - ::FILE *fp = ::fopen(filename, access); - C4_CHECK_MSG(fp != nullptr, "could not open file"); - ::fwrite(buf, 1, sz, fp); - ::fclose(fp); -} -C4_SUPPRESS_WARNING_MSVC_POP - -} // namespace sample diff --git a/thirdparty/ryml/samples/singleheader/amalgamate.cmake b/thirdparty/ryml/samples/singleheader/amalgamate.cmake deleted file mode 100644 index 6f2620a12..000000000 --- a/thirdparty/ryml/samples/singleheader/amalgamate.cmake +++ /dev/null @@ -1,18 +0,0 @@ -find_package(Python3 COMPONENTS Interpreter) - -# amalgamate ryml to get the single header -function(amalgamate_ryml header_dir header_file) - set(rymldir "${CMAKE_CURRENT_LIST_DIR}/../..") - set(singleheaderdir "${rymldir}/src_singleheader") - set(singleheader "${singleheaderdir}/ryml_all.hpp") - set(amscript "${rymldir}/tools/amalgamate.py") - file(GLOB_RECURSE srcfiles - LIST_DIRECTORIES FALSE - CONFIGURE_DEPENDS "${rymldir}/src") - add_custom_command(OUTPUT "${singleheader}" - COMMAND "${Python3_EXECUTABLE}" "${amscript}" "${singleheader}" - COMMENT "${Python3_EXECUTABLE} ${amscript} ${singleheader}" - DEPENDS ${srcfiles} "${amscript}" "${rymldir}/ext/c4core/cmake/amalgamate_utils.py") - set(${header_dir} "${singleheaderdir}" PARENT_SCOPE) - set(${header_file} "${singleheader}" PARENT_SCOPE) -endfunction() diff --git a/thirdparty/ryml/samples/singleheader/run.sh b/thirdparty/ryml/samples/singleheader/run.sh deleted file mode 100755 index fc2f7d858..000000000 --- a/thirdparty/ryml/samples/singleheader/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -x - -# take the build type from the command line, or default to release -cfg=${1:-Release} -# make sure to run from where this file is -cd $(dirname $0) -# configure the sample -cmake -S . -B ./build/$cfg -DCMAKE_BUILD_TYPE=$cfg -# build and run the sample -cmake --build ./build/$cfg --config $cfg --target run diff --git a/thirdparty/ryml/samples/singleheaderlib/lib.cpp b/thirdparty/ryml/samples/singleheaderlib/lib.cpp deleted file mode 100644 index aa33e6e17..000000000 --- a/thirdparty/ryml/samples/singleheaderlib/lib.cpp +++ /dev/null @@ -1,2 +0,0 @@ -#define RYML_SINGLE_HDR_DEFINE_NOW -#include diff --git a/thirdparty/ryml/samples/singleheaderlib/run_shared.sh b/thirdparty/ryml/samples/singleheaderlib/run_shared.sh deleted file mode 100755 index 372a36772..000000000 --- a/thirdparty/ryml/samples/singleheaderlib/run_shared.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -x - -# take the build type from the command line, or default to release -cfg=${1:-Release} -# make sure to run from where this file is -cd $(dirname $0) -# configure the sample -cmake -S . -B ./build/$cfg-shared -DCMAKE_BUILD_TYPE=$cfg -DBUILD_SHARED_LIBS=ON -# build and run the sample -cmake --build ./build/$cfg-shared --config $cfg --target run diff --git a/thirdparty/ryml/samples/singleheaderlib/run_static.sh b/thirdparty/ryml/samples/singleheaderlib/run_static.sh deleted file mode 100755 index d0e570215..000000000 --- a/thirdparty/ryml/samples/singleheaderlib/run_static.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -x - -# take the build type from the command line, or default to release -cfg=${1:-Release} -# make sure to run from where this file is -cd $(dirname $0) -# configure the sample -cmake -S . -B ./build/$cfg-static -DCMAKE_BUILD_TYPE=$cfg -DBUILD_SHARED_LIBS=OFF -# build and run the sample -cmake --build ./build/$cfg-static --config $cfg --target run diff --git a/thirdparty/ryml/setup.py b/thirdparty/ryml/setup.py deleted file mode 100644 index 47a6be656..000000000 --- a/thirdparty/ryml/setup.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: MIT - -import os -import shutil -import sys -import shlex - -from pathlib import Path -from distutils import log -from setuptools import setup, find_packages -from cmake_build_extension import BuildExtension, CMakeExtension - -TOP_DIR = (Path(__file__).parent).resolve() - -# where the Python library is actually found -PYTHON_DIR = "api/python" - - -def get_readme_for_python(): - with open(TOP_DIR / "README.md", "r", encoding="utf8") as fh: - marker = "" # get everything up to this tag - return fh.read().split(marker)[0] - - -def get_environment_cmake_flags(): - return shlex.split(os.environ.get("CMAKE_FLAGS", "")) - - -setup_kw = {} - -# read the module description from the README.md file -setup_kw['long_description'] = get_readme_for_python() -setup_kw['long_description_content_type'] = "text/markdown" - - -# read the package version when not in a git repository -VERSION_FILE = os.path.join(PYTHON_DIR, 'ryml', 'version.py') -if not (TOP_DIR / '.git').exists() and os.path.exists(VERSION_FILE): - exec(open(VERSION_FILE).read()) - setup_kw['version'] = version -else: - setup_kw['use_scm_version'] = { - "version_scheme": "post-release", - "local_scheme": "no-local-version", - "write_to": VERSION_FILE, - } - - -# define a CMake package -cmake_args = dict( - name='ryml.ryml', - install_prefix='', - source_dir='', - cmake_component='python', - cmake_configure_options=get_environment_cmake_flags() + [ - "-DRYML_BUILD_API:BOOL=ON", - # Force cmake to use the Python interpreter we are currently - # using to run setup.py - "-DPython3_EXECUTABLE:FILEPATH=" + sys.executable, - ], -) - - -try: - ext = CMakeExtension(**cmake_args) - log.info("Using standard CMakeExtension") -except TypeError: - log.info("Using custom CMakeExtension") - # If the CMakeExtension doesn't support `cmake_component` then we - # have to do some manual cleanup. - del cmake_args['cmake_component'] - ext = CMakeExtension(**cmake_args) - def _cleanup(path, mandatory): - if mandatory: - assert path.exists(), path - elif not path.exists(): - return - log.info("Removing everything under: %s", path) - shutil.rmtree(path) - _BuildExtension = BuildExtension - class BuildExtension(_BuildExtension): - def build_extension(self, ext): - _BuildExtension.build_extension(self, ext) - ext_dir = Path(self.get_ext_fullpath(ext.name)).parent.absolute() - cmake_install_prefix = ext_dir / ext.install_prefix - assert cmake_install_prefix.exists(), cmake_install_prefix - try: - _cleanup(cmake_install_prefix / "lib", mandatory=True) - _cleanup(cmake_install_prefix / "include", mandatory=True) - # Windows only - _cleanup(cmake_install_prefix / "cmake", mandatory=False) - except: - log.info('Found following installed files:') - for f in cmake_install_prefix.rglob("*"): - log.info(' - %s', f) - raise - -log.info('Compiling with CMake cfg:\n' + '\n'.join(ext.cmake_configure_options)) - -setup( - name='rapidyaml', - description='Rapid YAML - a library to parse and emit YAML, and do it fast', - url='https://github.com/biojppm/rapidyaml', - license='MIT', - license_files=['LICENSE.txt'], - author="Joao Paulo Magalhaes", - author_email="dev@jpmag.me", - # Package contents control - cmdclass={"build_ext": BuildExtension,}, - package_dir={"": PYTHON_DIR}, - packages=['ryml'], - ext_modules=[ext], - include_package_data=True, - # Requirements - python_requires=">=3.6", - setup_requires=[ - 'setuptools_scm', - 'setuptools-git', - 'setuptools', - ], - # Extra arguments - **setup_kw, -) diff --git a/thirdparty/ryml/src/c4/yml/common.cpp b/thirdparty/ryml/src/c4/yml/common.cpp deleted file mode 100644 index 75b873549..000000000 --- a/thirdparty/ryml/src/c4/yml/common.cpp +++ /dev/null @@ -1,117 +0,0 @@ -#include "c4/yml/common.hpp" - -#ifndef RYML_NO_DEFAULT_CALLBACKS -# include -# include -#endif // RYML_NO_DEFAULT_CALLBACKS - -namespace c4 { -namespace yml { - -namespace { -Callbacks s_default_callbacks; -} // anon namespace - -#ifndef RYML_NO_DEFAULT_CALLBACKS -void report_error_impl(const char* msg, size_t length, Location loc, FILE *f) -{ - if(!f) - f = stderr; - if(loc) - { - if(!loc.name.empty()) - { - fwrite(loc.name.str, 1, loc.name.len, f); - fputc(':', f); - } - fprintf(f, "%zu:", loc.line); - if(loc.col) - fprintf(f, "%zu:", loc.col); - if(loc.offset) - fprintf(f, " (%zuB):", loc.offset); - } - fprintf(f, "%.*s\n", (int)length, msg); - fflush(f); -} - -void error_impl(const char* msg, size_t length, Location loc, void * /*user_data*/) -{ - report_error_impl(msg, length, loc, nullptr); - ::abort(); -} - -void* allocate_impl(size_t length, void * /*hint*/, void * /*user_data*/) -{ - void *mem = ::malloc(length); - if(mem == nullptr) - { - const char msg[] = "could not allocate memory"; - error_impl(msg, sizeof(msg)-1, {}, nullptr); - } - return mem; -} - -void free_impl(void *mem, size_t /*length*/, void * /*user_data*/) -{ - ::free(mem); -} -#endif // RYML_NO_DEFAULT_CALLBACKS - - - -Callbacks::Callbacks() - : - m_user_data(nullptr), - #ifndef RYML_NO_DEFAULT_CALLBACKS - m_allocate(allocate_impl), - m_free(free_impl), - m_error(error_impl) - #else - m_allocate(nullptr), - m_free(nullptr), - m_error(nullptr) - #endif -{ -} - -Callbacks::Callbacks(void *user_data, pfn_allocate alloc_, pfn_free free_, pfn_error error_) - : - m_user_data(user_data), - #ifndef RYML_NO_DEFAULT_CALLBACKS - m_allocate(alloc_ ? alloc_ : allocate_impl), - m_free(free_ ? free_ : free_impl), - m_error(error_ ? error_ : error_impl) - #else - m_allocate(alloc_), - m_free(free_), - m_error(error_) - #endif -{ - C4_CHECK(m_allocate); - C4_CHECK(m_free); - C4_CHECK(m_error); -} - - -void set_callbacks(Callbacks const& c) -{ - s_default_callbacks = c; -} - -Callbacks const& get_callbacks() -{ - return s_default_callbacks; -} - -void reset_callbacks() -{ - set_callbacks(Callbacks()); -} - -void error(const char *msg, size_t msg_len, Location loc) -{ - s_default_callbacks.m_error(msg, msg_len, loc, s_default_callbacks.m_user_data); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/src/c4/yml/common.hpp b/thirdparty/ryml/src/c4/yml/common.hpp deleted file mode 100644 index 21aef6ba2..000000000 --- a/thirdparty/ryml/src/c4/yml/common.hpp +++ /dev/null @@ -1,278 +0,0 @@ -#ifndef _C4_YML_COMMON_HPP_ -#define _C4_YML_COMMON_HPP_ - -#include -#include -#include - - -#ifndef RYML_USE_ASSERT -# define RYML_USE_ASSERT C4_USE_ASSERT -#endif - - -#if RYML_USE_ASSERT -# define RYML_ASSERT(cond) RYML_CHECK(cond) -# define RYML_ASSERT_MSG(cond, msg) RYML_CHECK_MSG(cond, msg) -#else -# define RYML_ASSERT(cond) -# define RYML_ASSERT_MSG(cond, msg) -#endif - - -#if defined(NDEBUG) || defined(C4_NO_DEBUG_BREAK) -# define RYML_DEBUG_BREAK() -#else -# define RYML_DEBUG_BREAK() \ - { \ - if(c4::get_error_flags() & c4::ON_ERROR_DEBUGBREAK) \ - { \ - C4_DEBUG_BREAK(); \ - } \ - } -#endif - - -#define RYML_CHECK(cond) \ - do { \ - if(!(cond)) \ - { \ - RYML_DEBUG_BREAK() \ - c4::yml::error("check failed: " #cond, c4::yml::Location(__FILE__, __LINE__, 0)); \ - } \ - } while(0) - -#define RYML_CHECK_MSG(cond, msg) \ - do \ - { \ - if(!(cond)) \ - { \ - RYML_DEBUG_BREAK() \ - c4::yml::error(msg ": check failed: " #cond, c4::yml::Location(__FILE__, __LINE__, 0)); \ - } \ - } while(0) - - -#if C4_CPP >= 14 -# define RYML_DEPRECATED(msg) [[deprecated(msg)]] -#else -# if defined(_MSC_VER) -# define RYML_DEPRECATED(msg) __declspec(deprecated(msg)) -# else // defined(__GNUC__) || defined(__clang__) -# define RYML_DEPRECATED(msg) __attribute__((deprecated(msg))) -# endif -#endif - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace c4 { -namespace yml { - -enum : size_t { - /** a null position */ - npos = size_t(-1), - /** an index to none */ - NONE = size_t(-1) -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -//! holds a position into a source buffer -struct RYML_EXPORT LineCol -{ - //! number of bytes from the beginning of the source buffer - size_t offset; - //! line - size_t line; - //! column - size_t col; - - LineCol() : offset(), line(), col() {} - //! construct from line and column - LineCol(size_t l, size_t c) : offset(0), line(l), col(c) {} - //! construct from offset, line and column - LineCol(size_t o, size_t l, size_t c) : offset(o), line(l), col(c) {} -}; - - -//! a source file position -struct RYML_EXPORT Location : public LineCol -{ - csubstr name; - - operator bool () const { return !name.empty() || line != 0 || offset != 0; } - - Location() : LineCol(), name() {} - Location( size_t l, size_t c) : LineCol{ l, c}, name( ) {} - Location( csubstr n, size_t l, size_t c) : LineCol{ l, c}, name(n) {} - Location( csubstr n, size_t b, size_t l, size_t c) : LineCol{b, l, c}, name(n) {} - Location(const char *n, size_t l, size_t c) : LineCol{ l, c}, name(to_csubstr(n)) {} - Location(const char *n, size_t b, size_t l, size_t c) : LineCol{b, l, c}, name(to_csubstr(n)) {} -}; - - -//----------------------------------------------------------------------------- - -/** the type of the function used to report errors. This function must - * interrupt execution, either by raising an exception or calling - * std::abort(). - * - * @warning the error callback must never return: it must either abort - * or throw an exception. Otherwise, the parser will enter into an - * infinite loop, or the program may crash. */ -using pfn_error = void (*)(const char* msg, size_t msg_len, Location location, void *user_data); -/** the type of the function used to allocate memory */ -using pfn_allocate = void* (*)(size_t len, void* hint, void *user_data); -/** the type of the function used to free memory */ -using pfn_free = void (*)(void* mem, size_t size, void *user_data); - -/** trigger an error: call the current error callback. */ -RYML_EXPORT void error(const char *msg, size_t msg_len, Location loc); -/** @overload error */ -inline void error(const char *msg, size_t msg_len) -{ - error(msg, msg_len, Location{}); -} -/** @overload error */ -template -inline void error(const char (&msg)[N], Location loc) -{ - error(msg, N-1, loc); -} -/** @overload error */ -template -inline void error(const char (&msg)[N]) -{ - error(msg, N-1, Location{}); -} - -//----------------------------------------------------------------------------- - -/** a c-style callbacks class - * - * @warning the error callback must never return: it must either abort - * or throw an exception. Otherwise, the parser will enter into an - * infinite loop, or the program may crash. */ -struct RYML_EXPORT Callbacks -{ - void * m_user_data; - pfn_allocate m_allocate; - pfn_free m_free; - pfn_error m_error; - - Callbacks(); - Callbacks(void *user_data, pfn_allocate alloc, pfn_free free, pfn_error error_); - - bool operator!= (Callbacks const& that) const { return !operator==(that); } - bool operator== (Callbacks const& that) const - { - return (m_user_data == that.m_user_data && - m_allocate == that.m_allocate && - m_free == that.m_free && - m_error == that.m_error); - } -}; - -/** set the global callbacks. - * - * @warning the error callback must never return: it must either abort - * or throw an exception. Otherwise, the parser will enter into an - * infinite loop, or the program may crash. */ -RYML_EXPORT void set_callbacks(Callbacks const& c); -/// get the global callbacks -RYML_EXPORT Callbacks const& get_callbacks(); -/// set the global callbacks back to their defaults -RYML_EXPORT void reset_callbacks(); - -/// @cond dev -#define _RYML_CB_ERR(cb, msg_literal) \ -do \ -{ \ - const char msg[] = msg_literal; \ - RYML_DEBUG_BREAK() \ - (cb).m_error(msg, sizeof(msg), c4::yml::Location(__FILE__, 0, __LINE__, 0), (cb).m_user_data); \ -} while(0) -#define _RYML_CB_CHECK(cb, cond) \ - do \ - { \ - if(!(cond)) \ - { \ - const char msg[] = "check failed: " #cond; \ - RYML_DEBUG_BREAK() \ - (cb).m_error(msg, sizeof(msg), c4::yml::Location(__FILE__, 0, __LINE__, 0), (cb).m_user_data); \ - } \ - } while(0) -#ifdef RYML_USE_ASSERT -#define _RYML_CB_ASSERT(cb, cond) _RYML_CB_CHECK((cb), (cond)) -#else -#define _RYML_CB_ASSERT(cb, cond) do {} while(0) -#endif -#define _RYML_CB_ALLOC_HINT(cb, T, num, hint) (T*) (cb).m_allocate((num) * sizeof(T), (hint), (cb).m_user_data) -#define _RYML_CB_ALLOC(cb, T, num) _RYML_CB_ALLOC_HINT((cb), (T), (num), nullptr) -#define _RYML_CB_FREE(cb, buf, T, num) \ - do { \ - (cb).m_free((buf), (num) * sizeof(T), (cb).m_user_data); \ - (buf) = nullptr; \ - } while(0) - - - -namespace detail { -template -struct _charconstant_t - : public std::conditional::value, - std::integral_constant, - std::integral_constant>::type -{}; -#define _RYML_CHCONST(signedval, unsignedval) ::c4::yml::detail::_charconstant_t::value -} // namespace detail - - -namespace detail { -struct _SubstrWriter -{ - substr buf; - size_t pos; - _SubstrWriter(substr buf_, size_t pos_=0) : buf(buf_), pos(pos_) {} - void append(csubstr s) - { - C4_ASSERT(!s.overlaps(buf)); - if(pos + s.len <= buf.len) - memcpy(buf.str + pos, s.str, s.len); - pos += s.len; - } - void append(char c) - { - if(pos < buf.len) - buf.str[pos] = c; - ++pos; - } - void append_n(char c, size_t numtimes) - { - if(pos + numtimes < buf.len) - memset(buf.str + pos, c, numtimes); - pos += numtimes; - } - size_t slack() const { return pos <= buf.len ? buf.len - pos : 0; } - size_t excess() const { return pos > buf.len ? pos - buf.len : 0; } - //! get the part written so far - csubstr curr() const { return pos <= buf.len ? buf.first(pos) : buf; } - //! get the part that is still free to write to (the remainder) - substr rem() { return pos < buf.len ? buf.sub(pos) : buf.last(0); } - - size_t advance(size_t more) { pos += more; return pos; } -}; -} // namespace detail - -/// @endcond - -} // namespace yml -} // namespace c4 - -#endif /* _C4_YML_COMMON_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/detail/checks.hpp b/thirdparty/ryml/src/c4/yml/detail/checks.hpp deleted file mode 100644 index 39b49e856..000000000 --- a/thirdparty/ryml/src/c4/yml/detail/checks.hpp +++ /dev/null @@ -1,200 +0,0 @@ -#ifndef C4_YML_DETAIL_CHECKS_HPP_ -#define C4_YML_DETAIL_CHECKS_HPP_ - -#include "c4/yml/tree.hpp" - -#ifdef __clang__ -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wtype-limits" // error: comparison of unsigned expression >= 0 is always true -#elif defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4296/*expression is always 'boolean_value'*/) -#endif - -namespace c4 { -namespace yml { - - -void check_invariants(Tree const& t, size_t node=NONE); -void check_free_list(Tree const& t); -void check_arena(Tree const& t); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -inline void check_invariants(Tree const& t, size_t node) -{ - if(node == NONE) - { - if(t.size() == 0) return; - node = t.root_id(); - } - - auto const& n = *t._p(node); -#ifdef RYML_DBG - if(n.m_first_child != NONE || n.m_last_child != NONE) - { - printf("check(%zu): fc=%zu lc=%zu\n", node, n.m_first_child, n.m_last_child); - } - else - { - printf("check(%zu)\n", node); - } -#endif - - C4_CHECK(n.m_parent != node); - if(n.m_parent == NONE) - { - C4_CHECK(t.is_root(node)); - } - else //if(n.m_parent != NONE) - { - C4_CHECK(t.has_child(n.m_parent, node)); - - auto const& p = *t._p(n.m_parent); - if(n.m_prev_sibling == NONE) - { - C4_CHECK(p.m_first_child == node); - C4_CHECK(t.first_sibling(node) == node); - } - else - { - C4_CHECK(p.m_first_child != node); - C4_CHECK(t.first_sibling(node) != node); - } - - if(n.m_next_sibling == NONE) - { - C4_CHECK(p.m_last_child == node); - C4_CHECK(t.last_sibling(node) == node); - } - else - { - C4_CHECK(p.m_last_child != node); - C4_CHECK(t.last_sibling(node) != node); - } - } - - C4_CHECK(n.m_first_child != node); - C4_CHECK(n.m_last_child != node); - if(n.m_first_child != NONE || n.m_last_child != NONE) - { - C4_CHECK(n.m_first_child != NONE); - C4_CHECK(n.m_last_child != NONE); - } - - C4_CHECK(n.m_prev_sibling != node); - C4_CHECK(n.m_next_sibling != node); - if(n.m_prev_sibling != NONE) - { - C4_CHECK(t._p(n.m_prev_sibling)->m_next_sibling == node); - C4_CHECK(t._p(n.m_prev_sibling)->m_prev_sibling != node); - } - if(n.m_next_sibling != NONE) - { - C4_CHECK(t._p(n.m_next_sibling)->m_prev_sibling == node); - C4_CHECK(t._p(n.m_next_sibling)->m_next_sibling != node); - } - - size_t count = 0; - for(size_t i = n.m_first_child; i != NONE; i = t.next_sibling(i)) - { -#ifdef RYML_DBG - printf("check(%zu): descend to child[%zu]=%zu\n", node, count, i); -#endif - auto const& ch = *t._p(i); - C4_CHECK(ch.m_parent == node); - C4_CHECK(ch.m_next_sibling != i); - ++count; - } - C4_CHECK(count == t.num_children(node)); - - if(n.m_prev_sibling == NONE && n.m_next_sibling == NONE) - { - if(n.m_parent != NONE) - { - C4_CHECK(t.num_children(n.m_parent) == 1); - C4_CHECK(t.num_siblings(node) == 1); - } - } - - if(node == t.root_id()) - { - C4_CHECK(t.size() == t.m_size); - C4_CHECK(t.capacity() == t.m_cap); - C4_CHECK(t.m_cap == t.m_size + t.slack()); - check_free_list(t); - check_arena(t); - } - - for(size_t i = t.first_child(node); i != NONE; i = t.next_sibling(i)) - { - check_invariants(t, i); - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -inline void check_free_list(Tree const& t) -{ - if(t.m_free_head == NONE) - { - C4_CHECK(t.m_free_tail == t.m_free_head); - return; - } - - C4_CHECK(t.m_free_head >= 0 && t.m_free_head < t.m_cap); - C4_CHECK(t.m_free_tail >= 0 && t.m_free_tail < t.m_cap); - - auto const& head = *t._p(t.m_free_head); - //auto const& tail = *t._p(t.m_free_tail); - - //C4_CHECK(head.m_prev_sibling == NONE); - //C4_CHECK(tail.m_next_sibling == NONE); - - size_t count = 0; - for(size_t i = t.m_free_head, prev = NONE; i != NONE; i = t._p(i)->m_next_sibling) - { - auto const& elm = *t._p(i); - if(&elm != &head) - { - C4_CHECK(elm.m_prev_sibling == prev); - } - prev = i; - ++count; - } - C4_CHECK(count == t.slack()); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -inline void check_arena(Tree const& t) -{ - C4_CHECK(t.m_arena.len == 0 || (t.m_arena_pos >= 0 && t.m_arena_pos <= t.m_arena.len)); - C4_CHECK(t.arena_size() == t.m_arena_pos); - C4_CHECK(t.arena_slack() + t.m_arena_pos == t.m_arena.len); -} - - -} /* namespace yml */ -} /* namespace c4 */ - -#ifdef __clang__ -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#elif defined(_MSC_VER) -# pragma warning(pop) -#endif - -#endif /* C4_YML_DETAIL_CHECKS_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/detail/parser_dbg.hpp b/thirdparty/ryml/src/c4/yml/detail/parser_dbg.hpp deleted file mode 100644 index 457f1700d..000000000 --- a/thirdparty/ryml/src/c4/yml/detail/parser_dbg.hpp +++ /dev/null @@ -1,137 +0,0 @@ -#ifndef _C4_YML_DETAIL_PARSER_DBG_HPP_ -#define _C4_YML_DETAIL_PARSER_DBG_HPP_ - -#ifndef _C4_YML_COMMON_HPP_ -#include "../common.hpp" -#endif -#include - -//----------------------------------------------------------------------------- -// some debugging scaffolds - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4068/*unknown pragma*/) -#endif - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunknown-pragmas" -//#pragma GCC diagnostic ignored "-Wpragma-system-header-outside-header" -#pragma GCC system_header - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Werror" -#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" - -// some debugging scaffolds -#ifdef RYML_DBG -#include -namespace c4 { -inline void _dbg_dumper(csubstr s) { fwrite(s.str, 1, s.len, stdout); }; -template -void _dbg_printf(c4::csubstr fmt, Args&& ...args) -{ - static char writebuf[256]; - auto results = c4::format_dump_resume<&_dbg_dumper>(writebuf, fmt, std::forward(args)...); - // resume writing if the results failed to fit the buffer - if(C4_UNLIKELY(results.bufsize > sizeof(writebuf))) // bufsize will be that of the largest element serialized. Eg int(1), will require 1 byte. - { - results = format_dump_resume<&_dbg_dumper>(results, writebuf, fmt, std::forward(args)...); - if(C4_UNLIKELY(results.bufsize > sizeof(writebuf))) - { - results = format_dump_resume<&_dbg_dumper>(results, writebuf, fmt, std::forward(args)...); - } - } -} -} // namespace c4 - -# define _c4dbgt(fmt, ...) this->_dbg ("{}:{}: " fmt , __FILE__, __LINE__, ## __VA_ARGS__) -# define _c4dbgpf(fmt, ...) _dbg_printf("{}:{}: " fmt "\n", __FILE__, __LINE__, ## __VA_ARGS__) -# define _c4dbgp(msg) _dbg_printf("{}:{}: " msg "\n", __FILE__, __LINE__ ) -# define _c4dbgq(msg) _dbg_printf(msg "\n") -# define _c4err(fmt, ...) \ - do { if(c4::is_debugger_attached()) { C4_DEBUG_BREAK(); } \ - this->_err("ERROR:\n" "{}:{}: " fmt, __FILE__, __LINE__, ## __VA_ARGS__); } while(0) -#else -# define _c4dbgt(fmt, ...) -# define _c4dbgpf(fmt, ...) -# define _c4dbgp(msg) -# define _c4dbgq(msg) -# define _c4err(fmt, ...) \ - do { if(c4::is_debugger_attached()) { C4_DEBUG_BREAK(); } \ - this->_err("ERROR: " fmt, ## __VA_ARGS__); } while(0) -#endif - -#define _c4prsp(sp) sp -#define _c4presc(s) __c4presc(s.str, s.len) -inline c4::csubstr _c4prc(const char &C4_RESTRICT c) -{ - switch(c) - { - case '\n': return c4::csubstr("\\n"); - case '\t': return c4::csubstr("\\t"); - case '\0': return c4::csubstr("\\0"); - case '\r': return c4::csubstr("\\r"); - case '\f': return c4::csubstr("\\f"); - case '\b': return c4::csubstr("\\b"); - case '\v': return c4::csubstr("\\v"); - case '\a': return c4::csubstr("\\a"); - default: return c4::csubstr(&c, 1); - } -} -inline void __c4presc(const char *s, size_t len) -{ - size_t prev = 0; - for(size_t i = 0; i < len; ++i) - { - switch(s[i]) - { - case '\n' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('n'); putchar('\n'); prev = i+1; break; - case '\t' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('t'); prev = i+1; break; - case '\0' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('0'); prev = i+1; break; - case '\r' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('r'); prev = i+1; break; - case '\f' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('f'); prev = i+1; break; - case '\b' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('b'); prev = i+1; break; - case '\v' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('v'); prev = i+1; break; - case '\a' : fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('a'); prev = i+1; break; - case '\x1b': fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('e'); prev = i+1; break; - case -0x3e/*0xc2u*/: - if(i+1 < len) - { - if(s[i+1] == -0x60/*0xa0u*/) - { - fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('_'); prev = i+2; ++i; - } - else if(s[i+1] == -0x7b/*0x85u*/) - { - fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('N'); prev = i+2; ++i; - } - break; - } - case -0x1e/*0xe2u*/: - if(i+2 < len && s[i+1] == -0x80/*0x80u*/) - { - if(s[i+2] == -0x58/*0xa8u*/) - { - fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('L'); prev = i+3; i += 2; - } - else if(s[i+2] == -0x57/*0xa9u*/) - { - fwrite(s+prev, 1, i-prev, stdout); putchar('\\'); putchar('P'); prev = i+3; i += 2; - } - break; - } - } - } - fwrite(s + prev, 1, len - prev, stdout); -} - -#pragma clang diagnostic pop -#pragma GCC diagnostic pop - -#if defined(_MSC_VER) -# pragma warning(pop) -#endif - - -#endif /* _C4_YML_DETAIL_PARSER_DBG_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/detail/print.hpp b/thirdparty/ryml/src/c4/yml/detail/print.hpp deleted file mode 100644 index f88dc251d..000000000 --- a/thirdparty/ryml/src/c4/yml/detail/print.hpp +++ /dev/null @@ -1,128 +0,0 @@ -#ifndef C4_YML_DETAIL_PRINT_HPP_ -#define C4_YML_DETAIL_PRINT_HPP_ - -#include "c4/yml/tree.hpp" -#include "c4/yml/node.hpp" - - -namespace c4 { -namespace yml { - - -inline size_t print_node(Tree const& p, size_t node, int level, size_t count, bool print_children) -{ - printf("[%zd]%*s[%zd] %p", count, (2*level), "", node, (void*)p.get(node)); - if(p.is_root(node)) - { - printf(" [ROOT]"); - } - printf(" %s:", p.type_str(node)); - if(p.has_key(node)) - { - if(p.has_key_anchor(node)) - { - csubstr ka = p.key_anchor(node); - printf(" &%.*s", (int)ka.len, ka.str); - } - if(p.has_key_tag(node)) - { - csubstr kt = p.key_tag(node); - csubstr k = p.key(node); - printf(" %.*s '%.*s'", (int)kt.len, kt.str, (int)k.len, k.str); - } - else - { - csubstr k = p.key(node); - printf(" '%.*s'", (int)k.len, k.str); - } - } - else - { - RYML_ASSERT( ! p.has_key_tag(node)); - } - if(p.has_val(node)) - { - if(p.has_val_tag(node)) - { - csubstr vt = p.val_tag(node); - csubstr v = p.val(node); - printf(" %.*s '%.*s'", (int)vt.len, vt.str, (int)v.len, v.str); - } - else - { - csubstr v = p.val(node); - printf(" '%.*s'", (int)v.len, v.str); - } - } - else - { - if(p.has_val_tag(node)) - { - csubstr vt = p.val_tag(node); - printf(" %.*s", (int)vt.len, vt.str); - } - } - if(p.has_val_anchor(node)) - { - auto &a = p.val_anchor(node); - printf(" valanchor='&%.*s'", (int)a.len, a.str); - } - printf(" (%zd sibs)", p.num_siblings(node)); - - ++count; - - if(p.is_container(node)) - { - printf(" %zd children:\n", p.num_children(node)); - if(print_children) - { - for(size_t i = p.first_child(node); i != NONE; i = p.next_sibling(i)) - { - count = print_node(p, i, level+1, count, print_children); - } - } - } - else - { - printf("\n"); - } - - return count; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -inline void print_node(ConstNodeRef const& p, int level=0) -{ - print_node(*p.tree(), p.id(), level, 0, true); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -inline size_t print_tree(Tree const& p, size_t node=NONE) -{ - printf("--------------------------------------\n"); - size_t ret = 0; - if(!p.empty()) - { - if(node == NONE) - node = p.root_id(); - ret = print_node(p, node, 0, 0, true); - } - printf("#nodes=%zd vs #printed=%zd\n", p.size(), ret); - printf("--------------------------------------\n"); - return ret; -} - - -} /* namespace yml */ -} /* namespace c4 */ - - -#endif /* C4_YML_DETAIL_PRINT_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/detail/stack.hpp b/thirdparty/ryml/src/c4/yml/detail/stack.hpp deleted file mode 100644 index 95677ae27..000000000 --- a/thirdparty/ryml/src/c4/yml/detail/stack.hpp +++ /dev/null @@ -1,274 +0,0 @@ -#ifndef _C4_YML_DETAIL_STACK_HPP_ -#define _C4_YML_DETAIL_STACK_HPP_ - -#ifndef _C4_YML_COMMON_HPP_ -#include "../common.hpp" -#endif - -#ifdef RYML_DBG -# include -#endif - -#include - -namespace c4 { -namespace yml { -namespace detail { - -/** A lightweight contiguous stack with SSO. This avoids a dependency on std. */ -template -class stack -{ - static_assert(std::is_trivially_copyable::value, "T must be trivially copyable"); - static_assert(std::is_trivially_destructible::value, "T must be trivially destructible"); - - enum : size_t { sso_size = N }; - -public: - - T m_buf[N]; - T * m_stack; - size_t m_size; - size_t m_capacity; - Callbacks m_callbacks; - -public: - - constexpr static bool is_contiguous() { return true; } - - stack(Callbacks const& cb) - : m_buf() - , m_stack(m_buf) - , m_size(0) - , m_capacity(N) - , m_callbacks(cb) {} - stack() : stack(get_callbacks()) {} - ~stack() - { - _free(); - } - - stack(stack const& that) noexcept : stack(that.m_callbacks) - { - resize(that.m_size); - _cp(&that); - } - - stack(stack &&that) noexcept : stack(that.m_callbacks) - { - _mv(&that); - } - - stack& operator= (stack const& that) noexcept - { - _cb(that.m_callbacks); - resize(that.m_size); - _cp(&that); - return *this; - } - - stack& operator= (stack &&that) noexcept - { - _cb(that.m_callbacks); - _mv(&that); - return *this; - } - -public: - - size_t size() const { return m_size; } - size_t empty() const { return m_size == 0; } - size_t capacity() const { return m_capacity; } - - void clear() - { - m_size = 0; - } - - void resize(size_t sz) - { - reserve(sz); - m_size = sz; - } - - void reserve(size_t sz); - - void push(T const& C4_RESTRICT n) - { - RYML_ASSERT((const char*)&n + sizeof(T) < (const char*)m_stack || &n > m_stack + m_capacity); - if(m_size == m_capacity) - { - size_t cap = m_capacity == 0 ? N : 2 * m_capacity; - reserve(cap); - } - m_stack[m_size] = n; - ++m_size; - } - - void push_top() - { - RYML_ASSERT(m_size > 0); - if(m_size == m_capacity) - { - size_t cap = m_capacity == 0 ? N : 2 * m_capacity; - reserve(cap); - } - m_stack[m_size] = m_stack[m_size - 1]; - ++m_size; - } - - T const& C4_RESTRICT pop() - { - RYML_ASSERT(m_size > 0); - --m_size; - return m_stack[m_size]; - } - - C4_ALWAYS_INLINE T const& C4_RESTRICT top() const { RYML_ASSERT(m_size > 0); return m_stack[m_size - 1]; } - C4_ALWAYS_INLINE T & C4_RESTRICT top() { RYML_ASSERT(m_size > 0); return m_stack[m_size - 1]; } - - C4_ALWAYS_INLINE T const& C4_RESTRICT bottom() const { RYML_ASSERT(m_size > 0); return m_stack[0]; } - C4_ALWAYS_INLINE T & C4_RESTRICT bottom() { RYML_ASSERT(m_size > 0); return m_stack[0]; } - - C4_ALWAYS_INLINE T const& C4_RESTRICT top(size_t i) const { RYML_ASSERT(i < m_size); return m_stack[m_size - 1 - i]; } - C4_ALWAYS_INLINE T & C4_RESTRICT top(size_t i) { RYML_ASSERT(i < m_size); return m_stack[m_size - 1 - i]; } - - C4_ALWAYS_INLINE T const& C4_RESTRICT bottom(size_t i) const { RYML_ASSERT(i < m_size); return m_stack[i]; } - C4_ALWAYS_INLINE T & C4_RESTRICT bottom(size_t i) { RYML_ASSERT(i < m_size); return m_stack[i]; } - - C4_ALWAYS_INLINE T const& C4_RESTRICT operator[](size_t i) const { RYML_ASSERT(i < m_size); return m_stack[i]; } - C4_ALWAYS_INLINE T & C4_RESTRICT operator[](size_t i) { RYML_ASSERT(i < m_size); return m_stack[i]; } - -public: - - using iterator = T *; - using const_iterator = T const *; - - iterator begin() { return m_stack; } - iterator end () { return m_stack + m_size; } - - const_iterator begin() const { return (const_iterator)m_stack; } - const_iterator end () const { return (const_iterator)m_stack + m_size; } - -public: - void _free(); - void _cp(stack const* C4_RESTRICT that); - void _mv(stack * that); - void _cb(Callbacks const& cb); -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void stack::reserve(size_t sz) -{ - if(sz <= m_size) - return; - if(sz <= N) - { - m_stack = m_buf; - m_capacity = N; - return; - } - T *buf = (T*) m_callbacks.m_allocate(sz * sizeof(T), m_stack, m_callbacks.m_user_data); - memcpy(buf, m_stack, m_size * sizeof(T)); - if(m_stack != m_buf) - { - m_callbacks.m_free(m_stack, m_capacity * sizeof(T), m_callbacks.m_user_data); - } - m_stack = buf; - m_capacity = sz; -} - - -//----------------------------------------------------------------------------- - -template -void stack::_free() -{ - RYML_ASSERT(m_stack != nullptr); // this structure cannot be memset() to zero - if(m_stack != m_buf) - { - m_callbacks.m_free(m_stack, m_capacity * sizeof(T), m_callbacks.m_user_data); - m_stack = m_buf; - m_size = N; - m_capacity = N; - } - else - { - RYML_ASSERT(m_capacity == N); - } -} - - -//----------------------------------------------------------------------------- - -template -void stack::_cp(stack const* C4_RESTRICT that) -{ - if(that->m_stack != that->m_buf) - { - RYML_ASSERT(that->m_capacity > N); - RYML_ASSERT(that->m_size <= that->m_capacity); - } - else - { - RYML_ASSERT(that->m_capacity <= N); - RYML_ASSERT(that->m_size <= that->m_capacity); - } - memcpy(m_stack, that->m_stack, that->m_size * sizeof(T)); - m_size = that->m_size; - m_capacity = that->m_size < N ? N : that->m_size; - m_callbacks = that->m_callbacks; -} - - -//----------------------------------------------------------------------------- - -template -void stack::_mv(stack * that) -{ - if(that->m_stack != that->m_buf) - { - RYML_ASSERT(that->m_capacity > N); - RYML_ASSERT(that->m_size <= that->m_capacity); - m_stack = that->m_stack; - } - else - { - RYML_ASSERT(that->m_capacity <= N); - RYML_ASSERT(that->m_size <= that->m_capacity); - memcpy(m_buf, that->m_buf, that->m_size * sizeof(T)); - m_stack = m_buf; - } - m_size = that->m_size; - m_capacity = that->m_capacity; - m_callbacks = that->m_callbacks; - // make sure no deallocation happens on destruction - RYML_ASSERT(that->m_stack != m_buf); - that->m_stack = that->m_buf; - that->m_capacity = N; - that->m_size = 0; -} - - -//----------------------------------------------------------------------------- - -template -void stack::_cb(Callbacks const& cb) -{ - if(cb != m_callbacks) - { - _free(); - m_callbacks = cb; - } -} - -} // namespace detail -} // namespace yml -} // namespace c4 - -#endif /* _C4_YML_DETAIL_STACK_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/emit.def.hpp b/thirdparty/ryml/src/c4/yml/emit.def.hpp deleted file mode 100644 index d262a9e2a..000000000 --- a/thirdparty/ryml/src/c4/yml/emit.def.hpp +++ /dev/null @@ -1,960 +0,0 @@ -#ifndef _C4_YML_EMIT_DEF_HPP_ -#define _C4_YML_EMIT_DEF_HPP_ - -#ifndef _C4_YML_EMIT_HPP_ -#include "c4/yml/emit.hpp" -#endif - -namespace c4 { -namespace yml { - -template -substr Emitter::emit_as(EmitType_e type, Tree const& t, size_t id, bool error_on_excess) -{ - if(t.empty()) - { - _RYML_CB_ASSERT(t.callbacks(), id == NONE); - return {}; - } - _RYML_CB_CHECK(t.callbacks(), id < t.size()); - m_tree = &t; - if(type == EMIT_YAML) - _emit_yaml(id); - else if(type == EMIT_JSON) - _do_visit_json(id); - else - _RYML_CB_ERR(m_tree->callbacks(), "unknown emit type"); - return this->Writer::_get(error_on_excess); -} - -template -substr Emitter::emit_as(EmitType_e type, Tree const& t, bool error_on_excess) -{ - if(t.empty()) - return {}; - return this->emit_as(type, t, t.root_id(), error_on_excess); -} - -template -substr Emitter::emit_as(EmitType_e type, ConstNodeRef const& n, bool error_on_excess) -{ - _RYML_CB_CHECK(n.tree()->callbacks(), n.valid()); - return this->emit_as(type, *n.tree(), n.id(), error_on_excess); -} - - -//----------------------------------------------------------------------------- - -template -void Emitter::_emit_yaml(size_t id) -{ - // save branches in the visitor by doing the initial stream/doc - // logic here, sparing the need to check stream/val/keyval inside - // the visitor functions - auto dispatch = [this](size_t node){ - NodeType ty = m_tree->type(node); - if(ty.marked_flow_sl()) - _do_visit_flow_sl(node, 0); - else if(ty.marked_flow_ml()) - _do_visit_flow_ml(node, 0); - else - { - _do_visit_block(node, 0); - } - }; - if(!m_tree->is_root(id)) - { - if(m_tree->is_container(id) && !m_tree->type(id).marked_flow()) - { - size_t ilevel = 0; - if(m_tree->has_key(id)) - { - this->Writer::_do_write(m_tree->key(id)); - this->Writer::_do_write(":\n"); - ++ilevel; - } - _do_visit_block_container(id, ilevel, ilevel); - return; - } - } - - auto *btd = m_tree->tag_directives().b; - auto *etd = m_tree->tag_directives().e; - auto write_tag_directives = [&btd, etd, this](size_t next_node){ - auto end = btd; - while(end < etd) - { - if(end->next_node_id > next_node) - break; - ++end; - } - for( ; btd != end; ++btd) - { - if(next_node != m_tree->first_child(m_tree->parent(next_node))) - this->Writer::_do_write("...\n"); - this->Writer::_do_write("%TAG "); - this->Writer::_do_write(btd->handle); - this->Writer::_do_write(' '); - this->Writer::_do_write(btd->prefix); - this->Writer::_do_write('\n'); - } - }; - if(m_tree->is_stream(id)) - { - if(m_tree->first_child(id) != NONE) - write_tag_directives(m_tree->first_child(id)); - for(size_t child = m_tree->first_child(id); child != NONE; child = m_tree->next_sibling(child)) - { - dispatch(child); - if(m_tree->next_sibling(child) != NONE) - write_tag_directives(m_tree->next_sibling(child)); - } - } - else if(m_tree->is_container(id)) - { - dispatch(id); - } - else if(m_tree->is_doc(id)) - { - _RYML_CB_ASSERT(m_tree->callbacks(), !m_tree->is_container(id)); // checked above - _RYML_CB_ASSERT(m_tree->callbacks(), m_tree->is_val(id)); // so it must be a val - _write_doc(id); - } - else if(m_tree->is_keyval(id)) - { - _writek(id, 0); - this->Writer::_do_write(": "); - _writev(id, 0); - if(!m_tree->type(id).marked_flow()) - this->Writer::_do_write('\n'); - } - else if(m_tree->is_val(id)) - { - //this->Writer::_do_write("- "); - _writev(id, 0); - if(!m_tree->type(id).marked_flow()) - this->Writer::_do_write('\n'); - } - else if(m_tree->type(id) == NOTYPE) - { - ; - } - else - { - _RYML_CB_ERR(m_tree->callbacks(), "unknown type"); - } -} - -template -void Emitter::_write_doc(size_t id) -{ - RYML_ASSERT(m_tree->is_doc(id)); - if(!m_tree->is_root(id)) - { - RYML_ASSERT(m_tree->is_stream(m_tree->parent(id))); - this->Writer::_do_write("---"); - } - if(!m_tree->has_val(id)) // this is more frequent - { - if(m_tree->has_val_tag(id)) - { - if(!m_tree->is_root(id)) - this->Writer::_do_write(' '); - _write_tag(m_tree->val_tag(id)); - } - if(m_tree->has_val_anchor(id)) - { - if(!m_tree->is_root(id)) - this->Writer::_do_write(' '); - this->Writer::_do_write('&'); - this->Writer::_do_write(m_tree->val_anchor(id)); - } - } - else // docval - { - RYML_ASSERT(m_tree->has_val(id)); - RYML_ASSERT(!m_tree->has_key(id)); - if(!m_tree->is_root(id)) - this->Writer::_do_write(' '); - _writev(id, 0); - } - this->Writer::_do_write('\n'); -} - -template -void Emitter::_do_visit_flow_sl(size_t node, size_t ilevel) -{ - RYML_ASSERT(!m_tree->is_stream(node)); - RYML_ASSERT(m_tree->is_container(node) || m_tree->is_doc(node)); - RYML_ASSERT(m_tree->is_root(node) || (m_tree->parent_is_map(node) || m_tree->parent_is_seq(node))); - - if(m_tree->is_doc(node)) - { - _write_doc(node); - if(!m_tree->has_children(node)) - return; - } - else if(m_tree->is_container(node)) - { - RYML_ASSERT(m_tree->is_map(node) || m_tree->is_seq(node)); - - bool spc = false; // write a space - - if(m_tree->has_key(node)) - { - _writek(node, ilevel); - this->Writer::_do_write(':'); - spc = true; - } - - if(m_tree->has_val_tag(node)) - { - if(spc) - this->Writer::_do_write(' '); - _write_tag(m_tree->val_tag(node)); - spc = true; - } - - if(m_tree->has_val_anchor(node)) - { - if(spc) - this->Writer::_do_write(' '); - this->Writer::_do_write('&'); - this->Writer::_do_write(m_tree->val_anchor(node)); - spc = true; - } - - if(spc) - this->Writer::_do_write(' '); - - if(m_tree->is_map(node)) - { - this->Writer::_do_write('{'); - } - else - { - _RYML_CB_ASSERT(m_tree->callbacks(), m_tree->is_seq(node)); - this->Writer::_do_write('['); - } - } // container - - for(size_t child = m_tree->first_child(node), count = 0; child != NONE; child = m_tree->next_sibling(child)) - { - if(count++) - this->Writer::_do_write(','); - if(m_tree->is_keyval(child)) - { - _writek(child, ilevel); - this->Writer::_do_write(": "); - _writev(child, ilevel); - } - else if(m_tree->is_val(child)) - { - _writev(child, ilevel); - } - else - { - // with single-line flow, we can never go back to block - _do_visit_flow_sl(child, ilevel + 1); - } - } - - if(m_tree->is_map(node)) - { - this->Writer::_do_write('}'); - } - else if(m_tree->is_seq(node)) - { - this->Writer::_do_write(']'); - } -} - -template -void Emitter::_do_visit_flow_ml(size_t id, size_t ilevel, size_t do_indent) -{ - C4_UNUSED(id); - C4_UNUSED(ilevel); - C4_UNUSED(do_indent); - RYML_CHECK(false/*not implemented*/); -} - -template -void Emitter::_do_visit_block_container(size_t node, size_t next_level, size_t do_indent) -{ - RepC ind = indent_to(do_indent * next_level); - - if(m_tree->is_seq(node)) - { - for(size_t child = m_tree->first_child(node); child != NONE; child = m_tree->next_sibling(child)) - { - _RYML_CB_ASSERT(m_tree->callbacks(), !m_tree->has_key(child)); - if(m_tree->is_val(child)) - { - this->Writer::_do_write(ind); - this->Writer::_do_write("- "); - _writev(child, next_level); - this->Writer::_do_write('\n'); - } - else - { - _RYML_CB_ASSERT(m_tree->callbacks(), m_tree->is_container(child)); - NodeType ty = m_tree->type(child); - if(ty.marked_flow_sl()) - { - this->Writer::_do_write(ind); - this->Writer::_do_write("- "); - _do_visit_flow_sl(child, 0u); - this->Writer::_do_write('\n'); - } - else if(ty.marked_flow_ml()) - { - this->Writer::_do_write(ind); - this->Writer::_do_write("- "); - _do_visit_flow_ml(child, next_level, do_indent); - this->Writer::_do_write('\n'); - } - else - { - _do_visit_block(child, next_level, do_indent); - } - } - do_indent = true; - ind = indent_to(do_indent * next_level); - } - } - else // map - { - _RYML_CB_ASSERT(m_tree->callbacks(), m_tree->is_map(node)); - for(size_t ich = m_tree->first_child(node); ich != NONE; ich = m_tree->next_sibling(ich)) - { - _RYML_CB_ASSERT(m_tree->callbacks(), m_tree->has_key(ich)); - if(m_tree->is_keyval(ich)) - { - this->Writer::_do_write(ind); - _writek(ich, next_level); - this->Writer::_do_write(": "); - _writev(ich, next_level); - this->Writer::_do_write('\n'); - } - else - { - _RYML_CB_ASSERT(m_tree->callbacks(), m_tree->is_container(ich)); - NodeType ty = m_tree->type(ich); - if(ty.marked_flow_sl()) - { - this->Writer::_do_write(ind); - _do_visit_flow_sl(ich, 0u); - this->Writer::_do_write('\n'); - } - else if(ty.marked_flow_ml()) - { - this->Writer::_do_write(ind); - _do_visit_flow_ml(ich, 0u); - this->Writer::_do_write('\n'); - } - else - { - _do_visit_block(ich, next_level, do_indent); - } - } - do_indent = true; - ind = indent_to(do_indent * next_level); - } - } -} - -template -void Emitter::_do_visit_block(size_t node, size_t ilevel, size_t do_indent) -{ - RYML_ASSERT(!m_tree->is_stream(node)); - RYML_ASSERT(m_tree->is_container(node) || m_tree->is_doc(node)); - RYML_ASSERT(m_tree->is_root(node) || (m_tree->parent_is_map(node) || m_tree->parent_is_seq(node))); - RepC ind = indent_to(do_indent * ilevel); - - if(m_tree->is_doc(node)) - { - _write_doc(node); - if(!m_tree->has_children(node)) - return; - } - else if(m_tree->is_container(node)) - { - RYML_ASSERT(m_tree->is_map(node) || m_tree->is_seq(node)); - - bool spc = false; // write a space - bool nl = false; // write a newline - - if(m_tree->has_key(node)) - { - this->Writer::_do_write(ind); - _writek(node, ilevel); - this->Writer::_do_write(':'); - spc = true; - } - else if(!m_tree->is_root(node)) - { - this->Writer::_do_write(ind); - this->Writer::_do_write('-'); - spc = true; - } - - if(m_tree->has_val_tag(node)) - { - if(spc) - this->Writer::_do_write(' '); - _write_tag(m_tree->val_tag(node)); - spc = true; - nl = true; - } - - if(m_tree->has_val_anchor(node)) - { - if(spc) - this->Writer::_do_write(' '); - this->Writer::_do_write('&'); - this->Writer::_do_write(m_tree->val_anchor(node)); - spc = true; - nl = true; - } - - if(m_tree->has_children(node)) - { - if(m_tree->has_key(node)) - nl = true; - else - if(!m_tree->is_root(node) && !nl) - spc = true; - } - else - { - if(m_tree->is_seq(node)) - this->Writer::_do_write(" []\n"); - else if(m_tree->is_map(node)) - this->Writer::_do_write(" {}\n"); - return; - } - - if(spc && !nl) - this->Writer::_do_write(' '); - - do_indent = 0; - if(nl) - { - this->Writer::_do_write('\n'); - do_indent = 1; - } - } // container - - size_t next_level = ilevel + 1; - if(m_tree->is_root(node) || m_tree->is_doc(node)) - next_level = ilevel; // do not indent at top level - - _do_visit_block_container(node, next_level, do_indent); -} - -template -void Emitter::_do_visit_json(size_t id) -{ - _RYML_CB_CHECK(m_tree->callbacks(), !m_tree->is_stream(id)); // JSON does not have streams - if(m_tree->is_keyval(id)) - { - _writek_json(id); - this->Writer::_do_write(": "); - _writev_json(id); - } - else if(m_tree->is_val(id)) - { - _writev_json(id); - } - else if(m_tree->is_container(id)) - { - if(m_tree->has_key(id)) - { - _writek_json(id); - this->Writer::_do_write(": "); - } - if(m_tree->is_seq(id)) - this->Writer::_do_write('['); - else if(m_tree->is_map(id)) - this->Writer::_do_write('{'); - } // container - - for(size_t ich = m_tree->first_child(id); ich != NONE; ich = m_tree->next_sibling(ich)) - { - if(ich != m_tree->first_child(id)) - this->Writer::_do_write(','); - _do_visit_json(ich); - } - - if(m_tree->is_seq(id)) - this->Writer::_do_write(']'); - else if(m_tree->is_map(id)) - this->Writer::_do_write('}'); -} - -template -void Emitter::_write(NodeScalar const& C4_RESTRICT sc, NodeType flags, size_t ilevel) -{ - if( ! sc.tag.empty()) - { - _write_tag(sc.tag); - this->Writer::_do_write(' '); - } - if(flags.has_anchor()) - { - RYML_ASSERT(flags.is_ref() != flags.has_anchor()); - RYML_ASSERT( ! sc.anchor.empty()); - this->Writer::_do_write('&'); - this->Writer::_do_write(sc.anchor); - this->Writer::_do_write(' '); - } - else if(flags.is_ref()) - { - if(sc.anchor != "<<") - this->Writer::_do_write('*'); - this->Writer::_do_write(sc.anchor); - return; - } - - // ensure the style flags only have one of KEY or VAL - _RYML_CB_ASSERT(m_tree->callbacks(), ((flags & (_WIP_KEY_STYLE|_WIP_VAL_STYLE)) == 0) || (((flags&_WIP_KEY_STYLE) == 0) != ((flags&_WIP_VAL_STYLE) == 0))); - - auto style_marks = flags & (_WIP_KEY_STYLE|_WIP_VAL_STYLE); - if(style_marks & (_WIP_KEY_LITERAL|_WIP_VAL_LITERAL)) - { - _write_scalar_literal(sc.scalar, ilevel, flags.has_key()); - } - else if(style_marks & (_WIP_KEY_FOLDED|_WIP_VAL_FOLDED)) - { - _write_scalar_folded(sc.scalar, ilevel, flags.has_key()); - } - else if(style_marks & (_WIP_KEY_SQUO|_WIP_VAL_SQUO)) - { - _write_scalar_squo(sc.scalar, ilevel); - } - else if(style_marks & (_WIP_KEY_DQUO|_WIP_VAL_DQUO)) - { - _write_scalar_dquo(sc.scalar, ilevel); - } - else if(style_marks & (_WIP_KEY_PLAIN|_WIP_VAL_PLAIN)) - { - _write_scalar_plain(sc.scalar, ilevel); - } - else if(!style_marks) - { - size_t first_non_nl = sc.scalar.first_not_of('\n'); - bool all_newlines = first_non_nl == npos; - bool has_leading_ws = (!all_newlines) && sc.scalar.sub(first_non_nl).begins_with_any(" \t"); - bool do_literal = ((!sc.scalar.empty() && all_newlines) || (has_leading_ws && !sc.scalar.trim(' ').empty())); - if(do_literal) - { - _write_scalar_literal(sc.scalar, ilevel, flags.has_key(), /*explicit_indentation*/has_leading_ws); - } - else - { - for(size_t i = 0; i < sc.scalar.len; ++i) - { - if(sc.scalar.str[i] == '\n') - { - _write_scalar_literal(sc.scalar, ilevel, flags.has_key(), /*explicit_indentation*/has_leading_ws); - goto wrote_special; - } - // todo: check for escaped characters requiring double quotes - } - _write_scalar(sc.scalar, flags.is_quoted()); - wrote_special: - ; - } - } - else - { - _RYML_CB_ERR(m_tree->callbacks(), "not implemented"); - } -} -template -void Emitter::_write_json(NodeScalar const& C4_RESTRICT sc, NodeType flags) -{ - if(C4_UNLIKELY( ! sc.tag.empty())) - _RYML_CB_ERR(m_tree->callbacks(), "JSON does not have tags"); - if(C4_UNLIKELY(flags.has_anchor())) - _RYML_CB_ERR(m_tree->callbacks(), "JSON does not have anchors"); - _write_scalar_json(sc.scalar, flags.has_key(), flags.is_quoted()); -} - -#define _rymlindent_nextline() for(size_t lv = 0; lv < ilevel+1; ++lv) { this->Writer::_do_write(' '); this->Writer::_do_write(' '); } - -template -void Emitter::_write_scalar_literal(csubstr s, size_t ilevel, bool explicit_key, bool explicit_indentation) -{ - if(explicit_key) - this->Writer::_do_write("? "); - csubstr trimmed = s.trimr("\n\r"); - size_t numnewlines_at_end = s.len - trimmed.len - s.sub(trimmed.len).count('\r'); - // - if(!explicit_indentation) - this->Writer::_do_write('|'); - else - this->Writer::_do_write("|2"); - // - if(numnewlines_at_end > 1 || (trimmed.len == 0 && s.len > 0)/*only newlines*/) - this->Writer::_do_write("+\n"); - else if(numnewlines_at_end == 1) - this->Writer::_do_write('\n'); - else - this->Writer::_do_write("-\n"); - // - if(trimmed.len) - { - size_t pos = 0; // tracks the last character that was already written - for(size_t i = 0; i < trimmed.len; ++i) - { - if(trimmed[i] != '\n') - continue; - // write everything up to this point - csubstr since_pos = trimmed.range(pos, i+1); // include the newline - _rymlindent_nextline() - this->Writer::_do_write(since_pos); - pos = i+1; // already written - } - if(pos < trimmed.len) - { - _rymlindent_nextline() - this->Writer::_do_write(trimmed.sub(pos)); - } - if(numnewlines_at_end) - { - this->Writer::_do_write('\n'); - --numnewlines_at_end; - } - } - for(size_t i = 0; i < numnewlines_at_end; ++i) - { - _rymlindent_nextline() - if(i+1 < numnewlines_at_end || explicit_key) - this->Writer::_do_write('\n'); - } - if(explicit_key && !numnewlines_at_end) - this->Writer::_do_write('\n'); -} - -template -void Emitter::_write_scalar_folded(csubstr s, size_t ilevel, bool explicit_key) -{ - if(explicit_key) - { - this->Writer::_do_write("? "); - } - RYML_ASSERT(s.find("\r") == csubstr::npos); - csubstr trimmed = s.trimr('\n'); - size_t numnewlines_at_end = s.len - trimmed.len; - if(numnewlines_at_end == 0) - { - this->Writer::_do_write(">-\n"); - } - else if(numnewlines_at_end == 1) - { - this->Writer::_do_write(">\n"); - } - else if(numnewlines_at_end > 1) - { - this->Writer::_do_write(">+\n"); - } - if(trimmed.len) - { - size_t pos = 0; // tracks the last character that was already written - for(size_t i = 0; i < trimmed.len; ++i) - { - if(trimmed[i] != '\n') - continue; - // write everything up to this point - csubstr since_pos = trimmed.range(pos, i+1); // include the newline - pos = i+1; // because of the newline - _rymlindent_nextline() - this->Writer::_do_write(since_pos); - this->Writer::_do_write('\n'); // write the newline twice - } - if(pos < trimmed.len) - { - _rymlindent_nextline() - this->Writer::_do_write(trimmed.sub(pos)); - } - if(numnewlines_at_end) - { - this->Writer::_do_write('\n'); - --numnewlines_at_end; - } - } - for(size_t i = 0; i < numnewlines_at_end; ++i) - { - _rymlindent_nextline() - if(i+1 < numnewlines_at_end || explicit_key) - this->Writer::_do_write('\n'); - } - if(explicit_key && !numnewlines_at_end) - this->Writer::_do_write('\n'); -} - -template -void Emitter::_write_scalar_squo(csubstr s, size_t ilevel) -{ - size_t pos = 0; // tracks the last character that was already written - this->Writer::_do_write('\''); - for(size_t i = 0; i < s.len; ++i) - { - if(s[i] == '\n') - { - csubstr sub = s.range(pos, i+1); - this->Writer::_do_write(sub); // write everything up to (including) this char - this->Writer::_do_write('\n'); // write the character again - if(i + 1 < s.len) - _rymlindent_nextline() // indent the next line - pos = i+1; - } - else if(s[i] == '\'') - { - csubstr sub = s.range(pos, i+1); - this->Writer::_do_write(sub); // write everything up to (including) this char - this->Writer::_do_write('\''); // write the character again - pos = i+1; - } - } - // write missing characters at the end of the string - if(pos < s.len) - this->Writer::_do_write(s.sub(pos)); - this->Writer::_do_write('\''); -} - -template -void Emitter::_write_scalar_dquo(csubstr s, size_t ilevel) -{ - size_t pos = 0; // tracks the last character that was already written - this->Writer::_do_write('"'); - for(size_t i = 0; i < s.len; ++i) - { - const char curr = s.str[i]; - if(curr == '"' || curr == '\\') - { - csubstr sub = s.range(pos, i); - this->Writer::_do_write(sub); // write everything up to (excluding) this char - this->Writer::_do_write('\\'); // write the escape - this->Writer::_do_write(curr); // write the char - pos = i+1; - } - else if(s[i] == '\n') - { - csubstr sub = s.range(pos, i+1); - this->Writer::_do_write(sub); // write everything up to (including) this newline - this->Writer::_do_write('\n'); // write the newline again - if(i + 1 < s.len) - _rymlindent_nextline() // indent the next line - pos = i+1; - if(i+1 < s.len) // escape leading whitespace after the newline - { - const char next = s.str[i+1]; - if(next == ' ' || next == '\t') - this->Writer::_do_write('\\'); - } - } - else if(curr == ' ' || curr == '\t') - { - // escape trailing whitespace before a newline - size_t next = s.first_not_of(" \t\r", i); - if(next != npos && s[next] == '\n') - { - csubstr sub = s.range(pos, i); - this->Writer::_do_write(sub); // write everything up to (excluding) this char - this->Writer::_do_write('\\'); // escape the whitespace - pos = i; - } - } - else if(C4_UNLIKELY(curr == '\r')) - { - csubstr sub = s.range(pos, i); - this->Writer::_do_write(sub); // write everything up to (excluding) this char - this->Writer::_do_write("\\r"); // write the escaped char - pos = i+1; - } - } - // write missing characters at the end of the string - if(pos < s.len) - { - csubstr sub = s.sub(pos); - this->Writer::_do_write(sub); - } - this->Writer::_do_write('"'); -} - -template -void Emitter::_write_scalar_plain(csubstr s, size_t ilevel) -{ - size_t pos = 0; // tracks the last character that was already written - for(size_t i = 0; i < s.len; ++i) - { - const char curr = s.str[i]; - if(curr == '\n') - { - csubstr sub = s.range(pos, i+1); - this->Writer::_do_write(sub); // write everything up to (including) this newline - this->Writer::_do_write('\n'); // write the newline again - if(i + 1 < s.len) - _rymlindent_nextline() // indent the next line - pos = i+1; - } - } - // write missing characters at the end of the string - if(pos < s.len) - { - csubstr sub = s.sub(pos); - this->Writer::_do_write(sub); - } -} - -#undef _rymlindent_nextline - -template -void Emitter::_write_scalar(csubstr s, bool was_quoted) -{ - // this block of code needed to be moved to before the needs_quotes - // assignment to work around a g++ optimizer bug where (s.str != nullptr) - // was evaluated as true even if s.str was actually a nullptr (!!!) - if(s.len == size_t(0)) - { - if(was_quoted || s.str != nullptr) - this->Writer::_do_write("''"); - return; - } - - const bool needs_quotes = ( - was_quoted - || - ( - ( ! s.is_number()) - && - ( - // has leading whitespace - // looks like reference or anchor - // would be treated as a directive - // see https://www.yaml.info/learn/quote.html#noplain - s.begins_with_any(" \n\t\r*&%@`") - || - s.begins_with("<<") - || - // has trailing whitespace - s.ends_with_any(" \n\t\r") - || - // has special chars - (s.first_of("#:-?,\n{}[]'\"") != npos) - ) - ) - ); - - if( ! needs_quotes) - { - this->Writer::_do_write(s); - } - else - { - const bool has_dquotes = s.first_of( '"') != npos; - const bool has_squotes = s.first_of('\'') != npos; - if(!has_squotes && has_dquotes) - { - this->Writer::_do_write('\''); - this->Writer::_do_write(s); - this->Writer::_do_write('\''); - } - else if(has_squotes && !has_dquotes) - { - RYML_ASSERT(s.count('\n') == 0); - this->Writer::_do_write('"'); - this->Writer::_do_write(s); - this->Writer::_do_write('"'); - } - else - { - _write_scalar_squo(s, /*FIXME FIXME FIXME*/0); - } - } -} -template -void Emitter::_write_scalar_json(csubstr s, bool as_key, bool use_quotes) -{ - if((!use_quotes) - // json keys require quotes - && (!as_key) - && ( - // do not quote special cases - (s == "true" || s == "false" || s == "null") - || ( - // do not quote numbers - (s.is_number() - && ( - // quote integral numbers if they have a leading 0 - // https://github.com/biojppm/rapidyaml/issues/291 - (!(s.len > 1 && s.begins_with('0'))) - // do not quote reals with leading 0 - // https://github.com/biojppm/rapidyaml/issues/313 - || (s.find('.') != csubstr::npos) )) - ) - ) - ) - { - this->Writer::_do_write(s); - } - else - { - size_t pos = 0; - this->Writer::_do_write('"'); - for(size_t i = 0; i < s.len; ++i) - { - switch(s.str[i]) - { - case '"': - this->Writer ::_do_write(s.range(pos, i)); - this->Writer ::_do_write("\\\""); - pos = i + 1; - break; - case '\n': - this->Writer ::_do_write(s.range(pos, i)); - this->Writer ::_do_write("\\n"); - pos = i + 1; - break; - case '\t': - this->Writer ::_do_write(s.range(pos, i)); - this->Writer ::_do_write("\\t"); - pos = i + 1; - break; - case '\\': - this->Writer ::_do_write(s.range(pos, i)); - this->Writer ::_do_write("\\\\"); - pos = i + 1; - break; - case '\r': - this->Writer ::_do_write(s.range(pos, i)); - this->Writer ::_do_write("\\r"); - pos = i + 1; - break; - case '\b': - this->Writer ::_do_write(s.range(pos, i)); - this->Writer ::_do_write("\\b"); - pos = i + 1; - break; - case '\f': - this->Writer ::_do_write(s.range(pos, i)); - this->Writer ::_do_write("\\f"); - pos = i + 1; - break; - } - } - if(pos < s.len) - { - csubstr sub = s.sub(pos); - this->Writer::_do_write(sub); - } - this->Writer::_do_write('"'); - } -} - -} // namespace yml -} // namespace c4 - -#endif /* _C4_YML_EMIT_DEF_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/emit.hpp b/thirdparty/ryml/src/c4/yml/emit.hpp deleted file mode 100644 index c7cdd2a1a..000000000 --- a/thirdparty/ryml/src/c4/yml/emit.hpp +++ /dev/null @@ -1,490 +0,0 @@ -#ifndef _C4_YML_EMIT_HPP_ -#define _C4_YML_EMIT_HPP_ - -#ifndef _C4_YML_WRITER_HPP_ -#include "./writer.hpp" -#endif - -#ifndef _C4_YML_TREE_HPP_ -#include "./tree.hpp" -#endif - -#ifndef _C4_YML_NODE_HPP_ -#include "./node.hpp" -#endif - - -#define RYML_DEPRECATE_EMIT \ - RYML_DEPRECATED("use emit_yaml() instead. See https://github.com/biojppm/rapidyaml/issues/120") -#ifdef emit -#error "emit is defined, likely from a Qt include. This will cause a compilation error. See https://github.com/biojppm/rapidyaml/issues/120" -#endif -#define RYML_DEPRECATE_EMITRS \ - RYML_DEPRECATED("use emitrs_yaml() instead. See https://github.com/biojppm/rapidyaml/issues/120") - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace c4 { -namespace yml { - -template class Emitter; - -template -using EmitterOStream = Emitter>; -using EmitterFile = Emitter; -using EmitterBuf = Emitter; - -typedef enum { - EMIT_YAML = 0, - EMIT_JSON = 1 -} EmitType_e; - - -/** mark a tree or node to be emitted as json */ -struct as_json -{ - Tree const* tree; - size_t node; - as_json(Tree const& t) : tree(&t), node(t.empty() ? NONE : t.root_id()) {} - as_json(Tree const& t, size_t id) : tree(&t), node(id) {} - as_json(ConstNodeRef const& n) : tree(n.tree()), node(n.id()) {} -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -class Emitter : public Writer -{ -public: - - using Writer::Writer; - - /** emit! - * - * When writing to a buffer, returns a substr of the emitted YAML. - * If the given buffer has insufficient space, the returned span will - * be null and its size will be the needed space. No writes are done - * after the end of the buffer. - * - * When writing to a file, the returned substr will be null, but its - * length will be set to the number of bytes written. */ - substr emit_as(EmitType_e type, Tree const& t, size_t id, bool error_on_excess); - /** emit starting at the root node */ - substr emit_as(EmitType_e type, Tree const& t, bool error_on_excess=true); - /** emit the given node */ - substr emit_as(EmitType_e type, ConstNodeRef const& n, bool error_on_excess=true); - -private: - - Tree const* C4_RESTRICT m_tree; - - void _emit_yaml(size_t id); - void _do_visit_flow_sl(size_t id, size_t ilevel=0); - void _do_visit_flow_ml(size_t id, size_t ilevel=0, size_t do_indent=1); - void _do_visit_block(size_t id, size_t ilevel=0, size_t do_indent=1); - void _do_visit_block_container(size_t id, size_t next_level, size_t do_indent); - void _do_visit_json(size_t id); - -private: - - void _write(NodeScalar const& C4_RESTRICT sc, NodeType flags, size_t level); - void _write_json(NodeScalar const& C4_RESTRICT sc, NodeType flags); - - void _write_doc(size_t id); - void _write_scalar(csubstr s, bool was_quoted); - void _write_scalar_json(csubstr s, bool as_key, bool was_quoted); - void _write_scalar_literal(csubstr s, size_t level, bool as_key, bool explicit_indentation=false); - void _write_scalar_folded(csubstr s, size_t level, bool as_key); - void _write_scalar_squo(csubstr s, size_t level); - void _write_scalar_dquo(csubstr s, size_t level); - void _write_scalar_plain(csubstr s, size_t level); - - void _write_tag(csubstr tag) - { - if(!tag.begins_with('!')) - this->Writer::_do_write('!'); - this->Writer::_do_write(tag); - } - - enum : type_bits { - _keysc = (KEY|KEYREF|KEYANCH|KEYQUO|_WIP_KEY_STYLE) | ~(VAL|VALREF|VALANCH|VALQUO|_WIP_VAL_STYLE), - _valsc = ~(KEY|KEYREF|KEYANCH|KEYQUO|_WIP_KEY_STYLE) | (VAL|VALREF|VALANCH|VALQUO|_WIP_VAL_STYLE), - _keysc_json = (KEY) | ~(VAL), - _valsc_json = ~(KEY) | (VAL), - }; - - C4_ALWAYS_INLINE void _writek(size_t id, size_t level) { _write(m_tree->keysc(id), m_tree->_p(id)->m_type.type & ~_valsc, level); } - C4_ALWAYS_INLINE void _writev(size_t id, size_t level) { _write(m_tree->valsc(id), m_tree->_p(id)->m_type.type & ~_keysc, level); } - - C4_ALWAYS_INLINE void _writek_json(size_t id) { _write_json(m_tree->keysc(id), m_tree->_p(id)->m_type.type & ~(VAL)); } - C4_ALWAYS_INLINE void _writev_json(size_t id) { _write_json(m_tree->valsc(id), m_tree->_p(id)->m_type.type & ~(KEY)); } - -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** emit YAML to the given file. A null file defaults to stdout. - * Return the number of bytes written. */ -inline size_t emit_yaml(Tree const& t, size_t id, FILE *f) -{ - EmitterFile em(f); - return em.emit_as(EMIT_YAML, t, id, /*error_on_excess*/true).len; -} -RYML_DEPRECATE_EMIT inline size_t emit(Tree const& t, size_t id, FILE *f) -{ - return emit_yaml(t, id, f); -} - -/** emit JSON to the given file. A null file defaults to stdout. - * Return the number of bytes written. */ -inline size_t emit_json(Tree const& t, size_t id, FILE *f) -{ - EmitterFile em(f); - return em.emit_as(EMIT_JSON, t, id, /*error_on_excess*/true).len; -} - - -/** emit YAML to the given file. A null file defaults to stdout. - * Return the number of bytes written. - * @overload */ -inline size_t emit_yaml(Tree const& t, FILE *f=nullptr) -{ - EmitterFile em(f); - return em.emit_as(EMIT_YAML, t, /*error_on_excess*/true).len; -} -RYML_DEPRECATE_EMIT inline size_t emit(Tree const& t, FILE *f=nullptr) -{ - return emit_yaml(t, f); -} - -/** emit JSON to the given file. A null file defaults to stdout. - * Return the number of bytes written. - * @overload */ -inline size_t emit_json(Tree const& t, FILE *f=nullptr) -{ - EmitterFile em(f); - return em.emit_as(EMIT_JSON, t, /*error_on_excess*/true).len; -} - - -/** emit YAML to the given file. A null file defaults to stdout. - * Return the number of bytes written. - * @overload */ -inline size_t emit_yaml(ConstNodeRef const& r, FILE *f=nullptr) -{ - EmitterFile em(f); - return em.emit_as(EMIT_YAML, r, /*error_on_excess*/true).len; -} -RYML_DEPRECATE_EMIT inline size_t emit(ConstNodeRef const& r, FILE *f=nullptr) -{ - return emit_yaml(r, f); -} - -/** emit JSON to the given file. A null file defaults to stdout. - * Return the number of bytes written. - * @overload */ -inline size_t emit_json(ConstNodeRef const& r, FILE *f=nullptr) -{ - EmitterFile em(f); - return em.emit_as(EMIT_JSON, r, /*error_on_excess*/true).len; -} - - -//----------------------------------------------------------------------------- - -/** emit YAML to an STL-like ostream */ -template -inline OStream& operator<< (OStream& s, Tree const& t) -{ - EmitterOStream em(s); - em.emit_as(EMIT_YAML, t); - return s; -} - -/** emit YAML to an STL-like ostream - * @overload */ -template -inline OStream& operator<< (OStream& s, ConstNodeRef const& n) -{ - EmitterOStream em(s); - em.emit_as(EMIT_YAML, n); - return s; -} - -/** emit json to an STL-like stream */ -template -inline OStream& operator<< (OStream& s, as_json const& j) -{ - EmitterOStream em(s); - em.emit_as(EMIT_JSON, *j.tree, j.node, true); - return s; -} - - -//----------------------------------------------------------------------------- - - -/** emit YAML to the given buffer. Return a substr trimmed to the emitted YAML. - * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @overload */ -inline substr emit_yaml(Tree const& t, size_t id, substr buf, bool error_on_excess=true) -{ - EmitterBuf em(buf); - return em.emit_as(EMIT_YAML, t, id, error_on_excess); -} -RYML_DEPRECATE_EMIT inline substr emit(Tree const& t, size_t id, substr buf, bool error_on_excess=true) -{ - return emit_yaml(t, id, buf, error_on_excess); -} - -/** emit JSON to the given buffer. Return a substr trimmed to the emitted JSON. - * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @overload */ -inline substr emit_json(Tree const& t, size_t id, substr buf, bool error_on_excess=true) -{ - EmitterBuf em(buf); - return em.emit_as(EMIT_JSON, t, id, error_on_excess); -} - - -/** emit YAML to the given buffer. Return a substr trimmed to the emitted YAML. - * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @overload */ -inline substr emit_yaml(Tree const& t, substr buf, bool error_on_excess=true) -{ - EmitterBuf em(buf); - return em.emit_as(EMIT_YAML, t, error_on_excess); -} -RYML_DEPRECATE_EMIT inline substr emit(Tree const& t, substr buf, bool error_on_excess=true) -{ - return emit_yaml(t, buf, error_on_excess); -} - -/** emit JSON to the given buffer. Return a substr trimmed to the emitted JSON. - * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @overload */ -inline substr emit_json(Tree const& t, substr buf, bool error_on_excess=true) -{ - EmitterBuf em(buf); - return em.emit_as(EMIT_JSON, t, error_on_excess); -} - - -/** emit YAML to the given buffer. Return a substr trimmed to the emitted YAML. - * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @overload - */ -inline substr emit_yaml(ConstNodeRef const& r, substr buf, bool error_on_excess=true) -{ - EmitterBuf em(buf); - return em.emit_as(EMIT_YAML, r, error_on_excess); -} -RYML_DEPRECATE_EMIT inline substr emit(ConstNodeRef const& r, substr buf, bool error_on_excess=true) -{ - return emit_yaml(r, buf, error_on_excess); -} - -/** emit JSON to the given buffer. Return a substr trimmed to the emitted JSON. - * @param error_on_excess Raise an error if the space in the buffer is insufficient. - * @overload - */ -inline substr emit_json(ConstNodeRef const& r, substr buf, bool error_on_excess=true) -{ - EmitterBuf em(buf); - return em.emit_as(EMIT_JSON, r, error_on_excess); -} - - -//----------------------------------------------------------------------------- - -/** emit+resize: emit YAML to the given std::string/std::vector-like - * container, resizing it as needed to fit the emitted YAML. */ -template -substr emitrs_yaml(Tree const& t, size_t id, CharOwningContainer * cont) -{ - substr buf = to_substr(*cont); - substr ret = emit_yaml(t, id, buf, /*error_on_excess*/false); - if(ret.str == nullptr && ret.len > 0) - { - cont->resize(ret.len); - buf = to_substr(*cont); - ret = emit_yaml(t, id, buf, /*error_on_excess*/true); - } - return ret; -} -template -RYML_DEPRECATE_EMITRS substr emitrs(Tree const& t, size_t id, CharOwningContainer * cont) -{ - return emitrs_yaml(t, id, cont); -} - -/** emit+resize: emit JSON to the given std::string/std::vector-like - * container, resizing it as needed to fit the emitted JSON. */ -template -substr emitrs_json(Tree const& t, size_t id, CharOwningContainer * cont) -{ - substr buf = to_substr(*cont); - substr ret = emit_json(t, id, buf, /*error_on_excess*/false); - if(ret.str == nullptr && ret.len > 0) - { - cont->resize(ret.len); - buf = to_substr(*cont); - ret = emit_json(t, id, buf, /*error_on_excess*/true); - } - return ret; -} - - -/** emit+resize: emit YAML to the given std::string/std::vector-like - * container, resizing it as needed to fit the emitted YAML. */ -template -CharOwningContainer emitrs_yaml(Tree const& t, size_t id) -{ - CharOwningContainer c; - emitrs_yaml(t, id, &c); - return c; -} -template -RYML_DEPRECATE_EMITRS CharOwningContainer emitrs(Tree const& t, size_t id) -{ - CharOwningContainer c; - emitrs_yaml(t, id, &c); - return c; -} - -/** emit+resize: emit JSON to the given std::string/std::vector-like - * container, resizing it as needed to fit the emitted JSON. */ -template -CharOwningContainer emitrs_json(Tree const& t, size_t id) -{ - CharOwningContainer c; - emitrs_json(t, id, &c); - return c; -} - - -/** emit+resize: YAML to the given std::string/std::vector-like - * container, resizing it as needed to fit the emitted YAML. */ -template -substr emitrs_yaml(Tree const& t, CharOwningContainer * cont) -{ - if(t.empty()) - return {}; - return emitrs_yaml(t, t.root_id(), cont); -} -template -RYML_DEPRECATE_EMITRS substr emitrs(Tree const& t, CharOwningContainer * cont) -{ - return emitrs_yaml(t, cont); -} - -/** emit+resize: JSON to the given std::string/std::vector-like - * container, resizing it as needed to fit the emitted JSON. */ -template -substr emitrs_json(Tree const& t, CharOwningContainer * cont) -{ - if(t.empty()) - return {}; - return emitrs_json(t, t.root_id(), cont); -} - - -/** emit+resize: YAML to the given std::string/std::vector-like container, - * resizing it as needed to fit the emitted YAML. */ -template -CharOwningContainer emitrs_yaml(Tree const& t) -{ - CharOwningContainer c; - if(t.empty()) - return c; - emitrs_yaml(t, t.root_id(), &c); - return c; -} -template -RYML_DEPRECATE_EMITRS CharOwningContainer emitrs(Tree const& t) -{ - return emitrs_yaml(t); -} - -/** emit+resize: JSON to the given std::string/std::vector-like container, - * resizing it as needed to fit the emitted JSON. */ -template -CharOwningContainer emitrs_json(Tree const& t) -{ - CharOwningContainer c; - if(t.empty()) - return c; - emitrs_json(t, t.root_id(), &c); - return c; -} - - -/** emit+resize: YAML to the given std::string/std::vector-like container, - * resizing it as needed to fit the emitted YAML. */ -template -substr emitrs_yaml(ConstNodeRef const& n, CharOwningContainer * cont) -{ - _RYML_CB_CHECK(n.tree()->callbacks(), n.valid()); - return emitrs_yaml(*n.tree(), n.id(), cont); -} -template -RYML_DEPRECATE_EMITRS substr emitrs(ConstNodeRef const& n, CharOwningContainer * cont) -{ - return emitrs_yaml(n, cont); -} - -/** emit+resize: JSON to the given std::string/std::vector-like container, - * resizing it as needed to fit the emitted JSON. */ -template -substr emitrs_json(ConstNodeRef const& n, CharOwningContainer * cont) -{ - _RYML_CB_CHECK(n.tree()->callbacks(), n.valid()); - return emitrs_json(*n.tree(), n.id(), cont); -} - - -/** emit+resize: YAML to the given std::string/std::vector-like container, - * resizing it as needed to fit the emitted YAML. */ -template -CharOwningContainer emitrs_yaml(ConstNodeRef const& n) -{ - _RYML_CB_CHECK(n.tree()->callbacks(), n.valid()); - CharOwningContainer c; - emitrs_yaml(*n.tree(), n.id(), &c); - return c; -} -template -RYML_DEPRECATE_EMITRS CharOwningContainer emitrs(ConstNodeRef const& n) -{ - return emitrs_yaml(n); -} - -/** emit+resize: JSON to the given std::string/std::vector-like container, - * resizing it as needed to fit the emitted JSON. */ -template -CharOwningContainer emitrs_json(ConstNodeRef const& n) -{ - _RYML_CB_CHECK(n.tree()->callbacks(), n.valid()); - CharOwningContainer c; - emitrs_json(*n.tree(), n.id(), &c); - return c; -} - -} // namespace yml -} // namespace c4 - -#undef RYML_DEPRECATE_EMIT -#undef RYML_DEPRECATE_EMITRS - -#include "c4/yml/emit.def.hpp" - -#endif /* _C4_YML_EMIT_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/export.hpp b/thirdparty/ryml/src/c4/yml/export.hpp deleted file mode 100644 index 6b77f3f8d..000000000 --- a/thirdparty/ryml/src/c4/yml/export.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef C4_YML_EXPORT_HPP_ -#define C4_YML_EXPORT_HPP_ - -#ifdef _WIN32 - #ifdef RYML_SHARED - #ifdef RYML_EXPORTS - #define RYML_EXPORT __declspec(dllexport) - #else - #define RYML_EXPORT __declspec(dllimport) - #endif - #else - #define RYML_EXPORT - #endif -#else - #define RYML_EXPORT -#endif - -#endif /* C4_YML_EXPORT_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/node.cpp b/thirdparty/ryml/src/c4/yml/node.cpp deleted file mode 100644 index 50c7a0b60..000000000 --- a/thirdparty/ryml/src/c4/yml/node.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "c4/yml/node.hpp" - -namespace c4 { -namespace yml { - - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -size_t NodeRef::set_key_serialized(c4::fmt::const_base64_wrapper w) -{ - _apply_seed(); - csubstr encoded = this->to_arena(w); - this->set_key(encoded); - return encoded.len; -} - -size_t NodeRef::set_val_serialized(c4::fmt::const_base64_wrapper w) -{ - _apply_seed(); - csubstr encoded = this->to_arena(w); - this->set_val(encoded); - return encoded.len; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/src/c4/yml/node.hpp b/thirdparty/ryml/src/c4/yml/node.hpp deleted file mode 100644 index 42ed50442..000000000 --- a/thirdparty/ryml/src/c4/yml/node.hpp +++ /dev/null @@ -1,1276 +0,0 @@ -#ifndef _C4_YML_NODE_HPP_ -#define _C4_YML_NODE_HPP_ - -/** @file node.hpp - * @see NodeRef */ - -#include - -#include "c4/yml/tree.hpp" -#include "c4/base64.hpp" - -#ifdef __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wtype-limits" -#endif - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4251/*needs to have dll-interface to be used by clients of struct*/) -# pragma warning(disable: 4296/*expression is always 'boolean_value'*/) -#endif - -namespace c4 { -namespace yml { - -template struct Key { K & k; }; -template<> struct Key { fmt::const_base64_wrapper wrapper; }; -template<> struct Key { fmt::base64_wrapper wrapper; }; - -template C4_ALWAYS_INLINE Key key(K & k) { return Key{k}; } -C4_ALWAYS_INLINE Key key(fmt::const_base64_wrapper w) { return {w}; } -C4_ALWAYS_INLINE Key key(fmt::base64_wrapper w) { return {w}; } - -template void write(NodeRef *n, T const& v); - -template -typename std::enable_if< ! std::is_floating_point::value, bool>::type -read(NodeRef const& n, T *v); - -template -typename std::enable_if< std::is_floating_point::value, bool>::type -read(NodeRef const& n, T *v); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// forward decls -class NodeRef; -class ConstNodeRef; - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace detail { - -template -struct child_iterator -{ - using value_type = NodeRefType; - using tree_type = typename NodeRefType::tree_type; - - tree_type * C4_RESTRICT m_tree; - size_t m_child_id; - - child_iterator(tree_type * t, size_t id) : m_tree(t), m_child_id(id) {} - - child_iterator& operator++ () { RYML_ASSERT(m_child_id != NONE); m_child_id = m_tree->next_sibling(m_child_id); return *this; } - child_iterator& operator-- () { RYML_ASSERT(m_child_id != NONE); m_child_id = m_tree->prev_sibling(m_child_id); return *this; } - - NodeRefType operator* () const { return NodeRefType(m_tree, m_child_id); } - NodeRefType operator-> () const { return NodeRefType(m_tree, m_child_id); } - - bool operator!= (child_iterator that) const { RYML_ASSERT(m_tree == that.m_tree); return m_child_id != that.m_child_id; } - bool operator== (child_iterator that) const { RYML_ASSERT(m_tree == that.m_tree); return m_child_id == that.m_child_id; } -}; - -template -struct children_view_ -{ - using n_iterator = child_iterator; - - n_iterator b, e; - - inline children_view_(n_iterator const& C4_RESTRICT b_, - n_iterator const& C4_RESTRICT e_) : b(b_), e(e_) {} - - inline n_iterator begin() const { return b; } - inline n_iterator end () const { return e; } -}; - -template -bool _visit(NodeRefType &node, Visitor fn, size_t indentation_level, bool skip_root=false) -{ - size_t increment = 0; - if( ! (node.is_root() && skip_root)) - { - if(fn(node, indentation_level)) - return true; - ++increment; - } - if(node.has_children()) - { - for(auto ch : node.children()) - { - if(_visit(ch, fn, indentation_level + increment, false)) // no need to forward skip_root as it won't be root - { - return true; - } - } - } - return false; -} - -template -bool _visit_stacked(NodeRefType &node, Visitor fn, size_t indentation_level, bool skip_root=false) -{ - size_t increment = 0; - if( ! (node.is_root() && skip_root)) - { - if(fn(node, indentation_level)) - { - return true; - } - ++increment; - } - if(node.has_children()) - { - fn.push(node, indentation_level); - for(auto ch : node.children()) - { - if(_visit_stacked(ch, fn, indentation_level + increment, false)) // no need to forward skip_root as it won't be root - { - fn.pop(node, indentation_level); - return true; - } - } - fn.pop(node, indentation_level); - } - return false; -} - - -//----------------------------------------------------------------------------- - -/** a CRTP base for read-only node methods */ -template -struct RoNodeMethods -{ - C4_SUPPRESS_WARNING_GCC_CLANG_WITH_PUSH("-Wcast-align") - // helper CRTP macros, undefined at the end - #define tree_ ((ConstImpl const* C4_RESTRICT)this)->m_tree - #define id_ ((ConstImpl const* C4_RESTRICT)this)->m_id - #define tree__ ((Impl const* C4_RESTRICT)this)->m_tree - #define id__ ((Impl const* C4_RESTRICT)this)->m_id - // require valid - #define _C4RV() \ - RYML_ASSERT(tree_ != nullptr); \ - _RYML_CB_ASSERT(tree_->m_callbacks, id_ != NONE) - #define _C4_IF_MUTABLE(ty) typename std::enable_if::value, ty>::type - -public: - - /** @name node property getters */ - /** @{ */ - - /** returns the data or null when the id is NONE */ - C4_ALWAYS_INLINE C4_PURE NodeData const* get() const noexcept { RYML_ASSERT(tree_ != nullptr); return tree_->get(id_); } - /** returns the data or null when the id is NONE */ - template - C4_ALWAYS_INLINE C4_PURE auto get() noexcept -> _C4_IF_MUTABLE(NodeData*) { RYML_ASSERT(tree_ != nullptr); return tree__->get(id__); } - - C4_ALWAYS_INLINE C4_PURE NodeType type() const noexcept { _C4RV(); return tree_->type(id_); } - C4_ALWAYS_INLINE C4_PURE const char* type_str() const noexcept { return tree_->type_str(id_); } - - C4_ALWAYS_INLINE C4_PURE csubstr key() const noexcept { _C4RV(); return tree_->key(id_); } - C4_ALWAYS_INLINE C4_PURE csubstr key_tag() const noexcept { _C4RV(); return tree_->key_tag(id_); } - C4_ALWAYS_INLINE C4_PURE csubstr key_ref() const noexcept { _C4RV(); return tree_->key_ref(id_); } - C4_ALWAYS_INLINE C4_PURE csubstr key_anchor() const noexcept { _C4RV(); return tree_->key_anchor(id_); } - - C4_ALWAYS_INLINE C4_PURE csubstr val() const noexcept { _C4RV(); return tree_->val(id_); } - C4_ALWAYS_INLINE C4_PURE csubstr val_tag() const noexcept { _C4RV(); return tree_->val_tag(id_); } - C4_ALWAYS_INLINE C4_PURE csubstr val_ref() const noexcept { _C4RV(); return tree_->val_ref(id_); } - C4_ALWAYS_INLINE C4_PURE csubstr val_anchor() const noexcept { _C4RV(); return tree_->val_anchor(id_); } - - C4_ALWAYS_INLINE C4_PURE NodeScalar const& keysc() const noexcept { _C4RV(); return tree_->keysc(id_); } - C4_ALWAYS_INLINE C4_PURE NodeScalar const& valsc() const noexcept { _C4RV(); return tree_->valsc(id_); } - - C4_ALWAYS_INLINE C4_PURE bool key_is_null() const noexcept { _C4RV(); return tree_->key_is_null(id_); } - C4_ALWAYS_INLINE C4_PURE bool val_is_null() const noexcept { _C4RV(); return tree_->val_is_null(id_); } - - /** @} */ - -public: - - /** @name node property predicates */ - /** @{ */ - - C4_ALWAYS_INLINE C4_PURE bool empty() const noexcept { _C4RV(); return tree_->empty(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_stream() const noexcept { _C4RV(); return tree_->is_stream(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_doc() const noexcept { _C4RV(); return tree_->is_doc(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_container() const noexcept { _C4RV(); return tree_->is_container(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_map() const noexcept { _C4RV(); return tree_->is_map(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_seq() const noexcept { _C4RV(); return tree_->is_seq(id_); } - C4_ALWAYS_INLINE C4_PURE bool has_val() const noexcept { _C4RV(); return tree_->has_val(id_); } - C4_ALWAYS_INLINE C4_PURE bool has_key() const noexcept { _C4RV(); return tree_->has_key(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_val() const noexcept { _C4RV(); return tree_->is_val(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_keyval() const noexcept { _C4RV(); return tree_->is_keyval(id_); } - C4_ALWAYS_INLINE C4_PURE bool has_key_tag() const noexcept { _C4RV(); return tree_->has_key_tag(id_); } - C4_ALWAYS_INLINE C4_PURE bool has_val_tag() const noexcept { _C4RV(); return tree_->has_val_tag(id_); } - C4_ALWAYS_INLINE C4_PURE bool has_key_anchor() const noexcept { _C4RV(); return tree_->has_key_anchor(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_key_anchor() const noexcept { _C4RV(); return tree_->is_key_anchor(id_); } - C4_ALWAYS_INLINE C4_PURE bool has_val_anchor() const noexcept { _C4RV(); return tree_->has_val_anchor(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_val_anchor() const noexcept { _C4RV(); return tree_->is_val_anchor(id_); } - C4_ALWAYS_INLINE C4_PURE bool has_anchor() const noexcept { _C4RV(); return tree_->has_anchor(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_anchor() const noexcept { _C4RV(); return tree_->is_anchor(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_key_ref() const noexcept { _C4RV(); return tree_->is_key_ref(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_val_ref() const noexcept { _C4RV(); return tree_->is_val_ref(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_ref() const noexcept { _C4RV(); return tree_->is_ref(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_anchor_or_ref() const noexcept { _C4RV(); return tree_->is_anchor_or_ref(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_key_quoted() const noexcept { _C4RV(); return tree_->is_key_quoted(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_val_quoted() const noexcept { _C4RV(); return tree_->is_val_quoted(id_); } - C4_ALWAYS_INLINE C4_PURE bool is_quoted() const noexcept { _C4RV(); return tree_->is_quoted(id_); } - C4_ALWAYS_INLINE C4_PURE bool parent_is_seq() const noexcept { _C4RV(); return tree_->parent_is_seq(id_); } - C4_ALWAYS_INLINE C4_PURE bool parent_is_map() const noexcept { _C4RV(); return tree_->parent_is_map(id_); } - - /** @} */ - -public: - - /** @name hierarchy predicates */ - /** @{ */ - - C4_ALWAYS_INLINE C4_PURE bool is_root() const noexcept { _C4RV(); return tree_->is_root(id_); } - C4_ALWAYS_INLINE C4_PURE bool has_parent() const noexcept { _C4RV(); return tree_->has_parent(id_); } - - C4_ALWAYS_INLINE C4_PURE bool has_child(ConstImpl const& ch) const noexcept { _C4RV(); return tree_->has_child(id_, ch.m_id); } - C4_ALWAYS_INLINE C4_PURE bool has_child(csubstr name) const noexcept { _C4RV(); return tree_->has_child(id_, name); } - C4_ALWAYS_INLINE C4_PURE bool has_children() const noexcept { _C4RV(); return tree_->has_children(id_); } - - C4_ALWAYS_INLINE C4_PURE bool has_sibling(ConstImpl const& n) const noexcept { _C4RV(); return tree_->has_sibling(id_, n.m_id); } - C4_ALWAYS_INLINE C4_PURE bool has_sibling(csubstr name) const noexcept { _C4RV(); return tree_->has_sibling(id_, name); } - /** counts with this */ - C4_ALWAYS_INLINE C4_PURE bool has_siblings() const noexcept { _C4RV(); return tree_->has_siblings(id_); } - /** does not count with this */ - C4_ALWAYS_INLINE C4_PURE bool has_other_siblings() const noexcept { _C4RV(); return tree_->has_other_siblings(id_); } - - /** @} */ - -public: - - /** @name hierarchy getters */ - /** @{ */ - - - template - C4_ALWAYS_INLINE C4_PURE auto doc(size_t num) noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->doc(num)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl doc(size_t num) const noexcept { _C4RV(); return {tree_, tree_->doc(num)}; } - - - template - C4_ALWAYS_INLINE C4_PURE auto parent() noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->parent(id__)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl parent() const noexcept { _C4RV(); return {tree_, tree_->parent(id_)}; } - - - /** O(#num_children) */ - C4_ALWAYS_INLINE C4_PURE size_t child_pos(ConstImpl const& n) const noexcept { _C4RV(); return tree_->child_pos(id_, n.m_id); } - C4_ALWAYS_INLINE C4_PURE size_t num_children() const noexcept { _C4RV(); return tree_->num_children(id_); } - - template - C4_ALWAYS_INLINE C4_PURE auto first_child() noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->first_child(id__)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl first_child() const noexcept { _C4RV(); return {tree_, tree_->first_child(id_)}; } - - template - C4_ALWAYS_INLINE C4_PURE auto last_child() noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->last_child(id__)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl last_child () const noexcept { _C4RV(); return {tree_, tree_->last_child (id_)}; } - - template - C4_ALWAYS_INLINE C4_PURE auto child(size_t pos) noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->child(id__, pos)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl child(size_t pos) const noexcept { _C4RV(); return {tree_, tree_->child(id_, pos)}; } - - template - C4_ALWAYS_INLINE C4_PURE auto find_child(csubstr name) noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->find_child(id__, name)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl find_child(csubstr name) const noexcept { _C4RV(); return {tree_, tree_->find_child(id_, name)}; } - - - /** O(#num_siblings) */ - C4_ALWAYS_INLINE C4_PURE size_t num_siblings() const noexcept { _C4RV(); return tree_->num_siblings(id_); } - C4_ALWAYS_INLINE C4_PURE size_t num_other_siblings() const noexcept { _C4RV(); return tree_->num_other_siblings(id_); } - C4_ALWAYS_INLINE C4_PURE size_t sibling_pos(ConstImpl const& n) const noexcept { _C4RV(); return tree_->child_pos(tree_->parent(id_), n.m_id); } - - template - C4_ALWAYS_INLINE C4_PURE auto prev_sibling() noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->prev_sibling(id__)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl prev_sibling() const noexcept { _C4RV(); return {tree_, tree_->prev_sibling(id_)}; } - - template - C4_ALWAYS_INLINE C4_PURE auto next_sibling() noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->next_sibling(id__)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl next_sibling() const noexcept { _C4RV(); return {tree_, tree_->next_sibling(id_)}; } - - template - C4_ALWAYS_INLINE C4_PURE auto first_sibling() noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->first_sibling(id__)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl first_sibling() const noexcept { _C4RV(); return {tree_, tree_->first_sibling(id_)}; } - - template - C4_ALWAYS_INLINE C4_PURE auto last_sibling() noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->last_sibling(id__)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl last_sibling () const noexcept { _C4RV(); return {tree_, tree_->last_sibling(id_)}; } - - template - C4_ALWAYS_INLINE C4_PURE auto sibling(size_t pos) noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->sibling(id__, pos)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl sibling(size_t pos) const noexcept { _C4RV(); return {tree_, tree_->sibling(id_, pos)}; } - - template - C4_ALWAYS_INLINE C4_PURE auto find_sibling(csubstr name) noexcept -> _C4_IF_MUTABLE(Impl) { _C4RV(); return {tree__, tree__->find_sibling(id__, name)}; } - C4_ALWAYS_INLINE C4_PURE ConstImpl find_sibling(csubstr name) const noexcept { _C4RV(); return {tree_, tree_->find_sibling(id_, name)}; } - - - /** O(num_children) */ - C4_ALWAYS_INLINE C4_PURE ConstImpl operator[] (csubstr k) const noexcept - { - _C4RV(); - size_t ch = tree_->find_child(id_, k); - _RYML_CB_ASSERT(tree_->m_callbacks, ch != NONE); - return {tree_, ch}; - } - /** Find child by key. O(num_children). returns a seed node if no such child is found. */ - template - C4_ALWAYS_INLINE C4_PURE auto operator[] (csubstr k) noexcept -> _C4_IF_MUTABLE(Impl) - { - _C4RV(); - size_t ch = tree__->find_child(id__, k); - return ch != NONE ? Impl(tree__, ch) : NodeRef(tree__, id__, k); - } - - /** O(num_children) */ - C4_ALWAYS_INLINE C4_PURE ConstImpl operator[] (size_t pos) const noexcept - { - _C4RV(); - size_t ch = tree_->child(id_, pos); - _RYML_CB_ASSERT(tree_->m_callbacks, ch != NONE); - return {tree_, ch}; - } - - /** Find child by position. O(pos). returns a seed node if no such child is found. */ - template - C4_ALWAYS_INLINE C4_PURE auto operator[] (size_t pos) noexcept -> _C4_IF_MUTABLE(Impl) - { - _C4RV(); - size_t ch = tree__->child(id__, pos); - return ch != NONE ? Impl(tree__, ch) : NodeRef(tree__, id__, pos); - } - - /** @} */ - -public: - - /** deserialization */ - /** @{ */ - - template - ConstImpl const& operator>> (T &v) const - { - _C4RV(); - if( ! read((ConstImpl const&)*this, &v)) - _RYML_CB_ERR(tree_->m_callbacks, "could not deserialize value"); - return *((ConstImpl const*)this); - } - - /** deserialize the node's key to the given variable */ - template - ConstImpl const& operator>> (Key v) const - { - _C4RV(); - if( ! from_chars(key(), &v.k)) - _RYML_CB_ERR(tree_->m_callbacks, "could not deserialize key"); - return *((ConstImpl const*)this); - } - - /** deserialize the node's key as base64 */ - ConstImpl const& operator>> (Key w) const - { - deserialize_key(w.wrapper); - return *((ConstImpl const*)this); - } - - /** deserialize the node's val as base64 */ - ConstImpl const& operator>> (fmt::base64_wrapper w) const - { - deserialize_val(w); - return *((ConstImpl const*)this); - } - - /** decode the base64-encoded key and assign the - * decoded blob to the given buffer/ - * @return the size of base64-decoded blob */ - size_t deserialize_key(fmt::base64_wrapper v) const - { - _C4RV(); - return from_chars(key(), &v); - } - /** decode the base64-encoded key and assign the - * decoded blob to the given buffer/ - * @return the size of base64-decoded blob */ - size_t deserialize_val(fmt::base64_wrapper v) const - { - _C4RV(); - return from_chars(val(), &v); - }; - - template - bool get_if(csubstr name, T *var) const - { - auto ch = find_child(name); - if(!ch.valid()) - return false; - ch >> *var; - return true; - } - - template - bool get_if(csubstr name, T *var, T const& fallback) const - { - auto ch = find_child(name); - if(ch.valid()) - { - ch >> *var; - return true; - } - else - { - *var = fallback; - return false; - } - } - - /** @} */ - -public: - - #if defined(__clang__) - # pragma clang diagnostic push - # pragma clang diagnostic ignored "-Wnull-dereference" - #elif defined(__GNUC__) - # pragma GCC diagnostic push - # if __GNUC__ >= 6 - # pragma GCC diagnostic ignored "-Wnull-dereference" - # endif - #endif - - /** @name iteration */ - /** @{ */ - - using iterator = detail::child_iterator; - using const_iterator = detail::child_iterator; - using children_view = detail::children_view_; - using const_children_view = detail::children_view_; - - template - C4_ALWAYS_INLINE C4_PURE auto begin() noexcept -> _C4_IF_MUTABLE(iterator) { _C4RV(); return iterator(tree__, tree__->first_child(id__)); } - C4_ALWAYS_INLINE C4_PURE const_iterator begin() const noexcept { _C4RV(); return const_iterator(tree_, tree_->first_child(id_)); } - C4_ALWAYS_INLINE C4_PURE const_iterator cbegin() const noexcept { _C4RV(); return const_iterator(tree_, tree_->first_child(id_)); } - - template - C4_ALWAYS_INLINE C4_PURE auto end() noexcept -> _C4_IF_MUTABLE(iterator) { _C4RV(); return iterator(tree__, NONE); } - C4_ALWAYS_INLINE C4_PURE const_iterator end() const noexcept { _C4RV(); return const_iterator(tree_, NONE); } - C4_ALWAYS_INLINE C4_PURE const_iterator cend() const noexcept { _C4RV(); return const_iterator(tree_, tree_->first_child(id_)); } - - /** get an iterable view over children */ - template - C4_ALWAYS_INLINE C4_PURE auto children() noexcept -> _C4_IF_MUTABLE(children_view) { _C4RV(); return children_view(begin(), end()); } - /** get an iterable view over children */ - C4_ALWAYS_INLINE C4_PURE const_children_view children() const noexcept { _C4RV(); return const_children_view(begin(), end()); } - /** get an iterable view over children */ - C4_ALWAYS_INLINE C4_PURE const_children_view cchildren() const noexcept { _C4RV(); return const_children_view(begin(), end()); } - - /** get an iterable view over all siblings (including the calling node) */ - template - C4_ALWAYS_INLINE C4_PURE auto siblings() noexcept -> _C4_IF_MUTABLE(children_view) - { - _C4RV(); - NodeData const *nd = tree__->get(id__); - return (nd->m_parent != NONE) ? // does it have a parent? - children_view(iterator(tree__, tree_->get(nd->m_parent)->m_first_child), iterator(tree__, NONE)) - : - children_view(end(), end()); - } - /** get an iterable view over all siblings (including the calling node) */ - C4_ALWAYS_INLINE C4_PURE const_children_view siblings() const noexcept - { - _C4RV(); - NodeData const *nd = tree_->get(id_); - return (nd->m_parent != NONE) ? // does it have a parent? - const_children_view(const_iterator(tree_, tree_->get(nd->m_parent)->m_first_child), const_iterator(tree_, NONE)) - : - const_children_view(end(), end()); - } - /** get an iterable view over all siblings (including the calling node) */ - C4_ALWAYS_INLINE C4_PURE const_children_view csiblings() const noexcept { return siblings(); } - - /** visit every child node calling fn(node) */ - template - C4_ALWAYS_INLINE C4_PURE bool visit(Visitor fn, size_t indentation_level=0, bool skip_root=true) const noexcept - { - return detail::_visit(*(ConstImpl*)this, fn, indentation_level, skip_root); - } - /** visit every child node calling fn(node) */ - template - auto visit(Visitor fn, size_t indentation_level=0, bool skip_root=true) noexcept - -> _C4_IF_MUTABLE(bool) - { - return detail::_visit(*(Impl*)this, fn, indentation_level, skip_root); - } - - /** visit every child node calling fn(node, level) */ - template - C4_ALWAYS_INLINE C4_PURE bool visit_stacked(Visitor fn, size_t indentation_level=0, bool skip_root=true) const noexcept - { - return detail::_visit_stacked(*(ConstImpl*)this, fn, indentation_level, skip_root); - } - /** visit every child node calling fn(node, level) */ - template - auto visit_stacked(Visitor fn, size_t indentation_level=0, bool skip_root=true) noexcept - -> _C4_IF_MUTABLE(bool) - { - return detail::_visit_stacked(*(Impl*)this, fn, indentation_level, skip_root); - } - - /** @} */ - - #if defined(__clang__) - # pragma clang diagnostic pop - #elif defined(__GNUC__) - # pragma GCC diagnostic pop - #endif - - #undef _C4_IF_MUTABLE - #undef _C4RV - #undef tree_ - #undef tree__ - #undef id_ - #undef id__ - - C4_SUPPRESS_WARNING_GCC_CLANG_POP -}; - -} // namespace detail - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -class RYML_EXPORT ConstNodeRef : public detail::RoNodeMethods -{ -public: - - using tree_type = Tree const; - -public: - - Tree const* C4_RESTRICT m_tree; - size_t m_id; - - friend NodeRef; - friend struct detail::RoNodeMethods; - -public: - - /** @name construction */ - /** @{ */ - - ConstNodeRef() : m_tree(nullptr), m_id(NONE) {} - ConstNodeRef(Tree const &t) : m_tree(&t), m_id(t .root_id()) {} - ConstNodeRef(Tree const *t) : m_tree(t ), m_id(t->root_id()) {} - ConstNodeRef(Tree const *t, size_t id) : m_tree(t), m_id(id) {} - ConstNodeRef(std::nullptr_t) : m_tree(nullptr), m_id(NONE) {} - - ConstNodeRef(ConstNodeRef const&) = default; - ConstNodeRef(ConstNodeRef &&) = default; - - ConstNodeRef(NodeRef const&); - ConstNodeRef(NodeRef &&); - - /** @} */ - -public: - - /** @name assignment */ - /** @{ */ - - ConstNodeRef& operator= (std::nullptr_t) { m_tree = nullptr; m_id = NONE; return *this; } - - ConstNodeRef& operator= (ConstNodeRef const&) = default; - ConstNodeRef& operator= (ConstNodeRef &&) = default; - - ConstNodeRef& operator= (NodeRef const&); - ConstNodeRef& operator= (NodeRef &&); - - - /** @} */ - -public: - - /** @name state queries */ - /** @{ */ - - C4_ALWAYS_INLINE C4_PURE bool valid() const noexcept { return m_tree != nullptr && m_id != NONE; } - - /** @} */ - -public: - - /** @name member getters */ - /** @{ */ - - C4_ALWAYS_INLINE C4_PURE Tree const* tree() const noexcept { return m_tree; } - C4_ALWAYS_INLINE C4_PURE size_t id() const noexcept { return m_id; } - - /** @} */ - -public: - - /** @name comparisons */ - /** @{ */ - - C4_ALWAYS_INLINE C4_PURE bool operator== (ConstNodeRef const& that) const noexcept { RYML_ASSERT(that.m_tree == m_tree); return m_id == that.m_id; } - C4_ALWAYS_INLINE C4_PURE bool operator!= (ConstNodeRef const& that) const noexcept { RYML_ASSERT(that.m_tree == m_tree); return ! this->operator==(that); } - - C4_ALWAYS_INLINE C4_PURE bool operator== (std::nullptr_t) const noexcept { return m_tree == nullptr || m_id == NONE; } - C4_ALWAYS_INLINE C4_PURE bool operator!= (std::nullptr_t) const noexcept { return ! this->operator== (nullptr); } - - C4_ALWAYS_INLINE C4_PURE bool operator== (csubstr val) const noexcept { RYML_ASSERT(has_val()); return m_tree->val(m_id) == val; } - C4_ALWAYS_INLINE C4_PURE bool operator!= (csubstr val) const noexcept { RYML_ASSERT(has_val()); return m_tree->val(m_id) != val; } - - /** @} */ - -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** a reference to a node in an existing yaml tree, offering a more - * convenient API than the index-based API used in the tree. */ -class RYML_EXPORT NodeRef : public detail::RoNodeMethods -{ -public: - - using tree_type = Tree; - using base_type = detail::RoNodeMethods; - -private: - - Tree *C4_RESTRICT m_tree; - size_t m_id; - - /** This member is used to enable lazy operator[] writing. When a child - * with a key or index is not found, m_id is set to the id of the parent - * and the asked-for key or index are stored in this member until a write - * does happen. Then it is given as key or index for creating the child. - * When a key is used, the csubstr stores it (so the csubstr's string is - * non-null and the csubstr's size is different from NONE). When an index is - * used instead, the csubstr's string is set to null, and only the csubstr's - * size is set to a value different from NONE. Otherwise, when operator[] - * does find the child then this member is empty: the string is null and - * the size is NONE. */ - csubstr m_seed; - - friend ConstNodeRef; - friend struct detail::RoNodeMethods; - - // require valid: a helper macro, undefined at the end - #define _C4RV() \ - RYML_ASSERT(m_tree != nullptr); \ - _RYML_CB_ASSERT(m_tree->m_callbacks, m_id != NONE && !is_seed()) - -public: - - /** @name construction */ - /** @{ */ - - NodeRef() : m_tree(nullptr), m_id(NONE), m_seed() { _clear_seed(); } - NodeRef(Tree &t) : m_tree(&t), m_id(t .root_id()), m_seed() { _clear_seed(); } - NodeRef(Tree *t) : m_tree(t ), m_id(t->root_id()), m_seed() { _clear_seed(); } - NodeRef(Tree *t, size_t id) : m_tree(t), m_id(id), m_seed() { _clear_seed(); } - NodeRef(Tree *t, size_t id, size_t seed_pos) : m_tree(t), m_id(id), m_seed() { m_seed.str = nullptr; m_seed.len = seed_pos; } - NodeRef(Tree *t, size_t id, csubstr seed_key) : m_tree(t), m_id(id), m_seed(seed_key) {} - NodeRef(std::nullptr_t) : m_tree(nullptr), m_id(NONE), m_seed() {} - - /** @} */ - -public: - - /** @name assignment */ - /** @{ */ - - NodeRef(NodeRef const&) = default; - NodeRef(NodeRef &&) = default; - - NodeRef& operator= (NodeRef const&) = default; - NodeRef& operator= (NodeRef &&) = default; - - /** @} */ - -public: - - /** @name state queries */ - /** @{ */ - - inline bool valid() const { return m_tree != nullptr && m_id != NONE; } - inline bool is_seed() const { return m_seed.str != nullptr || m_seed.len != NONE; } - - inline void _clear_seed() { /*do this manually or an assert is triggered*/ m_seed.str = nullptr; m_seed.len = NONE; } - - /** @} */ - -public: - - /** @name comparisons */ - /** @{ */ - - inline bool operator== (NodeRef const& that) const { _C4RV(); RYML_ASSERT(that.valid() && !that.is_seed()); RYML_ASSERT(that.m_tree == m_tree); return m_id == that.m_id; } - inline bool operator!= (NodeRef const& that) const { return ! this->operator==(that); } - - inline bool operator== (ConstNodeRef const& that) const { _C4RV(); RYML_ASSERT(that.valid()); RYML_ASSERT(that.m_tree == m_tree); return m_id == that.m_id; } - inline bool operator!= (ConstNodeRef const& that) const { return ! this->operator==(that); } - - inline bool operator== (std::nullptr_t) const { return m_tree == nullptr || m_id == NONE || is_seed(); } - inline bool operator!= (std::nullptr_t) const { return m_tree != nullptr && m_id != NONE && !is_seed(); } - - inline bool operator== (csubstr val) const { _C4RV(); RYML_ASSERT(has_val()); return m_tree->val(m_id) == val; } - inline bool operator!= (csubstr val) const { _C4RV(); RYML_ASSERT(has_val()); return m_tree->val(m_id) != val; } - - //inline operator bool () const { return m_tree == nullptr || m_id == NONE || is_seed(); } - - /** @} */ - -public: - - /** @name node property getters */ - /** @{ */ - - C4_ALWAYS_INLINE C4_PURE Tree * tree() noexcept { return m_tree; } - C4_ALWAYS_INLINE C4_PURE Tree const* tree() const noexcept { return m_tree; } - - C4_ALWAYS_INLINE C4_PURE size_t id() const noexcept { return m_id; } - - /** @} */ - -public: - - /** @name node modifiers */ - /** @{ */ - - void change_type(NodeType t) { _C4RV(); m_tree->change_type(m_id, t); } - - void set_type(NodeType t) { _C4RV(); m_tree->_set_flags(m_id, t); } - void set_key(csubstr key) { _C4RV(); m_tree->_set_key(m_id, key); } - void set_val(csubstr val) { _C4RV(); m_tree->_set_val(m_id, val); } - void set_key_tag(csubstr key_tag) { _C4RV(); m_tree->set_key_tag(m_id, key_tag); } - void set_val_tag(csubstr val_tag) { _C4RV(); m_tree->set_val_tag(m_id, val_tag); } - void set_key_anchor(csubstr key_anchor) { _C4RV(); m_tree->set_key_anchor(m_id, key_anchor); } - void set_val_anchor(csubstr val_anchor) { _C4RV(); m_tree->set_val_anchor(m_id, val_anchor); } - void set_key_ref(csubstr key_ref) { _C4RV(); m_tree->set_key_ref(m_id, key_ref); } - void set_val_ref(csubstr val_ref) { _C4RV(); m_tree->set_val_ref(m_id, val_ref); } - - template - size_t set_key_serialized(T const& C4_RESTRICT k) - { - _C4RV(); - csubstr s = m_tree->to_arena(k); - m_tree->_set_key(m_id, s); - return s.len; - } - template - size_t set_val_serialized(T const& C4_RESTRICT v) - { - _C4RV(); - csubstr s = m_tree->to_arena(v); - m_tree->_set_val(m_id, s); - return s.len; - } - size_t set_val_serialized(std::nullptr_t) - { - _C4RV(); - m_tree->_set_val(m_id, csubstr{}); - return 0; - } - - /** encode a blob as base64, then assign the result to the node's key - * @return the size of base64-encoded blob */ - size_t set_key_serialized(fmt::const_base64_wrapper w); - /** encode a blob as base64, then assign the result to the node's val - * @return the size of base64-encoded blob */ - size_t set_val_serialized(fmt::const_base64_wrapper w); - -public: - - inline void clear() - { - if(is_seed()) - return; - m_tree->remove_children(m_id); - m_tree->_clear(m_id); - } - - inline void clear_key() - { - if(is_seed()) - return; - m_tree->_clear_key(m_id); - } - - inline void clear_val() - { - if(is_seed()) - return; - m_tree->_clear_val(m_id); - } - - inline void clear_children() - { - if(is_seed()) - return; - m_tree->remove_children(m_id); - } - - void create() { _apply_seed(); } - - inline void operator= (NodeType_e t) - { - _apply_seed(); - m_tree->_add_flags(m_id, t); - } - - inline void operator|= (NodeType_e t) - { - _apply_seed(); - m_tree->_add_flags(m_id, t); - } - - inline void operator= (NodeInit const& v) - { - _apply_seed(); - _apply(v); - } - - inline void operator= (NodeScalar const& v) - { - _apply_seed(); - _apply(v); - } - - inline void operator= (std::nullptr_t) - { - _apply_seed(); - _apply(csubstr{}); - } - - inline void operator= (csubstr v) - { - _apply_seed(); - _apply(v); - } - - template - inline void operator= (const char (&v)[N]) - { - _apply_seed(); - csubstr sv; - sv.assign(v); - _apply(sv); - } - - /** @} */ - -public: - - /** @name serialization */ - /** @{ */ - - /** serialize a variable to the arena */ - template - inline csubstr to_arena(T const& C4_RESTRICT s) - { - _C4RV(); - return m_tree->to_arena(s); - } - - /** serialize a variable, then assign the result to the node's val */ - inline NodeRef& operator<< (csubstr s) - { - // this overload is needed to prevent ambiguity (there's also - // operator<< for writing a substr to a stream) - _apply_seed(); - write(this, s); - RYML_ASSERT(val() == s); - return *this; - } - - template - inline NodeRef& operator<< (T const& C4_RESTRICT v) - { - _apply_seed(); - write(this, v); - return *this; - } - - /** serialize a variable, then assign the result to the node's key */ - template - inline NodeRef& operator<< (Key const& C4_RESTRICT v) - { - _apply_seed(); - set_key_serialized(v.k); - return *this; - } - - /** serialize a variable, then assign the result to the node's key */ - template - inline NodeRef& operator<< (Key const& C4_RESTRICT v) - { - _apply_seed(); - set_key_serialized(v.k); - return *this; - } - - NodeRef& operator<< (Key w) - { - set_key_serialized(w.wrapper); - return *this; - } - - NodeRef& operator<< (fmt::const_base64_wrapper w) - { - set_val_serialized(w); - return *this; - } - - /** @} */ - -private: - - void _apply_seed() - { - if(m_seed.str) // we have a seed key: use it to create the new child - { - //RYML_ASSERT(i.key.scalar.empty() || m_key == i.key.scalar || m_key.empty()); - m_id = m_tree->append_child(m_id); - m_tree->_set_key(m_id, m_seed); - m_seed.str = nullptr; - m_seed.len = NONE; - } - else if(m_seed.len != NONE) // we have a seed index: create a child at that position - { - RYML_ASSERT(m_tree->num_children(m_id) == m_seed.len); - m_id = m_tree->append_child(m_id); - m_seed.str = nullptr; - m_seed.len = NONE; - } - else - { - RYML_ASSERT(valid()); - } - } - - inline void _apply(csubstr v) - { - m_tree->_set_val(m_id, v); - } - - inline void _apply(NodeScalar const& v) - { - m_tree->_set_val(m_id, v); - } - - inline void _apply(NodeInit const& i) - { - m_tree->_set(m_id, i); - } - -public: - - /** @name modification of hierarchy */ - /** @{ */ - - inline NodeRef insert_child(NodeRef after) - { - _C4RV(); - RYML_ASSERT(after.m_tree == m_tree); - NodeRef r(m_tree, m_tree->insert_child(m_id, after.m_id)); - return r; - } - - inline NodeRef insert_child(NodeInit const& i, NodeRef after) - { - _C4RV(); - RYML_ASSERT(after.m_tree == m_tree); - NodeRef r(m_tree, m_tree->insert_child(m_id, after.m_id)); - r._apply(i); - return r; - } - - inline NodeRef prepend_child() - { - _C4RV(); - NodeRef r(m_tree, m_tree->insert_child(m_id, NONE)); - return r; - } - - inline NodeRef prepend_child(NodeInit const& i) - { - _C4RV(); - NodeRef r(m_tree, m_tree->insert_child(m_id, NONE)); - r._apply(i); - return r; - } - - inline NodeRef append_child() - { - _C4RV(); - NodeRef r(m_tree, m_tree->append_child(m_id)); - return r; - } - - inline NodeRef append_child(NodeInit const& i) - { - _C4RV(); - NodeRef r(m_tree, m_tree->append_child(m_id)); - r._apply(i); - return r; - } - -public: - - inline NodeRef insert_sibling(ConstNodeRef const& after) - { - _C4RV(); - RYML_ASSERT(after.m_tree == m_tree); - NodeRef r(m_tree, m_tree->insert_sibling(m_id, after.m_id)); - return r; - } - - inline NodeRef insert_sibling(NodeInit const& i, ConstNodeRef const& after) - { - _C4RV(); - RYML_ASSERT(after.m_tree == m_tree); - NodeRef r(m_tree, m_tree->insert_sibling(m_id, after.m_id)); - r._apply(i); - return r; - } - - inline NodeRef prepend_sibling() - { - _C4RV(); - NodeRef r(m_tree, m_tree->prepend_sibling(m_id)); - return r; - } - - inline NodeRef prepend_sibling(NodeInit const& i) - { - _C4RV(); - NodeRef r(m_tree, m_tree->prepend_sibling(m_id)); - r._apply(i); - return r; - } - - inline NodeRef append_sibling() - { - _C4RV(); - NodeRef r(m_tree, m_tree->append_sibling(m_id)); - return r; - } - - inline NodeRef append_sibling(NodeInit const& i) - { - _C4RV(); - NodeRef r(m_tree, m_tree->append_sibling(m_id)); - r._apply(i); - return r; - } - -public: - - inline void remove_child(NodeRef & child) - { - _C4RV(); - RYML_ASSERT(has_child(child)); - RYML_ASSERT(child.parent().id() == id()); - m_tree->remove(child.id()); - child.clear(); - } - - //! remove the nth child of this node - inline void remove_child(size_t pos) - { - _C4RV(); - RYML_ASSERT(pos >= 0 && pos < num_children()); - size_t child = m_tree->child(m_id, pos); - RYML_ASSERT(child != NONE); - m_tree->remove(child); - } - - //! remove a child by name - inline void remove_child(csubstr key) - { - _C4RV(); - size_t child = m_tree->find_child(m_id, key); - RYML_ASSERT(child != NONE); - m_tree->remove(child); - } - -public: - - /** change the node's position within its parent, placing it after - * @p after. To move to the first position in the parent, simply - * pass an empty or default-constructed reference like this: - * `n.move({})`. */ - inline void move(ConstNodeRef const& after) - { - _C4RV(); - m_tree->move(m_id, after.m_id); - } - - /** move the node to a different @p parent (which may belong to a - * different tree), placing it after @p after. When the - * destination parent is in a new tree, then this node's tree - * pointer is reset to the tree of the parent node. */ - inline void move(NodeRef const& parent, ConstNodeRef const& after) - { - _C4RV(); - if(parent.m_tree == m_tree) - { - m_tree->move(m_id, parent.m_id, after.m_id); - } - else - { - parent.m_tree->move(m_tree, m_id, parent.m_id, after.m_id); - m_tree = parent.m_tree; - } - } - - /** duplicate the current node somewhere within its parent, and - * place it after the node @p after. To place into the first - * position of the parent, simply pass an empty or - * default-constructed reference like this: `n.move({})`. */ - inline NodeRef duplicate(ConstNodeRef const& after) const - { - _C4RV(); - RYML_ASSERT(m_tree == after.m_tree || after.m_id == NONE); - size_t dup = m_tree->duplicate(m_id, m_tree->parent(m_id), after.m_id); - NodeRef r(m_tree, dup); - return r; - } - - /** duplicate the current node somewhere into a different @p parent - * (possibly from a different tree), and place it after the node - * @p after. To place into the first position of the parent, - * simply pass an empty or default-constructed reference like - * this: `n.move({})`. */ - inline NodeRef duplicate(NodeRef const& parent, ConstNodeRef const& after) const - { - _C4RV(); - RYML_ASSERT(parent.m_tree == after.m_tree || after.m_id == NONE); - if(parent.m_tree == m_tree) - { - size_t dup = m_tree->duplicate(m_id, parent.m_id, after.m_id); - NodeRef r(m_tree, dup); - return r; - } - else - { - size_t dup = parent.m_tree->duplicate(m_tree, m_id, parent.m_id, after.m_id); - NodeRef r(parent.m_tree, dup); - return r; - } - } - - inline void duplicate_children(NodeRef const& parent, ConstNodeRef const& after) const - { - _C4RV(); - RYML_ASSERT(parent.m_tree == after.m_tree); - if(parent.m_tree == m_tree) - { - m_tree->duplicate_children(m_id, parent.m_id, after.m_id); - } - else - { - parent.m_tree->duplicate_children(m_tree, m_id, parent.m_id, after.m_id); - } - } - - /** @} */ - -#undef _C4RV -}; - - -//----------------------------------------------------------------------------- - -inline ConstNodeRef::ConstNodeRef(NodeRef const& that) - : m_tree(that.m_tree) - , m_id(!that.is_seed() ? that.id() : NONE) -{ -} - -inline ConstNodeRef::ConstNodeRef(NodeRef && that) - : m_tree(that.m_tree) - , m_id(!that.is_seed() ? that.id() : NONE) -{ -} - - -inline ConstNodeRef& ConstNodeRef::operator= (NodeRef const& that) -{ - m_tree = (that.m_tree); - m_id = (!that.is_seed() ? that.id() : NONE); - return *this; -} - -inline ConstNodeRef& ConstNodeRef::operator= (NodeRef && that) -{ - m_tree = (that.m_tree); - m_id = (!that.is_seed() ? that.id() : NONE); - return *this; -} - - -//----------------------------------------------------------------------------- - -template -inline void write(NodeRef *n, T const& v) -{ - n->set_val_serialized(v); -} - -template -typename std::enable_if< ! std::is_floating_point::value, bool>::type -inline read(NodeRef const& n, T *v) -{ - return from_chars(n.val(), v); -} -template -typename std::enable_if< ! std::is_floating_point::value, bool>::type -inline read(ConstNodeRef const& n, T *v) -{ - return from_chars(n.val(), v); -} - -template -typename std::enable_if::value, bool>::type -inline read(NodeRef const& n, T *v) -{ - return from_chars_float(n.val(), v); -} -template -typename std::enable_if::value, bool>::type -inline read(ConstNodeRef const& n, T *v) -{ - return from_chars_float(n.val(), v); -} - - -} // namespace yml -} // namespace c4 - - -#if defined(_MSC_VER) -# pragma warning(pop) -#endif - -#ifdef __GNUC__ -# pragma GCC diagnostic pop -#endif - -#endif /* _C4_YML_NODE_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/parse.cpp b/thirdparty/ryml/src/c4/yml/parse.cpp deleted file mode 100644 index 7b038e672..000000000 --- a/thirdparty/ryml/src/c4/yml/parse.cpp +++ /dev/null @@ -1,5724 +0,0 @@ -#include "c4/yml/parse.hpp" -#include "c4/error.hpp" -#include "c4/utf.hpp" -#include - -#include -#include -#include - -#include "c4/yml/detail/parser_dbg.hpp" -#ifdef RYML_DBG -#include "c4/yml/detail/print.hpp" -#endif - -#ifndef RYML_ERRMSG_SIZE - #define RYML_ERRMSG_SIZE 1024 -#endif - -//#define RYML_WITH_TAB_TOKENS -#ifdef RYML_WITH_TAB_TOKENS -#define _RYML_WITH_TAB_TOKENS(...) __VA_ARGS__ -#define _RYML_WITH_OR_WITHOUT_TAB_TOKENS(with, without) with -#else -#define _RYML_WITH_TAB_TOKENS(...) -#define _RYML_WITH_OR_WITHOUT_TAB_TOKENS(with, without) without -#endif - - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4296/*expression is always 'boolean_value'*/) -#elif defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wtype-limits" // to remove a warning on an assertion that a size_t >= 0. Later on, this size_t will turn into a template argument, and then it can become < 0. -# pragma clang diagnostic ignored "-Wformat-nonliteral" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wtype-limits" // to remove a warning on an assertion that a size_t >= 0. Later on, this size_t will turn into a template argument, and then it can become < 0. -# pragma GCC diagnostic ignored "-Wformat-nonliteral" -# if __GNUC__ >= 7 -# pragma GCC diagnostic ignored "-Wduplicated-branches" -# endif -#endif - -namespace c4 { -namespace yml { - -namespace { - -template -void _parse_dump(DumpFn dumpfn, c4::csubstr fmt, Args&& ...args) -{ - char writebuf[256]; - auto results = c4::format_dump_resume(dumpfn, writebuf, fmt, std::forward(args)...); - // resume writing if the results failed to fit the buffer - if(C4_UNLIKELY(results.bufsize > sizeof(writebuf))) // bufsize will be that of the largest element serialized. Eg int(1), will require 1 byte. - { - results = format_dump_resume(dumpfn, results, writebuf, fmt, std::forward(args)...); - if(C4_UNLIKELY(results.bufsize > sizeof(writebuf))) - { - results = format_dump_resume(dumpfn, results, writebuf, fmt, std::forward(args)...); - } - } -} - -bool _is_scalar_next__runk(csubstr s) -{ - return !(s.begins_with(": ") || s.begins_with_any("#,{}[]%&") || s.begins_with("? ") || s == "-" || s.begins_with("- ") || s.begins_with(":\"") || s.begins_with(":'")); -} - -bool _is_scalar_next__rseq_rval(csubstr s) -{ - return !(s.begins_with_any("[{!&") || s.begins_with("? ") || s.begins_with("- ") || s == "-"); -} - -bool _is_scalar_next__rmap(csubstr s) -{ - return !(s.begins_with(": ") || s.begins_with_any("#,!&") || s.begins_with("? ") _RYML_WITH_TAB_TOKENS(|| s.begins_with(":\t"))); -} - -bool _is_scalar_next__rmap_val(csubstr s) -{ - return !(s.begins_with("- ") || s.begins_with_any("{[") || s == "-"); -} - -bool _is_doc_sep(csubstr s) -{ - constexpr const csubstr dashes = "---"; - constexpr const csubstr ellipsis = "..."; - constexpr const csubstr whitesp = " \t"; - if(s.begins_with(dashes)) - return s == dashes || s.sub(3).begins_with_any(whitesp); - else if(s.begins_with(ellipsis)) - return s == ellipsis || s.sub(3).begins_with_any(whitesp); - return false; -} - -/** @p i is set to the first non whitespace character after the line - * @return the number of empty lines after the initial position */ -size_t count_following_newlines(csubstr r, size_t *C4_RESTRICT i, size_t indentation) -{ - RYML_ASSERT(r[*i] == '\n'); - size_t numnl_following = 0; - ++(*i); - for( ; *i < r.len; ++(*i)) - { - if(r.str[*i] == '\n') - { - ++numnl_following; - if(indentation) // skip the indentation after the newline - { - size_t stop = *i + indentation; - for( ; *i < r.len; ++(*i)) - { - if(r.str[*i] != ' ' && r.str[*i] != '\r') - break; - RYML_ASSERT(*i < stop); - } - C4_UNUSED(stop); - } - } - else if(r.str[*i] == ' ' || r.str[*i] == '\t' || r.str[*i] == '\r') // skip leading whitespace - ; - else - break; - } - return numnl_following; -} - -} // anon namespace - - -//----------------------------------------------------------------------------- - -Parser::~Parser() -{ - _free(); - _clr(); -} - -Parser::Parser(Callbacks const& cb, ParserOptions opts) - : m_options(opts) - , m_file() - , m_buf() - , m_root_id(NONE) - , m_tree() - , m_stack(cb) - , m_state() - , m_key_tag_indentation(0) - , m_key_tag2_indentation(0) - , m_key_tag() - , m_key_tag2() - , m_val_tag_indentation(0) - , m_val_tag() - , m_key_anchor_was_before(false) - , m_key_anchor_indentation(0) - , m_key_anchor() - , m_val_anchor_indentation(0) - , m_val_anchor() - , m_filter_arena() - , m_newline_offsets() - , m_newline_offsets_size(0) - , m_newline_offsets_capacity(0) - , m_newline_offsets_buf() -{ - m_stack.push(State{}); - m_state = &m_stack.top(); -} - -Parser::Parser(Parser &&that) - : m_options(that.m_options) - , m_file(that.m_file) - , m_buf(that.m_buf) - , m_root_id(that.m_root_id) - , m_tree(that.m_tree) - , m_stack(std::move(that.m_stack)) - , m_state(&m_stack.top()) - , m_key_tag_indentation(that.m_key_tag_indentation) - , m_key_tag2_indentation(that.m_key_tag2_indentation) - , m_key_tag(that.m_key_tag) - , m_key_tag2(that.m_key_tag2) - , m_val_tag_indentation(that.m_val_tag_indentation) - , m_val_tag(that.m_val_tag) - , m_key_anchor_was_before(that.m_key_anchor_was_before) - , m_key_anchor_indentation(that.m_key_anchor_indentation) - , m_key_anchor(that.m_key_anchor) - , m_val_anchor_indentation(that.m_val_anchor_indentation) - , m_val_anchor(that.m_val_anchor) - , m_filter_arena(that.m_filter_arena) - , m_newline_offsets(that.m_newline_offsets) - , m_newline_offsets_size(that.m_newline_offsets_size) - , m_newline_offsets_capacity(that.m_newline_offsets_capacity) - , m_newline_offsets_buf(that.m_newline_offsets_buf) -{ - that._clr(); -} - -Parser::Parser(Parser const& that) - : m_options(that.m_options) - , m_file(that.m_file) - , m_buf(that.m_buf) - , m_root_id(that.m_root_id) - , m_tree(that.m_tree) - , m_stack(that.m_stack) - , m_state(&m_stack.top()) - , m_key_tag_indentation(that.m_key_tag_indentation) - , m_key_tag2_indentation(that.m_key_tag2_indentation) - , m_key_tag(that.m_key_tag) - , m_key_tag2(that.m_key_tag2) - , m_val_tag_indentation(that.m_val_tag_indentation) - , m_val_tag(that.m_val_tag) - , m_key_anchor_was_before(that.m_key_anchor_was_before) - , m_key_anchor_indentation(that.m_key_anchor_indentation) - , m_key_anchor(that.m_key_anchor) - , m_val_anchor_indentation(that.m_val_anchor_indentation) - , m_val_anchor(that.m_val_anchor) - , m_filter_arena() - , m_newline_offsets() - , m_newline_offsets_size() - , m_newline_offsets_capacity() - , m_newline_offsets_buf() -{ - if(that.m_newline_offsets_capacity) - { - _resize_locations(that.m_newline_offsets_capacity); - _RYML_CB_CHECK(m_stack.m_callbacks, m_newline_offsets_capacity == that.m_newline_offsets_capacity); - memcpy(m_newline_offsets, that.m_newline_offsets, that.m_newline_offsets_size * sizeof(size_t)); - m_newline_offsets_size = that.m_newline_offsets_size; - } - if(that.m_filter_arena.len) - { - _resize_filter_arena(that.m_filter_arena.len); - } -} - -Parser& Parser::operator=(Parser &&that) -{ - _free(); - m_options = (that.m_options); - m_file = (that.m_file); - m_buf = (that.m_buf); - m_root_id = (that.m_root_id); - m_tree = (that.m_tree); - m_stack = std::move(that.m_stack); - m_state = (&m_stack.top()); - m_key_tag_indentation = (that.m_key_tag_indentation); - m_key_tag2_indentation = (that.m_key_tag2_indentation); - m_key_tag = (that.m_key_tag); - m_key_tag2 = (that.m_key_tag2); - m_val_tag_indentation = (that.m_val_tag_indentation); - m_val_tag = (that.m_val_tag); - m_key_anchor_was_before = (that.m_key_anchor_was_before); - m_key_anchor_indentation = (that.m_key_anchor_indentation); - m_key_anchor = (that.m_key_anchor); - m_val_anchor_indentation = (that.m_val_anchor_indentation); - m_val_anchor = (that.m_val_anchor); - m_filter_arena = that.m_filter_arena; - m_newline_offsets = (that.m_newline_offsets); - m_newline_offsets_size = (that.m_newline_offsets_size); - m_newline_offsets_capacity = (that.m_newline_offsets_capacity); - m_newline_offsets_buf = (that.m_newline_offsets_buf); - that._clr(); - return *this; -} - -Parser& Parser::operator=(Parser const& that) -{ - _free(); - m_options = (that.m_options); - m_file = (that.m_file); - m_buf = (that.m_buf); - m_root_id = (that.m_root_id); - m_tree = (that.m_tree); - m_stack = that.m_stack; - m_state = &m_stack.top(); - m_key_tag_indentation = (that.m_key_tag_indentation); - m_key_tag2_indentation = (that.m_key_tag2_indentation); - m_key_tag = (that.m_key_tag); - m_key_tag2 = (that.m_key_tag2); - m_val_tag_indentation = (that.m_val_tag_indentation); - m_val_tag = (that.m_val_tag); - m_key_anchor_was_before = (that.m_key_anchor_was_before); - m_key_anchor_indentation = (that.m_key_anchor_indentation); - m_key_anchor = (that.m_key_anchor); - m_val_anchor_indentation = (that.m_val_anchor_indentation); - m_val_anchor = (that.m_val_anchor); - if(that.m_filter_arena.len > 0) - _resize_filter_arena(that.m_filter_arena.len); - if(that.m_newline_offsets_capacity > m_newline_offsets_capacity) - _resize_locations(that.m_newline_offsets_capacity); - _RYML_CB_CHECK(m_stack.m_callbacks, m_newline_offsets_capacity >= that.m_newline_offsets_capacity); - _RYML_CB_CHECK(m_stack.m_callbacks, m_newline_offsets_capacity >= that.m_newline_offsets_size); - memcpy(m_newline_offsets, that.m_newline_offsets, that.m_newline_offsets_size * sizeof(size_t)); - m_newline_offsets_size = that.m_newline_offsets_size; - m_newline_offsets_buf = that.m_newline_offsets_buf; - return *this; -} - -void Parser::_clr() -{ - m_options = {}; - m_file = {}; - m_buf = {}; - m_root_id = {}; - m_tree = {}; - m_stack.clear(); - m_state = {}; - m_key_tag_indentation = {}; - m_key_tag2_indentation = {}; - m_key_tag = {}; - m_key_tag2 = {}; - m_val_tag_indentation = {}; - m_val_tag = {}; - m_key_anchor_was_before = {}; - m_key_anchor_indentation = {}; - m_key_anchor = {}; - m_val_anchor_indentation = {}; - m_val_anchor = {}; - m_filter_arena = {}; - m_newline_offsets = {}; - m_newline_offsets_size = {}; - m_newline_offsets_capacity = {}; - m_newline_offsets_buf = {}; -} - -void Parser::_free() -{ - if(m_newline_offsets) - { - _RYML_CB_FREE(m_stack.m_callbacks, m_newline_offsets, size_t, m_newline_offsets_capacity); - m_newline_offsets = nullptr; - m_newline_offsets_size = 0u; - m_newline_offsets_capacity = 0u; - m_newline_offsets_buf = 0u; - } - if(m_filter_arena.len) - { - _RYML_CB_FREE(m_stack.m_callbacks, m_filter_arena.str, char, m_filter_arena.len); - m_filter_arena = {}; - } - m_stack._free(); -} - - -//----------------------------------------------------------------------------- -void Parser::_reset() -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_stack.size() == 1); - m_stack.clear(); - m_stack.push({}); - m_state = &m_stack.top(); - m_state->reset(m_file.str, m_root_id); - - m_key_tag_indentation = 0; - m_key_tag2_indentation = 0; - m_key_tag.clear(); - m_key_tag2.clear(); - m_val_tag_indentation = 0; - m_val_tag.clear(); - m_key_anchor_was_before = false; - m_key_anchor_indentation = 0; - m_key_anchor.clear(); - m_val_anchor_indentation = 0; - m_val_anchor.clear(); - - if(m_options.locations()) - { - _prepare_locations(); - } -} - -//----------------------------------------------------------------------------- -template -void Parser::_fmt_msg(DumpFn &&dumpfn) const -{ - auto const& lc = m_state->line_contents; - csubstr contents = lc.stripped; - if(contents.len) - { - // print the yaml src line - size_t offs = 3u + to_chars(substr{}, m_state->pos.line) + to_chars(substr{}, m_state->pos.col); - if(m_file.len) - { - _parse_dump(dumpfn, "{}:", m_file); - offs += m_file.len + 1; - } - _parse_dump(dumpfn, "{}:{}: ", m_state->pos.line, m_state->pos.col); - csubstr maybe_full_content = (contents.len < 80u ? contents : contents.first(80u)); - csubstr maybe_ellipsis = (contents.len < 80u ? csubstr{} : csubstr("...")); - _parse_dump(dumpfn, "{}{} (size={})\n", maybe_full_content, maybe_ellipsis, contents.len); - // highlight the remaining portion of the previous line - size_t firstcol = (size_t)(lc.rem.begin() - lc.full.begin()); - size_t lastcol = firstcol + lc.rem.len; - for(size_t i = 0; i < offs + firstcol; ++i) - dumpfn(" "); - dumpfn("^"); - for(size_t i = 1, e = (lc.rem.len < 80u ? lc.rem.len : 80u); i < e; ++i) - dumpfn("~"); - _parse_dump(dumpfn, "{} (cols {}-{})\n", maybe_ellipsis, firstcol+1, lastcol+1); - } - else - { - dumpfn("\n"); - } - -#ifdef RYML_DBG - // next line: print the state flags - { - char flagbuf_[64]; - _parse_dump(dumpfn, "top state: {}\n", _prfl(flagbuf_, m_state->flags)); - } -#endif -} - - -//----------------------------------------------------------------------------- -template -void Parser::_err(csubstr fmt, Args const& C4_RESTRICT ...args) const -{ - char errmsg[RYML_ERRMSG_SIZE]; - detail::_SubstrWriter writer(errmsg); - auto dumpfn = [&writer](csubstr s){ writer.append(s); }; - _parse_dump(dumpfn, fmt, args...); - writer.append('\n'); - _fmt_msg(dumpfn); - size_t len = writer.pos < RYML_ERRMSG_SIZE ? writer.pos : RYML_ERRMSG_SIZE; - m_tree->m_callbacks.m_error(errmsg, len, m_state->pos, m_tree->m_callbacks.m_user_data); -} - -//----------------------------------------------------------------------------- -#ifdef RYML_DBG -template -void Parser::_dbg(csubstr fmt, Args const& C4_RESTRICT ...args) const -{ - auto dumpfn = [](csubstr s){ fwrite(s.str, 1, s.len, stdout); }; - _parse_dump(dumpfn, fmt, args...); - dumpfn("\n"); - _fmt_msg(dumpfn); -} -#endif - -//----------------------------------------------------------------------------- -bool Parser::_finished_file() const -{ - bool ret = m_state->pos.offset >= m_buf.len; - if(ret) - { - _c4dbgp("finished file!!!"); - } - return ret; -} - -//----------------------------------------------------------------------------- -bool Parser::_finished_line() const -{ - return m_state->line_contents.rem.empty(); -} - -//----------------------------------------------------------------------------- -void Parser::parse_in_place(csubstr file, substr buf, Tree *t, size_t node_id) -{ - m_file = file; - m_buf = buf; - m_root_id = node_id; - m_tree = t; - _reset(); - while( ! _finished_file()) - { - _scan_line(); - while( ! _finished_line()) - _handle_line(); - if(_finished_file()) - break; // it may have finished because of multiline blocks - _line_ended(); - } - _handle_finished_file(); -} - -//----------------------------------------------------------------------------- -void Parser::_handle_finished_file() -{ - _end_stream(); -} - -//----------------------------------------------------------------------------- -void Parser::_handle_line() -{ - _c4dbgq("\n-----------"); - _c4dbgt("handling line={}, offset={}B", m_state->pos.line, m_state->pos.offset); - _RYML_CB_ASSERT(m_stack.m_callbacks, ! m_state->line_contents.rem.empty()); - if(has_any(RSEQ)) - { - if(has_any(FLOW)) - { - if(_handle_seq_flow()) - return; - } - else - { - if(_handle_seq_blck()) - return; - } - } - else if(has_any(RMAP)) - { - if(has_any(FLOW)) - { - if(_handle_map_flow()) - return; - } - else - { - if(_handle_map_blck()) - return; - } - } - else if(has_any(RUNK)) - { - if(_handle_unk()) - return; - } - - if(_handle_top()) - return; -} - - -//----------------------------------------------------------------------------- -bool Parser::_handle_unk() -{ - _c4dbgp("handle_unk"); - - csubstr rem = m_state->line_contents.rem; - const bool start_as_child = (node(m_state) == nullptr); - - if(C4_UNLIKELY(has_any(NDOC))) - { - if(rem == "---" || rem.begins_with("--- ")) - { - _start_new_doc(rem); - return true; - } - auto trimmed = rem.triml(' '); - if(trimmed == "---" || trimmed.begins_with("--- ")) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, rem.len >= trimmed.len); - _line_progressed(rem.len - trimmed.len); - _start_new_doc(trimmed); - _save_indentation(); - return true; - } - else if(trimmed.begins_with("...")) - { - _end_stream(); - } - else if(trimmed.first_of("#%") == csubstr::npos) // neither a doc nor a tag - { - _c4dbgpf("starting implicit doc to accomodate unexpected tokens: '{}'", rem); - size_t indref = m_state->indref; - _push_level(); - _start_doc(); - _set_indentation(indref); - } - _RYML_CB_ASSERT(m_stack.m_callbacks, !trimmed.empty()); - } - - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RNXT|RSEQ|RMAP)); - if(m_state->indref > 0) - { - csubstr ws = rem.left_of(rem.first_not_of(' ')); - if(m_state->indref <= ws.len) - { - _c4dbgpf("skipping base indentation of {}", m_state->indref); - _line_progressed(m_state->indref); - rem = rem.sub(m_state->indref); - } - } - - if(rem.begins_with("- ") _RYML_WITH_TAB_TOKENS( || rem.begins_with("-\t"))) - { - _c4dbgpf("it's a seq (as_child={})", start_as_child); - _move_key_anchor_to_val_anchor(); - _move_key_tag_to_val_tag(); - _push_level(); - _start_seq(start_as_child); - _save_indentation(); - _line_progressed(2); - return true; - } - else if(rem == '-') - { - _c4dbgpf("it's a seq (as_child={})", start_as_child); - _move_key_anchor_to_val_anchor(); - _move_key_tag_to_val_tag(); - _push_level(); - _start_seq(start_as_child); - _save_indentation(); - _line_progressed(1); - return true; - } - else if(rem.begins_with('[')) - { - _c4dbgpf("it's a seq, flow (as_child={})", start_as_child); - _move_key_anchor_to_val_anchor(); - _move_key_tag_to_val_tag(); - _push_level(/*explicit flow*/true); - _start_seq(start_as_child); - add_flags(FLOW); - _line_progressed(1); - return true; - } - else if(rem.begins_with('{')) - { - _c4dbgpf("it's a map, flow (as_child={})", start_as_child); - _move_key_anchor_to_val_anchor(); - _move_key_tag_to_val_tag(); - _push_level(/*explicit flow*/true); - _start_map(start_as_child); - addrem_flags(FLOW|RKEY, RVAL); - _line_progressed(1); - return true; - } - else if(rem.begins_with("? ")) - { - _c4dbgpf("it's a map (as_child={}) + this key is complex", start_as_child); - _move_key_anchor_to_val_anchor(); - _move_key_tag_to_val_tag(); - _push_level(); - _start_map(start_as_child); - addrem_flags(RKEY|QMRK, RVAL); - _save_indentation(); - _line_progressed(2); - return true; - } - else if(rem.begins_with(": ") && !has_all(SSCL)) - { - _c4dbgp("it's a map with an empty key"); - _move_key_anchor_to_val_anchor(); - _move_key_tag_to_val_tag(); - _push_level(); - _start_map(start_as_child); - _store_scalar_null(rem.str); - addrem_flags(RVAL, RKEY); - _save_indentation(); - _line_progressed(2); - return true; - } - else if(rem == ':' && !has_all(SSCL)) - { - _c4dbgp("it's a map with an empty key"); - _move_key_anchor_to_val_anchor(); - _move_key_tag_to_val_tag(); - _push_level(); - _start_map(start_as_child); - _store_scalar_null(rem.str); - addrem_flags(RVAL, RKEY); - _save_indentation(); - _line_progressed(1); - return true; - } - else if(_handle_types()) - { - return true; - } - else if(!rem.begins_with('*') && _handle_key_anchors_and_refs()) - { - return true; - } - else if(has_all(SSCL)) - { - _c4dbgpf("there's a stored scalar: '{}'", m_state->scalar); - - csubstr saved_scalar; - bool is_quoted; - if(_scan_scalar_unk(&saved_scalar, &is_quoted)) - { - rem = m_state->line_contents.rem; - _c4dbgpf("... and there's also a scalar next! '{}'", saved_scalar); - if(rem.begins_with_any(" \t")) - { - size_t n = rem.first_not_of(" \t"); - _c4dbgpf("skipping {} spaces/tabs", n); - rem = rem.sub(n); - _line_progressed(n); - } - } - - _c4dbgpf("rem='{}'", rem); - - if(rem.begins_with(", ")) - { - _c4dbgpf("got a ',' -- it's a seq (as_child={})", start_as_child); - _start_seq(start_as_child); - add_flags(FLOW); - _append_val(_consume_scalar()); - _line_progressed(2); - } - else if(rem.begins_with(',')) - { - _c4dbgpf("got a ',' -- it's a seq (as_child={})", start_as_child); - _start_seq(start_as_child); - add_flags(FLOW); - _append_val(_consume_scalar()); - _line_progressed(1); - } - else if(rem.begins_with(": ") _RYML_WITH_TAB_TOKENS( || rem.begins_with(":\t"))) - { - _c4dbgpf("got a ': ' -- it's a map (as_child={})", start_as_child); - _start_map_unk(start_as_child); // wait for the val scalar to append the key-val pair - _line_progressed(2); - } - else if(rem == ":" || rem.begins_with(":\"") || rem.begins_with(":'")) - { - if(rem == ":") { _c4dbgpf("got a ':' -- it's a map (as_child={})", start_as_child); } - else { _c4dbgpf("got a '{}' -- it's a map (as_child={})", rem.first(2), start_as_child); } - _start_map_unk(start_as_child); // wait for the val scalar to append the key-val pair - _line_progressed(1); // advance only 1 - } - else if(rem.begins_with('}')) - { - if(!has_all(RMAP|FLOW)) - { - _c4err("invalid token: not reading a map"); - } - if(!has_all(SSCL)) - { - _c4err("no scalar stored"); - } - _append_key_val(saved_scalar); - _stop_map(); - _line_progressed(1); - } - else if(rem.begins_with("...")) - { - _c4dbgp("got stream end '...'"); - _end_stream(); - _line_progressed(3); - } - else if(rem.begins_with('#')) - { - _c4dbgpf("it's a comment: '{}'", rem); - _scan_comment(); - return true; - } - else if(_handle_key_anchors_and_refs()) - { - return true; - } - else if(rem.begins_with(" ") || rem.begins_with("\t")) - { - size_t n = rem.first_not_of(" \t"); - if(n == npos) - n = rem.len; - _c4dbgpf("has {} spaces/tabs, skip...", n); - _line_progressed(n); - return true; - } - else if(rem.empty()) - { - // nothing to do - } - else if(rem == "---" || rem.begins_with("--- ")) - { - _c4dbgp("caught ---: starting doc"); - _start_new_doc(rem); - return true; - } - else if(rem.begins_with('%')) - { - _c4dbgp("caught a directive: ignoring..."); - _line_progressed(rem.len); - return true; - } - else - { - _c4err("parse error"); - } - - if( ! saved_scalar.empty()) - { - _store_scalar(saved_scalar, is_quoted); - } - - return true; - } - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, ! has_any(SSCL)); - csubstr scalar; - size_t indentation = m_state->line_contents.indentation; // save - bool is_quoted; - if(_scan_scalar_unk(&scalar, &is_quoted)) - { - _c4dbgpf("got a {} scalar", is_quoted ? "quoted" : ""); - rem = m_state->line_contents.rem; - { - size_t first = rem.first_not_of(" \t"); - if(first && first != npos) - { - _c4dbgpf("skip {} whitespace characters", first); - _line_progressed(first); - rem = rem.sub(first); - } - } - _store_scalar(scalar, is_quoted); - if(rem.begins_with(": ") _RYML_WITH_TAB_TOKENS( || rem.begins_with(":\t"))) - { - _c4dbgpf("got a ': ' next -- it's a map (as_child={})", start_as_child); - _push_level(); - _start_map(start_as_child); // wait for the val scalar to append the key-val pair - _set_indentation(indentation); - _line_progressed(2); // call this AFTER saving the indentation - } - else if(rem == ":") - { - _c4dbgpf("got a ':' next -- it's a map (as_child={})", start_as_child); - _push_level(); - _start_map(start_as_child); // wait for the val scalar to append the key-val pair - _set_indentation(indentation); - _line_progressed(1); // call this AFTER saving the indentation - } - else - { - // we still don't know whether it's a seq or a map - // so just store the scalar - } - return true; - } - else if(rem.begins_with_any(" \t")) - { - csubstr ws = rem.left_of(rem.first_not_of(" \t")); - rem = rem.right_of(ws); - if(has_all(RTOP) && rem.begins_with("---")) - { - _c4dbgp("there's a doc starting, and it's indented"); - _set_indentation(ws.len); - } - _c4dbgpf("skipping {} spaces/tabs", ws.len); - _line_progressed(ws.len); - return true; - } - } - - return false; -} - - -//----------------------------------------------------------------------------- -C4_ALWAYS_INLINE void Parser::_skipchars(char c) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->line_contents.rem.begins_with(c)); - size_t pos = m_state->line_contents.rem.first_not_of(c); - if(pos == npos) - pos = m_state->line_contents.rem.len; // maybe the line is just whitespace - _c4dbgpf("skip {} '{}'", pos, c); - _line_progressed(pos); -} - -template -C4_ALWAYS_INLINE void Parser::_skipchars(const char (&chars)[N]) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->line_contents.rem.begins_with_any(chars)); - size_t pos = m_state->line_contents.rem.first_not_of(chars); - if(pos == npos) - pos = m_state->line_contents.rem.len; // maybe the line is just whitespace - _c4dbgpf("skip {} characters", pos); - _line_progressed(pos); -} - - -//----------------------------------------------------------------------------- -bool Parser::_handle_seq_flow() -{ - _c4dbgpf("handle_seq_flow: node_id={} level={}", m_state->node_id, m_state->level); - csubstr rem = m_state->line_contents.rem; - - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RKEY)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(RSEQ|FLOW)); - - if(rem.begins_with(' ')) - { - // with explicit flow, indentation does not matter - _c4dbgp("starts with spaces"); - _skipchars(' '); - return true; - } - _RYML_WITH_TAB_TOKENS(else if(rem.begins_with('\t')) - { - _c4dbgp("starts with tabs"); - _skipchars('\t'); - return true; - }) - else if(rem.begins_with('#')) - { - _c4dbgp("it's a comment"); - rem = _scan_comment(); // also progresses the line - return true; - } - else if(rem.begins_with(']')) - { - _c4dbgp("end the sequence"); - _pop_level(); - _line_progressed(1); - if(has_all(RSEQIMAP)) - { - _stop_seqimap(); - _pop_level(); - } - return true; - } - - if(has_any(RVAL)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RNXT)); - bool is_quoted; - if(_scan_scalar_seq_flow(&rem, &is_quoted)) - { - _c4dbgp("it's a scalar"); - addrem_flags(RNXT, RVAL); - _append_val(rem, is_quoted); - return true; - } - else if(rem.begins_with('[')) - { - _c4dbgp("val is a child seq"); - addrem_flags(RNXT, RVAL); // before _push_level! - _push_level(/*explicit flow*/true); - _start_seq(); - add_flags(FLOW); - _line_progressed(1); - return true; - } - else if(rem.begins_with('{')) - { - _c4dbgp("val is a child map"); - addrem_flags(RNXT, RVAL); // before _push_level! - _push_level(/*explicit flow*/true); - _start_map(); - addrem_flags(FLOW|RKEY, RVAL); - _line_progressed(1); - return true; - } - else if(rem == ':') - { - _c4dbgpf("found ':' -- there's an implicit map in the seq node[{}]", m_state->node_id); - _start_seqimap(); - _line_progressed(1); - return true; - } - else if(rem.begins_with(": ") _RYML_WITH_TAB_TOKENS( || rem.begins_with(":\t"))) - { - _c4dbgpf("found ': ' -- there's an implicit map in the seq node[{}]", m_state->node_id); - _start_seqimap(); - _line_progressed(2); - return true; - } - else if(rem.begins_with("? ")) - { - _c4dbgpf("found '? ' -- there's an implicit map in the seq node[{}]", m_state->node_id); - _start_seqimap(); - _line_progressed(2); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(SSCL) && m_state->scalar == ""); - addrem_flags(QMRK|RKEY, RVAL|SSCL); - return true; - } - else if(_handle_types()) - { - return true; - } - else if(_handle_val_anchors_and_refs()) - { - return true; - } - else if(rem.begins_with(", ")) - { - _c4dbgp("found ',' -- the value was null"); - _append_val_null(rem.str - 1); - _line_progressed(2); - return true; - } - else if(rem.begins_with(',')) - { - _c4dbgp("found ',' -- the value was null"); - _append_val_null(rem.str - 1); - _line_progressed(1); - return true; - } - else if(rem.begins_with('\t')) - { - _skipchars('\t'); - return true; - } - else - { - _c4err("parse error"); - } - } - else if(has_any(RNXT)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RVAL)); - if(rem.begins_with(", ")) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(FLOW)); - _c4dbgp("seq: expect next val"); - addrem_flags(RVAL, RNXT); - _line_progressed(2); - return true; - } - else if(rem.begins_with(',')) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(FLOW)); - _c4dbgp("seq: expect next val"); - addrem_flags(RVAL, RNXT); - _line_progressed(1); - return true; - } - else if(rem == ':') - { - _c4dbgpf("found ':' -- there's an implicit map in the seq node[{}]", m_state->node_id); - _start_seqimap(); - _line_progressed(1); - return true; - } - else if(rem.begins_with(": ") _RYML_WITH_TAB_TOKENS( || rem.begins_with(":\t"))) - { - _c4dbgpf("found ': ' -- there's an implicit map in the seq node[{}]", m_state->node_id); - _start_seqimap(); - _line_progressed(2); - return true; - } - else - { - _c4err("was expecting a comma"); - } - } - else - { - _c4err("internal error"); - } - - return true; -} - -//----------------------------------------------------------------------------- -bool Parser::_handle_seq_blck() -{ - _c4dbgpf("handle_seq_impl: node_id={} level={}", m_state->node_id, m_state->level); - csubstr rem = m_state->line_contents.rem; - - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(RSEQ)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RKEY)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(FLOW)); - - if(rem.begins_with('#')) - { - _c4dbgp("it's a comment"); - rem = _scan_comment(); - return true; - } - if(has_any(RNXT)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RVAL)); - - if(_handle_indentation()) - return true; - - if(rem.begins_with("- ") _RYML_WITH_TAB_TOKENS( || rem.begins_with("-\t"))) - { - _c4dbgp("expect another val"); - addrem_flags(RVAL, RNXT); - _line_progressed(2); - return true; - } - else if(rem == '-') - { - _c4dbgp("expect another val"); - addrem_flags(RVAL, RNXT); - _line_progressed(1); - return true; - } - else if(rem.begins_with_any(" \t")) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, ! _at_line_begin()); - _skipchars(" \t"); - return true; - } - else if(rem.begins_with("...")) - { - _c4dbgp("got stream end '...'"); - _end_stream(); - _line_progressed(3); - return true; - } - else if(rem.begins_with("---")) - { - _c4dbgp("got document start '---'"); - _start_new_doc(rem); - return true; - } - else - { - _c4err("parse error"); - } - } - else if(has_any(RVAL)) - { - // there can be empty values - if(_handle_indentation()) - return true; - - csubstr s; - bool is_quoted; - if(_scan_scalar_seq_blck(&s, &is_quoted)) // this also progresses the line - { - _c4dbgpf("it's a{} scalar", is_quoted ? " quoted" : ""); - - rem = m_state->line_contents.rem; - if(_RYML_WITH_OR_WITHOUT_TAB_TOKENS(rem.begins_with_any(" \t"), rem.begins_with(' '))) - { - _c4dbgp("skipping whitespace..."); - size_t skip = rem.first_not_of(_RYML_WITH_OR_WITHOUT_TAB_TOKENS(" \t", ' ')); - if(skip == csubstr::npos) - skip = rem.len; // maybe the line is just whitespace - _line_progressed(skip); - rem = rem.sub(skip); - } - - _c4dbgpf("rem=[{}]~~~{}~~~", rem.len, rem); - if(!rem.begins_with('#') && (rem.ends_with(':') || rem.begins_with(": ") _RYML_WITH_TAB_TOKENS( || rem.begins_with(":\t")))) - { - _c4dbgp("actually, the scalar is the first key of a map, and it opens a new scope"); - if(m_key_anchor.empty()) - _move_val_anchor_to_key_anchor(); - if(m_key_tag.empty()) - _move_val_tag_to_key_tag(); - addrem_flags(RNXT, RVAL); // before _push_level! This prepares the current level for popping by setting it to RNXT - _push_level(); - _start_map(); - _store_scalar(s, is_quoted); - if( ! _maybe_set_indentation_from_anchor_or_tag()) - { - _c4dbgpf("set indentation from scalar: {}", m_state->scalar_col); - _set_indentation(m_state->scalar_col); // this is the column where the scalar starts - } - _move_key_tag2_to_key_tag(); - addrem_flags(RVAL, RKEY); - _line_progressed(1); - } - else - { - _c4dbgp("appending val to current seq"); - _append_val(s, is_quoted); - addrem_flags(RNXT, RVAL); - } - return true; - } - else if(rem.begins_with("- ") _RYML_WITH_TAB_TOKENS( || rem.begins_with("-\t"))) - { - if(_rval_dash_start_or_continue_seq()) - _line_progressed(2); - return true; - } - else if(rem == '-') - { - if(_rval_dash_start_or_continue_seq()) - _line_progressed(1); - return true; - } - else if(rem.begins_with('[')) - { - _c4dbgp("val is a child seq, flow"); - addrem_flags(RNXT, RVAL); // before _push_level! - _push_level(/*explicit flow*/true); - _start_seq(); - add_flags(FLOW); - _line_progressed(1); - return true; - } - else if(rem.begins_with('{')) - { - _c4dbgp("val is a child map, flow"); - addrem_flags(RNXT, RVAL); // before _push_level! - _push_level(/*explicit flow*/true); - _start_map(); - addrem_flags(FLOW|RKEY, RVAL); - _line_progressed(1); - return true; - } - else if(rem.begins_with("? ")) - { - _c4dbgp("val is a child map + this key is complex"); - addrem_flags(RNXT, RVAL); // before _push_level! - _push_level(); - _start_map(); - addrem_flags(QMRK|RKEY, RVAL); - _save_indentation(); - _line_progressed(2); - return true; - } - else if(rem.begins_with(' ')) - { - csubstr spc = rem.left_of(rem.first_not_of(' ')); - if(_at_line_begin()) - { - _c4dbgpf("skipping value indentation: {} spaces", spc.len); - _line_progressed(spc.len); - return true; - } - else - { - _c4dbgpf("skipping {} spaces", spc.len); - _line_progressed(spc.len); - return true; - } - } - else if(_handle_types()) - { - return true; - } - else if(_handle_val_anchors_and_refs()) - { - return true; - } - /* pathological case: - * - &key : val - * - &key : - * - : val - */ - else if((!has_all(SSCL)) && - (rem.begins_with(": ") || rem.left_of(rem.find("#")).trimr("\t") == ":")) - { - if(!m_val_anchor.empty() || !m_val_tag.empty()) - { - _c4dbgp("val is a child map + this key is empty, with anchors or tags"); - addrem_flags(RNXT, RVAL); // before _push_level! - _move_val_tag_to_key_tag(); - _move_val_anchor_to_key_anchor(); - _push_level(); - _start_map(); - _store_scalar_null(rem.str); - addrem_flags(RVAL, RKEY); - RYML_CHECK(_maybe_set_indentation_from_anchor_or_tag()); // one of them must exist - _line_progressed(rem.begins_with(": ") ? 2u : 1u); - return true; - } - else - { - _c4dbgp("val is a child map + this key is empty, no anchors or tags"); - addrem_flags(RNXT, RVAL); // before _push_level! - size_t ind = m_state->indref; - _push_level(); - _start_map(); - _store_scalar_null(rem.str); - addrem_flags(RVAL, RKEY); - _c4dbgpf("set indentation from map anchor: {}", ind + 2); - _set_indentation(ind + 2); // this is the column where the map starts - _line_progressed(rem.begins_with(": ") ? 2u : 1u); - return true; - } - } - else - { - _c4err("parse error"); - } - } - - return false; -} - -//----------------------------------------------------------------------------- - -bool Parser::_rval_dash_start_or_continue_seq() -{ - size_t ind = m_state->line_contents.current_col(); - _RYML_CB_ASSERT(m_stack.m_callbacks, ind >= m_state->indref); - size_t delta_ind = ind - m_state->indref; - if( ! delta_ind) - { - _c4dbgp("prev val was empty"); - addrem_flags(RNXT, RVAL); - _append_val_null(&m_state->line_contents.full[ind]); - return false; - } - _c4dbgp("val is a nested seq, indented"); - addrem_flags(RNXT, RVAL); // before _push_level! - _push_level(); - _start_seq(); - _save_indentation(); - return true; -} - -//----------------------------------------------------------------------------- -bool Parser::_handle_map_flow() -{ - // explicit flow, ie, inside {}, separated by commas - _c4dbgpf("handle_map_flow: node_id={} level={}", m_state->node_id, m_state->level); - csubstr rem = m_state->line_contents.rem; - - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(RMAP|FLOW)); - - if(rem.begins_with(' ')) - { - // with explicit flow, indentation does not matter - _c4dbgp("starts with spaces"); - _skipchars(' '); - return true; - } - _RYML_WITH_TAB_TOKENS(else if(rem.begins_with('\t')) - { - // with explicit flow, indentation does not matter - _c4dbgp("starts with tabs"); - _skipchars('\t'); - return true; - }) - else if(rem.begins_with('#')) - { - _c4dbgp("it's a comment"); - rem = _scan_comment(); // also progresses the line - return true; - } - else if(rem.begins_with('}')) - { - _c4dbgp("end the map"); - if(has_all(SSCL)) - { - _c4dbgp("the last val was null"); - _append_key_val_null(rem.str - 1); - rem_flags(RVAL); - } - _pop_level(); - _line_progressed(1); - if(has_all(RSEQIMAP)) - { - _c4dbgp("stopping implicitly nested 1x map"); - _stop_seqimap(); - _pop_level(); - } - return true; - } - - if(has_any(RNXT)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RKEY)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RVAL)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RSEQIMAP)); - - if(rem.begins_with(", ")) - { - _c4dbgp("seq: expect next keyval"); - addrem_flags(RKEY, RNXT); - _line_progressed(2); - return true; - } - else if(rem.begins_with(',')) - { - _c4dbgp("seq: expect next keyval"); - addrem_flags(RKEY, RNXT); - _line_progressed(1); - return true; - } - else - { - _c4err("parse error"); - } - } - else if(has_any(RKEY)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RNXT)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RVAL)); - - bool is_quoted; - if(has_none(SSCL) && _scan_scalar_map_flow(&rem, &is_quoted)) - { - _c4dbgp("it's a scalar"); - _store_scalar(rem, is_quoted); - rem = m_state->line_contents.rem; - csubstr trimmed = rem.triml(" \t"); - if(trimmed.len && (trimmed.begins_with(": ") || trimmed.begins_with_any(":,}") _RYML_WITH_TAB_TOKENS( || rem.begins_with(":\t")))) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, trimmed.str >= rem.str); - size_t num = static_cast(trimmed.str - rem.str); - _c4dbgpf("trimming {} whitespace after the scalar: '{}' --> '{}'", num, rem, rem.sub(num)); - rem = rem.sub(num); - _line_progressed(num); - } - } - - if(rem.begins_with(": ") _RYML_WITH_TAB_TOKENS( || rem.begins_with(":\t"))) - { - _c4dbgp("wait for val"); - addrem_flags(RVAL, RKEY|QMRK); - _line_progressed(2); - if(!has_all(SSCL)) - { - _c4dbgp("no key was found, defaulting to empty key ''"); - _store_scalar_null(rem.str); - } - return true; - } - else if(rem == ':') - { - _c4dbgp("wait for val"); - addrem_flags(RVAL, RKEY|QMRK); - _line_progressed(1); - if(!has_all(SSCL)) - { - _c4dbgp("no key was found, defaulting to empty key ''"); - _store_scalar_null(rem.str); - } - return true; - } - else if(rem.begins_with('?')) - { - _c4dbgp("complex key"); - add_flags(QMRK); - _line_progressed(1); - return true; - } - else if(rem.begins_with(',')) - { - _c4dbgp("prev scalar was a key with null value"); - _append_key_val_null(rem.str - 1); - _line_progressed(1); - return true; - } - else if(rem.begins_with('}')) - { - _c4dbgp("map terminates after a key..."); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(SSCL)); - _c4dbgp("the last val was null"); - _append_key_val_null(rem.str - 1); - rem_flags(RVAL); - if(has_all(RSEQIMAP)) - { - _c4dbgp("stopping implicitly nested 1x map"); - _stop_seqimap(); - _pop_level(); - } - _pop_level(); - _line_progressed(1); - return true; - } - else if(_handle_types()) - { - return true; - } - else if(_handle_key_anchors_and_refs()) - { - return true; - } - else if(rem == "") - { - return true; - } - else - { - size_t pos = rem.first_not_of(" \t"); - if(pos == csubstr::npos) - pos = 0; - rem = rem.sub(pos); - if(rem.begins_with(':')) - { - _c4dbgp("wait for val"); - addrem_flags(RVAL, RKEY|QMRK); - _line_progressed(pos + 1); - if(!has_all(SSCL)) - { - _c4dbgp("no key was found, defaulting to empty key ''"); - _store_scalar_null(rem.str); - } - return true; - } - else if(rem.begins_with('#')) - { - _c4dbgp("it's a comment"); - _line_progressed(pos); - rem = _scan_comment(); // also progresses the line - return true; - } - else - { - _c4err("parse error"); - } - } - } - else if(has_any(RVAL)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RNXT)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RKEY)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(SSCL)); - bool is_quoted; - if(_scan_scalar_map_flow(&rem, &is_quoted)) - { - _c4dbgp("it's a scalar"); - addrem_flags(RNXT, RVAL|RKEY); - _append_key_val(rem, is_quoted); - if(has_all(RSEQIMAP)) - { - _c4dbgp("stopping implicitly nested 1x map"); - _stop_seqimap(); - _pop_level(); - } - return true; - } - else if(rem.begins_with('[')) - { - _c4dbgp("val is a child seq"); - addrem_flags(RNXT, RVAL|RKEY); // before _push_level! - _push_level(/*explicit flow*/true); - _move_scalar_from_top(); - _start_seq(); - add_flags(FLOW); - _line_progressed(1); - return true; - } - else if(rem.begins_with('{')) - { - _c4dbgp("val is a child map"); - addrem_flags(RNXT, RVAL|RKEY); // before _push_level! - _push_level(/*explicit flow*/true); - _move_scalar_from_top(); - _start_map(); - addrem_flags(FLOW|RKEY, RNXT|RVAL); - _line_progressed(1); - return true; - } - else if(_handle_types()) - { - return true; - } - else if(_handle_val_anchors_and_refs()) - { - return true; - } - else if(rem.begins_with(',')) - { - _c4dbgp("appending empty val"); - _append_key_val_null(rem.str - 1); - addrem_flags(RKEY, RVAL); - _line_progressed(1); - if(has_any(RSEQIMAP)) - { - _c4dbgp("stopping implicitly nested 1x map"); - _stop_seqimap(); - _pop_level(); - } - return true; - } - else if(has_any(RSEQIMAP) && rem.begins_with(']')) - { - _c4dbgp("stopping implicitly nested 1x map"); - if(has_any(SSCL)) - { - _append_key_val_null(rem.str - 1); - } - _stop_seqimap(); - _pop_level(); - return true; - } - else - { - _c4err("parse error"); - } - } - else - { - _c4err("internal error"); - } - - return false; -} - -//----------------------------------------------------------------------------- -bool Parser::_handle_map_blck() -{ - _c4dbgpf("handle_map_blck: node_id={} level={}", m_state->node_id, m_state->level); - csubstr rem = m_state->line_contents.rem; - - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(RMAP)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(FLOW)); - - if(rem.begins_with('#')) - { - _c4dbgp("it's a comment"); - rem = _scan_comment(); - return true; - } - - if(has_any(RNXT)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RKEY)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RVAL)); - // actually, we don't need RNXT in indent-based maps. - addrem_flags(RKEY, RNXT); - } - - if(_handle_indentation()) - { - _c4dbgp("indentation token"); - return true; - } - - if(has_any(RKEY)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RNXT)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RVAL)); - - _c4dbgp("RMAP|RKEY read scalar?"); - bool is_quoted; - if(_scan_scalar_map_blck(&rem, &is_quoted)) // this also progresses the line - { - _c4dbgpf("it's a{} scalar", is_quoted ? " quoted" : ""); - if(has_all(QMRK|SSCL)) - { - _c4dbgpf("current key is QMRK; SSCL is set. so take store scalar='{}' as key and add an empty val", m_state->scalar); - _append_key_val_null(rem.str - 1); - } - _store_scalar(rem, is_quoted); - if(has_all(QMRK|RSET)) - { - _c4dbgp("it's a complex key, so use null value '~'"); - _append_key_val_null(rem.str); - } - rem = m_state->line_contents.rem; - - if(rem.begins_with(':')) - { - _c4dbgp("wait for val"); - addrem_flags(RVAL, RKEY|QMRK); - _line_progressed(1); - rem = m_state->line_contents.rem; - if(rem.begins_with_any(" \t")) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, ! _at_line_begin()); - rem = rem.left_of(rem.first_not_of(" \t")); - _c4dbgpf("skip {} spaces/tabs", rem.len); - _line_progressed(rem.len); - } - } - return true; - } - else if(rem.begins_with_any(" \t")) - { - size_t pos = rem.first_not_of(" \t"); - if(pos == npos) - pos = rem.len; - _c4dbgpf("skip {} spaces/tabs", pos); - _line_progressed(pos); - return true; - } - else if(rem == '?' || rem.begins_with("? ")) - { - _c4dbgp("it's a complex key"); - _line_progressed(rem.begins_with("? ") ? 2u : 1u); - if(has_any(SSCL)) - _append_key_val_null(rem.str - 1); - add_flags(QMRK); - return true; - } - else if(has_all(QMRK) && rem.begins_with(':')) - { - _c4dbgp("complex key finished"); - if(!has_any(SSCL)) - _store_scalar_null(rem.str); - addrem_flags(RVAL, RKEY|QMRK); - _line_progressed(1); - rem = m_state->line_contents.rem; - if(rem.begins_with(' ')) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, ! _at_line_begin()); - _skipchars(' '); - } - return true; - } - else if(rem == ':' || rem.begins_with(": ") _RYML_WITH_TAB_TOKENS( || rem.begins_with(":\t"))) - { - _c4dbgp("key finished"); - if(!has_all(SSCL)) - { - _c4dbgp("key was empty..."); - _store_scalar_null(rem.str); - rem_flags(QMRK); - } - addrem_flags(RVAL, RKEY); - _line_progressed(rem == ':' ? 1 : 2); - return true; - } - else if(rem.begins_with("...")) - { - _c4dbgp("end current document"); - _end_stream(); - _line_progressed(3); - return true; - } - else if(rem.begins_with("---")) - { - _c4dbgp("start new document '---'"); - _start_new_doc(rem); - return true; - } - else if(_handle_types()) - { - return true; - } - else if(_handle_key_anchors_and_refs()) - { - return true; - } - else - { - _c4err("parse error"); - } - } - else if(has_any(RVAL)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RNXT)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RKEY)); - - _c4dbgp("RMAP|RVAL read scalar?"); - csubstr s; - bool is_quoted; - if(_scan_scalar_map_blck(&s, &is_quoted)) // this also progresses the line - { - _c4dbgpf("it's a{} scalar", is_quoted ? " quoted" : ""); - - rem = m_state->line_contents.rem; - - if(rem.begins_with(": ")) - { - _c4dbgp("actually, the scalar is the first key of a map"); - addrem_flags(RKEY, RVAL); // before _push_level! This prepares the current level for popping by setting it to RNXT - _push_level(); - _move_scalar_from_top(); - _move_val_anchor_to_key_anchor(); - _start_map(); - _save_indentation(m_state->scalar_col); - addrem_flags(RVAL, RKEY); - _line_progressed(2); - } - else if(rem.begins_with(':')) - { - _c4dbgp("actually, the scalar is the first key of a map, and it opens a new scope"); - addrem_flags(RKEY, RVAL); // before _push_level! This prepares the current level for popping by setting it to RNXT - _push_level(); - _move_scalar_from_top(); - _move_val_anchor_to_key_anchor(); - _start_map(); - _save_indentation(/*behind*/s.len); - addrem_flags(RVAL, RKEY); - _line_progressed(1); - } - else - { - _c4dbgp("appending keyval to current map"); - _append_key_val(s, is_quoted); - addrem_flags(RKEY, RVAL); - } - return true; - } - else if(rem.begins_with("- ") _RYML_WITH_TAB_TOKENS( || rem.begins_with("-\t"))) - { - _c4dbgp("val is a nested seq, indented"); - addrem_flags(RKEY, RVAL); // before _push_level! - _push_level(); - _move_scalar_from_top(); - _start_seq(); - _save_indentation(); - _line_progressed(2); - return true; - } - else if(rem == '-') - { - _c4dbgp("maybe a seq. start unknown, indented"); - _start_unk(); - _save_indentation(); - _line_progressed(1); - return true; - } - else if(rem.begins_with('[')) - { - _c4dbgp("val is a child seq, flow"); - addrem_flags(RKEY, RVAL); // before _push_level! - _push_level(/*explicit flow*/true); - _move_scalar_from_top(); - _start_seq(); - add_flags(FLOW); - _line_progressed(1); - return true; - } - else if(rem.begins_with('{')) - { - _c4dbgp("val is a child map, flow"); - addrem_flags(RKEY, RVAL); // before _push_level! - _push_level(/*explicit flow*/true); - _move_scalar_from_top(); - _start_map(); - addrem_flags(FLOW|RKEY, RVAL); - _line_progressed(1); - return true; - } - else if(rem.begins_with(' ')) - { - csubstr spc = rem.left_of(rem.first_not_of(' ')); - if(_at_line_begin()) - { - _c4dbgpf("skipping value indentation: {} spaces", spc.len); - _line_progressed(spc.len); - return true; - } - else - { - _c4dbgpf("skipping {} spaces", spc.len); - _line_progressed(spc.len); - return true; - } - } - else if(_handle_types()) - { - return true; - } - else if(_handle_val_anchors_and_refs()) - { - return true; - } - else if(rem.begins_with("--- ") || rem == "---" || rem.begins_with("---\t")) - { - _start_new_doc(rem); - return true; - } - else if(rem.begins_with("...")) - { - _c4dbgp("end current document"); - _end_stream(); - _line_progressed(3); - return true; - } - else - { - _c4err("parse error"); - } - } - else - { - _c4err("internal error"); - } - - return false; -} - - -//----------------------------------------------------------------------------- -bool Parser::_handle_top() -{ - _c4dbgp("handle_top"); - csubstr rem = m_state->line_contents.rem; - - if(rem.begins_with('#')) - { - _c4dbgp("a comment line"); - _scan_comment(); - return true; - } - - csubstr trimmed = rem.triml(' '); - - if(trimmed.begins_with('%')) - { - _handle_directive(trimmed); - _line_progressed(rem.len); - return true; - } - else if(trimmed.begins_with("--- ") || trimmed == "---" || trimmed.begins_with("---\t")) - { - _start_new_doc(rem); - if(trimmed.len < rem.len) - { - _line_progressed(rem.len - trimmed.len); - _save_indentation(); - } - return true; - } - else if(trimmed.begins_with("...")) - { - _c4dbgp("end current document"); - _end_stream(); - if(trimmed.len < rem.len) - { - _line_progressed(rem.len - trimmed.len); - } - _line_progressed(3); - return true; - } - else - { - _c4err("parse error"); - } - - return false; -} - - -//----------------------------------------------------------------------------- - -bool Parser::_handle_key_anchors_and_refs() -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, !has_any(RVAL)); - const csubstr rem = m_state->line_contents.rem; - if(rem.begins_with('&')) - { - _c4dbgp("found a key anchor!!!"); - if(has_all(QMRK|SSCL)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RKEY)); - _c4dbgp("there is a stored key, so this anchor is for the next element"); - _append_key_val_null(rem.str - 1); - rem_flags(QMRK); - return true; - } - csubstr anchor = rem.left_of(rem.first_of(' ')); - _line_progressed(anchor.len); - anchor = anchor.sub(1); // skip the first character - _move_key_anchor_to_val_anchor(); - _c4dbgpf("key anchor value: '{}'", anchor); - m_key_anchor = anchor; - m_key_anchor_indentation = m_state->line_contents.current_col(rem); - return true; - } - else if(C4_UNLIKELY(rem.begins_with('*'))) - { - _c4err("not implemented - this should have been catched elsewhere"); - C4_NEVER_REACH(); - return false; - } - return false; -} - -bool Parser::_handle_val_anchors_and_refs() -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, !has_any(RKEY)); - const csubstr rem = m_state->line_contents.rem; - if(rem.begins_with('&')) - { - csubstr anchor = rem.left_of(rem.first_of(' ')); - _line_progressed(anchor.len); - anchor = anchor.sub(1); // skip the first character - _c4dbgpf("val: found an anchor: '{}', indentation={}!!!", anchor, m_state->line_contents.current_col(rem)); - if(m_val_anchor.empty()) - { - _c4dbgpf("save val anchor: '{}'", anchor); - m_val_anchor = anchor; - m_val_anchor_indentation = m_state->line_contents.current_col(rem); - } - else - { - _c4dbgpf("there is a pending val anchor '{}'", m_val_anchor); - if(m_tree->is_seq(m_state->node_id)) - { - if(m_tree->has_children(m_state->node_id)) - { - _c4dbgpf("current node={} is a seq, has {} children", m_state->node_id, m_tree->num_children(m_state->node_id)); - _c4dbgpf("... so take the new one as a key anchor '{}'", anchor); - m_key_anchor = anchor; - m_key_anchor_indentation = m_state->line_contents.current_col(rem); - } - else - { - _c4dbgpf("current node={} is a seq, has no children", m_state->node_id); - if(m_tree->has_val_anchor(m_state->node_id)) - { - _c4dbgpf("... node={} already has val anchor: '{}'", m_state->node_id, m_tree->val_anchor(m_state->node_id)); - _c4dbgpf("... so take the new one as a key anchor '{}'", anchor); - m_key_anchor = anchor; - m_key_anchor_indentation = m_state->line_contents.current_col(rem); - } - else - { - _c4dbgpf("... so set pending val anchor: '{}' on current node {}", m_val_anchor, m_state->node_id); - m_tree->set_val_anchor(m_state->node_id, m_val_anchor); - m_val_anchor = anchor; - m_val_anchor_indentation = m_state->line_contents.current_col(rem); - } - } - } - } - return true; - } - else if(C4_UNLIKELY(rem.begins_with('*'))) - { - _c4err("not implemented - this should have been catched elsewhere"); - C4_NEVER_REACH(); - return false; - } - return false; -} - -void Parser::_move_key_anchor_to_val_anchor() -{ - if(m_key_anchor.empty()) - return; - _c4dbgpf("move current key anchor to val slot: key='{}' -> val='{}'", m_key_anchor, m_val_anchor); - if(!m_val_anchor.empty()) - _c4err("triple-pending anchor"); - m_val_anchor = m_key_anchor; - m_val_anchor_indentation = m_key_anchor_indentation; - m_key_anchor = {}; - m_key_anchor_indentation = {}; -} - -void Parser::_move_val_anchor_to_key_anchor() -{ - if(m_val_anchor.empty()) - return; - if(!_token_is_from_this_line(m_val_anchor)) - return; - _c4dbgpf("move current val anchor to key slot: key='{}' <- val='{}'", m_key_anchor, m_val_anchor); - if(!m_key_anchor.empty()) - _c4err("triple-pending anchor"); - m_key_anchor = m_val_anchor; - m_key_anchor_indentation = m_val_anchor_indentation; - m_val_anchor = {}; - m_val_anchor_indentation = {}; -} - -void Parser::_move_key_tag_to_val_tag() -{ - if(m_key_tag.empty()) - return; - _c4dbgpf("move key tag to val tag: key='{}' -> val='{}'", m_key_tag, m_val_tag); - m_val_tag = m_key_tag; - m_val_tag_indentation = m_key_tag_indentation; - m_key_tag.clear(); - m_key_tag_indentation = 0; -} - -void Parser::_move_val_tag_to_key_tag() -{ - if(m_val_tag.empty()) - return; - if(!_token_is_from_this_line(m_val_tag)) - return; - _c4dbgpf("move val tag to key tag: key='{}' <- val='{}'", m_key_tag, m_val_tag); - m_key_tag = m_val_tag; - m_key_tag_indentation = m_val_tag_indentation; - m_val_tag.clear(); - m_val_tag_indentation = 0; -} - -void Parser::_move_key_tag2_to_key_tag() -{ - if(m_key_tag2.empty()) - return; - _c4dbgpf("move key tag2 to key tag: key='{}' <- key2='{}'", m_key_tag, m_key_tag2); - m_key_tag = m_key_tag2; - m_key_tag_indentation = m_key_tag2_indentation; - m_key_tag2.clear(); - m_key_tag2_indentation = 0; -} - - -//----------------------------------------------------------------------------- - -bool Parser::_handle_types() -{ - csubstr rem = m_state->line_contents.rem.triml(' '); - csubstr t; - - if(rem.begins_with("!!")) - { - _c4dbgp("begins with '!!'"); - t = rem.left_of(rem.first_of(" ,")); - _RYML_CB_ASSERT(m_stack.m_callbacks, t.len >= 2); - //t = t.sub(2); - if(t == "!!set") - add_flags(RSET); - } - else if(rem.begins_with("!<")) - { - _c4dbgp("begins with '!<'"); - t = rem.left_of(rem.first_of('>'), true); - _RYML_CB_ASSERT(m_stack.m_callbacks, t.len >= 2); - //t = t.sub(2, t.len-1); - } - else if(rem.begins_with("!h!")) - { - _c4dbgp("begins with '!h!'"); - t = rem.left_of(rem.first_of(' ')); - _RYML_CB_ASSERT(m_stack.m_callbacks, t.len >= 3); - //t = t.sub(3); - } - else if(rem.begins_with('!')) - { - _c4dbgp("begins with '!'"); - t = rem.left_of(rem.first_of(' ')); - _RYML_CB_ASSERT(m_stack.m_callbacks, t.len >= 1); - //t = t.sub(1); - } - - if(t.empty()) - return false; - - if(has_all(QMRK|SSCL)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RKEY)); - _c4dbgp("there is a stored key, so this tag is for the next element"); - _append_key_val_null(rem.str - 1); - rem_flags(QMRK); - } - - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED - const char *tag_beginning = rem.str; - #endif - size_t tag_indentation = m_state->line_contents.current_col(t); - _c4dbgpf("there was a tag: '{}', indentation={}", t, tag_indentation); - _RYML_CB_ASSERT(m_stack.m_callbacks, t.end() > m_state->line_contents.rem.begin()); - _line_progressed(static_cast(t.end() - m_state->line_contents.rem.begin())); - { - size_t pos = m_state->line_contents.rem.first_not_of(" \t"); - if(pos != csubstr::npos) - _line_progressed(pos); - } - - if(has_all(RMAP|RKEY)) - { - _c4dbgpf("saving map key tag '{}'", t); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_key_tag.empty()); - m_key_tag = t; - m_key_tag_indentation = tag_indentation; - } - else if(has_all(RMAP|RVAL)) - { - /* foo: !!str - * !!str : bar */ - rem = m_state->line_contents.rem; - rem = rem.left_of(rem.find("#")); - rem = rem.trimr(" \t"); - _c4dbgpf("rem='{}'", rem); - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED - if(rem == ':' || rem.begins_with(": ")) - { - _c4dbgp("the last val was null, and this is a tag from a null key"); - _append_key_val_null(tag_beginning - 1); - _store_scalar_null(rem.str - 1); - // do not change the flag to key, it is ~ - _RYML_CB_ASSERT(m_stack.m_callbacks, rem.begin() > m_state->line_contents.rem.begin()); - size_t token_len = rem == ':' ? 1 : 2; - _line_progressed(static_cast(token_len + rem.begin() - m_state->line_contents.rem.begin())); - } - #endif - _c4dbgpf("saving map val tag '{}'", t); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_val_tag.empty()); - m_val_tag = t; - m_val_tag_indentation = tag_indentation; - } - else if(has_all(RSEQ|RVAL) || has_all(RTOP|RUNK|NDOC)) - { - if(m_val_tag.empty()) - { - _c4dbgpf("saving seq/doc val tag '{}'", t); - m_val_tag = t; - m_val_tag_indentation = tag_indentation; - } - else - { - _c4dbgpf("saving seq/doc key tag '{}'", t); - m_key_tag = t; - m_key_tag_indentation = tag_indentation; - } - } - else if(has_all(RTOP|RUNK) || has_any(RUNK)) - { - rem = m_state->line_contents.rem; - rem = rem.left_of(rem.find("#")); - rem = rem.trimr(" \t"); - if(rem.empty()) - { - _c4dbgpf("saving val tag '{}'", t); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_val_tag.empty()); - m_val_tag = t; - m_val_tag_indentation = tag_indentation; - } - else - { - _c4dbgpf("saving key tag '{}'", t); - if(m_key_tag.empty()) - { - m_key_tag = t; - m_key_tag_indentation = tag_indentation; - } - else - { - /* handle this case: - * !!str foo: !!map - * !!int 1: !!float 20.0 - * !!int 3: !!float 40.0 - * - * (m_key_tag would be !!str and m_key_tag2 would be !!int) - */ - m_key_tag2 = t; - m_key_tag2_indentation = tag_indentation; - } - } - } - else - { - _c4err("internal error"); - } - - if(m_val_tag.not_empty()) - { - YamlTag_e tag = to_tag(t); - if(tag == TAG_STR) - { - _c4dbgpf("tag '{}' is a str-type tag", t); - if(has_all(RTOP|RUNK|NDOC)) - { - _c4dbgpf("docval. slurping the string. pos={}", m_state->pos.offset); - csubstr scalar = _slurp_doc_scalar(); - _c4dbgpf("docval. after slurp: {}, at node {}: '{}'", m_state->pos.offset, m_state->node_id, scalar); - m_tree->to_val(m_state->node_id, scalar, DOC); - _c4dbgpf("docval. val tag {} -> {}", m_val_tag, normalize_tag(m_val_tag)); - m_tree->set_val_tag(m_state->node_id, normalize_tag(m_val_tag)); - m_val_tag.clear(); - if(!m_val_anchor.empty()) - { - _c4dbgpf("setting val anchor[{}]='{}'", m_state->node_id, m_val_anchor); - m_tree->set_val_anchor(m_state->node_id, m_val_anchor); - m_val_anchor.clear(); - } - _end_stream(); - } - } - } - return true; -} - -//----------------------------------------------------------------------------- -csubstr Parser::_slurp_doc_scalar() -{ - csubstr s = m_state->line_contents.rem; - size_t pos = m_state->pos.offset; - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->line_contents.full.find("---") != csubstr::npos); - _c4dbgpf("slurp 0 '{}'. REM='{}'", s, m_buf.sub(m_state->pos.offset)); - if(s.len == 0) - { - _line_ended(); - _scan_line(); - s = m_state->line_contents.rem; - pos = m_state->pos.offset; - } - - size_t skipws = s.first_not_of(" \t"); - _c4dbgpf("slurp 1 '{}'. REM='{}'", s, m_buf.sub(m_state->pos.offset)); - if(skipws != npos) - { - _line_progressed(skipws); - s = m_state->line_contents.rem; - pos = m_state->pos.offset; - _c4dbgpf("slurp 2 '{}'. REM='{}'", s, m_buf.sub(m_state->pos.offset)); - } - - _RYML_CB_ASSERT(m_stack.m_callbacks, m_val_anchor.empty()); - _handle_val_anchors_and_refs(); - if(!m_val_anchor.empty()) - { - s = m_state->line_contents.rem; - skipws = s.first_not_of(" \t"); - if(skipws != npos) - { - _line_progressed(skipws); - } - s = m_state->line_contents.rem; - pos = m_state->pos.offset; - _c4dbgpf("slurp 3 '{}'. REM='{}'", s, m_buf.sub(m_state->pos.offset)); - } - - if(s.begins_with('\'')) - { - m_state->scalar_col = m_state->line_contents.current_col(s); - return _scan_squot_scalar(); - } - else if(s.begins_with('"')) - { - m_state->scalar_col = m_state->line_contents.current_col(s); - return _scan_dquot_scalar(); - } - else if(s.begins_with('|') || s.begins_with('>')) - { - return _scan_block(); - } - - _c4dbgpf("slurp 4 '{}'. REM='{}'", s, m_buf.sub(m_state->pos.offset)); - - m_state->scalar_col = m_state->line_contents.current_col(s); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.end() >= m_buf.begin() + pos); - _line_progressed(static_cast(s.end() - (m_buf.begin() + pos))); - - _c4dbgpf("slurp 5 '{}'. REM='{}'", s, m_buf.sub(m_state->pos.offset)); - - if(_at_line_end()) - { - _c4dbgpf("at line end. curr='{}'", s); - s = _extend_scanned_scalar(s); - } - - _c4dbgpf("scalar was '{}'", s); - - return s; -} - - -//----------------------------------------------------------------------------- - -bool Parser::_scan_scalar_seq_blck(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RSEQ)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RVAL)); - _RYML_CB_ASSERT(m_stack.m_callbacks, ! has_any(RKEY)); - _RYML_CB_ASSERT(m_stack.m_callbacks, ! has_any(FLOW)); - - csubstr s = m_state->line_contents.rem; - if(s.len == 0) - return false; - s = s.trim(" \t"); - if(s.len == 0) - return false; - - if(s.begins_with('\'')) - { - _c4dbgp("got a ': scanning single-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_squot_scalar(); - *quoted = true; - return true; - } - else if(s.begins_with('"')) - { - _c4dbgp("got a \": scanning double-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_dquot_scalar(); - *quoted = true; - return true; - } - else if(s.begins_with('|') || s.begins_with('>')) - { - *scalar = _scan_block(); - *quoted = true; - return true; - } - else if(has_any(RTOP) && _is_doc_sep(s)) - { - return false; - } - - _c4dbgp("RSEQ|RVAL"); - if( ! _is_scalar_next__rseq_rval(s)) - return false; - _RYML_WITH_TAB_TOKENS(else if(s.begins_with("-\t")) - return false; - ) - - if(s.ends_with(':')) - { - --s.len; - } - else - { - auto first = s.first_of_any(": " _RYML_WITH_TAB_TOKENS( , ":\t"), " #"); - if(first) - s.len = first.pos; - } - s = s.trimr(_RYML_WITH_OR_WITHOUT_TAB_TOKENS(" \t", ' ')); - - if(s.empty()) - return false; - - m_state->scalar_col = m_state->line_contents.current_col(s); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.str >= m_state->line_contents.rem.str); - _line_progressed(static_cast(s.str - m_state->line_contents.rem.str) + s.len); - - if(_at_line_end() && s != '~') - { - _c4dbgpf("at line end. curr='{}'", s); - s = _extend_scanned_scalar(s); - } - - _c4dbgpf("scalar was '{}'", s); - - *scalar = s; - *quoted = false; - return true; -} - -bool Parser::_scan_scalar_map_blck(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted) -{ - _c4dbgp("_scan_scalar_map_blck"); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RMAP)); - _RYML_CB_ASSERT(m_stack.m_callbacks, ! has_any(FLOW)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RKEY|RVAL)); - - csubstr s = m_state->line_contents.rem; - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED__OR_REFACTORED - if(s.len == 0) - return false; - #endif - s = s.trim(" \t"); - if(s.len == 0) - return false; - - if(s.begins_with('\'')) - { - _c4dbgp("got a ': scanning single-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_squot_scalar(); - *quoted = true; - return true; - } - else if(s.begins_with('"')) - { - _c4dbgp("got a \": scanning double-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_dquot_scalar(); - *quoted = true; - return true; - } - else if(s.begins_with('|') || s.begins_with('>')) - { - *scalar = _scan_block(); - *quoted = true; - return true; - } - else if(has_any(RTOP) && _is_doc_sep(s)) - { - return false; - } - - if( ! _is_scalar_next__rmap(s)) - return false; - - size_t colon_token = s.find(": "); - if(colon_token == npos) - { - _RYML_WITH_OR_WITHOUT_TAB_TOKENS( - // with tab tokens - colon_token = s.find(":\t"); - if(colon_token == npos) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, s.len > 0); - colon_token = s.find(':'); - if(colon_token != s.len-1) - colon_token = npos; - } - , - // without tab tokens - colon_token = s.find(':'); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.len > 0); - if(colon_token != s.len-1) - colon_token = npos; - ) - } - - if(has_all(RKEY)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, !s.begins_with(' ')); - if(has_any(QMRK)) - { - _c4dbgp("RMAP|RKEY|CPLX"); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RMAP)); - if(s.begins_with("? ") || s == '?') - return false; - s = s.left_of(colon_token); - s = s.left_of(s.first_of("#")); - s = s.trimr(" \t"); - if(s.begins_with("---")) - return false; - else if(s.begins_with("...")) - return false; - } - else - { - _c4dbgp("RMAP|RKEY"); - _RYML_CB_CHECK(m_stack.m_callbacks, !s.begins_with('{')); - if(s.begins_with("? ") || s == '?') - return false; - s = s.left_of(colon_token); - s = s.trimr(_RYML_WITH_OR_WITHOUT_TAB_TOKENS(" \t", ' ')); - if(s.begins_with("---")) - { - return false; - } - else if(s.begins_with("...")) - { - return false; - } - } - } - else if(has_all(RVAL)) - { - _c4dbgp("RMAP|RVAL"); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(QMRK)); - if( ! _is_scalar_next__rmap_val(s)) - return false; - _RYML_WITH_TAB_TOKENS( - else if(s.begins_with("-\t")) - return false; - ) - _c4dbgp("RMAP|RVAL: scalar"); - s = s.left_of(s.find(" #")); // is there a comment? - s = s.left_of(s.find("\t#")); // is there a comment? - s = s.trim(_RYML_WITH_OR_WITHOUT_TAB_TOKENS(" \t", ' ')); - if(s.begins_with("---")) - return false; - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED__OR_REFACTORED - else if(s.begins_with("...")) - return false; - #endif - } - - if(s.empty()) - return false; - - m_state->scalar_col = m_state->line_contents.current_col(s); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.str >= m_state->line_contents.rem.str); - _line_progressed(static_cast(s.str - m_state->line_contents.rem.str) + s.len); - - if(_at_line_end() && s != '~') - { - _c4dbgpf("at line end. curr='{}'", s); - s = _extend_scanned_scalar(s); - } - - _c4dbgpf("scalar was '{}'", s); - - *scalar = s; - *quoted = false; - return true; -} - -bool Parser::_scan_scalar_seq_flow(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RSEQ)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(FLOW)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RVAL)); - _RYML_CB_ASSERT(m_stack.m_callbacks, ! has_any(RKEY)); - - csubstr s = m_state->line_contents.rem; - if(s.len == 0) - return false; - s = s.trim(" \t"); - if(s.len == 0) - return false; - - if(s.begins_with('\'')) - { - _c4dbgp("got a ': scanning single-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_squot_scalar(); - *quoted = true; - return true; - } - else if(s.begins_with('"')) - { - _c4dbgp("got a \": scanning double-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_dquot_scalar(); - *quoted = true; - return true; - } - - if(has_all(RVAL)) - { - _c4dbgp("RSEQ|RVAL"); - if( ! _is_scalar_next__rseq_rval(s)) - return false; - _RYML_WITH_TAB_TOKENS(else if(s.begins_with("-\t")) - return false; - ) - _c4dbgp("RSEQ|RVAL|FLOW"); - s = s.left_of(s.first_of(",]")); - if(s.ends_with(':')) - { - --s.len; - } - else - { - auto first = s.first_of_any(": " _RYML_WITH_TAB_TOKENS( , ":\t"), " #"); - if(first) - s.len = first.pos; - } - s = s.trimr(_RYML_WITH_OR_WITHOUT_TAB_TOKENS(" \t", ' ')); - } - - if(s.empty()) - return false; - - m_state->scalar_col = m_state->line_contents.current_col(s); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.str >= m_state->line_contents.rem.str); - _line_progressed(static_cast(s.str - m_state->line_contents.rem.str) + s.len); - - if(_at_line_end() && s != '~') - { - _c4dbgpf("at line end. curr='{}'", s); - s = _extend_scanned_scalar(s); - } - - _c4dbgpf("scalar was '{}'", s); - - *scalar = s; - *quoted = false; - return true; -} - -bool Parser::_scan_scalar_map_flow(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RMAP)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(FLOW)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RKEY|RVAL)); - - csubstr s = m_state->line_contents.rem; - if(s.len == 0) - return false; - s = s.trim(" \t"); - if(s.len == 0) - return false; - - if(s.begins_with('\'')) - { - _c4dbgp("got a ': scanning single-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_squot_scalar(); - *quoted = true; - return true; - } - else if(s.begins_with('"')) - { - _c4dbgp("got a \": scanning double-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_dquot_scalar(); - *quoted = true; - return true; - } - - if( ! _is_scalar_next__rmap(s)) - return false; - - if(has_all(RKEY)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, !s.begins_with(' ')); - size_t colon_token = s.find(": "); - if(colon_token == npos) - { - _RYML_WITH_OR_WITHOUT_TAB_TOKENS( - // with tab tokens - colon_token = s.find(":\t"); - if(colon_token == npos) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, s.len > 0); - colon_token = s.find(':'); - if(colon_token != s.len-1) - colon_token = npos; - } - , - // without tab tokens - colon_token = s.find(':'); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.len > 0); - if(colon_token != s.len-1) - colon_token = npos; - ) - } - if(s.begins_with("? ") || s == '?') - return false; - if(has_any(QMRK)) - { - _c4dbgp("RMAP|RKEY|CPLX"); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RMAP)); - s = s.left_of(colon_token); - s = s.left_of(s.first_of("#")); - s = s.left_of(s.first_of(':')); - s = s.trimr(" \t"); - if(s.begins_with("---")) - return false; - else if(s.begins_with("...")) - return false; - } - else - { - _RYML_CB_CHECK(m_stack.m_callbacks, !s.begins_with('{')); - _c4dbgp("RMAP|RKEY"); - s = s.left_of(colon_token); - s = s.trimr(_RYML_WITH_OR_WITHOUT_TAB_TOKENS(" \t", ' ')); - _c4dbgpf("RMAP|RKEY|FLOW: '{}'", s); - s = s.left_of(s.first_of(",}")); - if(s.ends_with(':')) - --s.len; - } - } - else if(has_all(RVAL)) - { - _c4dbgp("RMAP|RVAL"); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(QMRK)); - if( ! _is_scalar_next__rmap_val(s)) - return false; - _RYML_WITH_TAB_TOKENS(else if(s.begins_with("-\t")) - return false; - ) - _c4dbgp("RMAP|RVAL|FLOW"); - if(has_none(RSEQIMAP)) - s = s.left_of(s.first_of(",}")); - else - s = s.left_of(s.first_of(",]")); - s = s.left_of(s.find(" #")); // is there a comment? - s = s.left_of(s.find("\t#")); // is there a comment? - s = s.trim(_RYML_WITH_OR_WITHOUT_TAB_TOKENS(" \t", ' ')); - } - - if(s.empty()) - return false; - - m_state->scalar_col = m_state->line_contents.current_col(s); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.str >= m_state->line_contents.rem.str); - _line_progressed(static_cast(s.str - m_state->line_contents.rem.str) + s.len); - - if(_at_line_end() && s != '~') - { - _c4dbgpf("at line end. curr='{}'", s); - s = _extend_scanned_scalar(s); - } - - _c4dbgpf("scalar was '{}'", s); - - *scalar = s; - *quoted = false; - return true; -} - -bool Parser::_scan_scalar_unk(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, has_any(RUNK)); - - csubstr s = m_state->line_contents.rem; - if(s.len == 0) - return false; - s = s.trim(" \t"); - if(s.len == 0) - return false; - - if(s.begins_with('\'')) - { - _c4dbgp("got a ': scanning single-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_squot_scalar(); - *quoted = true; - return true; - } - else if(s.begins_with('"')) - { - _c4dbgp("got a \": scanning double-quoted scalar"); - m_state->scalar_col = m_state->line_contents.current_col(s); - *scalar = _scan_dquot_scalar(); - *quoted = true; - return true; - } - else if(s.begins_with('|') || s.begins_with('>')) - { - *scalar = _scan_block(); - *quoted = true; - return true; - } - else if(has_any(RTOP) && _is_doc_sep(s)) - { - return false; - } - - _c4dbgpf("RUNK '[{}]~~~{}~~~", s.len, s); - if( ! _is_scalar_next__runk(s)) - { - _c4dbgp("RUNK: no scalar next"); - return false; - } - size_t pos = s.find(" #"); - if(pos != npos) - s = s.left_of(pos); - pos = s.find(": "); - if(pos != npos) - s = s.left_of(pos); - else if(s.ends_with(':')) - s = s.left_of(s.len-1); - _RYML_WITH_TAB_TOKENS( - else if((pos = s.find(":\t")) != npos) // TABS - s = s.left_of(pos); - ) - else - s = s.left_of(s.first_of(',')); - s = s.trim(" \t"); - _c4dbgpf("RUNK: scalar='{}'", s); - - if(s.empty()) - return false; - - m_state->scalar_col = m_state->line_contents.current_col(s); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.str >= m_state->line_contents.rem.str); - _line_progressed(static_cast(s.str - m_state->line_contents.rem.str) + s.len); - - if(_at_line_end() && s != '~') - { - _c4dbgpf("at line end. curr='{}'", s); - s = _extend_scanned_scalar(s); - } - - _c4dbgpf("scalar was '{}'", s); - - *scalar = s; - *quoted = false; - return true; -} - - -//----------------------------------------------------------------------------- - -csubstr Parser::_extend_scanned_scalar(csubstr s) -{ - if(has_all(RMAP|RKEY|QMRK)) - { - size_t scalar_indentation = has_any(FLOW) ? 0 : m_state->scalar_col; - _c4dbgpf("extend_scalar: explicit key! indref={} scalar_indentation={} scalar_col={}", m_state->indref, scalar_indentation, m_state->scalar_col); - csubstr n = _scan_to_next_nonempty_line(scalar_indentation); - if(!n.empty()) - { - substr full = _scan_complex_key(s, n).trimr(" \t\r\n"); - if(full != s) - s = _filter_plain_scalar(full, scalar_indentation); - } - } - // deal with plain (unquoted) scalars that continue to the next line - else if(!s.begins_with_any("*")) // cannot be a plain scalar if it starts with * (that's an anchor reference) - { - _c4dbgpf("extend_scalar: line ended, scalar='{}'", s); - if(has_none(FLOW)) - { - size_t scalar_indentation = m_state->indref + 1; - if(has_all(RUNK) && scalar_indentation == 1) - scalar_indentation = 0; - csubstr n = _scan_to_next_nonempty_line(scalar_indentation); - if(!n.empty()) - { - _c4dbgpf("rscalar[IMPL]: state_indref={} state_indentation={} scalar_indentation={}", m_state->indref, m_state->line_contents.indentation, scalar_indentation); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->line_contents.full.is_super(n)); - substr full = _scan_plain_scalar_blck(s, n, scalar_indentation); - if(full.len >= s.len) - s = _filter_plain_scalar(full, scalar_indentation); - } - } - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(FLOW)); - csubstr n = _scan_to_next_nonempty_line(/*indentation*/0); - if(!n.empty()) - { - _c4dbgp("rscalar[FLOW]"); - substr full = _scan_plain_scalar_flow(s, n); - s = _filter_plain_scalar(full, /*indentation*/0); - } - } - } - - return s; -} - - -//----------------------------------------------------------------------------- - -substr Parser::_scan_plain_scalar_flow(csubstr currscalar, csubstr peeked_line) -{ - static constexpr const csubstr chars = "[]{}?#,"; - size_t pos = peeked_line.first_of(chars); - bool first = true; - while(pos != 0) - { - if(has_all(RMAP|RKEY) || has_any(RUNK)) - { - csubstr tpkl = peeked_line.triml(' ').trimr("\r\n"); - if(tpkl.begins_with(": ") || tpkl == ':') - { - _c4dbgpf("rscalar[FLOW]: map value starts on the peeked line: '{}'", peeked_line); - peeked_line = peeked_line.first(0); - break; - } - else - { - auto colon_pos = peeked_line.first_of_any(": ", ":"); - if(colon_pos && colon_pos.pos < pos) - { - peeked_line = peeked_line.first(colon_pos.pos); - _c4dbgpf("rscalar[FLOW]: found colon at {}. peeked='{}'", colon_pos.pos, peeked_line); - _RYML_CB_ASSERT(m_stack.m_callbacks, peeked_line.end() >= m_state->line_contents.rem.begin()); - _line_progressed(static_cast(peeked_line.end() - m_state->line_contents.rem.begin())); - break; - } - } - } - if(pos != npos) - { - _c4dbgpf("rscalar[FLOW]: found special character '{}' at {}, stopping: '{}'", peeked_line[pos], pos, peeked_line.left_of(pos).trimr("\r\n")); - peeked_line = peeked_line.left_of(pos); - _RYML_CB_ASSERT(m_stack.m_callbacks, peeked_line.end() >= m_state->line_contents.rem.begin()); - _line_progressed(static_cast(peeked_line.end() - m_state->line_contents.rem.begin())); - break; - } - _c4dbgpf("rscalar[FLOW]: append another line, full: '{}'", peeked_line.trimr("\r\n")); - if(!first) - { - RYML_CHECK(_advance_to_peeked()); - } - peeked_line = _scan_to_next_nonempty_line(/*indentation*/0); - if(peeked_line.empty()) - { - _c4err("expected token or continuation"); - } - pos = peeked_line.first_of(chars); - first = false; - } - substr full(m_buf.str + (currscalar.str - m_buf.str), m_buf.begin() + m_state->pos.offset); - full = full.trimr("\n\r "); - return full; -} - - -//----------------------------------------------------------------------------- - -substr Parser::_scan_plain_scalar_blck(csubstr currscalar, csubstr peeked_line, size_t indentation) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf.is_super(currscalar)); - // NOTE. there's a problem with _scan_to_next_nonempty_line(), as it counts newlines twice - // size_t offs = m_state->pos.offset; // so we workaround by directly counting from the end of the given scalar - _RYML_CB_ASSERT(m_stack.m_callbacks, currscalar.end() >= m_buf.begin()); - size_t offs = static_cast(currscalar.end() - m_buf.begin()); - _RYML_CB_ASSERT(m_stack.m_callbacks, peeked_line.begins_with(' ', indentation)); - while(true) - { - _c4dbgpf("rscalar[IMPL]: continuing... ref_indentation={}", indentation); - if(peeked_line.begins_with("...") || peeked_line.begins_with("---")) - { - _c4dbgpf("rscalar[IMPL]: document termination next -- bail now '{}'", peeked_line.trimr("\r\n")); - break; - } - else if(( ! peeked_line.begins_with(' ', indentation))) // is the line deindented? - { - if(!peeked_line.trim(" \r\n\t").empty()) // is the line not blank? - { - _c4dbgpf("rscalar[IMPL]: deindented line, not blank -- bail now '{}'", peeked_line.trimr("\r\n")); - break; - } - _c4dbgpf("rscalar[IMPL]: line is blank and has less indentation: ref={} line={}: '{}'", indentation, peeked_line.first_not_of(' ') == csubstr::npos ? 0 : peeked_line.first_not_of(' '), peeked_line.trimr("\r\n")); - _c4dbgpf("rscalar[IMPL]: ... searching for a line starting at indentation {}", indentation); - csubstr next_peeked = _scan_to_next_nonempty_line(indentation); - if(next_peeked.empty()) - { - _c4dbgp("rscalar[IMPL]: ... finished."); - break; - } - _c4dbgp("rscalar[IMPL]: ... continuing."); - peeked_line = next_peeked; - } - - _c4dbgpf("rscalar[IMPL]: line contents: '{}'", peeked_line.right_of(indentation, true).trimr("\r\n")); - size_t token_pos; - if(peeked_line.find(": ") != npos) - { - _line_progressed(peeked_line.find(": ")); - _c4err("': ' is not a valid token in plain flow (unquoted) scalars"); - } - else if(peeked_line.ends_with(':')) - { - _line_progressed(peeked_line.find(':')); - _c4err("lines cannot end with ':' in plain flow (unquoted) scalars"); - } - else if((token_pos = peeked_line.find(" #")) != npos) - { - _line_progressed(token_pos); - break; - //_c4err("' #' is not a valid token in plain flow (unquoted) scalars"); - } - - _c4dbgpf("rscalar[IMPL]: append another line: (len={})'{}'", peeked_line.len, peeked_line.trimr("\r\n")); - if(!_advance_to_peeked()) - { - _c4dbgp("rscalar[IMPL]: file finishes after the scalar"); - break; - } - peeked_line = m_state->line_contents.rem; - } - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->pos.offset >= offs); - substr full(m_buf.str + (currscalar.str - m_buf.str), - currscalar.len + (m_state->pos.offset - offs)); - full = full.trimr("\r\n "); - return full; -} - -substr Parser::_scan_complex_key(csubstr currscalar, csubstr peeked_line) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf.is_super(currscalar)); - // NOTE. there's a problem with _scan_to_next_nonempty_line(), as it counts newlines twice - // size_t offs = m_state->pos.offset; // so we workaround by directly counting from the end of the given scalar - _RYML_CB_ASSERT(m_stack.m_callbacks, currscalar.end() >= m_buf.begin()); - size_t offs = static_cast(currscalar.end() - m_buf.begin()); - while(true) - { - _c4dbgp("rcplxkey: continuing..."); - if(peeked_line.begins_with("...") || peeked_line.begins_with("---")) - { - _c4dbgpf("rcplxkey: document termination next -- bail now '{}'", peeked_line.trimr("\r\n")); - break; - } - else - { - size_t pos = peeked_line.first_of("?:[]{}"); - if(pos == csubstr::npos) - { - pos = peeked_line.find("- "); - } - if(pos != csubstr::npos) - { - _c4dbgpf("rcplxkey: found special characters at pos={}: '{}'", pos, peeked_line.trimr("\r\n")); - _line_progressed(pos); - break; - } - } - - _c4dbgpf("rcplxkey: no special chars found '{}'", peeked_line.trimr("\r\n")); - csubstr next_peeked = _scan_to_next_nonempty_line(0); - if(next_peeked.empty()) - { - _c4dbgp("rcplxkey: empty ... finished."); - break; - } - _c4dbgp("rcplxkey: ... continuing."); - peeked_line = next_peeked; - - _c4dbgpf("rcplxkey: line contents: '{}'", peeked_line.trimr("\r\n")); - size_t colpos; - if((colpos = peeked_line.find(": ")) != npos) - { - _c4dbgp("rcplxkey: found ': ', stopping."); - _line_progressed(colpos); - break; - } - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED - else if((colpos = peeked_line.ends_with(':'))) - { - _c4dbgp("rcplxkey: ends with ':', stopping."); - _line_progressed(colpos); - break; - } - #endif - _c4dbgpf("rcplxkey: append another line: (len={})'{}'", peeked_line.len, peeked_line.trimr("\r\n")); - if(!_advance_to_peeked()) - { - _c4dbgp("rcplxkey: file finishes after the scalar"); - break; - } - peeked_line = m_state->line_contents.rem; - } - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->pos.offset >= offs); - substr full(m_buf.str + (currscalar.str - m_buf.str), - currscalar.len + (m_state->pos.offset - offs)); - return full; -} - -//! scans to the next non-blank line starting with the given indentation -csubstr Parser::_scan_to_next_nonempty_line(size_t indentation) -{ - csubstr next_peeked; - while(true) - { - _c4dbgpf("rscalar: ... curr offset: {} indentation={}", m_state->pos.offset, indentation); - next_peeked = _peek_next_line(m_state->pos.offset); - csubstr next_peeked_triml = next_peeked.triml(' '); - _c4dbgpf("rscalar: ... next peeked line='{}'", next_peeked.trimr("\r\n")); - if(next_peeked_triml.begins_with('#')) - { - _c4dbgp("rscalar: ... first non-space character is #"); - return {}; - } - else if(next_peeked.begins_with(' ', indentation)) - { - _c4dbgpf("rscalar: ... begins at same indentation {}, assuming continuation", indentation); - _advance_to_peeked(); - return next_peeked; - } - else // check for de-indentation - { - csubstr trimmed = next_peeked_triml.trimr("\t\r\n"); - _c4dbgpf("rscalar: ... deindented! trimmed='{}'", trimmed); - if(!trimmed.empty()) - { - _c4dbgp("rscalar: ... and not empty. bailing out."); - return {}; - } - } - if(!_advance_to_peeked()) - { - _c4dbgp("rscalar: file finished"); - return {}; - } - } - return {}; -} - -// returns false when the file finished -bool Parser::_advance_to_peeked() -{ - _line_progressed(m_state->line_contents.rem.len); - _line_ended(); // advances to the peeked-at line, consuming all remaining (probably newline) characters on the current line - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->line_contents.rem.first_of("\r\n") == csubstr::npos); - _c4dbgpf("advance to peeked: scan more... pos={} len={}", m_state->pos.offset, m_buf.len); - _scan_line(); // puts the peeked-at line in the buffer - if(_finished_file()) - { - _c4dbgp("rscalar: finished file!"); - return false; - } - return true; -} - -//----------------------------------------------------------------------------- - -C4_ALWAYS_INLINE size_t _extend_from_combined_newline(char nl, char following) -{ - return (nl == '\n' && following == '\r') || (nl == '\r' && following == '\n'); -} - -//! look for the next newline chars, and jump to the right of those -csubstr from_next_line(csubstr rem) -{ - size_t nlpos = rem.first_of("\r\n"); - if(nlpos == csubstr::npos) - return {}; - const char nl = rem[nlpos]; - rem = rem.right_of(nlpos); - if(rem.empty()) - return {}; - if(_extend_from_combined_newline(nl, rem.front())) - rem = rem.sub(1); - return rem; -} - -csubstr Parser::_peek_next_line(size_t pos) const -{ - csubstr rem{}; // declare here because of the goto - size_t nlpos{}; // declare here because of the goto - pos = pos == npos ? m_state->pos.offset : pos; - if(pos >= m_buf.len) - goto next_is_empty; - - // look for the next newline chars, and jump to the right of those - rem = from_next_line(m_buf.sub(pos)); - if(rem.empty()) - goto next_is_empty; - - // now get everything up to and including the following newline chars - nlpos = rem.first_of("\r\n"); - if((nlpos != csubstr::npos) && (nlpos + 1 < rem.len)) - nlpos += _extend_from_combined_newline(rem[nlpos], rem[nlpos+1]); - rem = rem.left_of(nlpos, /*include_pos*/true); - - _c4dbgpf("peek next line @ {}: (len={})'{}'", pos, rem.len, rem.trimr("\r\n")); - return rem; - -next_is_empty: - _c4dbgpf("peek next line @ {}: (len=0)''", pos); - return {}; -} - - -//----------------------------------------------------------------------------- -void Parser::LineContents::reset_with_next_line(csubstr buf, size_t offset) -{ - RYML_ASSERT(offset <= buf.len); - char const* C4_RESTRICT b = &buf[offset]; - char const* C4_RESTRICT e = b; - // get the current line stripped of newline chars - while(e < buf.end() && (*e != '\n' && *e != '\r')) - ++e; - RYML_ASSERT(e >= b); - const csubstr stripped_ = buf.sub(offset, static_cast(e - b)); - // advance pos to include the first line ending - if(e != buf.end() && *e == '\r') - ++e; - if(e != buf.end() && *e == '\n') - ++e; - RYML_ASSERT(e >= b); - const csubstr full_ = buf.sub(offset, static_cast(e - b)); - reset(full_, stripped_); -} - -void Parser::_scan_line() -{ - if(m_state->pos.offset >= m_buf.len) - { - m_state->line_contents.reset(m_buf.last(0), m_buf.last(0)); - return; - } - m_state->line_contents.reset_with_next_line(m_buf, m_state->pos.offset); -} - - -//----------------------------------------------------------------------------- -void Parser::_line_progressed(size_t ahead) -{ - _c4dbgpf("line[{}] ({} cols) progressed by {}: col {}-->{} offset {}-->{}", m_state->pos.line, m_state->line_contents.full.len, ahead, m_state->pos.col, m_state->pos.col+ahead, m_state->pos.offset, m_state->pos.offset+ahead); - m_state->pos.offset += ahead; - m_state->pos.col += ahead; - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->pos.col <= m_state->line_contents.stripped.len+1); - m_state->line_contents.rem = m_state->line_contents.rem.sub(ahead); -} - -void Parser::_line_ended() -{ - _c4dbgpf("line[{}] ({} cols) ended! offset {}-->{}", m_state->pos.line, m_state->line_contents.full.len, m_state->pos.offset, m_state->pos.offset+m_state->line_contents.full.len - m_state->line_contents.stripped.len); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->pos.col == m_state->line_contents.stripped.len+1); - m_state->pos.offset += m_state->line_contents.full.len - m_state->line_contents.stripped.len; - ++m_state->pos.line; - m_state->pos.col = 1; -} - -void Parser::_line_ended_undo() -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->pos.col == 1u); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->pos.line > 0u); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->pos.offset >= m_state->line_contents.full.len - m_state->line_contents.stripped.len); - size_t delta = m_state->line_contents.full.len - m_state->line_contents.stripped.len; - _c4dbgpf("line[{}] undo ended! line {}-->{}, offset {}-->{}", m_state->pos.line, m_state->pos.line, m_state->pos.line - 1, m_state->pos.offset, m_state->pos.offset - delta); - m_state->pos.offset -= delta; - --m_state->pos.line; - m_state->pos.col = m_state->line_contents.stripped.len + 1u; - // don't forget to undo also the changes to the remainder of the line - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->pos.offset >= m_buf.len || m_buf[m_state->pos.offset] == '\n' || m_buf[m_state->pos.offset] == '\r'); - m_state->line_contents.rem = m_buf.sub(m_state->pos.offset, 0); -} - - -//----------------------------------------------------------------------------- -void Parser::_set_indentation(size_t indentation) -{ - m_state->indref = indentation; - _c4dbgpf("state[{}]: saving indentation: {}", m_state-m_stack.begin(), m_state->indref); -} - -void Parser::_save_indentation(size_t behind) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->line_contents.rem.begin() >= m_state->line_contents.full.begin()); - m_state->indref = static_cast(m_state->line_contents.rem.begin() - m_state->line_contents.full.begin()); - _RYML_CB_ASSERT(m_stack.m_callbacks, behind <= m_state->indref); - m_state->indref -= behind; - _c4dbgpf("state[{}]: saving indentation: {}", m_state-m_stack.begin(), m_state->indref); -} - -bool Parser::_maybe_set_indentation_from_anchor_or_tag() -{ - if(m_key_anchor.not_empty()) - { - _c4dbgpf("set indentation from key anchor: {}", m_key_anchor_indentation); - _set_indentation(m_key_anchor_indentation); // this is the column where the anchor starts - return true; - } - else if(m_key_tag.not_empty()) - { - _c4dbgpf("set indentation from key tag: {}", m_key_tag_indentation); - _set_indentation(m_key_tag_indentation); // this is the column where the tag starts - return true; - } - return false; -} - - -//----------------------------------------------------------------------------- -void Parser::_write_key_anchor(size_t node_id) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_tree->has_key(node_id)); - if( ! m_key_anchor.empty()) - { - _c4dbgpf("node={}: set key anchor to '{}'", node_id, m_key_anchor); - m_tree->set_key_anchor(node_id, m_key_anchor); - m_key_anchor.clear(); - m_key_anchor_was_before = false; - m_key_anchor_indentation = 0; - } - else if( ! m_tree->is_key_quoted(node_id)) - { - csubstr r = m_tree->key(node_id); - if(r.begins_with('*')) - { - _c4dbgpf("node={}: set key reference: '{}'", node_id, r); - m_tree->set_key_ref(node_id, r.sub(1)); - } - else if(r == "<<") - { - m_tree->set_key_ref(node_id, r); - _c4dbgpf("node={}: it's an inheriting reference", node_id); - if(m_tree->is_seq(node_id)) - { - _c4dbgpf("node={}: inheriting from seq of {}", node_id, m_tree->num_children(node_id)); - for(size_t i = m_tree->first_child(node_id); i != NONE; i = m_tree->next_sibling(i)) - { - if( ! (m_tree->val(i).begins_with('*'))) - _c4err("malformed reference: '{}'", m_tree->val(i)); - } - } - else if( ! m_tree->val(node_id).begins_with('*')) - { - _c4err("malformed reference: '{}'", m_tree->val(node_id)); - } - //m_tree->set_key_ref(node_id, r); - } - } -} - -//----------------------------------------------------------------------------- -void Parser::_write_val_anchor(size_t node_id) -{ - if( ! m_val_anchor.empty()) - { - _c4dbgpf("node={}: set val anchor to '{}'", node_id, m_val_anchor); - m_tree->set_val_anchor(node_id, m_val_anchor); - m_val_anchor.clear(); - } - csubstr r = m_tree->has_val(node_id) ? m_tree->val(node_id) : ""; - if(!m_tree->is_val_quoted(node_id) && r.begins_with('*')) - { - _c4dbgpf("node={}: set val reference: '{}'", node_id, r); - RYML_CHECK(!m_tree->has_val_anchor(node_id)); - m_tree->set_val_ref(node_id, r.sub(1)); - } -} - -//----------------------------------------------------------------------------- -void Parser::_push_level(bool explicit_flow_chars) -{ - _c4dbgpf("pushing level! currnode={} currlevel={} stacksize={} stackcap={}", m_state->node_id, m_state->level, m_stack.size(), m_stack.capacity()); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state == &m_stack.top()); - if(node(m_state) == nullptr) - { - _c4dbgp("pushing level! actually no, current node is null"); - //_RYML_CB_ASSERT(m_stack.m_callbacks, ! explicit_flow_chars); - return; - } - flag_t st = RUNK; - if(explicit_flow_chars || has_all(FLOW)) - { - st |= FLOW; - } - m_stack.push_top(); - m_state = &m_stack.top(); - set_flags(st); - m_state->node_id = (size_t)NONE; - m_state->indref = (size_t)NONE; - ++m_state->level; - _c4dbgpf("pushing level: now, currlevel={}", m_state->level); -} - -void Parser::_pop_level() -{ - _c4dbgpf("popping level! currnode={} currlevel={}", m_state->node_id, m_state->level); - if(has_any(RMAP) || m_tree->is_map(m_state->node_id)) - { - _stop_map(); - } - if(has_any(RSEQ) || m_tree->is_seq(m_state->node_id)) - { - _stop_seq(); - } - if(m_tree->is_doc(m_state->node_id)) - { - _stop_doc(); - } - _RYML_CB_ASSERT(m_stack.m_callbacks, m_stack.size() > 1); - _prepare_pop(); - m_stack.pop(); - m_state = &m_stack.top(); - /*if(has_any(RMAP)) - { - _toggle_key_val(); - }*/ - if(m_state->line_contents.indentation == 0) - { - //_RYML_CB_ASSERT(m_stack.m_callbacks, has_none(RTOP)); - add_flags(RTOP); - } - _c4dbgpf("popping level: now, currnode={} currlevel={}", m_state->node_id, m_state->level); -} - -//----------------------------------------------------------------------------- -void Parser::_start_unk(bool /*as_child*/) -{ - _c4dbgp("start_unk"); - _push_level(); - _move_scalar_from_top(); -} - -//----------------------------------------------------------------------------- -void Parser::_start_doc(bool as_child) -{ - _c4dbgpf("start_doc (as child={})", as_child); - _RYML_CB_ASSERT(m_stack.m_callbacks, node(m_stack.bottom()) == node(m_root_id)); - size_t parent_id = m_stack.size() < 2 ? m_root_id : m_stack.top(1).node_id; - _RYML_CB_ASSERT(m_stack.m_callbacks, parent_id != NONE); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_tree->is_root(parent_id)); - _RYML_CB_ASSERT(m_stack.m_callbacks, node(m_state) == nullptr || node(m_state) == node(m_root_id)); - if(as_child) - { - _c4dbgpf("start_doc: parent={}", parent_id); - if( ! m_tree->is_stream(parent_id)) - { - _c4dbgp("start_doc: rearranging with root as STREAM"); - m_tree->set_root_as_stream(); - } - m_state->node_id = m_tree->append_child(parent_id); - m_tree->to_doc(m_state->node_id); - } - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, m_tree->is_seq(parent_id) || m_tree->empty(parent_id)); - m_state->node_id = parent_id; - if( ! m_tree->is_doc(parent_id)) - { - m_tree->to_doc(parent_id, DOC); - } - } - #endif - _c4dbgpf("start_doc: id={}", m_state->node_id); - add_flags(RUNK|RTOP|NDOC); - _handle_types(); - rem_flags(NDOC); -} - -void Parser::_stop_doc() -{ - size_t doc_node = m_state->node_id; - _c4dbgpf("stop_doc[{}]", doc_node); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_tree->is_doc(doc_node)); - if(!m_tree->is_seq(doc_node) && !m_tree->is_map(doc_node) && !m_tree->is_val(doc_node)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(SSCL)); - _c4dbgpf("stop_doc[{}]: there was nothing; adding null val", doc_node); - m_tree->to_val(doc_node, {}, DOC); - } -} - -void Parser::_end_stream() -{ - _c4dbgpf("end_stream, level={} node_id={}", m_state->level, m_state->node_id); - _RYML_CB_ASSERT(m_stack.m_callbacks, ! m_stack.empty()); - NodeData *added = nullptr; - if(has_any(SSCL)) - { - if(m_tree->is_seq(m_state->node_id)) - { - _c4dbgp("append val..."); - added = _append_val(_consume_scalar()); - } - else if(m_tree->is_map(m_state->node_id)) - { - _c4dbgp("append null key val..."); - added = _append_key_val_null(m_state->line_contents.rem.str); - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED - if(has_any(RSEQIMAP)) - { - _stop_seqimap(); - _pop_level(); - } - #endif - } - else if(m_tree->is_doc(m_state->node_id) || m_tree->type(m_state->node_id) == NOTYPE) - { - NodeType_e quoted = has_any(QSCL) ? VALQUO : NOTYPE; // do this before consuming the scalar - csubstr scalar = _consume_scalar(); - _c4dbgpf("node[{}]: to docval '{}'{}", m_state->node_id, scalar, quoted == VALQUO ? ", quoted" : ""); - m_tree->to_val(m_state->node_id, scalar, DOC|quoted); - added = m_tree->get(m_state->node_id); - } - else - { - _c4err("internal error"); - } - } - else if(has_all(RSEQ|RVAL) && has_none(FLOW)) - { - _c4dbgp("add last..."); - added = _append_val_null(m_state->line_contents.rem.str); - } - else if(!m_val_tag.empty() && (m_tree->is_doc(m_state->node_id) || m_tree->type(m_state->node_id) == NOTYPE)) - { - csubstr scalar = m_state->line_contents.rem.first(0); - _c4dbgpf("node[{}]: add null scalar as docval", m_state->node_id); - m_tree->to_val(m_state->node_id, scalar, DOC); - added = m_tree->get(m_state->node_id); - } - - if(added) - { - size_t added_id = m_tree->id(added); - if(m_tree->is_seq(m_state->node_id) || m_tree->is_doc(m_state->node_id)) - { - if(!m_key_anchor.empty()) - { - _c4dbgpf("node[{}]: move key to val anchor: '{}'", added_id, m_key_anchor); - m_val_anchor = m_key_anchor; - m_key_anchor = {}; - } - if(!m_key_tag.empty()) - { - _c4dbgpf("node[{}]: move key to val tag: '{}'", added_id, m_key_tag); - m_val_tag = m_key_tag; - m_key_tag = {}; - } - } - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED - if(!m_key_anchor.empty()) - { - _c4dbgpf("node[{}]: set key anchor='{}'", added_id, m_key_anchor); - m_tree->set_key_anchor(added_id, m_key_anchor); - m_key_anchor = {}; - } - #endif - if(!m_val_anchor.empty()) - { - _c4dbgpf("node[{}]: set val anchor='{}'", added_id, m_val_anchor); - m_tree->set_val_anchor(added_id, m_val_anchor); - m_val_anchor = {}; - } - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED - if(!m_key_tag.empty()) - { - _c4dbgpf("node[{}]: set key tag='{}' -> '{}'", added_id, m_key_tag, normalize_tag(m_key_tag)); - m_tree->set_key_tag(added_id, normalize_tag(m_key_tag)); - m_key_tag = {}; - } - #endif - if(!m_val_tag.empty()) - { - _c4dbgpf("node[{}]: set val tag='{}' -> '{}'", added_id, m_val_tag, normalize_tag(m_val_tag)); - m_tree->set_val_tag(added_id, normalize_tag(m_val_tag)); - m_val_tag = {}; - } - } - - while(m_stack.size() > 1) - { - _c4dbgpf("popping level: {} (stack sz={})", m_state->level, m_stack.size()); - _RYML_CB_ASSERT(m_stack.m_callbacks, ! has_any(SSCL, &m_stack.top())); - if(has_all(RSEQ|FLOW)) - _err("closing ] not found"); - _pop_level(); - } - add_flags(NDOC); -} - -void Parser::_start_new_doc(csubstr rem) -{ - _c4dbgp("_start_new_doc"); - _RYML_CB_ASSERT(m_stack.m_callbacks, rem.begins_with("---")); - C4_UNUSED(rem); - - _end_stream(); - - size_t indref = m_state->indref; - _c4dbgpf("start a document, indentation={}", indref); - _line_progressed(3); - _push_level(); - _start_doc(); - _set_indentation(indref); -} - - -//----------------------------------------------------------------------------- -void Parser::_start_map(bool as_child) -{ - _c4dbgpf("start_map (as child={})", as_child); - addrem_flags(RMAP|RVAL, RKEY|RUNK); - _RYML_CB_ASSERT(m_stack.m_callbacks, node(m_stack.bottom()) == node(m_root_id)); - size_t parent_id = m_stack.size() < 2 ? m_root_id : m_stack.top(1).node_id; - _RYML_CB_ASSERT(m_stack.m_callbacks, parent_id != NONE); - _RYML_CB_ASSERT(m_stack.m_callbacks, node(m_state) == nullptr || node(m_state) == node(m_root_id)); - if(as_child) - { - m_state->node_id = m_tree->append_child(parent_id); - if(has_all(SSCL)) - { - type_bits key_quoted = NOTYPE; - if(m_state->flags & QSCL) // before consuming the scalar - key_quoted |= KEYQUO; - csubstr key = _consume_scalar(); - m_tree->to_map(m_state->node_id, key, key_quoted); - _c4dbgpf("start_map: id={} key='{}'", m_state->node_id, m_tree->key(m_state->node_id)); - _write_key_anchor(m_state->node_id); - if( ! m_key_tag.empty()) - { - _c4dbgpf("node[{}]: set key tag='{}' -> '{}'", m_state->node_id, m_key_tag, normalize_tag(m_key_tag)); - m_tree->set_key_tag(m_state->node_id, normalize_tag(m_key_tag)); - m_key_tag.clear(); - } - } - else - { - m_tree->to_map(m_state->node_id); - _c4dbgpf("start_map: id={}", m_state->node_id); - } - m_tree->_p(m_state->node_id)->m_val.scalar.str = m_state->line_contents.rem.str; - _write_val_anchor(m_state->node_id); - } - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, parent_id != NONE); - m_state->node_id = parent_id; - _c4dbgpf("start_map: id={}", m_state->node_id); - type_bits as_doc = 0; - if(m_tree->is_doc(m_state->node_id)) - as_doc |= DOC; - if(!m_tree->is_map(parent_id)) - { - RYML_CHECK(!m_tree->has_children(parent_id)); - m_tree->to_map(parent_id, as_doc); - } - else - { - m_tree->_add_flags(parent_id, as_doc); - } - _move_scalar_from_top(); - if(m_key_anchor.not_empty()) - m_key_anchor_was_before = true; - _write_val_anchor(parent_id); - if(m_stack.size() >= 2) - { - State const& parent_state = m_stack.top(1); - if(parent_state.flags & RSET) - add_flags(RSET); - } - m_tree->_p(parent_id)->m_val.scalar.str = m_state->line_contents.rem.str; - } - if( ! m_val_tag.empty()) - { - _c4dbgpf("node[{}]: set val tag='{}' -> '{}'", m_state->node_id, m_val_tag, normalize_tag(m_val_tag)); - m_tree->set_val_tag(m_state->node_id, normalize_tag(m_val_tag)); - m_val_tag.clear(); - } -} - -void Parser::_start_map_unk(bool as_child) -{ - if(!m_key_anchor_was_before) - { - _c4dbgpf("stash key anchor before starting map... '{}'", m_key_anchor); - csubstr ka = m_key_anchor; - m_key_anchor = {}; - _start_map(as_child); - m_key_anchor = ka; - } - else - { - _start_map(as_child); - m_key_anchor_was_before = false; - } - if(m_key_tag2.not_empty()) - { - m_key_tag = m_key_tag2; - m_key_tag_indentation = m_key_tag2_indentation; - m_key_tag2.clear(); - m_key_tag2_indentation = 0; - } -} - -void Parser::_stop_map() -{ - _c4dbgpf("stop_map[{}]", m_state->node_id); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_tree->is_map(m_state->node_id)); - if(has_all(QMRK|RKEY) && !has_all(SSCL)) - { - _c4dbgpf("stop_map[{}]: RKEY", m_state->node_id); - _store_scalar_null(m_state->line_contents.rem.str); - _append_key_val_null(m_state->line_contents.rem.str); - } -} - - -//----------------------------------------------------------------------------- -void Parser::_start_seq(bool as_child) -{ - _c4dbgpf("start_seq (as child={})", as_child); - if(has_all(RTOP|RUNK)) - { - _c4dbgpf("start_seq: moving key tag to val tag: '{}'", m_key_tag); - m_val_tag = m_key_tag; - m_key_tag.clear(); - } - addrem_flags(RSEQ|RVAL, RUNK); - _RYML_CB_ASSERT(m_stack.m_callbacks, node(m_stack.bottom()) == node(m_root_id)); - size_t parent_id = m_stack.size() < 2 ? m_root_id : m_stack.top(1).node_id; - _RYML_CB_ASSERT(m_stack.m_callbacks, parent_id != NONE); - _RYML_CB_ASSERT(m_stack.m_callbacks, node(m_state) == nullptr || node(m_state) == node(m_root_id)); - if(as_child) - { - m_state->node_id = m_tree->append_child(parent_id); - if(has_all(SSCL)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, m_tree->is_map(parent_id)); - type_bits key_quoted = 0; - if(m_state->flags & QSCL) // before consuming the scalar - key_quoted |= KEYQUO; - csubstr key = _consume_scalar(); - m_tree->to_seq(m_state->node_id, key, key_quoted); - _c4dbgpf("start_seq: id={} name='{}'", m_state->node_id, m_tree->key(m_state->node_id)); - _write_key_anchor(m_state->node_id); - if( ! m_key_tag.empty()) - { - _c4dbgpf("start_seq[{}]: set key tag='{}' -> '{}'", m_state->node_id, m_key_tag, normalize_tag(m_key_tag)); - m_tree->set_key_tag(m_state->node_id, normalize_tag(m_key_tag)); - m_key_tag.clear(); - } - } - else - { - type_bits as_doc = 0; - _RYML_CB_ASSERT(m_stack.m_callbacks, !m_tree->is_doc(m_state->node_id)); - m_tree->to_seq(m_state->node_id, as_doc); - _c4dbgpf("start_seq: id={}{}", m_state->node_id, as_doc ? " as doc" : ""); - } - _write_val_anchor(m_state->node_id); - m_tree->_p(m_state->node_id)->m_val.scalar.str = m_state->line_contents.rem.str; - } - else - { - m_state->node_id = parent_id; - type_bits as_doc = 0; - if(m_tree->is_doc(m_state->node_id)) - as_doc |= DOC; - if(!m_tree->is_seq(parent_id)) - { - RYML_CHECK(!m_tree->has_children(parent_id)); - m_tree->to_seq(parent_id, as_doc); - } - else - { - m_tree->_add_flags(parent_id, as_doc); - } - _move_scalar_from_top(); - _c4dbgpf("start_seq: id={}{}", m_state->node_id, as_doc ? " as_doc" : ""); - _write_val_anchor(parent_id); - m_tree->_p(parent_id)->m_val.scalar.str = m_state->line_contents.rem.str; - } - if( ! m_val_tag.empty()) - { - _c4dbgpf("start_seq[{}]: set val tag='{}' -> '{}'", m_state->node_id, m_val_tag, normalize_tag(m_val_tag)); - m_tree->set_val_tag(m_state->node_id, normalize_tag(m_val_tag)); - m_val_tag.clear(); - } -} - -void Parser::_stop_seq() -{ - _c4dbgp("stop_seq"); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_tree->is_seq(m_state->node_id)); -} - - -//----------------------------------------------------------------------------- -void Parser::_start_seqimap() -{ - _c4dbgpf("start_seqimap at node={}. has_children={}", m_state->node_id, m_tree->has_children(m_state->node_id)); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(RSEQ|FLOW)); - // create a map, and turn the last scalar of this sequence - // into the key of the map's first child. This scalar was - // understood to be a value in the sequence, but it is - // actually a key of a map, implicitly opened here. - // Eg [val, key: val] - // - // Yep, YAML is crazy. - if(m_tree->has_children(m_state->node_id) && m_tree->has_val(m_tree->last_child(m_state->node_id))) - { - size_t prev = m_tree->last_child(m_state->node_id); - NodeType ty = m_tree->_p(prev)->m_type; // don't use type() because it masks out the quotes - NodeScalar tmp = m_tree->valsc(prev); - _c4dbgpf("has children and last child={} has val. saving the scalars, val='{}' quoted={}", prev, tmp.scalar, ty.is_val_quoted()); - m_tree->remove(prev); - _push_level(); - _start_map(); - _store_scalar(tmp.scalar, ty.is_val_quoted()); - m_key_anchor = tmp.anchor; - m_key_tag = tmp.tag; - } - else - { - _c4dbgpf("node {} has no children yet, using empty key", m_state->node_id); - _push_level(); - _start_map(); - _store_scalar_null(m_state->line_contents.rem.str); - } - add_flags(RSEQIMAP|FLOW); -} - -void Parser::_stop_seqimap() -{ - _c4dbgp("stop_seqimap"); - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(RSEQIMAP)); -} - - -//----------------------------------------------------------------------------- -NodeData* Parser::_append_val(csubstr val, flag_t quoted) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, ! has_all(SSCL)); - _RYML_CB_ASSERT(m_stack.m_callbacks, node(m_state) != nullptr); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_tree->is_seq(m_state->node_id)); - type_bits additional_flags = quoted ? VALQUO : NOTYPE; - _c4dbgpf("append val: '{}' to parent id={} (level={}){}", val, m_state->node_id, m_state->level, quoted ? " VALQUO!" : ""); - size_t nid = m_tree->append_child(m_state->node_id); - m_tree->to_val(nid, val, additional_flags); - - _c4dbgpf("append val: id={} val='{}'", nid, m_tree->get(nid)->m_val.scalar); - if( ! m_val_tag.empty()) - { - _c4dbgpf("append val[{}]: set val tag='{}' -> '{}'", nid, m_val_tag, normalize_tag(m_val_tag)); - m_tree->set_val_tag(nid, normalize_tag(m_val_tag)); - m_val_tag.clear(); - } - _write_val_anchor(nid); - return m_tree->get(nid); -} - -NodeData* Parser::_append_key_val(csubstr val, flag_t val_quoted) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, m_tree->is_map(m_state->node_id)); - type_bits additional_flags = 0; - if(m_state->flags & QSCL) - additional_flags |= KEYQUO; - if(val_quoted) - additional_flags |= VALQUO; - - csubstr key = _consume_scalar(); - _c4dbgpf("append keyval: '{}' '{}' to parent id={} (level={}){}{}", key, val, m_state->node_id, m_state->level, (additional_flags & KEYQUO) ? " KEYQUO!" : "", (additional_flags & VALQUO) ? " VALQUO!" : ""); - size_t nid = m_tree->append_child(m_state->node_id); - m_tree->to_keyval(nid, key, val, additional_flags); - _c4dbgpf("append keyval: id={} key='{}' val='{}'", nid, m_tree->key(nid), m_tree->val(nid)); - if( ! m_key_tag.empty()) - { - _c4dbgpf("append keyval[{}]: set key tag='{}' -> '{}'", nid, m_key_tag, normalize_tag(m_key_tag)); - m_tree->set_key_tag(nid, normalize_tag(m_key_tag)); - m_key_tag.clear(); - } - if( ! m_val_tag.empty()) - { - _c4dbgpf("append keyval[{}]: set val tag='{}' -> '{}'", nid, m_val_tag, normalize_tag(m_val_tag)); - m_tree->set_val_tag(nid, normalize_tag(m_val_tag)); - m_val_tag.clear(); - } - _write_key_anchor(nid); - _write_val_anchor(nid); - rem_flags(QMRK); - return m_tree->get(nid); -} - - -//----------------------------------------------------------------------------- -void Parser::_store_scalar(csubstr s, flag_t is_quoted) -{ - _c4dbgpf("state[{}]: storing scalar '{}' (flag: {}) (old scalar='{}')", - m_state-m_stack.begin(), s, m_state->flags & SSCL, m_state->scalar); - RYML_CHECK(has_none(SSCL)); - add_flags(SSCL | (is_quoted * QSCL)); - m_state->scalar = s; -} - -csubstr Parser::_consume_scalar() -{ - _c4dbgpf("state[{}]: consuming scalar '{}' (flag: {}))", m_state-m_stack.begin(), m_state->scalar, m_state->flags & SSCL); - RYML_CHECK(m_state->flags & SSCL); - csubstr s = m_state->scalar; - rem_flags(SSCL | QSCL); - m_state->scalar.clear(); - return s; -} - -void Parser::_move_scalar_from_top() -{ - if(m_stack.size() < 2) return; - State &prev = m_stack.top(1); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state == &m_stack.top()); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state != &prev); - if(prev.flags & SSCL) - { - _c4dbgpf("moving scalar '{}' from state[{}] to state[{}] (overwriting '{}')", prev.scalar, &prev-m_stack.begin(), m_state-m_stack.begin(), m_state->scalar); - add_flags(prev.flags & (SSCL | QSCL)); - m_state->scalar = prev.scalar; - rem_flags(SSCL | QSCL, &prev); - prev.scalar.clear(); - } -} - -//----------------------------------------------------------------------------- -/** @todo this function is a monster and needs love. Likely, it needs - * to be split like _scan_scalar_*() */ -bool Parser::_handle_indentation() -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(FLOW)); - if( ! _at_line_begin()) - return false; - - size_t ind = m_state->line_contents.indentation; - csubstr rem = m_state->line_contents.rem; - /** @todo instead of trimming, we should use the indentation index from above */ - csubstr remt = rem.triml(' '); - - if(remt.empty() || remt.begins_with('#')) // this is a blank or comment line - { - _line_progressed(rem.size()); - return true; - } - - _c4dbgpf("indentation? ind={} indref={}", ind, m_state->indref); - if(ind == m_state->indref) - { - _c4dbgpf("same indentation: {}", ind); - if(!rem.sub(ind).begins_with('-')) - { - _c4dbgp("does not begin with -"); - if(has_any(RMAP)) - { - if(has_all(SSCL|RVAL)) - { - _c4dbgp("add with null val"); - _append_key_val_null(rem.str + ind - 1); - addrem_flags(RKEY, RVAL); - } - } - else if(has_any(RSEQ)) - { - if(m_stack.size() > 2) // do not pop to root level - { - if(has_any(RNXT)) - { - _c4dbgp("end the indentless seq"); - _pop_level(); - return true; - } - else if(has_any(RVAL)) - { - _c4dbgp("add with null val"); - _append_val_null(rem.str); - _c4dbgp("end the indentless seq"); - _pop_level(); - return true; - } - } - } - } - _line_progressed(ind); - return ind > 0; - } - else if(ind < m_state->indref) - { - _c4dbgpf("smaller indentation ({} < {})!!!", ind, m_state->indref); - if(has_all(RVAL)) - { - _c4dbgp("there was an empty val -- appending"); - if(has_all(RMAP)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_all(SSCL)); - _append_key_val_null(rem.sub(ind).str - 1); - } - else if(has_all(RSEQ)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, has_none(SSCL)); - _append_val_null(rem.sub(ind).str - 1); - } - } - // search the stack frame to jump to based on its indentation - State const* popto = nullptr; - _RYML_CB_ASSERT(m_stack.m_callbacks, m_stack.is_contiguous()); // this search relies on the stack being contiguous - for(State const* s = m_state-1; s >= m_stack.begin(); --s) - { - _c4dbgpf("searching for state with indentation {}. curr={} (level={},node={})", ind, s->indref, s->level, s->node_id); - if(s->indref == ind) - { - _c4dbgpf("gotit!!! level={} node={}", s->level, s->node_id); - popto = s; - // while it may be tempting to think we're done at this - // point, we must still determine whether we're jumping to a - // parent with the same indentation. Consider this case with - // an indentless sequence: - // - // product: - // - sku: BL394D - // quantity: 4 - // description: Basketball - // price: 450.00 - // - sku: BL4438H - // quantity: 1 - // description: Super Hoop - // price: 2392.00 # jumping one level here would be wrong. - // tax: 1234.5 # we must jump two levels - if(popto > m_stack.begin()) - { - auto parent = popto - 1; - if(parent->indref == popto->indref) - { - _c4dbgpf("the parent (level={},node={}) has the same indentation ({}). is this in an indentless sequence?", parent->level, parent->node_id, popto->indref); - _c4dbgpf("isseq(popto)={} ismap(parent)={}", m_tree->is_seq(popto->node_id), m_tree->is_map(parent->node_id)); - if(m_tree->is_seq(popto->node_id) && m_tree->is_map(parent->node_id)) - { - if( ! remt.begins_with('-')) - { - _c4dbgp("this is an indentless sequence"); - popto = parent; - } - else - { - _c4dbgp("not an indentless sequence"); - } - } - } - } - break; - } - } - if(!popto || popto >= m_state || popto->level >= m_state->level) - { - _c4err("parse error: incorrect indentation?"); - } - _c4dbgpf("popping {} levels: from level {} to level {}", m_state->level-popto->level, m_state->level, popto->level); - while(m_state != popto) - { - _c4dbgpf("popping level {} (indentation={})", m_state->level, m_state->indref); - _pop_level(); - } - _RYML_CB_ASSERT(m_stack.m_callbacks, ind == m_state->indref); - _line_progressed(ind); - return true; - } - else - { - _c4dbgpf("larger indentation ({} > {})!!!", ind, m_state->indref); - _RYML_CB_ASSERT(m_stack.m_callbacks, ind > m_state->indref); - if(has_all(RMAP|RVAL)) - { - if(_is_scalar_next__rmap_val(remt) && remt.first_of(":?") == npos) - { - _c4dbgpf("actually it seems a value: '{}'", remt); - } - else - { - addrem_flags(RKEY, RVAL); - _start_unk(); - //_move_scalar_from_top(); - _line_progressed(ind); - _save_indentation(); - return true; - } - } - else if(has_all(RSEQ|RVAL)) - { - // nothing to do here - } - else - { - _c4err("parse error - indentation should not increase at this point"); - } - } - - return false; -} - -//----------------------------------------------------------------------------- -csubstr Parser::_scan_comment() -{ - csubstr s = m_state->line_contents.rem; - _RYML_CB_ASSERT(m_stack.m_callbacks, s.begins_with('#')); - _line_progressed(s.len); - // skip the # character - s = s.sub(1); - // skip leading whitespace - s = s.right_of(s.first_not_of(' '), /*include_pos*/true); - _c4dbgpf("comment was '{}'", s); - return s; -} - -//----------------------------------------------------------------------------- -csubstr Parser::_scan_squot_scalar() -{ - // quoted scalars can spread over multiple lines! - // nice explanation here: http://yaml-multiline.info/ - - // a span to the end of the file - size_t b = m_state->pos.offset; - substr s = m_buf.sub(b); - if(s.begins_with(' ')) - { - s = s.triml(' '); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf.sub(b).is_super(s)); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.begin() >= m_buf.sub(b).begin()); - _line_progressed((size_t)(s.begin() - m_buf.sub(b).begin())); - } - b = m_state->pos.offset; // take this into account - _RYML_CB_ASSERT(m_stack.m_callbacks, s.begins_with('\'')); - - // skip the opening quote - _line_progressed(1); - s = s.sub(1); - - bool needs_filter = false; - - size_t numlines = 1; // we already have one line - size_t pos = npos; // find the pos of the matching quote - while( ! _finished_file()) - { - const csubstr line = m_state->line_contents.rem; - bool line_is_blank = true; - _c4dbgpf("scanning single quoted scalar @ line[{}]: ~~~{}~~~", m_state->pos.line, line); - for(size_t i = 0; i < line.len; ++i) - { - const char curr = line.str[i]; - if(curr == '\'') // single quotes are escaped with two single quotes - { - const char next = i+1 < line.len ? line.str[i+1] : '~'; - if(next != '\'') // so just look for the first quote - { // without another after it - pos = i; - break; - } - else - { - needs_filter = true; // needs filter to remove escaped quotes - ++i; // skip the escaped quote - } - } - else if(curr != ' ') - { - line_is_blank = false; - } - } - - // leading whitespace also needs filtering - needs_filter = needs_filter - || (numlines > 1) - || line_is_blank - || (_at_line_begin() && line.begins_with(' ')); - - if(pos == npos) - { - _line_progressed(line.len); - ++numlines; - } - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, pos >= 0 && pos < m_buf.len); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf[m_state->pos.offset + pos] == '\''); - _line_progressed(pos + 1); // progress beyond the quote - pos = m_state->pos.offset - b - 1; // but we stop before it - break; - } - - _line_ended(); - _scan_line(); - } - - if(pos == npos) - { - _c4err("reached end of file while looking for closing quote"); - } - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, pos > 0); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.end() >= m_buf.begin() && s.end() <= m_buf.end()); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.end() == m_buf.end() || *s.end() == '\''); - s = s.sub(0, pos-1); - } - - if(needs_filter) - { - csubstr ret = _filter_squot_scalar(s); - _RYML_CB_ASSERT(m_stack.m_callbacks, ret.len <= s.len || s.empty() || s.trim(' ').empty()); - _c4dbgpf("final scalar: \"{}\"", ret); - return ret; - } - - _c4dbgpf("final scalar: \"{}\"", s); - - return s; -} - -//----------------------------------------------------------------------------- -csubstr Parser::_scan_dquot_scalar() -{ - // quoted scalars can spread over multiple lines! - // nice explanation here: http://yaml-multiline.info/ - - // a span to the end of the file - size_t b = m_state->pos.offset; - substr s = m_buf.sub(b); - if(s.begins_with(' ')) - { - s = s.triml(' '); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf.sub(b).is_super(s)); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.begin() >= m_buf.sub(b).begin()); - _line_progressed((size_t)(s.begin() - m_buf.sub(b).begin())); - } - b = m_state->pos.offset; // take this into account - _RYML_CB_ASSERT(m_stack.m_callbacks, s.begins_with('"')); - - // skip the opening quote - _line_progressed(1); - s = s.sub(1); - - bool needs_filter = false; - - size_t numlines = 1; // we already have one line - size_t pos = npos; // find the pos of the matching quote - while( ! _finished_file()) - { - const csubstr line = m_state->line_contents.rem; - bool line_is_blank = true; - _c4dbgpf("scanning double quoted scalar @ line[{}]: line='{}'", m_state->pos.line, line); - for(size_t i = 0; i < line.len; ++i) - { - const char curr = line.str[i]; - if(curr != ' ') - line_is_blank = false; - // every \ is an escape - if(curr == '\\') - { - const char next = i+1 < line.len ? line.str[i+1] : '~'; - needs_filter = true; - if(next == '"' || next == '\\') - ++i; - } - else if(curr == '"') - { - pos = i; - break; - } - } - - // leading whitespace also needs filtering - needs_filter = needs_filter - || (numlines > 1) - || line_is_blank - || (_at_line_begin() && line.begins_with(' ')); - - if(pos == npos) - { - _line_progressed(line.len); - ++numlines; - } - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, pos >= 0 && pos < m_buf.len); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf[m_state->pos.offset + pos] == '"'); - _line_progressed(pos + 1); // progress beyond the quote - pos = m_state->pos.offset - b - 1; // but we stop before it - break; - } - - _line_ended(); - _scan_line(); - } - - if(pos == npos) - { - _c4err("reached end of file looking for closing quote"); - } - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, pos > 0); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.end() == m_buf.end() || *s.end() == '"'); - _RYML_CB_ASSERT(m_stack.m_callbacks, s.end() >= m_buf.begin() && s.end() <= m_buf.end()); - s = s.sub(0, pos-1); - } - - if(needs_filter) - { - csubstr ret = _filter_dquot_scalar(s); - _c4dbgpf("final scalar: [{}]\"{}\"", ret.len, ret); - _RYML_CB_ASSERT(m_stack.m_callbacks, ret.len <= s.len || s.empty() || s.trim(' ').empty()); - return ret; - } - - _c4dbgpf("final scalar: \"{}\"", s); - - return s; -} - -//----------------------------------------------------------------------------- -csubstr Parser::_scan_block() -{ - // nice explanation here: http://yaml-multiline.info/ - csubstr s = m_state->line_contents.rem; - csubstr trimmed = s.triml(' '); - if(trimmed.str > s.str) - { - _c4dbgp("skipping whitespace"); - _RYML_CB_ASSERT(m_stack.m_callbacks, trimmed.str >= s.str); - _line_progressed(static_cast(trimmed.str - s.str)); - s = trimmed; - } - _RYML_CB_ASSERT(m_stack.m_callbacks, s.begins_with('|') || s.begins_with('>')); - - _c4dbgpf("scanning block: specs=\"{}\"", s); - - // parse the spec - BlockStyle_e newline = s.begins_with('>') ? BLOCK_FOLD : BLOCK_LITERAL; - BlockChomp_e chomp = CHOMP_CLIP; // default to clip unless + or - are used - size_t indentation = npos; // have to find out if no spec is given - csubstr digits; - if(s.len > 1) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, s.begins_with_any("|>")); - csubstr t = s.sub(1); - _c4dbgpf("scanning block: spec is multichar: '{}'", t); - _RYML_CB_ASSERT(m_stack.m_callbacks, t.len >= 1); - size_t pos = t.first_of("-+"); - _c4dbgpf("scanning block: spec chomp char at {}", pos); - if(pos != npos) - { - if(t[pos] == '-') - chomp = CHOMP_STRIP; - else if(t[pos] == '+') - chomp = CHOMP_KEEP; - if(pos == 0) - t = t.sub(1); - else - t = t.first(pos); - } - // from here to the end, only digits are considered - digits = t.left_of(t.first_not_of("0123456789")); - if( ! digits.empty()) - { - if( ! c4::atou(digits, &indentation)) - _c4err("parse error: could not read decimal"); - _c4dbgpf("scanning block: indentation specified: {}. add {} from curr state -> {}", indentation, m_state->indref, indentation+m_state->indref); - indentation += m_state->indref; - } - } - - // finish the current line - _line_progressed(s.len); - _line_ended(); - _scan_line(); - - _c4dbgpf("scanning block: style={} chomp={} indentation={}", newline==BLOCK_FOLD ? "fold" : "literal", chomp==CHOMP_CLIP ? "clip" : (chomp==CHOMP_STRIP ? "strip" : "keep"), indentation); - - // start with a zero-length block, already pointing at the right place - substr raw_block(m_buf.data() + m_state->pos.offset, size_t(0));// m_state->line_contents.full.sub(0, 0); - _RYML_CB_ASSERT(m_stack.m_callbacks, raw_block.begin() == m_state->line_contents.full.begin()); - - // read every full line into a raw block, - // from which newlines are to be stripped as needed. - // - // If no explicit indentation was given, pick it from the first - // non-empty line. See - // https://yaml.org/spec/1.2.2/#8111-block-indentation-indicator - size_t num_lines = 0, first = m_state->pos.line, provisional_indentation = npos; - LineContents lc; - while(( ! _finished_file())) - { - // peek next line, but do not advance immediately - lc.reset_with_next_line(m_buf, m_state->pos.offset); - _c4dbgpf("scanning block: peeking at '{}'", lc.stripped); - // evaluate termination conditions - if(indentation != npos) - { - // stop when the line is deindented and not empty - if(lc.indentation < indentation && ( ! lc.rem.trim(" \t\r\n").empty())) - { - _c4dbgpf("scanning block: indentation decreased ref={} thisline={}", indentation, lc.indentation); - break; - } - else if(indentation == 0) - { - if((lc.rem == "..." || lc.rem.begins_with("... ")) - || - (lc.rem == "---" || lc.rem.begins_with("--- "))) - { - _c4dbgp("scanning block: stop. indentation=0 and stream ended"); - break; - } - } - } - else - { - _c4dbgpf("scanning block: indentation ref not set. firstnonws={}", lc.stripped.first_not_of(' ')); - if(lc.stripped.first_not_of(' ') != npos) // non-empty line - { - _c4dbgpf("scanning block: line not empty. indref={} indprov={} indentation={}", m_state->indref, provisional_indentation, lc.indentation); - if(provisional_indentation == npos) - { - if(lc.indentation < m_state->indref) - { - _c4dbgpf("scanning block: block terminated indentation={} < indref={}", lc.indentation, m_state->indref); - if(raw_block.len == 0) - { - _c4dbgp("scanning block: was empty, undo next line"); - _line_ended_undo(); - } - break; - } - else if(lc.indentation == m_state->indref) - { - if(has_any(RSEQ|RMAP)) - { - _c4dbgpf("scanning block: block terminated. reading container and indentation={}==indref={}", lc.indentation, m_state->indref); - break; - } - } - _c4dbgpf("scanning block: set indentation ref from this line: ref={}", lc.indentation); - indentation = lc.indentation; - } - else - { - if(lc.indentation >= provisional_indentation) - { - _c4dbgpf("scanning block: set indentation ref from provisional indentation: provisional_ref={}, thisline={}", provisional_indentation, lc.indentation); - //indentation = provisional_indentation ? provisional_indentation : lc.indentation; - indentation = lc.indentation; - } - else - { - break; - //_c4err("parse error: first non-empty block line should have at least the original indentation"); - } - } - } - else // empty line - { - _c4dbgpf("scanning block: line empty or {} spaces. line_indentation={} prov_indentation={}", lc.stripped.len, lc.indentation, provisional_indentation); - if(provisional_indentation != npos) - { - if(lc.stripped.len >= provisional_indentation) - { - _c4dbgpf("scanning block: increase provisional_ref {} -> {}", provisional_indentation, lc.stripped.len); - provisional_indentation = lc.stripped.len; - } - #ifdef RYML_NO_COVERAGE__TO_BE_DELETED - else if(lc.indentation >= provisional_indentation && lc.indentation != npos) - { - _c4dbgpf("scanning block: increase provisional_ref {} -> {}", provisional_indentation, lc.indentation); - provisional_indentation = lc.indentation; - } - #endif - } - else - { - provisional_indentation = lc.indentation ? lc.indentation : has_any(RSEQ|RVAL); - _c4dbgpf("scanning block: initialize provisional_ref={}", provisional_indentation); - if(provisional_indentation == npos) - { - provisional_indentation = lc.stripped.len ? lc.stripped.len : has_any(RSEQ|RVAL); - _c4dbgpf("scanning block: initialize provisional_ref={}", provisional_indentation); - } - } - } - } - // advance now that we know the folded scalar continues - m_state->line_contents = lc; - _c4dbgpf("scanning block: append '{}'", m_state->line_contents.rem); - raw_block.len += m_state->line_contents.full.len; - _line_progressed(m_state->line_contents.rem.len); - _line_ended(); - ++num_lines; - } - _RYML_CB_ASSERT(m_stack.m_callbacks, m_state->pos.line == (first + num_lines) || (raw_block.len == 0)); - C4_UNUSED(num_lines); - C4_UNUSED(first); - - if(indentation == npos) - { - _c4dbgpf("scanning block: set indentation from provisional: {}", provisional_indentation); - indentation = provisional_indentation; - } - - if(num_lines) - _line_ended_undo(); - - _c4dbgpf("scanning block: raw=~~~{}~~~", raw_block); - - // ok! now we strip the newlines and spaces according to the specs - s = _filter_block_scalar(raw_block, newline, chomp, indentation); - - _c4dbgpf("scanning block: final=~~~{}~~~", s); - - return s; -} - - -//----------------------------------------------------------------------------- - -template -bool Parser::_filter_nl(substr r, size_t *C4_RESTRICT i, size_t *C4_RESTRICT pos, size_t indentation) -{ - // a debugging scaffold: - #if 0 - #define _c4dbgfnl(fmt, ...) _c4dbgpf("filter_nl[{}]: " fmt, *i, __VA_ARGS__) - #else - #define _c4dbgfnl(...) - #endif - - const char curr = r[*i]; - bool replaced = false; - - _RYML_CB_ASSERT(m_stack.m_callbacks, indentation != npos); - _RYML_CB_ASSERT(m_stack.m_callbacks, curr == '\n'); - - _c4dbgfnl("found newline. sofar=[{}]~~~{}~~~", *pos, m_filter_arena.first(*pos)); - size_t ii = *i; - size_t numnl_following = count_following_newlines(r, &ii, indentation); - if(numnl_following) - { - _c4dbgfnl("{} consecutive (empty) lines {} in the middle. totalws={}", 1+numnl_following, ii < r.len ? "in the middle" : "at the end", ii - *i); - for(size_t j = 0; j < numnl_following; ++j) - m_filter_arena.str[(*pos)++] = '\n'; - } - else - { - if(r.first_not_of(" \t", *i+1) != npos) - { - m_filter_arena.str[(*pos)++] = ' '; - _c4dbgfnl("single newline. convert to space. ii={}/{}. sofar=[{}]~~~{}~~~", ii, r.len, *pos, m_filter_arena.first(*pos)); - replaced = true; - } - else - { - if C4_IF_CONSTEXPR (keep_trailing_whitespace) - { - m_filter_arena.str[(*pos)++] = ' '; - _c4dbgfnl("single newline. convert to space. ii={}/{}. sofar=[{}]~~~{}~~~", ii, r.len, *pos, m_filter_arena.first(*pos)); - replaced = true; - } - else - { - _c4dbgfnl("last newline, everything else is whitespace. ii={}/{}", ii, r.len); - *i = r.len; - } - } - if C4_IF_CONSTEXPR (backslash_is_escape) - { - if(ii < r.len && r.str[ii] == '\\') - { - const char next = ii+1 < r.len ? r.str[ii+1] : '\0'; - if(next == ' ' || next == '\t') - { - _c4dbgfnl("extend skip to backslash{}", ""); - ++ii; - } - } - } - } - *i = ii - 1; // correct for the loop increment - - #undef _c4dbgfnl - - return replaced; -} - - -//----------------------------------------------------------------------------- - -template -void Parser::_filter_ws(substr r, size_t *C4_RESTRICT i, size_t *C4_RESTRICT pos) -{ - // a debugging scaffold: - #if 0 - #define _c4dbgfws(fmt, ...) _c4dbgpf("filt_nl[{}]: " fmt, *i, __VA_ARGS__) - #else - #define _c4dbgfws(...) - #endif - - const char curr = r[*i]; - _c4dbgfws("found whitespace '{}'", _c4prc(curr)); - _RYML_CB_ASSERT(m_stack.m_callbacks, curr == ' ' || curr == '\t'); - - size_t first = *i > 0 ? r.first_not_of(" \t", *i) : r.first_not_of(' ', *i); - if(first != npos) - { - if(r[first] == '\n' || r[first] == '\r') // skip trailing whitespace - { - _c4dbgfws("whitespace is trailing on line. firstnonws='{}'@{}", _c4prc(r[first]), first); - *i = first - 1; // correct for the loop increment - } - else // a legit whitespace - { - m_filter_arena.str[(*pos)++] = curr; - _c4dbgfws("legit whitespace. sofar=[{}]~~~{}~~~", *pos, m_filter_arena.first(*pos)); - } - } - else - { - _c4dbgfws("... everything else is trailing whitespace{}", ""); - if C4_IF_CONSTEXPR (keep_trailing_whitespace) - for(size_t j = *i; j < r.len; ++j) - m_filter_arena.str[(*pos)++] = r[j]; - *i = r.len; - } - - #undef _c4dbgfws -} - - -//----------------------------------------------------------------------------- -csubstr Parser::_filter_plain_scalar(substr s, size_t indentation) -{ - // a debugging scaffold: - #if 0 - #define _c4dbgfps(...) _c4dbgpf("filt_plain_scalar" __VA_ARGS__) - #else - #define _c4dbgfps(...) - #endif - - _c4dbgfps("before=~~~{}~~~", s); - - substr r = s.triml(" \t"); - _grow_filter_arena(r.len); - size_t pos = 0; // the filtered size - bool filtered_chars = false; - for(size_t i = 0; i < r.len; ++i) - { - const char curr = r.str[i]; - _c4dbgfps("[{}]: '{}'", i, _c4prc(curr)); - if(curr == ' ' || curr == '\t') - { - _filter_ws(r, &i, &pos); - } - else if(curr == '\n') - { - filtered_chars = _filter_nl(r, &i, &pos, indentation); - } - else if(curr == '\r') // skip \r --- https://stackoverflow.com/questions/1885900 - { - ; - } - else - { - m_filter_arena.str[pos++] = r[i]; - } - } - - _RYML_CB_ASSERT(m_stack.m_callbacks, pos <= m_filter_arena.len); - if(pos < r.len || filtered_chars) - { - r = _finish_filter_arena(r, pos); - } - - _RYML_CB_ASSERT(m_stack.m_callbacks, s.len >= r.len); - _c4dbgfps("#filteredchars={} after=~~~{}~~~", s.len - r.len, r); - - #undef _c4dbgfps - return r; -} - - -//----------------------------------------------------------------------------- -csubstr Parser::_filter_squot_scalar(substr s) -{ - // a debugging scaffold: - #if 0 - #define _c4dbgfsq(...) _c4dbgpf("filt_squo_scalar") - #else - #define _c4dbgfsq(...) - #endif - - // from the YAML spec for double-quoted scalars: - // https://yaml.org/spec/1.2-old/spec.html#style/flow/single-quoted - - _c4dbgfsq(": before=~~~{}~~~", s); - - _grow_filter_arena(s.len); - substr r = s; - size_t pos = 0; // the filtered size - bool filtered_chars = false; - for(size_t i = 0; i < r.len; ++i) - { - const char curr = r[i]; - _c4dbgfsq("[{}]: '{}'", i, _c4prc(curr)); - if(curr == ' ' || curr == '\t') - { - _filter_ws(r, &i, &pos); - } - else if(curr == '\n') - { - filtered_chars = _filter_nl(r, &i, &pos, /*indentation*/0); - } - else if(curr == '\r') // skip \r --- https://stackoverflow.com/questions/1885900 - { - ; - } - else if(curr == '\'') - { - char next = i+1 < r.len ? r[i+1] : '\0'; - if(next == '\'') - { - _c4dbgfsq("[{}]: two consecutive quotes", i); - filtered_chars = true; - m_filter_arena.str[pos++] = '\''; - ++i; - } - } - else - { - m_filter_arena.str[pos++] = curr; - } - } - - _RYML_CB_ASSERT(m_stack.m_callbacks, pos <= m_filter_arena.len); - if(pos < r.len || filtered_chars) - { - r = _finish_filter_arena(r, pos); - } - - _RYML_CB_ASSERT(m_stack.m_callbacks, s.len >= r.len); - _c4dbgpf(": #filteredchars={} after=~~~{}~~~", s.len - r.len, r); - - #undef _c4dbgfsq - return r; -} - - -//----------------------------------------------------------------------------- -csubstr Parser::_filter_dquot_scalar(substr s) -{ - // a debugging scaffold: - #if 0 - #define _c4dbgfdq(...) _c4dbgpf("filt_dquo_scalar" __VA_ARGS__) - #else - #define _c4dbgfdq(...) - #endif - - _c4dbgfdq(": before=~~~{}~~~", s); - - // from the YAML spec for double-quoted scalars: - // https://yaml.org/spec/1.2-old/spec.html#style/flow/double-quoted - // - // All leading and trailing white space characters are excluded - // from the content. Each continuation line must therefore contain - // at least one non-space character. Empty lines, if any, are - // consumed as part of the line folding. - - _grow_filter_arena(s.len + 2u * s.count('\\')); - substr r = s; - size_t pos = 0; // the filtered size - bool filtered_chars = false; - for(size_t i = 0; i < r.len; ++i) - { - const char curr = r[i]; - _c4dbgfdq("[{}]: '{}'", i, _c4prc(curr)); - if(curr == ' ' || curr == '\t') - { - _filter_ws(r, &i, &pos); - } - else if(curr == '\n') - { - filtered_chars = _filter_nl(r, &i, &pos, /*indentation*/0); - } - else if(curr == '\r') // skip \r --- https://stackoverflow.com/questions/1885900 - { - ; - } - else if(curr == '\\') - { - char next = i+1 < r.len ? r[i+1] : '\0'; - _c4dbgfdq("[{}]: backslash, next='{}'", i, _c4prc(next)); - filtered_chars = true; - if(next == '\r') - { - if(i+2 < r.len && r[i+2] == '\n') - { - ++i; // newline escaped with \ -- skip both (add only one as i is loop-incremented) - next = '\n'; - _c4dbgfdq("[{}]: was \\r\\n, now next='\\n'", i); - } - } - // remember the loop will also increment i - if(next == '\n') - { - size_t ii = i + 2; - for( ; ii < r.len; ++ii) - { - if(r.str[ii] == ' ' || r.str[ii] == '\t') // skip leading whitespace - ; - else - break; - } - i += ii - i - 1; - } - else if(next == '"' || next == '/' || next == ' ' || next == '\t') // escapes for json compatibility - { - m_filter_arena.str[pos++] = next; - ++i; - } - else if(next == '\r') - { - //++i; - } - else if(next == 'n') - { - m_filter_arena.str[pos++] = '\n'; - ++i; - } - else if(next == 'r') - { - m_filter_arena.str[pos++] = '\r'; - ++i; // skip - } - else if(next == 't') - { - m_filter_arena.str[pos++] = '\t'; - ++i; - } - else if(next == '\\') - { - m_filter_arena.str[pos++] = '\\'; - ++i; - } - else if(next == 'x') // UTF8 - { - if(i + 1u + 2u >= r.len) - _c4err("\\x requires 2 hex digits"); - uint8_t byteval = {}; - if(!read_hex(r.sub(i + 2u, 2u), &byteval)) - _c4err("failed to read \\x codepoint"); - m_filter_arena.str[pos++] = *(char*)&byteval; - i += 1u + 2u; - } - else if(next == 'u') // UTF16 - { - if(i + 1u + 4u >= r.len) - _c4err("\\u requires 4 hex digits"); - char readbuf[8]; - csubstr codepoint = r.sub(i + 2u, 4u); - uint32_t codepoint_val = {}; - if(!read_hex(codepoint, &codepoint_val)) - _c4err("failed to parse \\u codepoint"); - size_t numbytes = decode_code_point((uint8_t*)readbuf, sizeof(readbuf), codepoint_val); - C4_ASSERT(numbytes <= 4); - memcpy(m_filter_arena.str + pos, readbuf, numbytes); - pos += numbytes; - i += 1u + 4u; - } - else if(next == 'U') // UTF32 - { - if(i + 1u + 8u >= r.len) - _c4err("\\U requires 8 hex digits"); - char readbuf[8]; - csubstr codepoint = r.sub(i + 2u, 8u); - uint32_t codepoint_val = {}; - if(!read_hex(codepoint, &codepoint_val)) - _c4err("failed to parse \\U codepoint"); - size_t numbytes = decode_code_point((uint8_t*)readbuf, sizeof(readbuf), codepoint_val); - C4_ASSERT(numbytes <= 4); - memcpy(m_filter_arena.str + pos, readbuf, numbytes); - pos += numbytes; - i += 1u + 8u; - } - // https://yaml.org/spec/1.2.2/#rule-c-ns-esc-char - else if(next == '0') - { - m_filter_arena.str[pos++] = '\0'; - ++i; - } - else if(next == 'b') // backspace - { - m_filter_arena.str[pos++] = '\b'; - ++i; - } - else if(next == 'f') // form feed - { - m_filter_arena.str[pos++] = '\f'; - ++i; - } - else if(next == 'a') // bell character - { - m_filter_arena.str[pos++] = '\a'; - ++i; - } - else if(next == 'v') // vertical tab - { - m_filter_arena.str[pos++] = '\v'; - ++i; - } - else if(next == 'e') // escape character - { - m_filter_arena.str[pos++] = '\x1b'; - ++i; - } - else if(next == '_') // unicode non breaking space \u00a0 - { - // https://www.compart.com/en/unicode/U+00a0 - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x3e, 0xc2); - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x60, 0xa0); - ++i; - } - else if(next == 'N') // unicode next line \u0085 - { - // https://www.compart.com/en/unicode/U+0085 - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x3e, 0xc2); - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x7b, 0x85); - ++i; - } - else if(next == 'L') // unicode line separator \u2028 - { - // https://www.utf8-chartable.de/unicode-utf8-table.pl?start=8192&number=1024&names=-&utf8=0x&unicodeinhtml=hex - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x1e, 0xe2); - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x80, 0x80); - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x58, 0xa8); - ++i; - } - else if(next == 'P') // unicode paragraph separator \u2029 - { - // https://www.utf8-chartable.de/unicode-utf8-table.pl?start=8192&number=1024&names=-&utf8=0x&unicodeinhtml=hex - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x1e, 0xe2); - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x80, 0x80); - m_filter_arena.str[pos++] = _RYML_CHCONST(-0x57, 0xa9); - ++i; - } - _c4dbgfdq("[{}]: backslash...sofar=[{}]~~~{}~~~", i, pos, m_filter_arena.first(pos)); - } - else - { - m_filter_arena.str[pos++] = curr; - } - } - - _RYML_CB_ASSERT(m_stack.m_callbacks, pos <= m_filter_arena.len); - if(pos < r.len || filtered_chars) - { - r = _finish_filter_arena(r, pos); - } - - _RYML_CB_ASSERT(m_stack.m_callbacks, s.len >= r.len); - _c4dbgpf(": #filteredchars={} after=~~~{}~~~", s.len - r.len, r); - - #undef _c4dbgfdq - - return r; -} - - -//----------------------------------------------------------------------------- -bool Parser::_apply_chomp(substr buf, size_t *C4_RESTRICT pos, BlockChomp_e chomp) -{ - substr trimmed = buf.first(*pos).trimr('\n'); - bool added_newline = false; - switch(chomp) - { - case CHOMP_KEEP: - if(trimmed.len == *pos) - { - _c4dbgpf("chomp=KEEP: add missing newline @{}", *pos); - //m_filter_arena.str[(*pos)++] = '\n'; - added_newline = true; - } - break; - case CHOMP_CLIP: - if(trimmed.len == *pos) - { - _c4dbgpf("chomp=CLIP: add missing newline @{}", *pos); - m_filter_arena.str[(*pos)++] = '\n'; - added_newline = true; - } - else - { - _c4dbgpf("chomp=CLIP: include single trailing newline @{}", trimmed.len+1); - *pos = trimmed.len + 1; - } - break; - case CHOMP_STRIP: - _c4dbgpf("chomp=STRIP: strip {}-{}-{} newlines", *pos, trimmed.len, *pos-trimmed.len); - *pos = trimmed.len; - break; - default: - _c4err("unknown chomp style"); - } - return added_newline; -} - - -//----------------------------------------------------------------------------- -csubstr Parser::_filter_block_scalar(substr s, BlockStyle_e style, BlockChomp_e chomp, size_t indentation) -{ - // a debugging scaffold: - #if 0 - #define _c4dbgfbl(fmt, ...) _c4dbgpf("filt_block" fmt, __VA_ARGS__) - #else - #define _c4dbgfbl(...) - #endif - - _c4dbgfbl(": indentation={} before=[{}]~~~{}~~~", indentation, s.len, s); - - if(chomp != CHOMP_KEEP && s.trim(" \n\r").len == 0u) - { - _c4dbgp("filt_block: empty scalar"); - return s.first(0); - } - - substr r = s; - - switch(style) - { - case BLOCK_LITERAL: - { - _c4dbgp("filt_block: style=literal"); - // trim leading whitespace up to indentation - { - size_t numws = r.first_not_of(' '); - if(numws != npos) - { - if(numws > indentation) - r = r.sub(indentation); - else - r = r.sub(numws); - _c4dbgfbl(": after triml=[{}]~~~{}~~~", r.len, r); - } - else - { - if(chomp != CHOMP_KEEP || r.len == 0) - { - _c4dbgfbl(": all spaces {}, return empty", r.len); - return r.first(0); - } - else - { - r[0] = '\n'; - return r.first(1); - } - } - } - _grow_filter_arena(s.len + 2u); // use s.len! because we may need to add a newline at the end, so the leading indentation will allow space for that newline - size_t pos = 0; // the filtered size - for(size_t i = 0; i < r.len; ++i) - { - const char curr = r.str[i]; - _c4dbgfbl("[{}]='{}' pos={}", i, _c4prc(curr), pos); - if(curr == '\r') - continue; - m_filter_arena.str[pos++] = curr; - if(curr == '\n') - { - _c4dbgfbl("[{}]: found newline", i); - // skip indentation on the next line - csubstr rem = r.sub(i+1); - size_t first = rem.first_not_of(' '); - if(first != npos) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, first < rem.len); - _RYML_CB_ASSERT(m_stack.m_callbacks, i+1+first < r.len); - _c4dbgfbl("[{}]: {} spaces follow before next nonws character @ [{}]='{}'", i, first, i+1+first, rem.str[first]); - if(first < indentation) - { - _c4dbgfbl("[{}]: skip {}<{} spaces from indentation", i, first, indentation); - i += first; - } - else - { - _c4dbgfbl("[{}]: skip {} spaces from indentation", i, indentation); - i += indentation; - } - } - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, i+1 <= r.len); - first = rem.len; - _c4dbgfbl("[{}]: {} spaces to the end", i, first); - if(first) - { - if(first < indentation) - { - _c4dbgfbl("[{}]: skip everything", i); - --pos; - break; - } - else - { - _c4dbgfbl("[{}]: skip {} spaces from indentation", i, indentation); - i += indentation; - } - } - else if(i+1 == r.len) - { - if(chomp == CHOMP_STRIP) - --pos; - break; - } - } - } - } - _RYML_CB_ASSERT(m_stack.m_callbacks, s.len >= pos); - _c4dbgfbl(": #filteredchars={} after=~~~{}~~~", s.len - r.len, r); - bool changed = _apply_chomp(m_filter_arena, &pos, chomp); - _RYML_CB_ASSERT(m_stack.m_callbacks, pos <= m_filter_arena.len); - _RYML_CB_ASSERT(m_stack.m_callbacks, pos <= s.len); - if(pos < r.len || changed) - { - r = _finish_filter_arena(s, pos); // write into s - } - break; - } - case BLOCK_FOLD: - { - _c4dbgp("filt_block: style=fold"); - _grow_filter_arena(r.len + 2); - size_t pos = 0; // the filtered size - bool filtered_chars = false; - bool started = false; - bool is_indented = false; - size_t i = r.first_not_of(' '); - _c4dbgfbl(": first non space at {}", i); - if(i > indentation) - { - is_indented = true; - i = indentation; - } - _c4dbgfbl(": start folding at {}, is_indented={}", i, (int)is_indented); - auto on_change_indentation = [&](size_t numnl_following, size_t last_newl, size_t first_non_whitespace){ - _c4dbgfbl("[{}]: add 1+{} newlines", i, numnl_following); - for(size_t j = 0; j < 1 + numnl_following; ++j) - m_filter_arena.str[pos++] = '\n'; - for(i = last_newl + 1 + indentation; i < first_non_whitespace; ++i) - { - if(r.str[i] == '\r') - continue; - _c4dbgfbl("[{}]: add '{}'", i, _c4prc(r.str[i])); - m_filter_arena.str[pos++] = r.str[i]; - } - --i; - }; - for( ; i < r.len; ++i) - { - const char curr = r.str[i]; - _c4dbgfbl("[{}]='{}'", i, _c4prc(curr)); - if(curr == '\n') - { - filtered_chars = true; - // skip indentation on the next line, and advance over the next non-indented blank lines as well - size_t first_non_whitespace; - size_t numnl_following = (size_t)-1; - while(r[i] == '\n') - { - ++numnl_following; - csubstr rem = r.sub(i+1); - size_t first = rem.first_not_of(' '); - _c4dbgfbl("[{}]: found newline. first={} rem.len={}", i, first, rem.len); - if(first != npos) - { - first_non_whitespace = first + i+1; - while(first_non_whitespace < r.len && r[first_non_whitespace] == '\r') - ++first_non_whitespace; - _RYML_CB_ASSERT(m_stack.m_callbacks, first < rem.len); - _RYML_CB_ASSERT(m_stack.m_callbacks, i+1+first < r.len); - _c4dbgfbl("[{}]: {} spaces follow before next nonws character @ [{}]='{}'", i, first, i+1+first, _c4prc(rem.str[first])); - if(first < indentation) - { - _c4dbgfbl("[{}]: skip {}<{} spaces from indentation", i, first, indentation); - i += first; - } - else - { - _c4dbgfbl("[{}]: skip {} spaces from indentation", i, indentation); - i += indentation; - if(first > indentation) - { - _c4dbgfbl("[{}]: {} further indented than {}, stop newlining", i, first, indentation); - goto finished_counting_newlines; - } - } - // prepare the next while loop iteration - // by setting i at the next newline after - // an empty line - if(r[first_non_whitespace] == '\n') - i = first_non_whitespace; - else - goto finished_counting_newlines; - } - else - { - _RYML_CB_ASSERT(m_stack.m_callbacks, i+1 <= r.len); - first = rem.len; - first_non_whitespace = first + i+1; - if(first) - { - _c4dbgfbl("[{}]: {} spaces to the end", i, first); - if(first < indentation) - { - _c4dbgfbl("[{}]: skip everything", i); - i += first; - } - else - { - _c4dbgfbl("[{}]: skip {} spaces from indentation", i, indentation); - i += indentation; - if(first > indentation) - { - _c4dbgfbl("[{}]: {} spaces missing. not done yet", i, indentation - first); - goto finished_counting_newlines; - } - } - } - else // if(i+1 == r.len) - { - _c4dbgfbl("[{}]: it's the final newline", i); - _RYML_CB_ASSERT(m_stack.m_callbacks, i+1 == r.len); - _RYML_CB_ASSERT(m_stack.m_callbacks, rem.len == 0); - } - goto end_of_scalar; - } - } - end_of_scalar: - // Write all the trailing newlines. Since we're - // at the end no folding is needed, so write every - // newline (add 1). - _c4dbgfbl("[{}]: add {} trailing newlines", i, 1+numnl_following); - for(size_t j = 0; j < 1 + numnl_following; ++j) - m_filter_arena.str[pos++] = '\n'; - break; - finished_counting_newlines: - _c4dbgfbl("[{}]: #newlines={} firstnonws={}", i, numnl_following, first_non_whitespace); - while(first_non_whitespace < r.len && r[first_non_whitespace] == '\t') - ++first_non_whitespace; - _c4dbgfbl("[{}]: #newlines={} firstnonws={}", i, numnl_following, first_non_whitespace); - _RYML_CB_ASSERT(m_stack.m_callbacks, first_non_whitespace <= r.len); - size_t last_newl = r.last_of('\n', first_non_whitespace); - size_t this_indentation = first_non_whitespace - last_newl - 1; - _c4dbgfbl("[{}]: #newlines={} firstnonws={} lastnewl={} this_indentation={} vs indentation={}", i, numnl_following, first_non_whitespace, last_newl, this_indentation, indentation); - _RYML_CB_ASSERT(m_stack.m_callbacks, first_non_whitespace >= last_newl + 1); - _RYML_CB_ASSERT(m_stack.m_callbacks, this_indentation >= indentation); - if(!started) - { - _c4dbgfbl("[{}]: #newlines={}. write all leading newlines", i, numnl_following); - for(size_t j = 0; j < 1 + numnl_following; ++j) - m_filter_arena.str[pos++] = '\n'; - if(this_indentation > indentation) - { - is_indented = true; - _c4dbgfbl("[{}]: advance ->{}", i, last_newl + indentation); - i = last_newl + indentation; - } - else - { - i = first_non_whitespace - 1; - _c4dbgfbl("[{}]: advance ->{}", i, first_non_whitespace); - } - } - else if(this_indentation == indentation) - { - _c4dbgfbl("[{}]: same indentation", i); - if(!is_indented) - { - if(numnl_following == 0) - { - _c4dbgfbl("[{}]: fold!", i); - m_filter_arena.str[pos++] = ' '; - } - else - { - _c4dbgfbl("[{}]: add {} newlines", i, 1 + numnl_following); - for(size_t j = 0; j < numnl_following; ++j) - m_filter_arena.str[pos++] = '\n'; - } - i = first_non_whitespace - 1; - _c4dbgfbl("[{}]: advance {}->{}", i, i, first_non_whitespace); - } - else - { - _c4dbgfbl("[{}]: back to ref indentation", i); - is_indented = false; - on_change_indentation(numnl_following, last_newl, first_non_whitespace); - _c4dbgfbl("[{}]: advance {}->{}", i, i, first_non_whitespace); - } - } - else - { - _c4dbgfbl("[{}]: increased indentation.", i); - is_indented = true; - _RYML_CB_ASSERT(m_stack.m_callbacks, this_indentation > indentation); - on_change_indentation(numnl_following, last_newl, first_non_whitespace); - _c4dbgfbl("[{}]: advance {}->{}", i, i, first_non_whitespace); - } - } - else if(curr != '\r') - { - if(curr != '\t') - started = true; - m_filter_arena.str[pos++] = curr; - } - } - _RYML_CB_ASSERT(m_stack.m_callbacks, pos <= m_filter_arena.len); - _c4dbgfbl(": #filteredchars={} after=[{}]~~~{}~~~", (int)s.len - (int)pos, pos, m_filter_arena.first(pos)); - bool changed = _apply_chomp(m_filter_arena, &pos, chomp); - if(pos < r.len || filtered_chars || changed) - { - r = _finish_filter_arena(s, pos); // write into s - } - } - break; - default: - _c4err("unknown block style"); - } - - _c4dbgfbl(": final=[{}]~~~{}~~~", r.len, r); - - #undef _c4dbgfbl - - return r; -} - -//----------------------------------------------------------------------------- -size_t Parser::_count_nlines(csubstr src) -{ - return 1 + src.count('\n'); -} - -//----------------------------------------------------------------------------- -void Parser::_handle_directive(csubstr directive_) -{ - csubstr directive = directive_; - if(directive.begins_with("%TAG")) - { - TagDirective td; - _c4dbgpf("%TAG directive: {}", directive_); - directive = directive.sub(4); - if(!directive.begins_with(' ')) - _c4err("malformed tag directive: {}", directive_); - directive = directive.triml(' '); - size_t pos = directive.find(' '); - if(pos == npos) - _c4err("malformed tag directive: {}", directive_); - td.handle = directive.first(pos); - directive = directive.sub(td.handle.len).triml(' '); - pos = directive.find(' '); - if(pos != npos) - directive = directive.first(pos); - td.prefix = directive; - td.next_node_id = m_tree->size(); - if(m_tree->size() > 0) - { - size_t prev = m_tree->size() - 1; - if(m_tree->is_root(prev) && m_tree->type(prev) != NOTYPE && !m_tree->is_stream(prev)) - ++td.next_node_id; - } - _c4dbgpf("%TAG: handle={} prefix={} next_node={}", td.handle, td.prefix, td.next_node_id); - m_tree->add_tag_directive(td); - } - else if(directive.begins_with("%YAML")) - { - _c4dbgpf("%YAML directive! ignoring...: {}", directive); - } -} - -//----------------------------------------------------------------------------- -void Parser::set_flags(flag_t f, State * s) -{ -#ifdef RYML_DBG - char buf1_[64], buf2_[64]; - csubstr buf1 = _prfl(buf1_, f); - csubstr buf2 = _prfl(buf2_, s->flags); - _c4dbgpf("state[{}]: setting flags to {}: before={}", s-m_stack.begin(), buf1, buf2); -#endif - s->flags = f; -} - -void Parser::add_flags(flag_t on, State * s) -{ -#ifdef RYML_DBG - char buf1_[64], buf2_[64], buf3_[64]; - csubstr buf1 = _prfl(buf1_, on); - csubstr buf2 = _prfl(buf2_, s->flags); - csubstr buf3 = _prfl(buf3_, s->flags|on); - _c4dbgpf("state[{}]: adding flags {}: before={} after={}", s-m_stack.begin(), buf1, buf2, buf3); -#endif - s->flags |= on; -} - -void Parser::addrem_flags(flag_t on, flag_t off, State * s) -{ -#ifdef RYML_DBG - char buf1_[64], buf2_[64], buf3_[64], buf4_[64]; - csubstr buf1 = _prfl(buf1_, on); - csubstr buf2 = _prfl(buf2_, off); - csubstr buf3 = _prfl(buf3_, s->flags); - csubstr buf4 = _prfl(buf4_, ((s->flags|on)&(~off))); - _c4dbgpf("state[{}]: adding flags {} / removing flags {}: before={} after={}", s-m_stack.begin(), buf1, buf2, buf3, buf4); -#endif - s->flags |= on; - s->flags &= ~off; -} - -void Parser::rem_flags(flag_t off, State * s) -{ -#ifdef RYML_DBG - char buf1_[64], buf2_[64], buf3_[64]; - csubstr buf1 = _prfl(buf1_, off); - csubstr buf2 = _prfl(buf2_, s->flags); - csubstr buf3 = _prfl(buf3_, s->flags&(~off)); - _c4dbgpf("state[{}]: removing flags {}: before={} after={}", s-m_stack.begin(), buf1, buf2, buf3); -#endif - s->flags &= ~off; -} - -//----------------------------------------------------------------------------- - -csubstr Parser::_prfl(substr buf, flag_t flags) -{ - size_t pos = 0; - bool gotone = false; - - #define _prflag(fl) \ - if((flags & fl) == (fl)) \ - { \ - if(gotone) \ - { \ - if(pos + 1 < buf.len) \ - buf[pos] = '|'; \ - ++pos; \ - } \ - csubstr fltxt = #fl; \ - if(pos + fltxt.len <= buf.len) \ - memcpy(buf.str + pos, fltxt.str, fltxt.len); \ - pos += fltxt.len; \ - gotone = true; \ - } - - _prflag(RTOP); - _prflag(RUNK); - _prflag(RMAP); - _prflag(RSEQ); - _prflag(FLOW); - _prflag(QMRK); - _prflag(RKEY); - _prflag(RVAL); - _prflag(RNXT); - _prflag(SSCL); - _prflag(QSCL); - _prflag(RSET); - _prflag(NDOC); - _prflag(RSEQIMAP); - - #undef _prflag - - RYML_ASSERT(pos <= buf.len); - - return buf.first(pos); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void Parser::_grow_filter_arena(size_t num_characters_needed) -{ - _c4dbgpf("grow: arena={} numchars={}", m_filter_arena.len, num_characters_needed); - if(num_characters_needed <= m_filter_arena.len) - return; - size_t sz = m_filter_arena.len << 1; - _c4dbgpf("grow: sz={}", sz); - sz = num_characters_needed > sz ? num_characters_needed : sz; - _c4dbgpf("grow: sz={}", sz); - sz = sz < 128u ? 128u : sz; - _c4dbgpf("grow: sz={}", sz); - _RYML_CB_ASSERT(m_stack.m_callbacks, sz >= num_characters_needed); - _resize_filter_arena(sz); -} - -void Parser::_resize_filter_arena(size_t num_characters) -{ - if(num_characters > m_filter_arena.len) - { - _c4dbgpf("resize: sz={}", num_characters); - char *prev = m_filter_arena.str; - if(m_filter_arena.str) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, m_filter_arena.len > 0); - _RYML_CB_FREE(m_stack.m_callbacks, m_filter_arena.str, char, m_filter_arena.len); - } - m_filter_arena.str = _RYML_CB_ALLOC_HINT(m_stack.m_callbacks, char, num_characters, prev); - m_filter_arena.len = num_characters; - } -} - -substr Parser::_finish_filter_arena(substr dst, size_t pos) -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, pos <= m_filter_arena.len); - _RYML_CB_ASSERT(m_stack.m_callbacks, pos <= dst.len); - memcpy(dst.str, m_filter_arena.str, pos); - return dst.first(pos); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -csubstr Parser::location_contents(Location const& loc) const -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, loc.offset < m_buf.len); - return m_buf.sub(loc.offset); -} - -Location Parser::location(ConstNodeRef node) const -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, node.valid()); - return location(*node.tree(), node.id()); -} - -Location Parser::location(Tree const& tree, size_t node) const -{ - // try hard to avoid getting the location from a null string. - Location loc; - if(_location_from_node(tree, node, &loc, 0)) - return loc; - return val_location(m_buf.str); -} - -bool Parser::_location_from_node(Tree const& tree, size_t node, Location *C4_RESTRICT loc, size_t level) const -{ - if(tree.has_key(node)) - { - csubstr k = tree.key(node); - if(C4_LIKELY(k.str != nullptr)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, k.is_sub(m_buf)); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf.is_super(k)); - *loc = val_location(k.str); - return true; - } - } - - if(tree.has_val(node)) - { - csubstr v = tree.val(node); - if(C4_LIKELY(v.str != nullptr)) - { - _RYML_CB_ASSERT(m_stack.m_callbacks, v.is_sub(m_buf)); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf.is_super(v)); - *loc = val_location(v.str); - return true; - } - } - - if(tree.is_container(node)) - { - if(_location_from_cont(tree, node, loc)) - return true; - } - - if(tree.type(node) != NOTYPE && level == 0) - { - // try the prev sibling - { - const size_t prev = tree.prev_sibling(node); - if(prev != NONE) - { - if(_location_from_node(tree, prev, loc, level+1)) - return true; - } - } - // try the next sibling - { - const size_t next = tree.next_sibling(node); - if(next != NONE) - { - if(_location_from_node(tree, next, loc, level+1)) - return true; - } - } - // try the parent - { - const size_t parent = tree.parent(node); - if(parent != NONE) - { - if(_location_from_node(tree, parent, loc, level+1)) - return true; - } - } - } - - return false; -} - -bool Parser::_location_from_cont(Tree const& tree, size_t node, Location *C4_RESTRICT loc) const -{ - _RYML_CB_ASSERT(m_stack.m_callbacks, tree.is_container(node)); - if(!tree.is_stream(node)) - { - const char *node_start = tree._p(node)->m_val.scalar.str; // this was stored in the container - if(tree.has_children(node)) - { - size_t child = tree.first_child(node); - if(tree.has_key(child)) - { - // when a map starts, the container was set after the key - csubstr k = tree.key(child); - if(k.str && node_start > k.str) - node_start = k.str; - } - } - *loc = val_location(node_start); - return true; - } - else // it's a stream - { - *loc = val_location(m_buf.str); // just return the front of the buffer - } - return true; -} - - -Location Parser::val_location(const char *val) const -{ - if(C4_UNLIKELY(val == nullptr)) - return {m_file, 0, 0, 0}; - - _RYML_CB_CHECK(m_stack.m_callbacks, m_options.locations()); - // NOTE: if any of these checks fails, the parser needs to be - // instantiated with locations enabled. - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf.str == m_newline_offsets_buf.str); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_buf.len == m_newline_offsets_buf.len); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_options.locations()); - _RYML_CB_ASSERT(m_stack.m_callbacks, !_locations_dirty()); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_newline_offsets != nullptr); - _RYML_CB_ASSERT(m_stack.m_callbacks, m_newline_offsets_size > 0); - // NOTE: the pointer needs to belong to the buffer that was used to parse. - csubstr src = m_buf; - _RYML_CB_CHECK(m_stack.m_callbacks, val != nullptr || src.str == nullptr); - _RYML_CB_CHECK(m_stack.m_callbacks, (val >= src.begin() && val <= src.end()) || (src.str == nullptr && val == nullptr)); - // ok. search the first stored newline after the given ptr - using lineptr_type = size_t const* C4_RESTRICT; - lineptr_type lineptr = nullptr; - size_t offset = (size_t)(val - src.begin()); - if(m_newline_offsets_size < 30) // TODO magic number - { - // just do a linear search if the size is small. - for(lineptr_type curr = m_newline_offsets, last = m_newline_offsets + m_newline_offsets_size; curr < last; ++curr) - { - if(*curr > offset) - { - lineptr = curr; - break; - } - } - } - else - { - // do a bisection search if the size is not small. - // - // We could use std::lower_bound but this is simple enough and - // spares the include of . - size_t count = m_newline_offsets_size; - size_t step; - lineptr_type it; - lineptr = m_newline_offsets; - while(count) - { - step = count >> 1; - it = lineptr + step; - if(*it < offset) - { - lineptr = ++it; - count -= step + 1; - } - else - { - count = step; - } - } - } - _RYML_CB_ASSERT(m_stack.m_callbacks, lineptr >= m_newline_offsets); - _RYML_CB_ASSERT(m_stack.m_callbacks, lineptr <= m_newline_offsets + m_newline_offsets_size); - _RYML_CB_ASSERT(m_stack.m_callbacks, *lineptr > offset); - Location loc; - loc.name = m_file; - loc.offset = offset; - loc.line = (size_t)(lineptr - m_newline_offsets); - if(lineptr > m_newline_offsets) - loc.col = (offset - *(lineptr-1) - 1u); - else - loc.col = offset; - return loc; -} - -void Parser::_prepare_locations() -{ - m_newline_offsets_buf = m_buf; - size_t numnewlines = 1u + m_buf.count('\n'); - _resize_locations(numnewlines); - m_newline_offsets_size = 0; - for(size_t i = 0; i < m_buf.len; i++) - if(m_buf[i] == '\n') - m_newline_offsets[m_newline_offsets_size++] = i; - m_newline_offsets[m_newline_offsets_size++] = m_buf.len; - _RYML_CB_ASSERT(m_stack.m_callbacks, m_newline_offsets_size == numnewlines); -} - -void Parser::_resize_locations(size_t numnewlines) -{ - if(numnewlines > m_newline_offsets_capacity) - { - if(m_newline_offsets) - _RYML_CB_FREE(m_stack.m_callbacks, m_newline_offsets, size_t, m_newline_offsets_capacity); - m_newline_offsets = _RYML_CB_ALLOC_HINT(m_stack.m_callbacks, size_t, numnewlines, m_newline_offsets); - m_newline_offsets_capacity = numnewlines; - } -} - -bool Parser::_locations_dirty() const -{ - return !m_newline_offsets_size; -} - -} // namespace yml -} // namespace c4 - - -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/src/c4/yml/parse.hpp b/thirdparty/ryml/src/c4/yml/parse.hpp deleted file mode 100644 index 659edf7e0..000000000 --- a/thirdparty/ryml/src/c4/yml/parse.hpp +++ /dev/null @@ -1,706 +0,0 @@ -#ifndef _C4_YML_PARSE_HPP_ -#define _C4_YML_PARSE_HPP_ - -#ifndef _C4_YML_TREE_HPP_ -#include "c4/yml/tree.hpp" -#endif - -#ifndef _C4_YML_NODE_HPP_ -#include "c4/yml/node.hpp" -#endif - -#ifndef _C4_YML_DETAIL_STACK_HPP_ -#include "c4/yml/detail/stack.hpp" -#endif - -#include - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4251/*needs to have dll-interface to be used by clients of struct*/) -#endif - -namespace c4 { -namespace yml { - -struct RYML_EXPORT ParserOptions -{ -private: - - typedef enum : uint32_t { - LOCATIONS = (1 << 0), - DEFAULTS = 0, - } Flags_e; - - uint32_t flags = DEFAULTS; -public: - ParserOptions() = default; - - /** @name source location tracking */ - /** @{ */ - - /** enable/disable source location tracking */ - ParserOptions& locations(bool enabled) - { - if(enabled) - flags |= LOCATIONS; - else - flags &= ~LOCATIONS; - return *this; - } - bool locations() const { return (flags & LOCATIONS) != 0u; } - - /** @} */ -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -class RYML_EXPORT Parser -{ -public: - - /** @name construction and assignment */ - /** @{ */ - - Parser(Callbacks const& cb, ParserOptions opts={}); - Parser(ParserOptions opts={}) : Parser(get_callbacks(), opts) {} - ~Parser(); - - Parser(Parser &&); - Parser(Parser const&); - Parser& operator=(Parser &&); - Parser& operator=(Parser const&); - - /** @} */ - -public: - - /** @name modifiers */ - /** @{ */ - - /** Reserve a certain capacity for the parsing stack. - * This should be larger than the expected depth of the parsed - * YAML tree. - * - * The parsing stack is the only (potential) heap memory used by - * the parser. - * - * If the requested capacity is below the default - * stack size of 16, the memory is used directly in the parser - * object; otherwise it will be allocated from the heap. - * - * @note this reserves memory only for the parser itself; all the - * allocations for the parsed tree will go through the tree's - * allocator. - * - * @note the tree and the arena can (and should) also be reserved. */ - void reserve_stack(size_t capacity) - { - m_stack.reserve(capacity); - } - - /** Reserve a certain capacity for the array used to track node - * locations in the source buffer. */ - void reserve_locations(size_t num_source_lines) - { - _resize_locations(num_source_lines); - } - - /** Reserve a certain capacity for the character arena used to - * filter scalars. */ - void reserve_filter_arena(size_t num_characters) - { - _resize_filter_arena(num_characters); - } - - /** @} */ - -public: - - /** @name getters and modifiers */ - /** @{ */ - - /** Get the current callbacks in the parser. */ - Callbacks callbacks() const { return m_stack.m_callbacks; } - - /** Get the name of the latest file parsed by this object. */ - csubstr filename() const { return m_file; } - - /** Get the latest YAML buffer parsed by this object. */ - csubstr source() const { return m_buf; } - - size_t stack_capacity() const { return m_stack.capacity(); } - size_t locations_capacity() const { return m_newline_offsets_capacity; } - size_t filter_arena_capacity() const { return m_filter_arena.len; } - - ParserOptions const& options() const { return m_options; } - - /** @} */ - -public: - - /** @name parse_in_place */ - /** @{ */ - - /** Create a new tree and parse into its root. - * The tree is created with the callbacks currently in the parser. */ - Tree parse_in_place(csubstr filename, substr src) - { - Tree t(callbacks()); - t.reserve(_estimate_capacity(src)); - this->parse_in_place(filename, src, &t, t.root_id()); - return t; - } - - /** Parse into an existing tree, starting at its root node. - * The callbacks in the tree are kept, and used to allocate - * the tree members, if any allocation is required. */ - void parse_in_place(csubstr filename, substr src, Tree *t) - { - this->parse_in_place(filename, src, t, t->root_id()); - } - - /** Parse into an existing node. - * The callbacks in the tree are kept, and used to allocate - * the tree members, if any allocation is required. */ - void parse_in_place(csubstr filename, substr src, Tree *t, size_t node_id); - // ^^^^^^^^^^^^^ this is the workhorse overload; everything else is syntactic candy - - /** Parse into an existing node. - * The callbacks in the tree are kept, and used to allocate - * the tree members, if any allocation is required. */ - void parse_in_place(csubstr filename, substr src, NodeRef node) - { - this->parse_in_place(filename, src, node.tree(), node.id()); - } - - RYML_DEPRECATED("use parse_in_place() instead") Tree parse(csubstr filename, substr src) { return parse_in_place(filename, src); } - RYML_DEPRECATED("use parse_in_place() instead") void parse(csubstr filename, substr src, Tree *t) { parse_in_place(filename, src, t); } - RYML_DEPRECATED("use parse_in_place() instead") void parse(csubstr filename, substr src, Tree *t, size_t node_id) { parse_in_place(filename, src, t, node_id); } - RYML_DEPRECATED("use parse_in_place() instead") void parse(csubstr filename, substr src, NodeRef node) { parse_in_place(filename, src, node); } - - /** @} */ - -public: - - /** @name parse_in_arena: copy the YAML source buffer to the - * tree's arena, then parse the copy in situ - * - * @note overloads receiving a substr YAML buffer are intentionally - * left undefined, such that calling parse_in_arena() with a substr - * will cause a linker error. This is to prevent an accidental - * copy of the source buffer to the tree's arena, because substr - * is implicitly convertible to csubstr. If you really intend to parse - * a mutable buffer in the tree's arena, convert it first to immutable - * by assigning the substr to a csubstr prior to calling parse_in_arena(). - * This is not needed for parse_in_place() because csubstr is not - * implicitly convertible to substr. */ - /** @{ */ - - // READ THE NOTE ABOVE! - #define RYML_DONT_PARSE_SUBSTR_IN_ARENA "Do not pass a (mutable) substr to parse_in_arena(); if you have a substr, it should be parsed in place. Consider using parse_in_place() instead, or convert the buffer to csubstr prior to calling. This function is deliberately left undefined and will cause a linker error." - RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) Tree parse_in_arena(csubstr filename, substr csrc); - RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) void parse_in_arena(csubstr filename, substr csrc, Tree *t); - RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) void parse_in_arena(csubstr filename, substr csrc, Tree *t, size_t node_id); - RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) void parse_in_arena(csubstr filename, substr csrc, NodeRef node); - - /** Create a new tree and parse into its root. - * The immutable YAML source is first copied to the tree's arena, - * and parsed from there. - * The callbacks in the tree are kept, and used to allocate - * the tree members, if any allocation is required. */ - Tree parse_in_arena(csubstr filename, csubstr csrc) - { - Tree t(callbacks()); - substr src = t.copy_to_arena(csrc); - t.reserve(_estimate_capacity(csrc)); - this->parse_in_place(filename, src, &t, t.root_id()); - return t; - } - - /** Parse into an existing tree, starting at its root node. - * The immutable YAML source is first copied to the tree's arena, - * and parsed from there. - * The callbacks in the tree are kept, and used to allocate - * the tree members, if any allocation is required. */ - void parse_in_arena(csubstr filename, csubstr csrc, Tree *t) - { - substr src = t->copy_to_arena(csrc); - this->parse_in_place(filename, src, t, t->root_id()); - } - - /** Parse into a specific node in an existing tree. - * The immutable YAML source is first copied to the tree's arena, - * and parsed from there. - * The callbacks in the tree are kept, and used to allocate - * the tree members, if any allocation is required. */ - void parse_in_arena(csubstr filename, csubstr csrc, Tree *t, size_t node_id) - { - substr src = t->copy_to_arena(csrc); - this->parse_in_place(filename, src, t, node_id); - } - - /** Parse into a specific node in an existing tree. - * The immutable YAML source is first copied to the tree's arena, - * and parsed from there. - * The callbacks in the tree are kept, and used to allocate - * the tree members, if any allocation is required. */ - void parse_in_arena(csubstr filename, csubstr csrc, NodeRef node) - { - substr src = node.tree()->copy_to_arena(csrc); - this->parse_in_place(filename, src, node.tree(), node.id()); - } - - RYML_DEPRECATED("use parse_in_arena() instead") Tree parse(csubstr filename, csubstr csrc) { return parse_in_arena(filename, csrc); } - RYML_DEPRECATED("use parse_in_arena() instead") void parse(csubstr filename, csubstr csrc, Tree *t) { parse_in_arena(filename, csrc, t); } - RYML_DEPRECATED("use parse_in_arena() instead") void parse(csubstr filename, csubstr csrc, Tree *t, size_t node_id) { parse_in_arena(filename, csrc, t, node_id); } - RYML_DEPRECATED("use parse_in_arena() instead") void parse(csubstr filename, csubstr csrc, NodeRef node) { parse_in_arena(filename, csrc, node); } - - /** @} */ - -public: - - /** @name locations */ - /** @{ */ - - /** Get the location of a node of the last tree to be parsed by this parser. */ - Location location(Tree const& tree, size_t node_id) const; - /** Get the location of a node of the last tree to be parsed by this parser. */ - Location location(ConstNodeRef node) const; - /** Get the string starting at a particular location, to the end - * of the parsed source buffer. */ - csubstr location_contents(Location const& loc) const; - /** Given a pointer to a buffer position, get the location. @p val - * must be pointing to somewhere in the source buffer that was - * last parsed by this object. */ - Location val_location(const char *val) const; - - /** @} */ - -private: - - typedef enum { - BLOCK_LITERAL, //!< keep newlines (|) - BLOCK_FOLD //!< replace newline with single space (>) - } BlockStyle_e; - - typedef enum { - CHOMP_CLIP, //!< single newline at end (default) - CHOMP_STRIP, //!< no newline at end (-) - CHOMP_KEEP //!< all newlines from end (+) - } BlockChomp_e; - -private: - - using flag_t = int; - - static size_t _estimate_capacity(csubstr src) { size_t c = _count_nlines(src); c = c >= 16 ? c : 16; return c; } - - void _reset(); - - bool _finished_file() const; - bool _finished_line() const; - - csubstr _peek_next_line(size_t pos=npos) const; - bool _advance_to_peeked(); - void _scan_line(); - - csubstr _slurp_doc_scalar(); - - /** - * @param [out] quoted - * Will only be written to if this method returns true. - * Will be set to true if the scanned scalar was quoted, by '', "", > or |. - */ - bool _scan_scalar_seq_blck(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted); - bool _scan_scalar_map_blck(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted); - bool _scan_scalar_seq_flow(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted); - bool _scan_scalar_map_flow(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted); - bool _scan_scalar_unk(csubstr *C4_RESTRICT scalar, bool *C4_RESTRICT quoted); - - csubstr _scan_comment(); - csubstr _scan_squot_scalar(); - csubstr _scan_dquot_scalar(); - csubstr _scan_block(); - substr _scan_plain_scalar_blck(csubstr currscalar, csubstr peeked_line, size_t indentation); - substr _scan_plain_scalar_flow(csubstr currscalar, csubstr peeked_line); - substr _scan_complex_key(csubstr currscalar, csubstr peeked_line); - csubstr _scan_to_next_nonempty_line(size_t indentation); - csubstr _extend_scanned_scalar(csubstr currscalar); - - csubstr _filter_squot_scalar(const substr s); - csubstr _filter_dquot_scalar(substr s); - csubstr _filter_plain_scalar(substr s, size_t indentation); - csubstr _filter_block_scalar(substr s, BlockStyle_e style, BlockChomp_e chomp, size_t indentation); - template - bool _filter_nl(substr scalar, size_t *C4_RESTRICT pos, size_t *C4_RESTRICT filter_arena_pos, size_t indentation); - template - void _filter_ws(substr scalar, size_t *C4_RESTRICT pos, size_t *C4_RESTRICT filter_arena_pos); - bool _apply_chomp(substr buf, size_t *C4_RESTRICT pos, BlockChomp_e chomp); - - void _handle_finished_file(); - void _handle_line(); - - bool _handle_indentation(); - - bool _handle_unk(); - bool _handle_map_flow(); - bool _handle_map_blck(); - bool _handle_seq_flow(); - bool _handle_seq_blck(); - bool _handle_top(); - bool _handle_types(); - bool _handle_key_anchors_and_refs(); - bool _handle_val_anchors_and_refs(); - void _move_val_tag_to_key_tag(); - void _move_key_tag_to_val_tag(); - void _move_key_tag2_to_key_tag(); - void _move_val_anchor_to_key_anchor(); - void _move_key_anchor_to_val_anchor(); - - void _push_level(bool explicit_flow_chars = false); - void _pop_level(); - - void _start_unk(bool as_child=true); - - void _start_map(bool as_child=true); - void _start_map_unk(bool as_child); - void _stop_map(); - - void _start_seq(bool as_child=true); - void _stop_seq(); - - void _start_seqimap(); - void _stop_seqimap(); - - void _start_doc(bool as_child=true); - void _stop_doc(); - void _start_new_doc(csubstr rem); - void _end_stream(); - - NodeData* _append_val(csubstr val, flag_t quoted=false); - NodeData* _append_key_val(csubstr val, flag_t val_quoted=false); - bool _rval_dash_start_or_continue_seq(); - - void _store_scalar(csubstr s, flag_t is_quoted); - csubstr _consume_scalar(); - void _move_scalar_from_top(); - - inline NodeData* _append_val_null(const char *str) { _RYML_CB_ASSERT(m_stack.m_callbacks, str >= m_buf.begin() && str <= m_buf.end()); return _append_val({nullptr, size_t(0)}); } - inline NodeData* _append_key_val_null(const char *str) { _RYML_CB_ASSERT(m_stack.m_callbacks, str >= m_buf.begin() && str <= m_buf.end()); return _append_key_val({nullptr, size_t(0)}); } - inline void _store_scalar_null(const char *str) { _RYML_CB_ASSERT(m_stack.m_callbacks, str >= m_buf.begin() && str <= m_buf.end()); _store_scalar({nullptr, size_t(0)}, false); } - - void _set_indentation(size_t behind); - void _save_indentation(size_t behind=0); - bool _maybe_set_indentation_from_anchor_or_tag(); - - void _write_key_anchor(size_t node_id); - void _write_val_anchor(size_t node_id); - - void _handle_directive(csubstr directive); - - void _skipchars(char c); - template - void _skipchars(const char (&chars)[N]); - -private: - - static size_t _count_nlines(csubstr src); - -private: - - typedef enum : flag_t { - RTOP = 0x01 << 0, ///< reading at top level - RUNK = 0x01 << 1, ///< reading an unknown: must determine whether scalar, map or seq - RMAP = 0x01 << 2, ///< reading a map - RSEQ = 0x01 << 3, ///< reading a seq - FLOW = 0x01 << 4, ///< reading is inside explicit flow chars: [] or {} - QMRK = 0x01 << 5, ///< reading an explicit key (`? key`) - RKEY = 0x01 << 6, ///< reading a scalar as key - RVAL = 0x01 << 7, ///< reading a scalar as val - RNXT = 0x01 << 8, ///< read next val or keyval - SSCL = 0x01 << 9, ///< there's a stored scalar - QSCL = 0x01 << 10, ///< stored scalar was quoted - RSET = 0x01 << 11, ///< the (implicit) map being read is a !!set. @see https://yaml.org/type/set.html - NDOC = 0x01 << 12, ///< no document mode. a document has ended and another has not started yet. - //! reading an implicit map nested in an explicit seq. - //! eg, {key: [key2: value2, key3: value3]} - //! is parsed as {key: [{key2: value2}, {key3: value3}]} - RSEQIMAP = 0x01 << 13, - } State_e; - - struct LineContents - { - csubstr full; ///< the full line, including newlines on the right - csubstr stripped; ///< the stripped line, excluding newlines on the right - csubstr rem; ///< the stripped line remainder; initially starts at the first non-space character - size_t indentation; ///< the number of spaces on the beginning of the line - - LineContents() : full(), stripped(), rem(), indentation() {} - - void reset_with_next_line(csubstr buf, size_t pos); - - void reset(csubstr full_, csubstr stripped_) - { - full = full_; - stripped = stripped_; - rem = stripped_; - // find the first column where the character is not a space - indentation = full.first_not_of(' '); - } - - size_t current_col() const - { - return current_col(rem); - } - - size_t current_col(csubstr s) const - { - RYML_ASSERT(s.str >= full.str); - RYML_ASSERT(full.is_super(s)); - size_t col = static_cast(s.str - full.str); - return col; - } - }; - - struct State - { - flag_t flags; - size_t level; - size_t node_id; // don't hold a pointer to the node as it will be relocated during tree resizes - csubstr scalar; - size_t scalar_col; // the column where the scalar (or its quotes) begin - - Location pos; - LineContents line_contents; - size_t indref; - - State() : flags(), level(), node_id(), scalar(), scalar_col(), pos(), line_contents(), indref() {} - - void reset(const char *file, size_t node_id_) - { - flags = RUNK|RTOP; - level = 0; - pos.name = to_csubstr(file); - pos.offset = 0; - pos.line = 1; - pos.col = 1; - node_id = node_id_; - scalar_col = 0; - scalar.clear(); - indref = 0; - } - }; - - void _line_progressed(size_t ahead); - void _line_ended(); - void _line_ended_undo(); - - void _prepare_pop() - { - RYML_ASSERT(m_stack.size() > 1); - State const& curr = m_stack.top(); - State & next = m_stack.top(1); - next.pos = curr.pos; - next.line_contents = curr.line_contents; - next.scalar = curr.scalar; - } - - inline bool _at_line_begin() const - { - return m_state->line_contents.rem.begin() == m_state->line_contents.full.begin(); - } - inline bool _at_line_end() const - { - csubstr r = m_state->line_contents.rem; - return r.empty() || r.begins_with(' ', r.len); - } - inline bool _token_is_from_this_line(csubstr token) const - { - return token.is_sub(m_state->line_contents.full); - } - - inline NodeData * node(State const* s) const { return m_tree->get(s->node_id); } - inline NodeData * node(State const& s) const { return m_tree->get(s .node_id); } - inline NodeData * node(size_t node_id) const { return m_tree->get( node_id); } - - inline bool has_all(flag_t f) const { return (m_state->flags & f) == f; } - inline bool has_any(flag_t f) const { return (m_state->flags & f) != 0; } - inline bool has_none(flag_t f) const { return (m_state->flags & f) == 0; } - - static inline bool has_all(flag_t f, State const* s) { return (s->flags & f) == f; } - static inline bool has_any(flag_t f, State const* s) { return (s->flags & f) != 0; } - static inline bool has_none(flag_t f, State const* s) { return (s->flags & f) == 0; } - - inline void set_flags(flag_t f) { set_flags(f, m_state); } - inline void add_flags(flag_t on) { add_flags(on, m_state); } - inline void addrem_flags(flag_t on, flag_t off) { addrem_flags(on, off, m_state); } - inline void rem_flags(flag_t off) { rem_flags(off, m_state); } - - void set_flags(flag_t f, State * s); - void add_flags(flag_t on, State * s); - void addrem_flags(flag_t on, flag_t off, State * s); - void rem_flags(flag_t off, State * s); - - void _resize_filter_arena(size_t num_characters); - void _grow_filter_arena(size_t num_characters); - substr _finish_filter_arena(substr dst, size_t pos); - - void _prepare_locations(); - void _resize_locations(size_t sz); - bool _locations_dirty() const; - - bool _location_from_cont(Tree const& tree, size_t node, Location *C4_RESTRICT loc) const; - bool _location_from_node(Tree const& tree, size_t node, Location *C4_RESTRICT loc, size_t level) const; - -private: - - void _free(); - void _clr(); - void _cp(Parser const* that); - void _mv(Parser *that); - -#ifdef RYML_DBG - template void _dbg(csubstr fmt, Args const& C4_RESTRICT ...args) const; -#endif - template void _err(csubstr fmt, Args const& C4_RESTRICT ...args) const; - template void _fmt_msg(DumpFn &&dumpfn) const; - static csubstr _prfl(substr buf, flag_t v); - -private: - - ParserOptions m_options; - - csubstr m_file; - substr m_buf; - - size_t m_root_id; - Tree * m_tree; - - detail::stack m_stack; - State * m_state; - - size_t m_key_tag_indentation; - size_t m_key_tag2_indentation; - csubstr m_key_tag; - csubstr m_key_tag2; - size_t m_val_tag_indentation; - csubstr m_val_tag; - - bool m_key_anchor_was_before; - size_t m_key_anchor_indentation; - csubstr m_key_anchor; - size_t m_val_anchor_indentation; - csubstr m_val_anchor; - - substr m_filter_arena; - - size_t *m_newline_offsets; - size_t m_newline_offsets_size; - size_t m_newline_offsets_capacity; - csubstr m_newline_offsets_buf; -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** @name parse_in_place - * - * @desc parse a mutable YAML source buffer. - * - * @note These freestanding functions use a temporary parser object, - * and are convenience functions to easily parse YAML without the need - * to instantiate a separate parser. Note that some properties - * (notably node locations in the original source code) are only - * available through the parser object after it has parsed the - * code. If you need access to any of these properties, use - * Parser::parse_in_place() */ -/** @{ */ - -inline Tree parse_in_place( substr yaml ) { Parser np; return np.parse_in_place({} , yaml); } //!< parse in-situ a modifiable YAML source buffer. -inline Tree parse_in_place(csubstr filename, substr yaml ) { Parser np; return np.parse_in_place(filename, yaml); } //!< parse in-situ a modifiable YAML source buffer, providing a filename for error messages. -inline void parse_in_place( substr yaml, Tree *t ) { Parser np; np.parse_in_place({} , yaml, t); } //!< reusing the YAML tree, parse in-situ a modifiable YAML source buffer -inline void parse_in_place(csubstr filename, substr yaml, Tree *t ) { Parser np; np.parse_in_place(filename, yaml, t); } //!< reusing the YAML tree, parse in-situ a modifiable YAML source buffer, providing a filename for error messages. -inline void parse_in_place( substr yaml, Tree *t, size_t node_id) { Parser np; np.parse_in_place({} , yaml, t, node_id); } //!< reusing the YAML tree, parse in-situ a modifiable YAML source buffer -inline void parse_in_place(csubstr filename, substr yaml, Tree *t, size_t node_id) { Parser np; np.parse_in_place(filename, yaml, t, node_id); } //!< reusing the YAML tree, parse in-situ a modifiable YAML source buffer, providing a filename for error messages. -inline void parse_in_place( substr yaml, NodeRef node ) { Parser np; np.parse_in_place({} , yaml, node); } //!< reusing the YAML tree, parse in-situ a modifiable YAML source buffer -inline void parse_in_place(csubstr filename, substr yaml, NodeRef node ) { Parser np; np.parse_in_place(filename, yaml, node); } //!< reusing the YAML tree, parse in-situ a modifiable YAML source buffer, providing a filename for error messages. - -RYML_DEPRECATED("use parse_in_place() instead") inline Tree parse( substr yaml ) { Parser np; return np.parse_in_place({} , yaml); } -RYML_DEPRECATED("use parse_in_place() instead") inline Tree parse(csubstr filename, substr yaml ) { Parser np; return np.parse_in_place(filename, yaml); } -RYML_DEPRECATED("use parse_in_place() instead") inline void parse( substr yaml, Tree *t ) { Parser np; np.parse_in_place({} , yaml, t); } -RYML_DEPRECATED("use parse_in_place() instead") inline void parse(csubstr filename, substr yaml, Tree *t ) { Parser np; np.parse_in_place(filename, yaml, t); } -RYML_DEPRECATED("use parse_in_place() instead") inline void parse( substr yaml, Tree *t, size_t node_id) { Parser np; np.parse_in_place({} , yaml, t, node_id); } -RYML_DEPRECATED("use parse_in_place() instead") inline void parse(csubstr filename, substr yaml, Tree *t, size_t node_id) { Parser np; np.parse_in_place(filename, yaml, t, node_id); } -RYML_DEPRECATED("use parse_in_place() instead") inline void parse( substr yaml, NodeRef node ) { Parser np; np.parse_in_place({} , yaml, node); } -RYML_DEPRECATED("use parse_in_place() instead") inline void parse(csubstr filename, substr yaml, NodeRef node ) { Parser np; np.parse_in_place(filename, yaml, node); } - -/** @} */ - - -//----------------------------------------------------------------------------- - -/** @name parse_in_arena - * @desc parse a read-only YAML source buffer, copying it first to the tree's arena. - * - * @note These freestanding functions use a temporary parser object, - * and are convenience functions to easily parse YAML without the need - * to instantiate a separate parser. Note that some properties - * (notably node locations in the original source code) are only - * available through the parser object after it has parsed the - * code. If you need access to any of these properties, use - * Parser::parse_in_arena(). - * - * @note overloads receiving a substr YAML buffer are intentionally - * left undefined, such that calling parse_in_arena() with a substr - * will cause a linker error. This is to prevent an accidental - * copy of the source buffer to the tree's arena, because substr - * is implicitly convertible to csubstr. If you really intend to parse - * a mutable buffer in the tree's arena, convert it first to immutable - * by assigning the substr to a csubstr prior to calling parse_in_arena(). - * This is not needed for parse_in_place() because csubstr is not - * implicitly convertible to substr. */ -/** @{ */ - -/* READ THE NOTE ABOVE! */ -RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) Tree parse_in_arena( substr yaml ); -RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) Tree parse_in_arena(csubstr filename, substr yaml ); -RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) void parse_in_arena( substr yaml, Tree *t ); -RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) void parse_in_arena(csubstr filename, substr yaml, Tree *t ); -RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) void parse_in_arena( substr yaml, Tree *t, size_t node_id); -RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) void parse_in_arena(csubstr filename, substr yaml, Tree *t, size_t node_id); -RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) void parse_in_arena( substr yaml, NodeRef node ); -RYML_DEPRECATED(RYML_DONT_PARSE_SUBSTR_IN_ARENA) void parse_in_arena(csubstr filename, substr yaml, NodeRef node ); - -inline Tree parse_in_arena( csubstr yaml ) { Parser np; return np.parse_in_arena({} , yaml); } //!< parse a read-only YAML source buffer, copying it first to the tree's source arena. -inline Tree parse_in_arena(csubstr filename, csubstr yaml ) { Parser np; return np.parse_in_arena(filename, yaml); } //!< parse a read-only YAML source buffer, copying it first to the tree's source arena, providing a filename for error messages. -inline void parse_in_arena( csubstr yaml, Tree *t ) { Parser np; np.parse_in_arena({} , yaml, t); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena. -inline void parse_in_arena(csubstr filename, csubstr yaml, Tree *t ) { Parser np; np.parse_in_arena(filename, yaml, t); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena, providing a filename for error messages. -inline void parse_in_arena( csubstr yaml, Tree *t, size_t node_id) { Parser np; np.parse_in_arena({} , yaml, t, node_id); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena. -inline void parse_in_arena(csubstr filename, csubstr yaml, Tree *t, size_t node_id) { Parser np; np.parse_in_arena(filename, yaml, t, node_id); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena, providing a filename for error messages. -inline void parse_in_arena( csubstr yaml, NodeRef node ) { Parser np; np.parse_in_arena({} , yaml, node); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena. -inline void parse_in_arena(csubstr filename, csubstr yaml, NodeRef node ) { Parser np; np.parse_in_arena(filename, yaml, node); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena, providing a filename for error messages. - -RYML_DEPRECATED("use parse_in_arena() instead") inline Tree parse( csubstr yaml ) { Parser np; return np.parse_in_arena({} , yaml); } //!< parse a read-only YAML source buffer, copying it first to the tree's source arena. -RYML_DEPRECATED("use parse_in_arena() instead") inline Tree parse(csubstr filename, csubstr yaml ) { Parser np; return np.parse_in_arena(filename, yaml); } //!< parse a read-only YAML source buffer, copying it first to the tree's source arena, providing a filename for error messages. -RYML_DEPRECATED("use parse_in_arena() instead") inline void parse( csubstr yaml, Tree *t ) { Parser np; np.parse_in_arena({} , yaml, t); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena. -RYML_DEPRECATED("use parse_in_arena() instead") inline void parse(csubstr filename, csubstr yaml, Tree *t ) { Parser np; np.parse_in_arena(filename, yaml, t); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena, providing a filename for error messages. -RYML_DEPRECATED("use parse_in_arena() instead") inline void parse( csubstr yaml, Tree *t, size_t node_id) { Parser np; np.parse_in_arena({} , yaml, t, node_id); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena. -RYML_DEPRECATED("use parse_in_arena() instead") inline void parse(csubstr filename, csubstr yaml, Tree *t, size_t node_id) { Parser np; np.parse_in_arena(filename, yaml, t, node_id); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena, providing a filename for error messages. -RYML_DEPRECATED("use parse_in_arena() instead") inline void parse( csubstr yaml, NodeRef node ) { Parser np; np.parse_in_arena({} , yaml, node); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena. -RYML_DEPRECATED("use parse_in_arena() instead") inline void parse(csubstr filename, csubstr yaml, NodeRef node ) { Parser np; np.parse_in_arena(filename, yaml, node); } //!< reusing the YAML tree, parse a read-only YAML source buffer, copying it first to the tree's source arena, providing a filename for error messages. - -/** @} */ - -} // namespace yml -} // namespace c4 - -#if defined(_MSC_VER) -# pragma warning(pop) -#endif - -#endif /* _C4_YML_PARSE_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/preprocess.cpp b/thirdparty/ryml/src/c4/yml/preprocess.cpp deleted file mode 100644 index 2e92dce14..000000000 --- a/thirdparty/ryml/src/c4/yml/preprocess.cpp +++ /dev/null @@ -1,110 +0,0 @@ -#include "c4/yml/preprocess.hpp" -#include "c4/yml/detail/parser_dbg.hpp" - -/** @file preprocess.hpp Functions for preprocessing YAML prior to parsing. */ - -namespace c4 { -namespace yml { - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -namespace { -C4_ALWAYS_INLINE bool _is_idchar(char c) -{ - return (c >= 'a' && c <= 'z') - || (c >= 'A' && c <= 'Z') - || (c >= '0' && c <= '9') - || (c == '_' || c == '-' || c == '~' || c == '$'); -} - -typedef enum { kReadPending = 0, kKeyPending = 1, kValPending = 2 } _ppstate; -C4_ALWAYS_INLINE _ppstate _next(_ppstate s) -{ - int n = (int)s + 1; - return (_ppstate)(n <= (int)kValPending ? n : 0); -} -} // empty namespace - - -//----------------------------------------------------------------------------- - -size_t preprocess_rxmap(csubstr s, substr buf) -{ - detail::_SubstrWriter writer(buf); - _ppstate state = kReadPending; - size_t last = 0; - - if(s.begins_with('{')) - { - RYML_CHECK(s.ends_with('}')); - s = s.offs(1, 1); - } - - writer.append('{'); - - for(size_t i = 0; i < s.len; ++i) - { - const char curr = s[i]; - const char next = i+1 < s.len ? s[i+1] : '\0'; - - if(curr == '\'' || curr == '"') - { - csubstr ss = s.sub(i).pair_range_esc(curr, '\\'); - i += static_cast(ss.end() - (s.str + i)); - state = _next(state); - } - else if(state == kReadPending && _is_idchar(curr)) - { - state = _next(state); - } - - switch(state) - { - case kKeyPending: - { - if(curr == ':' && next == ' ') - { - state = _next(state); - } - else if(curr == ',' && next == ' ') - { - writer.append(s.range(last, i)); - writer.append(": 1, "); - last = i + 2; - } - break; - } - case kValPending: - { - if(curr == '[' || curr == '{' || curr == '(') - { - csubstr ss = s.sub(i).pair_range_nested(curr, '\\'); - i += static_cast(ss.end() - (s.str + i)); - state = _next(state); - } - else if(curr == ',' && next == ' ') - { - state = _next(state); - } - break; - } - default: - // nothing to do - break; - } - } - - writer.append(s.sub(last)); - if(state == kKeyPending) - writer.append(": 1"); - writer.append('}'); - - return writer.pos; -} - - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/src/c4/yml/preprocess.hpp b/thirdparty/ryml/src/c4/yml/preprocess.hpp deleted file mode 100644 index def066389..000000000 --- a/thirdparty/ryml/src/c4/yml/preprocess.hpp +++ /dev/null @@ -1,99 +0,0 @@ -#ifndef _C4_YML_PREPROCESS_HPP_ -#define _C4_YML_PREPROCESS_HPP_ - -/** @file preprocess.hpp Functions for preprocessing YAML prior to parsing. */ - -/** @defgroup Preprocessors Preprocessor functions - * - * These are the existing preprocessors: - * - * @code{.cpp} - * size_t preprocess_json(csubstr json, substr buf) - * size_t preprocess_rxmap(csubstr json, substr buf) - * @endcode - */ - -#ifndef _C4_YML_COMMON_HPP_ -#include "./common.hpp" -#endif -#include - - -namespace c4 { -namespace yml { - -namespace detail { -using Preprocessor = size_t(csubstr, substr); -template -substr preprocess_into_container(csubstr input, CharContainer *out) -{ - // try to write once. the preprocessor will stop writing at the end of - // the container, but will process all the input to determine the - // required container size. - size_t sz = PP(input, to_substr(*out)); - // if the container size is not enough, resize, and run again in the - // resized container - if(sz > out->size()) - { - out->resize(sz); - sz = PP(input, to_substr(*out)); - } - return to_substr(*out).first(sz); -} -} // namespace detail - - -//----------------------------------------------------------------------------- - -/** @name preprocess_rxmap - * Convert flow-type relaxed maps (with implicit bools) into strict YAML - * flow map. - * - * @code{.yaml} - * {a, b, c, d: [e, f], g: {a, b}} - * # is converted into this: - * {a: 1, b: 1, c: 1, d: [e, f], g: {a, b}} - * @endcode - - * @note this is NOT recursive - conversion happens only in the top-level map - * @param rxmap A relaxed map - * @param buf output buffer - * @param out output container - */ - -//@{ - -/** Write into a given output buffer. This function is safe to call with - * empty or small buffers; it won't write beyond the end of the buffer. - * - * @return the number of characters required for output - */ -RYML_EXPORT size_t preprocess_rxmap(csubstr rxmap, substr buf); - - -/** Write into an existing container. It is resized to contained the output. - * @return a substr of the container - * @overload preprocess_rxmap */ -template -substr preprocess_rxmap(csubstr rxmap, CharContainer *out) -{ - return detail::preprocess_into_container(rxmap, out); -} - - -/** Create a container with the result. - * @overload preprocess_rxmap */ -template -CharContainer preprocess_rxmap(csubstr rxmap) -{ - CharContainer out; - preprocess_rxmap(rxmap, &out); - return out; -} - -//@} - -} // namespace yml -} // namespace c4 - -#endif /* _C4_YML_PREPROCESS_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/std/map.hpp b/thirdparty/ryml/src/c4/yml/std/map.hpp deleted file mode 100644 index fc48dc5e6..000000000 --- a/thirdparty/ryml/src/c4/yml/std/map.hpp +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef _C4_YML_STD_MAP_HPP_ -#define _C4_YML_STD_MAP_HPP_ - -/** @file map.hpp write/read std::map to/from a YAML tree. */ - -#include "c4/yml/node.hpp" -#include - -namespace c4 { -namespace yml { - -// std::map requires child nodes in the data -// tree hierarchy (a MAP node in ryml parlance). -// So it should be serialized via write()/read(). - -template -void write(c4::yml::NodeRef *n, std::map const& m) -{ - *n |= c4::yml::MAP; - for(auto const& C4_RESTRICT p : m) - { - auto ch = n->append_child(); - ch << c4::yml::key(p.first); - ch << p.second; - } -} - -template -bool read(c4::yml::ConstNodeRef const& n, std::map * m) -{ - K k{}; - V v{}; - for(auto const& C4_RESTRICT ch : n) - { - ch >> c4::yml::key(k); - ch >> v; - m->emplace(std::make_pair(std::move(k), std::move(v))); - } - return true; -} - -} // namespace yml -} // namespace c4 - -#endif // _C4_YML_STD_MAP_HPP_ diff --git a/thirdparty/ryml/src/c4/yml/std/std.hpp b/thirdparty/ryml/src/c4/yml/std/std.hpp deleted file mode 100644 index 08e80d155..000000000 --- a/thirdparty/ryml/src/c4/yml/std/std.hpp +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef _C4_YML_STD_STD_HPP_ -#define _C4_YML_STD_STD_HPP_ - -#include "c4/yml/std/string.hpp" -#include "c4/yml/std/vector.hpp" -#include "c4/yml/std/map.hpp" - -#endif // _C4_YML_STD_STD_HPP_ diff --git a/thirdparty/ryml/src/c4/yml/std/string.hpp b/thirdparty/ryml/src/c4/yml/std/string.hpp deleted file mode 100644 index e3318f91c..000000000 --- a/thirdparty/ryml/src/c4/yml/std/string.hpp +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef C4_YML_STD_STRING_HPP_ -#define C4_YML_STD_STRING_HPP_ - -/** @file string.hpp substring conversions for/from std::string */ - -// everything we need is implemented here: -#include - -#endif // C4_YML_STD_STRING_HPP_ diff --git a/thirdparty/ryml/src/c4/yml/std/vector.hpp b/thirdparty/ryml/src/c4/yml/std/vector.hpp deleted file mode 100644 index 1b6a4610a..000000000 --- a/thirdparty/ryml/src/c4/yml/std/vector.hpp +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef _C4_YML_STD_VECTOR_HPP_ -#define _C4_YML_STD_VECTOR_HPP_ - -#include "c4/yml/node.hpp" -#include -#include - -namespace c4 { -namespace yml { - -// vector is a sequence-like type, and it requires child nodes -// in the data tree hierarchy (a SEQ node in ryml parlance). -// So it should be serialized via write()/read(). - - -template -void write(c4::yml::NodeRef *n, std::vector const& vec) -{ - *n |= c4::yml::SEQ; - for(auto const& v : vec) - n->append_child() << v; -} - -template -bool read(c4::yml::ConstNodeRef const& n, std::vector *vec) -{ - vec->resize(n.num_children()); - size_t pos = 0; - for(auto const ch : n) - ch >> (*vec)[pos++]; - return true; -} - -/** specialization: std::vector uses std::vector::reference as - * the return value of its operator[]. */ -template -bool read(c4::yml::ConstNodeRef const& n, std::vector *vec) -{ - vec->resize(n.num_children()); - size_t pos = 0; - bool tmp; - for(auto const ch : n) - { - ch >> tmp; - (*vec)[pos++] = tmp; - } - return true; -} - -} // namespace yml -} // namespace c4 - -#endif // _C4_YML_STD_VECTOR_HPP_ diff --git a/thirdparty/ryml/src/c4/yml/tree.cpp b/thirdparty/ryml/src/c4/yml/tree.cpp deleted file mode 100644 index b16ff8f5d..000000000 --- a/thirdparty/ryml/src/c4/yml/tree.cpp +++ /dev/null @@ -1,2183 +0,0 @@ -#include "c4/yml/tree.hpp" -#include "c4/yml/detail/parser_dbg.hpp" -#include "c4/yml/node.hpp" -#include "c4/yml/detail/stack.hpp" - - -C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wtype-limits") -C4_SUPPRESS_WARNING_MSVC_WITH_PUSH(4296/*expression is always 'boolean_value'*/) - -namespace c4 { -namespace yml { - - -csubstr normalize_tag(csubstr tag) -{ - YamlTag_e t = to_tag(tag); - if(t != TAG_NONE) - return from_tag(t); - if(tag.begins_with("!<")) - tag = tag.sub(1); - if(tag.begins_with(""}; - case TAG_OMAP: - return {""}; - case TAG_PAIRS: - return {""}; - case TAG_SET: - return {""}; - case TAG_SEQ: - return {""}; - case TAG_BINARY: - return {""}; - case TAG_BOOL: - return {""}; - case TAG_FLOAT: - return {""}; - case TAG_INT: - return {""}; - case TAG_MERGE: - return {""}; - case TAG_NULL: - return {""}; - case TAG_STR: - return {""}; - case TAG_TIMESTAMP: - return {""}; - case TAG_VALUE: - return {""}; - case TAG_YAML: - return {""}; - case TAG_NONE: - return {""}; - } - return {""}; -} - -csubstr from_tag(YamlTag_e tag) -{ - switch(tag) - { - case TAG_MAP: - return {"!!map"}; - case TAG_OMAP: - return {"!!omap"}; - case TAG_PAIRS: - return {"!!pairs"}; - case TAG_SET: - return {"!!set"}; - case TAG_SEQ: - return {"!!seq"}; - case TAG_BINARY: - return {"!!binary"}; - case TAG_BOOL: - return {"!!bool"}; - case TAG_FLOAT: - return {"!!float"}; - case TAG_INT: - return {"!!int"}; - case TAG_MERGE: - return {"!!merge"}; - case TAG_NULL: - return {"!!null"}; - case TAG_STR: - return {"!!str"}; - case TAG_TIMESTAMP: - return {"!!timestamp"}; - case TAG_VALUE: - return {"!!value"}; - case TAG_YAML: - return {"!!yaml"}; - case TAG_NONE: - return {""}; - } - return {""}; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -const char* NodeType::type_str(NodeType_e ty) -{ - switch(ty & _TYMASK) - { - case KEYVAL: - return "KEYVAL"; - case KEY: - return "KEY"; - case VAL: - return "VAL"; - case MAP: - return "MAP"; - case SEQ: - return "SEQ"; - case KEYMAP: - return "KEYMAP"; - case KEYSEQ: - return "KEYSEQ"; - case DOCSEQ: - return "DOCSEQ"; - case DOCMAP: - return "DOCMAP"; - case DOCVAL: - return "DOCVAL"; - case DOC: - return "DOC"; - case STREAM: - return "STREAM"; - case NOTYPE: - return "NOTYPE"; - default: - if((ty & KEYVAL) == KEYVAL) - return "KEYVAL***"; - if((ty & KEYMAP) == KEYMAP) - return "KEYMAP***"; - if((ty & KEYSEQ) == KEYSEQ) - return "KEYSEQ***"; - if((ty & DOCSEQ) == DOCSEQ) - return "DOCSEQ***"; - if((ty & DOCMAP) == DOCMAP) - return "DOCMAP***"; - if((ty & DOCVAL) == DOCVAL) - return "DOCVAL***"; - if(ty & KEY) - return "KEY***"; - if(ty & VAL) - return "VAL***"; - if(ty & MAP) - return "MAP***"; - if(ty & SEQ) - return "SEQ***"; - if(ty & DOC) - return "DOC***"; - return "(unk)"; - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -NodeRef Tree::rootref() -{ - return NodeRef(this, root_id()); -} -ConstNodeRef Tree::rootref() const -{ - return ConstNodeRef(this, root_id()); -} - -ConstNodeRef Tree::crootref() -{ - return ConstNodeRef(this, root_id()); -} -ConstNodeRef Tree::crootref() const -{ - return ConstNodeRef(this, root_id()); -} - -NodeRef Tree::ref(size_t id) -{ - _RYML_CB_ASSERT(m_callbacks, id != NONE && id >= 0 && id < m_size); - return NodeRef(this, id); -} -ConstNodeRef Tree::ref(size_t id) const -{ - _RYML_CB_ASSERT(m_callbacks, id != NONE && id >= 0 && id < m_size); - return ConstNodeRef(this, id); -} - -ConstNodeRef Tree::cref(size_t id) -{ - _RYML_CB_ASSERT(m_callbacks, id != NONE && id >= 0 && id < m_size); - return ConstNodeRef(this, id); -} -ConstNodeRef Tree::cref(size_t id) const -{ - _RYML_CB_ASSERT(m_callbacks, id != NONE && id >= 0 && id < m_size); - return ConstNodeRef(this, id); -} - -NodeRef Tree::operator[] (csubstr key) -{ - return rootref()[key]; -} -ConstNodeRef Tree::operator[] (csubstr key) const -{ - return rootref()[key]; -} - -NodeRef Tree::operator[] (size_t i) -{ - return rootref()[i]; -} -ConstNodeRef Tree::operator[] (size_t i) const -{ - return rootref()[i]; -} - -NodeRef Tree::docref(size_t i) -{ - return ref(doc(i)); -} -ConstNodeRef Tree::docref(size_t i) const -{ - return cref(doc(i)); -} - - -//----------------------------------------------------------------------------- -Tree::Tree(Callbacks const& cb) - : m_buf(nullptr) - , m_cap(0) - , m_size(0) - , m_free_head(NONE) - , m_free_tail(NONE) - , m_arena() - , m_arena_pos(0) - , m_callbacks(cb) -{ -} - -Tree::Tree(size_t node_capacity, size_t arena_capacity, Callbacks const& cb) - : Tree(cb) -{ - reserve(node_capacity); - reserve_arena(arena_capacity); -} - -Tree::~Tree() -{ - _free(); -} - - -Tree::Tree(Tree const& that) noexcept : Tree(that.m_callbacks) -{ - _copy(that); -} - -Tree& Tree::operator= (Tree const& that) noexcept -{ - _free(); - m_callbacks = that.m_callbacks; - _copy(that); - return *this; -} - -Tree::Tree(Tree && that) noexcept : Tree(that.m_callbacks) -{ - _move(that); -} - -Tree& Tree::operator= (Tree && that) noexcept -{ - _free(); - m_callbacks = that.m_callbacks; - _move(that); - return *this; -} - -void Tree::_free() -{ - if(m_buf) - { - _RYML_CB_ASSERT(m_callbacks, m_cap > 0); - _RYML_CB_FREE(m_callbacks, m_buf, NodeData, m_cap); - } - if(m_arena.str) - { - _RYML_CB_ASSERT(m_callbacks, m_arena.len > 0); - _RYML_CB_FREE(m_callbacks, m_arena.str, char, m_arena.len); - } - _clear(); -} - - -C4_SUPPRESS_WARNING_GCC_PUSH -#if defined(__GNUC__) && __GNUC__>= 8 - C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wclass-memaccess") // error: ‘void* memset(void*, int, size_t)’ clearing an object of type ‘class c4::yml::Tree’ with no trivial copy-assignment; use assignment or value-initialization instead -#endif - -void Tree::_clear() -{ - m_buf = nullptr; - m_cap = 0; - m_size = 0; - m_free_head = 0; - m_free_tail = 0; - m_arena = {}; - m_arena_pos = 0; - for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i) - m_tag_directives[i] = {}; -} - -void Tree::_copy(Tree const& that) -{ - _RYML_CB_ASSERT(m_callbacks, m_buf == nullptr); - _RYML_CB_ASSERT(m_callbacks, m_arena.str == nullptr); - _RYML_CB_ASSERT(m_callbacks, m_arena.len == 0); - m_buf = _RYML_CB_ALLOC_HINT(m_callbacks, NodeData, that.m_cap, that.m_buf); - memcpy(m_buf, that.m_buf, that.m_cap * sizeof(NodeData)); - m_cap = that.m_cap; - m_size = that.m_size; - m_free_head = that.m_free_head; - m_free_tail = that.m_free_tail; - m_arena_pos = that.m_arena_pos; - m_arena = that.m_arena; - if(that.m_arena.str) - { - _RYML_CB_ASSERT(m_callbacks, that.m_arena.len > 0); - substr arena; - arena.str = _RYML_CB_ALLOC_HINT(m_callbacks, char, that.m_arena.len, that.m_arena.str); - arena.len = that.m_arena.len; - _relocate(arena); // does a memcpy of the arena and updates nodes using the old arena - m_arena = arena; - } - for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i) - m_tag_directives[i] = that.m_tag_directives[i]; -} - -void Tree::_move(Tree & that) -{ - _RYML_CB_ASSERT(m_callbacks, m_buf == nullptr); - _RYML_CB_ASSERT(m_callbacks, m_arena.str == nullptr); - _RYML_CB_ASSERT(m_callbacks, m_arena.len == 0); - m_buf = that.m_buf; - m_cap = that.m_cap; - m_size = that.m_size; - m_free_head = that.m_free_head; - m_free_tail = that.m_free_tail; - m_arena = that.m_arena; - m_arena_pos = that.m_arena_pos; - for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i) - m_tag_directives[i] = that.m_tag_directives[i]; - that._clear(); -} - -void Tree::_relocate(substr next_arena) -{ - _RYML_CB_ASSERT(m_callbacks, next_arena.not_empty()); - _RYML_CB_ASSERT(m_callbacks, next_arena.len >= m_arena.len); - memcpy(next_arena.str, m_arena.str, m_arena_pos); - for(NodeData *C4_RESTRICT n = m_buf, *e = m_buf + m_cap; n != e; ++n) - { - if(in_arena(n->m_key.scalar)) - n->m_key.scalar = _relocated(n->m_key.scalar, next_arena); - if(in_arena(n->m_key.tag)) - n->m_key.tag = _relocated(n->m_key.tag, next_arena); - if(in_arena(n->m_key.anchor)) - n->m_key.anchor = _relocated(n->m_key.anchor, next_arena); - if(in_arena(n->m_val.scalar)) - n->m_val.scalar = _relocated(n->m_val.scalar, next_arena); - if(in_arena(n->m_val.tag)) - n->m_val.tag = _relocated(n->m_val.tag, next_arena); - if(in_arena(n->m_val.anchor)) - n->m_val.anchor = _relocated(n->m_val.anchor, next_arena); - } - for(TagDirective &C4_RESTRICT td : m_tag_directives) - { - if(in_arena(td.prefix)) - td.prefix = _relocated(td.prefix, next_arena); - if(in_arena(td.handle)) - td.handle = _relocated(td.handle, next_arena); - } -} - - -//----------------------------------------------------------------------------- -void Tree::reserve(size_t cap) -{ - if(cap > m_cap) - { - NodeData *buf = _RYML_CB_ALLOC_HINT(m_callbacks, NodeData, cap, m_buf); - if(m_buf) - { - memcpy(buf, m_buf, m_cap * sizeof(NodeData)); - _RYML_CB_FREE(m_callbacks, m_buf, NodeData, m_cap); - } - size_t first = m_cap, del = cap - m_cap; - m_cap = cap; - m_buf = buf; - _clear_range(first, del); - if(m_free_head != NONE) - { - _RYML_CB_ASSERT(m_callbacks, m_buf != nullptr); - _RYML_CB_ASSERT(m_callbacks, m_free_tail != NONE); - m_buf[m_free_tail].m_next_sibling = first; - m_buf[first].m_prev_sibling = m_free_tail; - m_free_tail = cap-1; - } - else - { - _RYML_CB_ASSERT(m_callbacks, m_free_tail == NONE); - m_free_head = first; - m_free_tail = cap-1; - } - _RYML_CB_ASSERT(m_callbacks, m_free_head == NONE || (m_free_head >= 0 && m_free_head < cap)); - _RYML_CB_ASSERT(m_callbacks, m_free_tail == NONE || (m_free_tail >= 0 && m_free_tail < cap)); - - if( ! m_size) - _claim_root(); - } -} - - -//----------------------------------------------------------------------------- -void Tree::clear() -{ - _clear_range(0, m_cap); - m_size = 0; - if(m_buf) - { - _RYML_CB_ASSERT(m_callbacks, m_cap >= 0); - m_free_head = 0; - m_free_tail = m_cap-1; - _claim_root(); - } - else - { - m_free_head = NONE; - m_free_tail = NONE; - } - for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i) - m_tag_directives[i] = {}; -} - -void Tree::_claim_root() -{ - size_t r = _claim(); - _RYML_CB_ASSERT(m_callbacks, r == 0); - _set_hierarchy(r, NONE, NONE); -} - - -//----------------------------------------------------------------------------- -void Tree::_clear_range(size_t first, size_t num) -{ - if(num == 0) - return; // prevent overflow when subtracting - _RYML_CB_ASSERT(m_callbacks, first >= 0 && first + num <= m_cap); - memset(m_buf + first, 0, num * sizeof(NodeData)); // TODO we should not need this - for(size_t i = first, e = first + num; i < e; ++i) - { - _clear(i); - NodeData *n = m_buf + i; - n->m_prev_sibling = i - 1; - n->m_next_sibling = i + 1; - } - m_buf[first + num - 1].m_next_sibling = NONE; -} - -C4_SUPPRESS_WARNING_GCC_POP - - -//----------------------------------------------------------------------------- -void Tree::_release(size_t i) -{ - _RYML_CB_ASSERT(m_callbacks, i >= 0 && i < m_cap); - - _rem_hierarchy(i); - _free_list_add(i); - _clear(i); - - --m_size; -} - -//----------------------------------------------------------------------------- -// add to the front of the free list -void Tree::_free_list_add(size_t i) -{ - _RYML_CB_ASSERT(m_callbacks, i >= 0 && i < m_cap); - NodeData &C4_RESTRICT w = m_buf[i]; - - w.m_parent = NONE; - w.m_next_sibling = m_free_head; - w.m_prev_sibling = NONE; - if(m_free_head != NONE) - m_buf[m_free_head].m_prev_sibling = i; - m_free_head = i; - if(m_free_tail == NONE) - m_free_tail = m_free_head; -} - -void Tree::_free_list_rem(size_t i) -{ - if(m_free_head == i) - m_free_head = _p(i)->m_next_sibling; - _rem_hierarchy(i); -} - -//----------------------------------------------------------------------------- -size_t Tree::_claim() -{ - if(m_free_head == NONE || m_buf == nullptr) - { - size_t sz = 2 * m_cap; - sz = sz ? sz : 16; - reserve(sz); - _RYML_CB_ASSERT(m_callbacks, m_free_head != NONE); - } - - _RYML_CB_ASSERT(m_callbacks, m_size < m_cap); - _RYML_CB_ASSERT(m_callbacks, m_free_head >= 0 && m_free_head < m_cap); - - size_t ichild = m_free_head; - NodeData *child = m_buf + ichild; - - ++m_size; - m_free_head = child->m_next_sibling; - if(m_free_head == NONE) - { - m_free_tail = NONE; - _RYML_CB_ASSERT(m_callbacks, m_size == m_cap); - } - - _clear(ichild); - - return ichild; -} - -//----------------------------------------------------------------------------- - -C4_SUPPRESS_WARNING_GCC_PUSH -C4_SUPPRESS_WARNING_CLANG_PUSH -C4_SUPPRESS_WARNING_CLANG("-Wnull-dereference") -#if defined(__GNUC__) && (__GNUC__ >= 6) -C4_SUPPRESS_WARNING_GCC("-Wnull-dereference") -#endif - -void Tree::_set_hierarchy(size_t ichild, size_t iparent, size_t iprev_sibling) -{ - _RYML_CB_ASSERT(m_callbacks, iparent == NONE || (iparent >= 0 && iparent < m_cap)); - _RYML_CB_ASSERT(m_callbacks, iprev_sibling == NONE || (iprev_sibling >= 0 && iprev_sibling < m_cap)); - - NodeData *C4_RESTRICT child = get(ichild); - - child->m_parent = iparent; - child->m_prev_sibling = NONE; - child->m_next_sibling = NONE; - - if(iparent == NONE) - { - _RYML_CB_ASSERT(m_callbacks, ichild == 0); - _RYML_CB_ASSERT(m_callbacks, iprev_sibling == NONE); - } - - if(iparent == NONE) - return; - - size_t inext_sibling = iprev_sibling != NONE ? next_sibling(iprev_sibling) : first_child(iparent); - NodeData *C4_RESTRICT parent = get(iparent); - NodeData *C4_RESTRICT psib = get(iprev_sibling); - NodeData *C4_RESTRICT nsib = get(inext_sibling); - - if(psib) - { - _RYML_CB_ASSERT(m_callbacks, next_sibling(iprev_sibling) == id(nsib)); - child->m_prev_sibling = id(psib); - psib->m_next_sibling = id(child); - _RYML_CB_ASSERT(m_callbacks, psib->m_prev_sibling != psib->m_next_sibling || psib->m_prev_sibling == NONE); - } - - if(nsib) - { - _RYML_CB_ASSERT(m_callbacks, prev_sibling(inext_sibling) == id(psib)); - child->m_next_sibling = id(nsib); - nsib->m_prev_sibling = id(child); - _RYML_CB_ASSERT(m_callbacks, nsib->m_prev_sibling != nsib->m_next_sibling || nsib->m_prev_sibling == NONE); - } - - if(parent->m_first_child == NONE) - { - _RYML_CB_ASSERT(m_callbacks, parent->m_last_child == NONE); - parent->m_first_child = id(child); - parent->m_last_child = id(child); - } - else - { - if(child->m_next_sibling == parent->m_first_child) - parent->m_first_child = id(child); - - if(child->m_prev_sibling == parent->m_last_child) - parent->m_last_child = id(child); - } -} - -C4_SUPPRESS_WARNING_GCC_POP -C4_SUPPRESS_WARNING_CLANG_POP - - -//----------------------------------------------------------------------------- -void Tree::_rem_hierarchy(size_t i) -{ - _RYML_CB_ASSERT(m_callbacks, i >= 0 && i < m_cap); - - NodeData &C4_RESTRICT w = m_buf[i]; - - // remove from the parent - if(w.m_parent != NONE) - { - NodeData &C4_RESTRICT p = m_buf[w.m_parent]; - if(p.m_first_child == i) - { - p.m_first_child = w.m_next_sibling; - } - if(p.m_last_child == i) - { - p.m_last_child = w.m_prev_sibling; - } - } - - // remove from the used list - if(w.m_prev_sibling != NONE) - { - NodeData *C4_RESTRICT prev = get(w.m_prev_sibling); - prev->m_next_sibling = w.m_next_sibling; - } - if(w.m_next_sibling != NONE) - { - NodeData *C4_RESTRICT next = get(w.m_next_sibling); - next->m_prev_sibling = w.m_prev_sibling; - } -} - -//----------------------------------------------------------------------------- -void Tree::reorder() -{ - size_t r = root_id(); - _do_reorder(&r, 0); -} - -//----------------------------------------------------------------------------- -size_t Tree::_do_reorder(size_t *node, size_t count) -{ - // swap this node if it's not in place - if(*node != count) - { - _swap(*node, count); - *node = count; - } - ++count; // bump the count from this node - - // now descend in the hierarchy - for(size_t i = first_child(*node); i != NONE; i = next_sibling(i)) - { - // this child may have been relocated to a different index, - // so get an updated version - count = _do_reorder(&i, count); - } - return count; -} - -//----------------------------------------------------------------------------- -void Tree::_swap(size_t n_, size_t m_) -{ - _RYML_CB_ASSERT(m_callbacks, (parent(n_) != NONE) || type(n_) == NOTYPE); - _RYML_CB_ASSERT(m_callbacks, (parent(m_) != NONE) || type(m_) == NOTYPE); - NodeType tn = type(n_); - NodeType tm = type(m_); - if(tn != NOTYPE && tm != NOTYPE) - { - _swap_props(n_, m_); - _swap_hierarchy(n_, m_); - } - else if(tn == NOTYPE && tm != NOTYPE) - { - _copy_props(n_, m_); - _free_list_rem(n_); - _copy_hierarchy(n_, m_); - _clear(m_); - _free_list_add(m_); - } - else if(tn != NOTYPE && tm == NOTYPE) - { - _copy_props(m_, n_); - _free_list_rem(m_); - _copy_hierarchy(m_, n_); - _clear(n_); - _free_list_add(n_); - } - else - { - C4_NEVER_REACH(); - } -} - -//----------------------------------------------------------------------------- -void Tree::_swap_hierarchy(size_t ia, size_t ib) -{ - if(ia == ib) return; - - for(size_t i = first_child(ia); i != NONE; i = next_sibling(i)) - { - if(i == ib || i == ia) - continue; - _p(i)->m_parent = ib; - } - - for(size_t i = first_child(ib); i != NONE; i = next_sibling(i)) - { - if(i == ib || i == ia) - continue; - _p(i)->m_parent = ia; - } - - auto & C4_RESTRICT a = *_p(ia); - auto & C4_RESTRICT b = *_p(ib); - auto & C4_RESTRICT pa = *_p(a.m_parent); - auto & C4_RESTRICT pb = *_p(b.m_parent); - - if(&pa == &pb) - { - if((pa.m_first_child == ib && pa.m_last_child == ia) - || - (pa.m_first_child == ia && pa.m_last_child == ib)) - { - std::swap(pa.m_first_child, pa.m_last_child); - } - else - { - bool changed = false; - if(pa.m_first_child == ia) - { - pa.m_first_child = ib; - changed = true; - } - if(pa.m_last_child == ia) - { - pa.m_last_child = ib; - changed = true; - } - if(pb.m_first_child == ib && !changed) - { - pb.m_first_child = ia; - } - if(pb.m_last_child == ib && !changed) - { - pb.m_last_child = ia; - } - } - } - else - { - if(pa.m_first_child == ia) - pa.m_first_child = ib; - if(pa.m_last_child == ia) - pa.m_last_child = ib; - if(pb.m_first_child == ib) - pb.m_first_child = ia; - if(pb.m_last_child == ib) - pb.m_last_child = ia; - } - std::swap(a.m_first_child , b.m_first_child); - std::swap(a.m_last_child , b.m_last_child); - - if(a.m_prev_sibling != ib && b.m_prev_sibling != ia && - a.m_next_sibling != ib && b.m_next_sibling != ia) - { - if(a.m_prev_sibling != NONE && a.m_prev_sibling != ib) - _p(a.m_prev_sibling)->m_next_sibling = ib; - if(a.m_next_sibling != NONE && a.m_next_sibling != ib) - _p(a.m_next_sibling)->m_prev_sibling = ib; - if(b.m_prev_sibling != NONE && b.m_prev_sibling != ia) - _p(b.m_prev_sibling)->m_next_sibling = ia; - if(b.m_next_sibling != NONE && b.m_next_sibling != ia) - _p(b.m_next_sibling)->m_prev_sibling = ia; - std::swap(a.m_prev_sibling, b.m_prev_sibling); - std::swap(a.m_next_sibling, b.m_next_sibling); - } - else - { - if(a.m_next_sibling == ib) // n will go after m - { - _RYML_CB_ASSERT(m_callbacks, b.m_prev_sibling == ia); - if(a.m_prev_sibling != NONE) - { - _RYML_CB_ASSERT(m_callbacks, a.m_prev_sibling != ib); - _p(a.m_prev_sibling)->m_next_sibling = ib; - } - if(b.m_next_sibling != NONE) - { - _RYML_CB_ASSERT(m_callbacks, b.m_next_sibling != ia); - _p(b.m_next_sibling)->m_prev_sibling = ia; - } - size_t ns = b.m_next_sibling; - b.m_prev_sibling = a.m_prev_sibling; - b.m_next_sibling = ia; - a.m_prev_sibling = ib; - a.m_next_sibling = ns; - } - else if(a.m_prev_sibling == ib) // m will go after n - { - _RYML_CB_ASSERT(m_callbacks, b.m_next_sibling == ia); - if(b.m_prev_sibling != NONE) - { - _RYML_CB_ASSERT(m_callbacks, b.m_prev_sibling != ia); - _p(b.m_prev_sibling)->m_next_sibling = ia; - } - if(a.m_next_sibling != NONE) - { - _RYML_CB_ASSERT(m_callbacks, a.m_next_sibling != ib); - _p(a.m_next_sibling)->m_prev_sibling = ib; - } - size_t ns = b.m_prev_sibling; - a.m_prev_sibling = b.m_prev_sibling; - a.m_next_sibling = ib; - b.m_prev_sibling = ia; - b.m_next_sibling = ns; - } - else - { - C4_NEVER_REACH(); - } - } - _RYML_CB_ASSERT(m_callbacks, a.m_next_sibling != ia); - _RYML_CB_ASSERT(m_callbacks, a.m_prev_sibling != ia); - _RYML_CB_ASSERT(m_callbacks, b.m_next_sibling != ib); - _RYML_CB_ASSERT(m_callbacks, b.m_prev_sibling != ib); - - if(a.m_parent != ib && b.m_parent != ia) - { - std::swap(a.m_parent, b.m_parent); - } - else - { - if(a.m_parent == ib && b.m_parent != ia) - { - a.m_parent = b.m_parent; - b.m_parent = ia; - } - else if(a.m_parent != ib && b.m_parent == ia) - { - b.m_parent = a.m_parent; - a.m_parent = ib; - } - else - { - C4_NEVER_REACH(); - } - } -} - -//----------------------------------------------------------------------------- -void Tree::_copy_hierarchy(size_t dst_, size_t src_) -{ - auto const& C4_RESTRICT src = *_p(src_); - auto & C4_RESTRICT dst = *_p(dst_); - auto & C4_RESTRICT prt = *_p(src.m_parent); - for(size_t i = src.m_first_child; i != NONE; i = next_sibling(i)) - { - _p(i)->m_parent = dst_; - } - if(src.m_prev_sibling != NONE) - { - _p(src.m_prev_sibling)->m_next_sibling = dst_; - } - if(src.m_next_sibling != NONE) - { - _p(src.m_next_sibling)->m_prev_sibling = dst_; - } - if(prt.m_first_child == src_) - { - prt.m_first_child = dst_; - } - if(prt.m_last_child == src_) - { - prt.m_last_child = dst_; - } - dst.m_parent = src.m_parent; - dst.m_first_child = src.m_first_child; - dst.m_last_child = src.m_last_child; - dst.m_prev_sibling = src.m_prev_sibling; - dst.m_next_sibling = src.m_next_sibling; -} - -//----------------------------------------------------------------------------- -void Tree::_swap_props(size_t n_, size_t m_) -{ - NodeData &C4_RESTRICT n = *_p(n_); - NodeData &C4_RESTRICT m = *_p(m_); - std::swap(n.m_type, m.m_type); - std::swap(n.m_key, m.m_key); - std::swap(n.m_val, m.m_val); -} - -//----------------------------------------------------------------------------- -void Tree::move(size_t node, size_t after) -{ - _RYML_CB_ASSERT(m_callbacks, node != NONE); - _RYML_CB_ASSERT(m_callbacks, node != after); - _RYML_CB_ASSERT(m_callbacks, ! is_root(node)); - _RYML_CB_ASSERT(m_callbacks, (after == NONE) || (has_sibling(node, after) && has_sibling(after, node))); - - _rem_hierarchy(node); - _set_hierarchy(node, parent(node), after); -} - -//----------------------------------------------------------------------------- - -void Tree::move(size_t node, size_t new_parent, size_t after) -{ - _RYML_CB_ASSERT(m_callbacks, node != NONE); - _RYML_CB_ASSERT(m_callbacks, node != after); - _RYML_CB_ASSERT(m_callbacks, new_parent != NONE); - _RYML_CB_ASSERT(m_callbacks, new_parent != node); - _RYML_CB_ASSERT(m_callbacks, new_parent != after); - _RYML_CB_ASSERT(m_callbacks, ! is_root(node)); - - _rem_hierarchy(node); - _set_hierarchy(node, new_parent, after); -} - -size_t Tree::move(Tree *src, size_t node, size_t new_parent, size_t after) -{ - _RYML_CB_ASSERT(m_callbacks, src != nullptr); - _RYML_CB_ASSERT(m_callbacks, node != NONE); - _RYML_CB_ASSERT(m_callbacks, new_parent != NONE); - _RYML_CB_ASSERT(m_callbacks, new_parent != after); - - size_t dup = duplicate(src, node, new_parent, after); - src->remove(node); - return dup; -} - -void Tree::set_root_as_stream() -{ - size_t root = root_id(); - if(is_stream(root)) - return; - // don't use _add_flags() because it's checked and will fail - if(!has_children(root)) - { - if(is_val(root)) - { - _p(root)->m_type.add(SEQ); - size_t next_doc = append_child(root); - _copy_props_wo_key(next_doc, root); - _p(next_doc)->m_type.add(DOC); - _p(next_doc)->m_type.rem(SEQ); - } - _p(root)->m_type = STREAM; - return; - } - _RYML_CB_ASSERT(m_callbacks, !has_key(root)); - size_t next_doc = append_child(root); - _copy_props_wo_key(next_doc, root); - _add_flags(next_doc, DOC); - for(size_t prev = NONE, ch = first_child(root), next = next_sibling(ch); ch != NONE; ) - { - if(ch == next_doc) - break; - move(ch, next_doc, prev); - prev = ch; - ch = next; - next = next_sibling(next); - } - _p(root)->m_type = STREAM; -} - - -//----------------------------------------------------------------------------- -void Tree::remove_children(size_t node) -{ - _RYML_CB_ASSERT(m_callbacks, get(node) != nullptr); - size_t ich = get(node)->m_first_child; - while(ich != NONE) - { - remove_children(ich); - _RYML_CB_ASSERT(m_callbacks, get(ich) != nullptr); - size_t next = get(ich)->m_next_sibling; - _release(ich); - if(ich == get(node)->m_last_child) - break; - ich = next; - } -} - -bool Tree::change_type(size_t node, NodeType type) -{ - _RYML_CB_ASSERT(m_callbacks, type.is_val() || type.is_map() || type.is_seq()); - _RYML_CB_ASSERT(m_callbacks, type.is_val() + type.is_map() + type.is_seq() == 1); - _RYML_CB_ASSERT(m_callbacks, type.has_key() == has_key(node) || (has_key(node) && !type.has_key())); - NodeData *d = _p(node); - if(type.is_map() && is_map(node)) - return false; - else if(type.is_seq() && is_seq(node)) - return false; - else if(type.is_val() && is_val(node)) - return false; - d->m_type = (d->m_type & (~(MAP|SEQ|VAL))) | type; - remove_children(node); - return true; -} - - -//----------------------------------------------------------------------------- -size_t Tree::duplicate(size_t node, size_t parent, size_t after) -{ - return duplicate(this, node, parent, after); -} - -size_t Tree::duplicate(Tree const* src, size_t node, size_t parent, size_t after) -{ - _RYML_CB_ASSERT(m_callbacks, src != nullptr); - _RYML_CB_ASSERT(m_callbacks, node != NONE); - _RYML_CB_ASSERT(m_callbacks, parent != NONE); - _RYML_CB_ASSERT(m_callbacks, ! src->is_root(node)); - - size_t copy = _claim(); - - _copy_props(copy, src, node); - _set_hierarchy(copy, parent, after); - duplicate_children(src, node, copy, NONE); - - return copy; -} - -//----------------------------------------------------------------------------- -size_t Tree::duplicate_children(size_t node, size_t parent, size_t after) -{ - return duplicate_children(this, node, parent, after); -} - -size_t Tree::duplicate_children(Tree const* src, size_t node, size_t parent, size_t after) -{ - _RYML_CB_ASSERT(m_callbacks, src != nullptr); - _RYML_CB_ASSERT(m_callbacks, node != NONE); - _RYML_CB_ASSERT(m_callbacks, parent != NONE); - _RYML_CB_ASSERT(m_callbacks, after == NONE || has_child(parent, after)); - - size_t prev = after; - for(size_t i = src->first_child(node); i != NONE; i = src->next_sibling(i)) - { - prev = duplicate(src, i, parent, prev); - } - - return prev; -} - -//----------------------------------------------------------------------------- -void Tree::duplicate_contents(size_t node, size_t where) -{ - duplicate_contents(this, node, where); -} - -void Tree::duplicate_contents(Tree const *src, size_t node, size_t where) -{ - _RYML_CB_ASSERT(m_callbacks, src != nullptr); - _RYML_CB_ASSERT(m_callbacks, node != NONE); - _RYML_CB_ASSERT(m_callbacks, where != NONE); - _copy_props_wo_key(where, src, node); - duplicate_children(src, node, where, last_child(where)); -} - -//----------------------------------------------------------------------------- -size_t Tree::duplicate_children_no_rep(size_t node, size_t parent, size_t after) -{ - return duplicate_children_no_rep(this, node, parent, after); -} - -size_t Tree::duplicate_children_no_rep(Tree const *src, size_t node, size_t parent, size_t after) -{ - _RYML_CB_ASSERT(m_callbacks, node != NONE); - _RYML_CB_ASSERT(m_callbacks, parent != NONE); - _RYML_CB_ASSERT(m_callbacks, after == NONE || has_child(parent, after)); - - // don't loop using pointers as there may be a relocation - - // find the position where "after" is - size_t after_pos = NONE; - if(after != NONE) - { - for(size_t i = first_child(parent), icount = 0; i != NONE; ++icount, i = next_sibling(i)) - { - if(i == after) - { - after_pos = icount; - break; - } - } - _RYML_CB_ASSERT(m_callbacks, after_pos != NONE); - } - - // for each child to be duplicated... - size_t prev = after; - for(size_t i = src->first_child(node), icount = 0; i != NONE; ++icount, i = src->next_sibling(i)) - { - if(is_seq(parent)) - { - prev = duplicate(i, parent, prev); - } - else - { - _RYML_CB_ASSERT(m_callbacks, is_map(parent)); - // does the parent already have a node with key equal to that of the current duplicate? - size_t rep = NONE, rep_pos = NONE; - for(size_t j = first_child(parent), jcount = 0; j != NONE; ++jcount, j = next_sibling(j)) - { - if(key(j) == key(i)) - { - rep = j; - rep_pos = jcount; - break; - } - } - if(rep == NONE) // there is no repetition; just duplicate - { - prev = duplicate(src, i, parent, prev); - } - else // yes, there is a repetition - { - if(after_pos != NONE && rep_pos < after_pos) - { - // rep is located before the node which will be inserted, - // and will be overridden by the duplicate. So replace it. - remove(rep); - prev = duplicate(src, i, parent, prev); - } - else if(prev == NONE) - { - // first iteration with prev = after = NONE and repetition - prev = rep; - } - else if(rep != prev) - { - // rep is located after the node which will be inserted - // and overrides it. So move the rep into this node's place. - move(rep, prev); - prev = rep; - } - } // there's a repetition - } - } - - return prev; -} - - -//----------------------------------------------------------------------------- - -void Tree::merge_with(Tree const *src, size_t src_node, size_t dst_node) -{ - _RYML_CB_ASSERT(m_callbacks, src != nullptr); - if(src_node == NONE) - src_node = src->root_id(); - if(dst_node == NONE) - dst_node = root_id(); - _RYML_CB_ASSERT(m_callbacks, src->has_val(src_node) || src->is_seq(src_node) || src->is_map(src_node)); - - if(src->has_val(src_node)) - { - if( ! has_val(dst_node)) - { - if(has_children(dst_node)) - remove_children(dst_node); - } - if(src->is_keyval(src_node)) - _copy_props(dst_node, src, src_node); - else if(src->is_val(src_node)) - _copy_props_wo_key(dst_node, src, src_node); - else - C4_NEVER_REACH(); - } - else if(src->is_seq(src_node)) - { - if( ! is_seq(dst_node)) - { - if(has_children(dst_node)) - remove_children(dst_node); - _clear_type(dst_node); - if(src->has_key(src_node)) - to_seq(dst_node, src->key(src_node)); - else - to_seq(dst_node); - } - for(size_t sch = src->first_child(src_node); sch != NONE; sch = src->next_sibling(sch)) - { - size_t dch = append_child(dst_node); - _copy_props_wo_key(dch, src, sch); - merge_with(src, sch, dch); - } - } - else if(src->is_map(src_node)) - { - if( ! is_map(dst_node)) - { - if(has_children(dst_node)) - remove_children(dst_node); - _clear_type(dst_node); - if(src->has_key(src_node)) - to_map(dst_node, src->key(src_node)); - else - to_map(dst_node); - } - for(size_t sch = src->first_child(src_node); sch != NONE; sch = src->next_sibling(sch)) - { - size_t dch = find_child(dst_node, src->key(sch)); - if(dch == NONE) - { - dch = append_child(dst_node); - _copy_props(dch, src, sch); - } - merge_with(src, sch, dch); - } - } - else - { - C4_NEVER_REACH(); - } -} - - -//----------------------------------------------------------------------------- - -namespace detail { -/** @todo make this part of the public API, refactoring as appropriate - * to be able to use the same resolver to handle multiple trees (one - * at a time) */ -struct ReferenceResolver -{ - struct refdata - { - NodeType type; - size_t node; - size_t prev_anchor; - size_t target; - size_t parent_ref; - size_t parent_ref_sibling; - }; - - Tree *t; - /** from the specs: "an alias node refers to the most recent - * node in the serialization having the specified anchor". So - * we need to start looking upward from ref nodes. - * - * @see http://yaml.org/spec/1.2/spec.html#id2765878 */ - stack refs; - - ReferenceResolver(Tree *t_) : t(t_), refs(t_->callbacks()) - { - resolve(); - } - - void store_anchors_and_refs() - { - // minimize (re-)allocations by counting first - size_t num_anchors_and_refs = count_anchors_and_refs(t->root_id()); - if(!num_anchors_and_refs) - return; - refs.reserve(num_anchors_and_refs); - - // now descend through the hierarchy - _store_anchors_and_refs(t->root_id()); - - // finally connect the reference list - size_t prev_anchor = npos; - size_t count = 0; - for(auto &rd : refs) - { - rd.prev_anchor = prev_anchor; - if(rd.type.is_anchor()) - prev_anchor = count; - ++count; - } - } - - size_t count_anchors_and_refs(size_t n) - { - size_t c = 0; - c += t->has_key_anchor(n); - c += t->has_val_anchor(n); - c += t->is_key_ref(n); - c += t->is_val_ref(n); - for(size_t ch = t->first_child(n); ch != NONE; ch = t->next_sibling(ch)) - c += count_anchors_and_refs(ch); - return c; - } - - void _store_anchors_and_refs(size_t n) - { - if(t->is_key_ref(n) || t->is_val_ref(n) || (t->has_key(n) && t->key(n) == "<<")) - { - if(t->is_seq(n)) - { - // for merging multiple inheritance targets - // <<: [ *CENTER, *BIG ] - for(size_t ich = t->first_child(n); ich != NONE; ich = t->next_sibling(ich)) - { - RYML_ASSERT(t->num_children(ich) == 0); - refs.push({VALREF, ich, npos, npos, n, t->next_sibling(n)}); - } - return; - } - if(t->is_key_ref(n) && t->key(n) != "<<") // insert key refs BEFORE inserting val refs - { - RYML_CHECK((!t->has_key(n)) || t->key(n).ends_with(t->key_ref(n))); - refs.push({KEYREF, n, npos, npos, NONE, NONE}); - } - if(t->is_val_ref(n)) - { - RYML_CHECK((!t->has_val(n)) || t->val(n).ends_with(t->val_ref(n))); - refs.push({VALREF, n, npos, npos, NONE, NONE}); - } - } - if(t->has_key_anchor(n)) - { - RYML_CHECK(t->has_key(n)); - refs.push({KEYANCH, n, npos, npos, NONE, NONE}); - } - if(t->has_val_anchor(n)) - { - RYML_CHECK(t->has_val(n) || t->is_container(n)); - refs.push({VALANCH, n, npos, npos, NONE, NONE}); - } - for(size_t ch = t->first_child(n); ch != NONE; ch = t->next_sibling(ch)) - { - _store_anchors_and_refs(ch); - } - } - - size_t lookup_(refdata *C4_RESTRICT ra) - { - RYML_ASSERT(ra->type.is_key_ref() || ra->type.is_val_ref()); - RYML_ASSERT(ra->type.is_key_ref() != ra->type.is_val_ref()); - csubstr refname; - if(ra->type.is_val_ref()) - { - refname = t->val_ref(ra->node); - } - else - { - RYML_ASSERT(ra->type.is_key_ref()); - refname = t->key_ref(ra->node); - } - while(ra->prev_anchor != npos) - { - ra = &refs[ra->prev_anchor]; - if(t->has_anchor(ra->node, refname)) - return ra->node; - } - - #ifndef RYML_ERRMSG_SIZE - #define RYML_ERRMSG_SIZE 1024 - #endif - - char errmsg[RYML_ERRMSG_SIZE]; - snprintf(errmsg, RYML_ERRMSG_SIZE, "anchor does not exist: '%.*s'", - static_cast(refname.size()), refname.data()); - c4::yml::error(errmsg); - return NONE; - } - - void resolve() - { - store_anchors_and_refs(); - if(refs.empty()) - return; - - /* from the specs: "an alias node refers to the most recent - * node in the serialization having the specified anchor". So - * we need to start looking upward from ref nodes. - * - * @see http://yaml.org/spec/1.2/spec.html#id2765878 */ - for(size_t i = 0, e = refs.size(); i < e; ++i) - { - auto &C4_RESTRICT rd = refs.top(i); - if( ! rd.type.is_ref()) - continue; - rd.target = lookup_(&rd); - } - } - -}; // ReferenceResolver -} // namespace detail - -void Tree::resolve() -{ - if(m_size == 0) - return; - - detail::ReferenceResolver rr(this); - - // insert the resolved references - size_t prev_parent_ref = NONE; - size_t prev_parent_ref_after = NONE; - for(auto const& C4_RESTRICT rd : rr.refs) - { - if( ! rd.type.is_ref()) - continue; - if(rd.parent_ref != NONE) - { - _RYML_CB_ASSERT(m_callbacks, is_seq(rd.parent_ref)); - size_t after, p = parent(rd.parent_ref); - if(prev_parent_ref != rd.parent_ref) - { - after = rd.parent_ref;//prev_sibling(rd.parent_ref_sibling); - prev_parent_ref_after = after; - } - else - { - after = prev_parent_ref_after; - } - prev_parent_ref = rd.parent_ref; - prev_parent_ref_after = duplicate_children_no_rep(rd.target, p, after); - remove(rd.node); - } - else - { - if(has_key(rd.node) && is_key_ref(rd.node) && key(rd.node) == "<<") - { - _RYML_CB_ASSERT(m_callbacks, is_keyval(rd.node)); - size_t p = parent(rd.node); - size_t after = prev_sibling(rd.node); - duplicate_children_no_rep(rd.target, p, after); - remove(rd.node); - } - else if(rd.type.is_key_ref()) - { - _RYML_CB_ASSERT(m_callbacks, is_key_ref(rd.node)); - _RYML_CB_ASSERT(m_callbacks, has_key_anchor(rd.target) || has_val_anchor(rd.target)); - if(has_val_anchor(rd.target) && val_anchor(rd.target) == key_ref(rd.node)) - { - _RYML_CB_CHECK(m_callbacks, !is_container(rd.target)); - _RYML_CB_CHECK(m_callbacks, has_val(rd.target)); - _p(rd.node)->m_key.scalar = val(rd.target); - _add_flags(rd.node, KEY); - } - else - { - _RYML_CB_CHECK(m_callbacks, key_anchor(rd.target) == key_ref(rd.node)); - _p(rd.node)->m_key.scalar = key(rd.target); - _add_flags(rd.node, VAL); - } - } - else - { - _RYML_CB_ASSERT(m_callbacks, rd.type.is_val_ref()); - if(has_key_anchor(rd.target) && key_anchor(rd.target) == val_ref(rd.node)) - { - _RYML_CB_CHECK(m_callbacks, !is_container(rd.target)); - _RYML_CB_CHECK(m_callbacks, has_val(rd.target)); - _p(rd.node)->m_val.scalar = key(rd.target); - _add_flags(rd.node, VAL); - } - else - { - duplicate_contents(rd.target, rd.node); - } - } - } - } - - // clear anchors and refs - for(auto const& C4_RESTRICT ar : rr.refs) - { - rem_anchor_ref(ar.node); - if(ar.parent_ref != NONE) - if(type(ar.parent_ref) != NOTYPE) - remove(ar.parent_ref); - } - -} - -//----------------------------------------------------------------------------- - -size_t Tree::num_children(size_t node) const -{ - size_t count = 0; - for(size_t i = first_child(node); i != NONE; i = next_sibling(i)) - ++count; - return count; -} - -size_t Tree::child(size_t node, size_t pos) const -{ - _RYML_CB_ASSERT(m_callbacks, node != NONE); - size_t count = 0; - for(size_t i = first_child(node); i != NONE; i = next_sibling(i)) - { - if(count++ == pos) - return i; - } - return NONE; -} - -size_t Tree::child_pos(size_t node, size_t ch) const -{ - size_t count = 0; - for(size_t i = first_child(node); i != NONE; i = next_sibling(i)) - { - if(i == ch) - return count; - ++count; - } - return npos; -} - -#if defined(__clang__) -# pragma clang diagnostic push -# pragma GCC diagnostic ignored "-Wnull-dereference" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# if __GNUC__ >= 6 -# pragma GCC diagnostic ignored "-Wnull-dereference" -# endif -#endif - -size_t Tree::find_child(size_t node, csubstr const& name) const -{ - _RYML_CB_ASSERT(m_callbacks, node != NONE); - _RYML_CB_ASSERT(m_callbacks, is_map(node)); - if(get(node)->m_first_child == NONE) - { - _RYML_CB_ASSERT(m_callbacks, _p(node)->m_last_child == NONE); - return NONE; - } - else - { - _RYML_CB_ASSERT(m_callbacks, _p(node)->m_last_child != NONE); - } - for(size_t i = first_child(node); i != NONE; i = next_sibling(i)) - { - if(_p(i)->m_key.scalar == name) - { - return i; - } - } - return NONE; -} - -#if defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - - -//----------------------------------------------------------------------------- - -void Tree::to_val(size_t node, csubstr val, type_bits more_flags) -{ - _RYML_CB_ASSERT(m_callbacks, ! has_children(node)); - _RYML_CB_ASSERT(m_callbacks, parent(node) == NONE || ! parent_is_map(node)); - _set_flags(node, VAL|more_flags); - _p(node)->m_key.clear(); - _p(node)->m_val = val; -} - -void Tree::to_keyval(size_t node, csubstr key, csubstr val, type_bits more_flags) -{ - _RYML_CB_ASSERT(m_callbacks, ! has_children(node)); - _RYML_CB_ASSERT(m_callbacks, parent(node) == NONE || parent_is_map(node)); - _set_flags(node, KEYVAL|more_flags); - _p(node)->m_key = key; - _p(node)->m_val = val; -} - -void Tree::to_map(size_t node, type_bits more_flags) -{ - _RYML_CB_ASSERT(m_callbacks, ! has_children(node)); - _RYML_CB_ASSERT(m_callbacks, parent(node) == NONE || ! parent_is_map(node)); // parent must not have children with keys - _set_flags(node, MAP|more_flags); - _p(node)->m_key.clear(); - _p(node)->m_val.clear(); -} - -void Tree::to_map(size_t node, csubstr key, type_bits more_flags) -{ - _RYML_CB_ASSERT(m_callbacks, ! has_children(node)); - _RYML_CB_ASSERT(m_callbacks, parent(node) == NONE || parent_is_map(node)); - _set_flags(node, KEY|MAP|more_flags); - _p(node)->m_key = key; - _p(node)->m_val.clear(); -} - -void Tree::to_seq(size_t node, type_bits more_flags) -{ - _RYML_CB_ASSERT(m_callbacks, ! has_children(node)); - _RYML_CB_ASSERT(m_callbacks, parent(node) == NONE || parent_is_seq(node)); - _set_flags(node, SEQ|more_flags); - _p(node)->m_key.clear(); - _p(node)->m_val.clear(); -} - -void Tree::to_seq(size_t node, csubstr key, type_bits more_flags) -{ - _RYML_CB_ASSERT(m_callbacks, ! has_children(node)); - _RYML_CB_ASSERT(m_callbacks, parent(node) == NONE || parent_is_map(node)); - _set_flags(node, KEY|SEQ|more_flags); - _p(node)->m_key = key; - _p(node)->m_val.clear(); -} - -void Tree::to_doc(size_t node, type_bits more_flags) -{ - _RYML_CB_ASSERT(m_callbacks, ! has_children(node)); - _set_flags(node, DOC|more_flags); - _p(node)->m_key.clear(); - _p(node)->m_val.clear(); -} - -void Tree::to_stream(size_t node, type_bits more_flags) -{ - _RYML_CB_ASSERT(m_callbacks, ! has_children(node)); - _set_flags(node, STREAM|more_flags); - _p(node)->m_key.clear(); - _p(node)->m_val.clear(); -} - - -//----------------------------------------------------------------------------- -size_t Tree::num_tag_directives() const -{ - // this assumes we have a very small number of tag directives - for(size_t i = 0; i < RYML_MAX_TAG_DIRECTIVES; ++i) - if(m_tag_directives[i].handle.empty()) - return i; - return RYML_MAX_TAG_DIRECTIVES; -} - -void Tree::clear_tag_directives() -{ - for(TagDirective &td : m_tag_directives) - td = {}; -} - -size_t Tree::add_tag_directive(TagDirective const& td) -{ - _RYML_CB_CHECK(m_callbacks, !td.handle.empty()); - _RYML_CB_CHECK(m_callbacks, !td.prefix.empty()); - _RYML_CB_ASSERT(m_callbacks, td.handle.begins_with('!')); - _RYML_CB_ASSERT(m_callbacks, td.handle.ends_with('!')); - // https://yaml.org/spec/1.2.2/#rule-ns-word-char - _RYML_CB_ASSERT(m_callbacks, td.handle == '!' || td.handle == "!!" || td.handle.trim('!').first_not_of("01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ-") == npos); - size_t pos = num_tag_directives(); - _RYML_CB_CHECK(m_callbacks, pos < RYML_MAX_TAG_DIRECTIVES); - m_tag_directives[pos] = td; - return pos; -} - -size_t Tree::resolve_tag(substr output, csubstr tag, size_t node_id) const -{ - // lookup from the end. We want to find the first directive that - // matches the tag and has a target node id leq than the given - // node_id. - for(size_t i = RYML_MAX_TAG_DIRECTIVES-1; i != (size_t)-1; --i) - { - auto const& td = m_tag_directives[i]; - if(td.handle.empty()) - continue; - if(tag.begins_with(td.handle) && td.next_node_id <= node_id) - { - _RYML_CB_ASSERT(m_callbacks, tag.len >= td.handle.len); - csubstr rest = tag.sub(td.handle.len); - size_t len = 1u + td.prefix.len + rest.len + 1u; - size_t numpc = rest.count('%'); - if(numpc == 0) - { - if(len <= output.len) - { - output.str[0] = '<'; - memcpy(1u + output.str, td.prefix.str, td.prefix.len); - memcpy(1u + output.str + td.prefix.len, rest.str, rest.len); - output.str[1u + td.prefix.len + rest.len] = '>'; - } - } - else - { - // need to decode URI % sequences - size_t pos = rest.find('%'); - _RYML_CB_ASSERT(m_callbacks, pos != npos); - do { - size_t next = rest.first_not_of("0123456789abcdefABCDEF", pos+1); - if(next == npos) - next = rest.len; - _RYML_CB_CHECK(m_callbacks, pos+1 < next); - _RYML_CB_CHECK(m_callbacks, pos+1 + 2 <= next); - size_t delta = next - (pos+1); - len -= delta; - pos = rest.find('%', pos+1); - } while(pos != npos); - if(len <= output.len) - { - size_t prev = 0, wpos = 0; - auto appendstr = [&](csubstr s) { memcpy(output.str + wpos, s.str, s.len); wpos += s.len; }; - auto appendchar = [&](char c) { output.str[wpos++] = c; }; - appendchar('<'); - appendstr(td.prefix); - pos = rest.find('%'); - _RYML_CB_ASSERT(m_callbacks, pos != npos); - do { - size_t next = rest.first_not_of("0123456789abcdefABCDEF", pos+1); - if(next == npos) - next = rest.len; - _RYML_CB_CHECK(m_callbacks, pos+1 < next); - _RYML_CB_CHECK(m_callbacks, pos+1 + 2 <= next); - uint8_t val; - if(C4_UNLIKELY(!read_hex(rest.range(pos+1, next), &val) || val > 127)) - _RYML_CB_ERR(m_callbacks, "invalid URI character"); - appendstr(rest.range(prev, pos)); - appendchar((char)val); - prev = next; - pos = rest.find('%', pos+1); - } while(pos != npos); - _RYML_CB_ASSERT(m_callbacks, pos == npos); - _RYML_CB_ASSERT(m_callbacks, prev > 0); - _RYML_CB_ASSERT(m_callbacks, rest.len >= prev); - appendstr(rest.sub(prev)); - appendchar('>'); - _RYML_CB_ASSERT(m_callbacks, wpos == len); - } - } - return len; - } - } - return 0; // return 0 to signal that the tag is local and cannot be resolved -} - -namespace { -csubstr _transform_tag(Tree *t, csubstr tag, size_t node) -{ - size_t required_size = t->resolve_tag(substr{}, tag, node); - if(!required_size) - return tag; - const char *prev_arena = t->arena().str; - substr buf = t->alloc_arena(required_size); - _RYML_CB_ASSERT(t->m_callbacks, t->arena().str == prev_arena); - size_t actual_size = t->resolve_tag(buf, tag, node); - _RYML_CB_ASSERT(t->m_callbacks, actual_size <= required_size); - return buf.first(actual_size); -} -void _resolve_tags(Tree *t, size_t node) -{ - for(size_t child = t->first_child(node); child != NONE; child = t->next_sibling(child)) - { - if(t->has_key(child) && t->has_key_tag(child)) - t->set_key_tag(child, _transform_tag(t, t->key_tag(child), child)); - if(t->has_val(child) && t->has_val_tag(child)) - t->set_val_tag(child, _transform_tag(t, t->val_tag(child), child)); - _resolve_tags(t, child); - } -} -size_t _count_resolved_tags_size(Tree const* t, size_t node) -{ - size_t sz = 0; - for(size_t child = t->first_child(node); child != NONE; child = t->next_sibling(child)) - { - if(t->has_key(child) && t->has_key_tag(child)) - sz += t->resolve_tag(substr{}, t->key_tag(child), child); - if(t->has_val(child) && t->has_val_tag(child)) - sz += t->resolve_tag(substr{}, t->val_tag(child), child); - sz += _count_resolved_tags_size(t, child); - } - return sz; -} -} // namespace - -void Tree::resolve_tags() -{ - if(empty()) - return; - if(num_tag_directives() == 0) - return; - size_t needed_size = _count_resolved_tags_size(this, root_id()); - if(needed_size) - reserve_arena(arena_size() + needed_size); - _resolve_tags(this, root_id()); -} - - -//----------------------------------------------------------------------------- - -csubstr Tree::lookup_result::resolved() const -{ - csubstr p = path.first(path_pos); - if(p.ends_with('.')) - p = p.first(p.len-1); - return p; -} - -csubstr Tree::lookup_result::unresolved() const -{ - return path.sub(path_pos); -} - -void Tree::_advance(lookup_result *r, size_t more) const -{ - r->path_pos += more; - if(r->path.sub(r->path_pos).begins_with('.')) - ++r->path_pos; -} - -Tree::lookup_result Tree::lookup_path(csubstr path, size_t start) const -{ - if(start == NONE) - start = root_id(); - lookup_result r(path, start); - if(path.empty()) - return r; - _lookup_path(&r); - if(r.target == NONE && r.closest == start) - r.closest = NONE; - return r; -} - -size_t Tree::lookup_path_or_modify(csubstr default_value, csubstr path, size_t start) -{ - size_t target = _lookup_path_or_create(path, start); - if(parent_is_map(target)) - to_keyval(target, key(target), default_value); - else - to_val(target, default_value); - return target; -} - -size_t Tree::lookup_path_or_modify(Tree const *src, size_t src_node, csubstr path, size_t start) -{ - size_t target = _lookup_path_or_create(path, start); - merge_with(src, src_node, target); - return target; -} - -size_t Tree::_lookup_path_or_create(csubstr path, size_t start) -{ - if(start == NONE) - start = root_id(); - lookup_result r(path, start); - _lookup_path(&r); - if(r.target != NONE) - { - C4_ASSERT(r.unresolved().empty()); - return r.target; - } - _lookup_path_modify(&r); - return r.target; -} - -void Tree::_lookup_path(lookup_result *r) const -{ - C4_ASSERT( ! r->unresolved().empty()); - _lookup_path_token parent{"", type(r->closest)}; - size_t node; - do - { - node = _next_node(r, &parent); - if(node != NONE) - r->closest = node; - if(r->unresolved().empty()) - { - r->target = node; - return; - } - } while(node != NONE); -} - -void Tree::_lookup_path_modify(lookup_result *r) -{ - C4_ASSERT( ! r->unresolved().empty()); - _lookup_path_token parent{"", type(r->closest)}; - size_t node; - do - { - node = _next_node_modify(r, &parent); - if(node != NONE) - r->closest = node; - if(r->unresolved().empty()) - { - r->target = node; - return; - } - } while(node != NONE); -} - -size_t Tree::_next_node(lookup_result * r, _lookup_path_token *parent) const -{ - _lookup_path_token token = _next_token(r, *parent); - if( ! token) - return NONE; - - size_t node = NONE; - csubstr prev = token.value; - if(token.type == MAP || token.type == SEQ) - { - _RYML_CB_ASSERT(m_callbacks, !token.value.begins_with('[')); - //_RYML_CB_ASSERT(m_callbacks, is_container(r->closest) || r->closest == NONE); - _RYML_CB_ASSERT(m_callbacks, is_map(r->closest)); - node = find_child(r->closest, token.value); - } - else if(token.type == KEYVAL) - { - _RYML_CB_ASSERT(m_callbacks, r->unresolved().empty()); - if(is_map(r->closest)) - node = find_child(r->closest, token.value); - } - else if(token.type == KEY) - { - _RYML_CB_ASSERT(m_callbacks, token.value.begins_with('[') && token.value.ends_with(']')); - token.value = token.value.offs(1, 1).trim(' '); - size_t idx = 0; - _RYML_CB_CHECK(m_callbacks, from_chars(token.value, &idx)); - node = child(r->closest, idx); - } - else - { - C4_NEVER_REACH(); - } - - if(node != NONE) - { - *parent = token; - } - else - { - csubstr p = r->path.sub(r->path_pos > 0 ? r->path_pos - 1 : r->path_pos); - r->path_pos -= prev.len; - if(p.begins_with('.')) - r->path_pos -= 1u; - } - - return node; -} - -size_t Tree::_next_node_modify(lookup_result * r, _lookup_path_token *parent) -{ - _lookup_path_token token = _next_token(r, *parent); - if( ! token) - return NONE; - - size_t node = NONE; - if(token.type == MAP || token.type == SEQ) - { - _RYML_CB_ASSERT(m_callbacks, !token.value.begins_with('[')); - //_RYML_CB_ASSERT(m_callbacks, is_container(r->closest) || r->closest == NONE); - if( ! is_container(r->closest)) - { - if(has_key(r->closest)) - to_map(r->closest, key(r->closest)); - else - to_map(r->closest); - } - else - { - if(is_map(r->closest)) - node = find_child(r->closest, token.value); - else - { - size_t pos = NONE; - _RYML_CB_CHECK(m_callbacks, c4::atox(token.value, &pos)); - _RYML_CB_ASSERT(m_callbacks, pos != NONE); - node = child(r->closest, pos); - } - } - if(node == NONE) - { - _RYML_CB_ASSERT(m_callbacks, is_map(r->closest)); - node = append_child(r->closest); - NodeData *n = _p(node); - n->m_key.scalar = token.value; - n->m_type.add(KEY); - } - } - else if(token.type == KEYVAL) - { - _RYML_CB_ASSERT(m_callbacks, r->unresolved().empty()); - if(is_map(r->closest)) - { - node = find_child(r->closest, token.value); - if(node == NONE) - node = append_child(r->closest); - } - else - { - _RYML_CB_ASSERT(m_callbacks, !is_seq(r->closest)); - _add_flags(r->closest, MAP); - node = append_child(r->closest); - } - NodeData *n = _p(node); - n->m_key.scalar = token.value; - n->m_val.scalar = ""; - n->m_type.add(KEYVAL); - } - else if(token.type == KEY) - { - _RYML_CB_ASSERT(m_callbacks, token.value.begins_with('[') && token.value.ends_with(']')); - token.value = token.value.offs(1, 1).trim(' '); - size_t idx; - if( ! from_chars(token.value, &idx)) - return NONE; - if( ! is_container(r->closest)) - { - if(has_key(r->closest)) - { - csubstr k = key(r->closest); - _clear_type(r->closest); - to_seq(r->closest, k); - } - else - { - _clear_type(r->closest); - to_seq(r->closest); - } - } - _RYML_CB_ASSERT(m_callbacks, is_container(r->closest)); - node = child(r->closest, idx); - if(node == NONE) - { - _RYML_CB_ASSERT(m_callbacks, num_children(r->closest) <= idx); - for(size_t i = num_children(r->closest); i <= idx; ++i) - { - node = append_child(r->closest); - if(i < idx) - { - if(is_map(r->closest)) - to_keyval(node, /*"~"*/{}, /*"~"*/{}); - else if(is_seq(r->closest)) - to_val(node, /*"~"*/{}); - } - } - } - } - else - { - C4_NEVER_REACH(); - } - - _RYML_CB_ASSERT(m_callbacks, node != NONE); - *parent = token; - return node; -} - -/** types of tokens: - * - seeing "map." ---> "map"/MAP - * - finishing "scalar" ---> "scalar"/KEYVAL - * - seeing "seq[n]" ---> "seq"/SEQ (--> "[n]"/KEY) - * - seeing "[n]" ---> "[n]"/KEY - */ -Tree::_lookup_path_token Tree::_next_token(lookup_result *r, _lookup_path_token const& parent) const -{ - csubstr unres = r->unresolved(); - if(unres.empty()) - return {}; - - // is it an indexation like [0], [1], etc? - if(unres.begins_with('[')) - { - size_t pos = unres.find(']'); - if(pos == csubstr::npos) - return {}; - csubstr idx = unres.first(pos + 1); - _advance(r, pos + 1); - return {idx, KEY}; - } - - // no. so it must be a name - size_t pos = unres.first_of(".["); - if(pos == csubstr::npos) - { - _advance(r, unres.len); - NodeType t; - if(( ! parent) || parent.type.is_seq()) - return {unres, VAL}; - return {unres, KEYVAL}; - } - - // it's either a map or a seq - _RYML_CB_ASSERT(m_callbacks, unres[pos] == '.' || unres[pos] == '['); - if(unres[pos] == '.') - { - _RYML_CB_ASSERT(m_callbacks, pos != 0); - _advance(r, pos + 1); - return {unres.first(pos), MAP}; - } - - _RYML_CB_ASSERT(m_callbacks, unres[pos] == '['); - _advance(r, pos); - return {unres.first(pos), SEQ}; -} - - -} // namespace ryml -} // namespace c4 - - -C4_SUPPRESS_WARNING_GCC_POP -C4_SUPPRESS_WARNING_MSVC_POP diff --git a/thirdparty/ryml/src/c4/yml/tree.hpp b/thirdparty/ryml/src/c4/yml/tree.hpp deleted file mode 100644 index 5adc5583a..000000000 --- a/thirdparty/ryml/src/c4/yml/tree.hpp +++ /dev/null @@ -1,1495 +0,0 @@ -#ifndef _C4_YML_TREE_HPP_ -#define _C4_YML_TREE_HPP_ - - -#include "c4/error.hpp" -#include "c4/types.hpp" -#ifndef _C4_YML_COMMON_HPP_ -#include "c4/yml/common.hpp" -#endif - -#include -#include -#include - - -C4_SUPPRESS_WARNING_MSVC_PUSH -C4_SUPPRESS_WARNING_MSVC(4251) // needs to have dll-interface to be used by clients of struct -C4_SUPPRESS_WARNING_MSVC(4296) // expression is always 'boolean_value' -C4_SUPPRESS_WARNING_GCC_CLANG_PUSH -C4_SUPPRESS_WARNING_GCC("-Wtype-limits") - - -namespace c4 { -namespace yml { - -struct NodeScalar; -struct NodeInit; -struct NodeData; -class NodeRef; -class ConstNodeRef; -class Tree; - - -/** encode a floating point value to a string. */ -template -size_t to_chars_float(substr buf, T val) -{ - C4_SUPPRESS_WARNING_GCC_CLANG_WITH_PUSH("-Wfloat-equal"); - static_assert(std::is_floating_point::value, "must be floating point"); - if(C4_UNLIKELY(std::isnan(val))) - return to_chars(buf, csubstr(".nan")); - else if(C4_UNLIKELY(val == std::numeric_limits::infinity())) - return to_chars(buf, csubstr(".inf")); - else if(C4_UNLIKELY(val == -std::numeric_limits::infinity())) - return to_chars(buf, csubstr("-.inf")); - return to_chars(buf, val); - C4_SUPPRESS_WARNING_GCC_CLANG_POP -} - - -/** decode a floating point from string. Accepts special values: .nan, - * .inf, -.inf */ -template -bool from_chars_float(csubstr buf, T *C4_RESTRICT val) -{ - static_assert(std::is_floating_point::value, "must be floating point"); - if(C4_LIKELY(from_chars(buf, val))) - { - return true; - } - else if(C4_UNLIKELY(buf == ".nan" || buf == ".NaN" || buf == ".NAN")) - { - *val = std::numeric_limits::quiet_NaN(); - return true; - } - else if(C4_UNLIKELY(buf == ".inf" || buf == ".Inf" || buf == ".INF")) - { - *val = std::numeric_limits::infinity(); - return true; - } - else if(C4_UNLIKELY(buf == "-.inf" || buf == "-.Inf" || buf == "-.INF")) - { - *val = -std::numeric_limits::infinity(); - return true; - } - else - { - return false; - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** the integral type necessary to cover all the bits marking node tags */ -using tag_bits = uint16_t; - -/** a bit mask for marking tags for types */ -typedef enum : tag_bits { - // container types - TAG_NONE = 0, - TAG_MAP = 1, /**< !!map Unordered set of key: value pairs without duplicates. @see https://yaml.org/type/map.html */ - TAG_OMAP = 2, /**< !!omap Ordered sequence of key: value pairs without duplicates. @see https://yaml.org/type/omap.html */ - TAG_PAIRS = 3, /**< !!pairs Ordered sequence of key: value pairs allowing duplicates. @see https://yaml.org/type/pairs.html */ - TAG_SET = 4, /**< !!set Unordered set of non-equal values. @see https://yaml.org/type/set.html */ - TAG_SEQ = 5, /**< !!seq Sequence of arbitrary values. @see https://yaml.org/type/seq.html */ - // scalar types - TAG_BINARY = 6, /**< !!binary A sequence of zero or more octets (8 bit values). @see https://yaml.org/type/binary.html */ - TAG_BOOL = 7, /**< !!bool Mathematical Booleans. @see https://yaml.org/type/bool.html */ - TAG_FLOAT = 8, /**< !!float Floating-point approximation to real numbers. https://yaml.org/type/float.html */ - TAG_INT = 9, /**< !!float Mathematical integers. https://yaml.org/type/int.html */ - TAG_MERGE = 10, /**< !!merge Specify one or more mapping to be merged with the current one. https://yaml.org/type/merge.html */ - TAG_NULL = 11, /**< !!null Devoid of value. https://yaml.org/type/null.html */ - TAG_STR = 12, /**< !!str A sequence of zero or more Unicode characters. https://yaml.org/type/str.html */ - TAG_TIMESTAMP = 13, /**< !!timestamp A point in time https://yaml.org/type/timestamp.html */ - TAG_VALUE = 14, /**< !!value Specify the default value of a mapping https://yaml.org/type/value.html */ - TAG_YAML = 15, /**< !!yaml Specify the default value of a mapping https://yaml.org/type/yaml.html */ -} YamlTag_e; - -YamlTag_e to_tag(csubstr tag); -csubstr from_tag(YamlTag_e tag); -csubstr from_tag_long(YamlTag_e tag); -csubstr normalize_tag(csubstr tag); -csubstr normalize_tag_long(csubstr tag); - -struct TagDirective -{ - /** Eg `!e!` in `%TAG !e! tag:example.com,2000:app/` */ - csubstr handle; - /** Eg `tag:example.com,2000:app/` in `%TAG !e! tag:example.com,2000:app/` */ - csubstr prefix; - /** The next node to which this tag directive applies */ - size_t next_node_id; -}; - -#ifndef RYML_MAX_TAG_DIRECTIVES -/** the maximum number of tag directives in a Tree */ -#define RYML_MAX_TAG_DIRECTIVES 4 -#endif - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -/** the integral type necessary to cover all the bits marking node types */ -using type_bits = uint64_t; - - -/** a bit mask for marking node types */ -typedef enum : type_bits { - // a convenience define, undefined below - #define c4bit(v) (type_bits(1) << v) - NOTYPE = 0, ///< no node type is set - VAL = c4bit(0), ///< a leaf node, has a (possibly empty) value - KEY = c4bit(1), ///< is member of a map, must have non-empty key - MAP = c4bit(2), ///< a map: a parent of keyvals - SEQ = c4bit(3), ///< a seq: a parent of vals - DOC = c4bit(4), ///< a document - STREAM = c4bit(5)|SEQ, ///< a stream: a seq of docs - KEYREF = c4bit(6), ///< a *reference: the key references an &anchor - VALREF = c4bit(7), ///< a *reference: the val references an &anchor - KEYANCH = c4bit(8), ///< the key has an &anchor - VALANCH = c4bit(9), ///< the val has an &anchor - KEYTAG = c4bit(10), ///< the key has an explicit tag/type - VALTAG = c4bit(11), ///< the val has an explicit tag/type - _TYMASK = c4bit(12)-1, // all the bits up to here - VALQUO = c4bit(12), ///< the val is quoted by '', "", > or | - KEYQUO = c4bit(13), ///< the key is quoted by '', "", > or | - KEYVAL = KEY|VAL, - KEYSEQ = KEY|SEQ, - KEYMAP = KEY|MAP, - DOCMAP = DOC|MAP, - DOCSEQ = DOC|SEQ, - DOCVAL = DOC|VAL, - _KEYMASK = KEY | KEYQUO | KEYANCH | KEYREF | KEYTAG, - _VALMASK = VAL | VALQUO | VALANCH | VALREF | VALTAG, - // these flags are from a work in progress and should not be used yet - _WIP_STYLE_FLOW_SL = c4bit(14), ///< mark container with single-line flow format (seqs as '[val1,val2], maps as '{key: val, key2: val2}') - _WIP_STYLE_FLOW_ML = c4bit(15), ///< mark container with multi-line flow format (seqs as '[val1,\nval2], maps as '{key: val,\nkey2: val2}') - _WIP_STYLE_BLOCK = c4bit(16), ///< mark container with block format (seqs as '- val\n', maps as 'key: val') - _WIP_KEY_LITERAL = c4bit(17), ///< mark key scalar as multiline, block literal | - _WIP_VAL_LITERAL = c4bit(18), ///< mark val scalar as multiline, block literal | - _WIP_KEY_FOLDED = c4bit(19), ///< mark key scalar as multiline, block folded > - _WIP_VAL_FOLDED = c4bit(20), ///< mark val scalar as multiline, block folded > - _WIP_KEY_SQUO = c4bit(21), ///< mark key scalar as single quoted - _WIP_VAL_SQUO = c4bit(22), ///< mark val scalar as single quoted - _WIP_KEY_DQUO = c4bit(23), ///< mark key scalar as double quoted - _WIP_VAL_DQUO = c4bit(24), ///< mark val scalar as double quoted - _WIP_KEY_PLAIN = c4bit(25), ///< mark key scalar as plain scalar (unquoted, even when multiline) - _WIP_VAL_PLAIN = c4bit(26), ///< mark val scalar as plain scalar (unquoted, even when multiline) - _WIP_KEY_STYLE = _WIP_KEY_LITERAL|_WIP_KEY_FOLDED|_WIP_KEY_SQUO|_WIP_KEY_DQUO|_WIP_KEY_PLAIN, - _WIP_VAL_STYLE = _WIP_VAL_LITERAL|_WIP_VAL_FOLDED|_WIP_VAL_SQUO|_WIP_VAL_DQUO|_WIP_VAL_PLAIN, - _WIP_KEY_FT_NL = c4bit(27), ///< features: mark key scalar as having \n in its contents - _WIP_VAL_FT_NL = c4bit(28), ///< features: mark val scalar as having \n in its contents - _WIP_KEY_FT_SQ = c4bit(29), ///< features: mark key scalar as having single quotes in its contents - _WIP_VAL_FT_SQ = c4bit(30), ///< features: mark val scalar as having single quotes in its contents - _WIP_KEY_FT_DQ = c4bit(31), ///< features: mark key scalar as having double quotes in its contents - _WIP_VAL_FT_DQ = c4bit(32), ///< features: mark val scalar as having double quotes in its contents - #undef c4bit -} NodeType_e; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** wraps a NodeType_e element with some syntactic sugar and predicates */ -struct NodeType -{ -public: - - NodeType_e type; - -public: - - C4_ALWAYS_INLINE NodeType() : type(NOTYPE) {} - C4_ALWAYS_INLINE NodeType(NodeType_e t) : type(t) {} - C4_ALWAYS_INLINE NodeType(type_bits t) : type((NodeType_e)t) {} - - C4_ALWAYS_INLINE const char *type_str() const { return type_str(type); } - static const char* type_str(NodeType_e t); - - C4_ALWAYS_INLINE void set(NodeType_e t) { type = t; } - C4_ALWAYS_INLINE void set(type_bits t) { type = (NodeType_e)t; } - - C4_ALWAYS_INLINE void add(NodeType_e t) { type = (NodeType_e)(type|t); } - C4_ALWAYS_INLINE void add(type_bits t) { type = (NodeType_e)(type|t); } - - C4_ALWAYS_INLINE void rem(NodeType_e t) { type = (NodeType_e)(type & ~t); } - C4_ALWAYS_INLINE void rem(type_bits t) { type = (NodeType_e)(type & ~t); } - - C4_ALWAYS_INLINE void clear() { type = NOTYPE; } - -public: - - C4_ALWAYS_INLINE operator NodeType_e & C4_RESTRICT () { return type; } - C4_ALWAYS_INLINE operator NodeType_e const& C4_RESTRICT () const { return type; } - - C4_ALWAYS_INLINE bool operator== (NodeType_e t) const { return type == t; } - C4_ALWAYS_INLINE bool operator!= (NodeType_e t) const { return type != t; } - -public: - - #if defined(__clang__) - # pragma clang diagnostic push - # pragma clang diagnostic ignored "-Wnull-dereference" - #elif defined(__GNUC__) - # pragma GCC diagnostic push - # if __GNUC__ >= 6 - # pragma GCC diagnostic ignored "-Wnull-dereference" - # endif - #endif - - C4_ALWAYS_INLINE bool is_notype() const { return type == NOTYPE; } - C4_ALWAYS_INLINE bool is_stream() const { return ((type & STREAM) == STREAM) != 0; } - C4_ALWAYS_INLINE bool is_doc() const { return (type & DOC) != 0; } - C4_ALWAYS_INLINE bool is_container() const { return (type & (MAP|SEQ|STREAM)) != 0; } - C4_ALWAYS_INLINE bool is_map() const { return (type & MAP) != 0; } - C4_ALWAYS_INLINE bool is_seq() const { return (type & SEQ) != 0; } - C4_ALWAYS_INLINE bool has_key() const { return (type & KEY) != 0; } - C4_ALWAYS_INLINE bool has_val() const { return (type & VAL) != 0; } - C4_ALWAYS_INLINE bool is_val() const { return (type & KEYVAL) == VAL; } - C4_ALWAYS_INLINE bool is_keyval() const { return (type & KEYVAL) == KEYVAL; } - C4_ALWAYS_INLINE bool has_key_tag() const { return (type & (KEY|KEYTAG)) == (KEY|KEYTAG); } - C4_ALWAYS_INLINE bool has_val_tag() const { return ((type & VALTAG) && (type & (VAL|MAP|SEQ))); } - C4_ALWAYS_INLINE bool has_key_anchor() const { return (type & (KEY|KEYANCH)) == (KEY|KEYANCH); } - C4_ALWAYS_INLINE bool is_key_anchor() const { return (type & (KEY|KEYANCH)) == (KEY|KEYANCH); } - C4_ALWAYS_INLINE bool has_val_anchor() const { return (type & VALANCH) != 0 && (type & (VAL|SEQ|MAP)) != 0; } - C4_ALWAYS_INLINE bool is_val_anchor() const { return (type & VALANCH) != 0 && (type & (VAL|SEQ|MAP)) != 0; } - C4_ALWAYS_INLINE bool has_anchor() const { return (type & (KEYANCH|VALANCH)) != 0; } - C4_ALWAYS_INLINE bool is_anchor() const { return (type & (KEYANCH|VALANCH)) != 0; } - C4_ALWAYS_INLINE bool is_key_ref() const { return (type & KEYREF) != 0; } - C4_ALWAYS_INLINE bool is_val_ref() const { return (type & VALREF) != 0; } - C4_ALWAYS_INLINE bool is_ref() const { return (type & (KEYREF|VALREF)) != 0; } - C4_ALWAYS_INLINE bool is_anchor_or_ref() const { return (type & (KEYANCH|VALANCH|KEYREF|VALREF)) != 0; } - C4_ALWAYS_INLINE bool is_key_quoted() const { return (type & (KEY|KEYQUO)) == (KEY|KEYQUO); } - C4_ALWAYS_INLINE bool is_val_quoted() const { return (type & (VAL|VALQUO)) == (VAL|VALQUO); } - C4_ALWAYS_INLINE bool is_quoted() const { return (type & (KEY|KEYQUO)) == (KEY|KEYQUO) || (type & (VAL|VALQUO)) == (VAL|VALQUO); } - - // these predicates are a work in progress and subject to change. Don't use yet. - C4_ALWAYS_INLINE bool default_block() const { return (type & (_WIP_STYLE_BLOCK|_WIP_STYLE_FLOW_ML|_WIP_STYLE_FLOW_SL)) == 0; } - C4_ALWAYS_INLINE bool marked_block() const { return (type & (_WIP_STYLE_BLOCK)) != 0; } - C4_ALWAYS_INLINE bool marked_flow_sl() const { return (type & (_WIP_STYLE_FLOW_SL)) != 0; } - C4_ALWAYS_INLINE bool marked_flow_ml() const { return (type & (_WIP_STYLE_FLOW_ML)) != 0; } - C4_ALWAYS_INLINE bool marked_flow() const { return (type & (_WIP_STYLE_FLOW_ML|_WIP_STYLE_FLOW_SL)) != 0; } - C4_ALWAYS_INLINE bool key_marked_literal() const { return (type & (_WIP_KEY_LITERAL)) != 0; } - C4_ALWAYS_INLINE bool val_marked_literal() const { return (type & (_WIP_VAL_LITERAL)) != 0; } - C4_ALWAYS_INLINE bool key_marked_folded() const { return (type & (_WIP_KEY_FOLDED)) != 0; } - C4_ALWAYS_INLINE bool val_marked_folded() const { return (type & (_WIP_VAL_FOLDED)) != 0; } - C4_ALWAYS_INLINE bool key_marked_squo() const { return (type & (_WIP_KEY_SQUO)) != 0; } - C4_ALWAYS_INLINE bool val_marked_squo() const { return (type & (_WIP_VAL_SQUO)) != 0; } - C4_ALWAYS_INLINE bool key_marked_dquo() const { return (type & (_WIP_KEY_DQUO)) != 0; } - C4_ALWAYS_INLINE bool val_marked_dquo() const { return (type & (_WIP_VAL_DQUO)) != 0; } - C4_ALWAYS_INLINE bool key_marked_plain() const { return (type & (_WIP_KEY_PLAIN)) != 0; } - C4_ALWAYS_INLINE bool val_marked_plain() const { return (type & (_WIP_VAL_PLAIN)) != 0; } - - #if defined(__clang__) - # pragma clang diagnostic pop - #elif defined(__GNUC__) - # pragma GCC diagnostic pop - #endif - -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** a node scalar is a csubstr, which may be tagged and anchored. */ -struct NodeScalar -{ - csubstr tag; - csubstr scalar; - csubstr anchor; - -public: - - /// initialize as an empty scalar - inline NodeScalar() noexcept : tag(), scalar(), anchor() {} - - /// initialize as an untagged scalar - template - inline NodeScalar(const char (&s)[N]) noexcept : tag(), scalar(s), anchor() {} - inline NodeScalar(csubstr s ) noexcept : tag(), scalar(s), anchor() {} - - /// initialize as a tagged scalar - template - inline NodeScalar(const char (&t)[N], const char (&s)[N]) noexcept : tag(t), scalar(s), anchor() {} - inline NodeScalar(csubstr t , csubstr s ) noexcept : tag(t), scalar(s), anchor() {} - -public: - - ~NodeScalar() noexcept = default; - NodeScalar(NodeScalar &&) noexcept = default; - NodeScalar(NodeScalar const&) noexcept = default; - NodeScalar& operator= (NodeScalar &&) noexcept = default; - NodeScalar& operator= (NodeScalar const&) noexcept = default; - -public: - - bool empty() const noexcept { return tag.empty() && scalar.empty() && anchor.empty(); } - - void clear() noexcept { tag.clear(); scalar.clear(); anchor.clear(); } - - void set_ref_maybe_replacing_scalar(csubstr ref, bool has_scalar) noexcept - { - csubstr trimmed = ref.begins_with('*') ? ref.sub(1) : ref; - anchor = trimmed; - if((!has_scalar) || !scalar.ends_with(trimmed)) - scalar = ref; - } -}; -C4_MUST_BE_TRIVIAL_COPY(NodeScalar); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** convenience class to initialize nodes */ -struct NodeInit -{ - - NodeType type; - NodeScalar key; - NodeScalar val; - -public: - - /// initialize as an empty node - NodeInit() : type(NOTYPE), key(), val() {} - /// initialize as a typed node - NodeInit(NodeType_e t) : type(t), key(), val() {} - /// initialize as a sequence member - NodeInit(NodeScalar const& v) : type(VAL), key(), val(v) { _add_flags(); } - /// initialize as a mapping member - NodeInit( NodeScalar const& k, NodeScalar const& v) : type(KEYVAL), key(k.tag, k.scalar), val(v.tag, v.scalar) { _add_flags(); } - /// initialize as a mapping member with explicit type - NodeInit(NodeType_e t, NodeScalar const& k, NodeScalar const& v) : type(t ), key(k.tag, k.scalar), val(v.tag, v.scalar) { _add_flags(); } - /// initialize as a mapping member with explicit type (eg SEQ or MAP) - NodeInit(NodeType_e t, NodeScalar const& k ) : type(t ), key(k.tag, k.scalar), val( ) { _add_flags(KEY); } - -public: - - void clear() - { - type.clear(); - key.clear(); - val.clear(); - } - - void _add_flags(type_bits more_flags=0) - { - type = (type|more_flags); - if( ! key.tag.empty()) - type = (type|KEYTAG); - if( ! val.tag.empty()) - type = (type|VALTAG); - if( ! key.anchor.empty()) - type = (type|KEYANCH); - if( ! val.anchor.empty()) - type = (type|VALANCH); - } - - bool _check() const - { - // key cannot be empty - RYML_ASSERT(key.scalar.empty() == ((type & KEY) == 0)); - // key tag cannot be empty - RYML_ASSERT(key.tag.empty() == ((type & KEYTAG) == 0)); - // val may be empty even though VAL is set. But when VAL is not set, val must be empty - RYML_ASSERT(((type & VAL) != 0) || val.scalar.empty()); - // val tag cannot be empty - RYML_ASSERT(val.tag.empty() == ((type & VALTAG) == 0)); - return true; - } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** contains the data for each YAML node. */ -struct NodeData -{ - NodeType m_type; - - NodeScalar m_key; - NodeScalar m_val; - - size_t m_parent; - size_t m_first_child; - size_t m_last_child; - size_t m_next_sibling; - size_t m_prev_sibling; -}; -C4_MUST_BE_TRIVIAL_COPY(NodeData); - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -class RYML_EXPORT Tree -{ -public: - - /** @name construction and assignment */ - /** @{ */ - - Tree() : Tree(get_callbacks()) {} - Tree(Callbacks const& cb); - Tree(size_t node_capacity, size_t arena_capacity=0) : Tree(node_capacity, arena_capacity, get_callbacks()) {} - Tree(size_t node_capacity, size_t arena_capacity, Callbacks const& cb); - - ~Tree(); - - Tree(Tree const& that) noexcept; - Tree(Tree && that) noexcept; - - Tree& operator= (Tree const& that) noexcept; - Tree& operator= (Tree && that) noexcept; - - /** @} */ - -public: - - /** @name memory and sizing */ - /** @{ */ - - void reserve(size_t node_capacity); - - /** clear the tree and zero every node - * @note does NOT clear the arena - * @see clear_arena() */ - void clear(); - inline void clear_arena() { m_arena_pos = 0; } - - inline bool empty() const { return m_size == 0; } - - inline size_t size() const { return m_size; } - inline size_t capacity() const { return m_cap; } - inline size_t slack() const { RYML_ASSERT(m_cap >= m_size); return m_cap - m_size; } - - Callbacks const& callbacks() const { return m_callbacks; } - void callbacks(Callbacks const& cb) { m_callbacks = cb; } - - /** @} */ - -public: - - /** @name node getters */ - /** @{ */ - - //! get the index of a node belonging to this tree. - //! @p n can be nullptr, in which case a - size_t id(NodeData const* n) const - { - if( ! n) - { - return NONE; - } - RYML_ASSERT(n >= m_buf && n < m_buf + m_cap); - return static_cast(n - m_buf); - } - - //! get a pointer to a node's NodeData. - //! i can be NONE, in which case a nullptr is returned - inline NodeData *get(size_t i) - { - if(i == NONE) - return nullptr; - RYML_ASSERT(i >= 0 && i < m_cap); - return m_buf + i; - } - //! get a pointer to a node's NodeData. - //! i can be NONE, in which case a nullptr is returned. - inline NodeData const *get(size_t i) const - { - if(i == NONE) - return nullptr; - RYML_ASSERT(i >= 0 && i < m_cap); - return m_buf + i; - } - - //! An if-less form of get() that demands a valid node index. - //! This function is implementation only; use at your own risk. - inline NodeData * _p(size_t i) { RYML_ASSERT(i != NONE && i >= 0 && i < m_cap); return m_buf + i; } - //! An if-less form of get() that demands a valid node index. - //! This function is implementation only; use at your own risk. - inline NodeData const * _p(size_t i) const { RYML_ASSERT(i != NONE && i >= 0 && i < m_cap); return m_buf + i; } - - //! Get the id of the root node - size_t root_id() { if(m_cap == 0) { reserve(16); } RYML_ASSERT(m_cap > 0 && m_size > 0); return 0; } - //! Get the id of the root node - size_t root_id() const { RYML_ASSERT(m_cap > 0 && m_size > 0); return 0; } - - //! Get a NodeRef of a node by id - NodeRef ref(size_t id); - //! Get a NodeRef of a node by id - ConstNodeRef ref(size_t id) const; - //! Get a NodeRef of a node by id - ConstNodeRef cref(size_t id); - //! Get a NodeRef of a node by id - ConstNodeRef cref(size_t id) const; - - //! Get the root as a NodeRef - NodeRef rootref(); - //! Get the root as a NodeRef - ConstNodeRef rootref() const; - //! Get the root as a NodeRef - ConstNodeRef crootref(); - //! Get the root as a NodeRef - ConstNodeRef crootref() const; - - //! find a root child by name, return it as a NodeRef - //! @note requires the root to be a map. - NodeRef operator[] (csubstr key); - //! find a root child by name, return it as a NodeRef - //! @note requires the root to be a map. - ConstNodeRef operator[] (csubstr key) const; - - //! find a root child by index: return the root node's @p i-th child as a NodeRef - //! @note @i is NOT the node id, but the child's position - NodeRef operator[] (size_t i); - //! find a root child by index: return the root node's @p i-th child as a NodeRef - //! @note @i is NOT the node id, but the child's position - ConstNodeRef operator[] (size_t i) const; - - //! get the i-th document of the stream - //! @note @i is NOT the node id, but the doc position within the stream - NodeRef docref(size_t i); - //! get the i-th document of the stream - //! @note @i is NOT the node id, but the doc position within the stream - ConstNodeRef docref(size_t i) const; - - /** @} */ - -public: - - /** @name node property getters */ - /** @{ */ - - NodeType type(size_t node) const { return _p(node)->m_type; } - const char* type_str(size_t node) const { return NodeType::type_str(_p(node)->m_type); } - - csubstr const& key (size_t node) const { RYML_ASSERT(has_key(node)); return _p(node)->m_key.scalar; } - csubstr const& key_tag (size_t node) const { RYML_ASSERT(has_key_tag(node)); return _p(node)->m_key.tag; } - csubstr const& key_ref (size_t node) const { RYML_ASSERT(is_key_ref(node) && ! has_key_anchor(node)); return _p(node)->m_key.anchor; } - csubstr const& key_anchor(size_t node) const { RYML_ASSERT( ! is_key_ref(node) && has_key_anchor(node)); return _p(node)->m_key.anchor; } - NodeScalar const& keysc (size_t node) const { RYML_ASSERT(has_key(node)); return _p(node)->m_key; } - - csubstr const& val (size_t node) const { RYML_ASSERT(has_val(node)); return _p(node)->m_val.scalar; } - csubstr const& val_tag (size_t node) const { RYML_ASSERT(has_val_tag(node)); return _p(node)->m_val.tag; } - csubstr const& val_ref (size_t node) const { RYML_ASSERT(is_val_ref(node) && ! has_val_anchor(node)); return _p(node)->m_val.anchor; } - csubstr const& val_anchor(size_t node) const { RYML_ASSERT( ! is_val_ref(node) && has_val_anchor(node)); return _p(node)->m_val.anchor; } - NodeScalar const& valsc (size_t node) const { RYML_ASSERT(has_val(node)); return _p(node)->m_val; } - - /** @} */ - -public: - - /** @name node predicates */ - /** @{ */ - - C4_ALWAYS_INLINE bool is_stream(size_t node) const { return _p(node)->m_type.is_stream(); } - C4_ALWAYS_INLINE bool is_doc(size_t node) const { return _p(node)->m_type.is_doc(); } - C4_ALWAYS_INLINE bool is_container(size_t node) const { return _p(node)->m_type.is_container(); } - C4_ALWAYS_INLINE bool is_map(size_t node) const { return _p(node)->m_type.is_map(); } - C4_ALWAYS_INLINE bool is_seq(size_t node) const { return _p(node)->m_type.is_seq(); } - C4_ALWAYS_INLINE bool has_key(size_t node) const { return _p(node)->m_type.has_key(); } - C4_ALWAYS_INLINE bool has_val(size_t node) const { return _p(node)->m_type.has_val(); } - C4_ALWAYS_INLINE bool is_val(size_t node) const { return _p(node)->m_type.is_val(); } - C4_ALWAYS_INLINE bool is_keyval(size_t node) const { return _p(node)->m_type.is_keyval(); } - C4_ALWAYS_INLINE bool has_key_tag(size_t node) const { return _p(node)->m_type.has_key_tag(); } - C4_ALWAYS_INLINE bool has_val_tag(size_t node) const { return _p(node)->m_type.has_val_tag(); } - C4_ALWAYS_INLINE bool has_key_anchor(size_t node) const { return _p(node)->m_type.has_key_anchor(); } - C4_ALWAYS_INLINE bool is_key_anchor(size_t node) const { return _p(node)->m_type.is_key_anchor(); } - C4_ALWAYS_INLINE bool has_val_anchor(size_t node) const { return _p(node)->m_type.has_val_anchor(); } - C4_ALWAYS_INLINE bool is_val_anchor(size_t node) const { return _p(node)->m_type.is_val_anchor(); } - C4_ALWAYS_INLINE bool has_anchor(size_t node) const { return _p(node)->m_type.has_anchor(); } - C4_ALWAYS_INLINE bool is_anchor(size_t node) const { return _p(node)->m_type.is_anchor(); } - C4_ALWAYS_INLINE bool is_key_ref(size_t node) const { return _p(node)->m_type.is_key_ref(); } - C4_ALWAYS_INLINE bool is_val_ref(size_t node) const { return _p(node)->m_type.is_val_ref(); } - C4_ALWAYS_INLINE bool is_ref(size_t node) const { return _p(node)->m_type.is_ref(); } - C4_ALWAYS_INLINE bool is_anchor_or_ref(size_t node) const { return _p(node)->m_type.is_anchor_or_ref(); } - C4_ALWAYS_INLINE bool is_key_quoted(size_t node) const { return _p(node)->m_type.is_key_quoted(); } - C4_ALWAYS_INLINE bool is_val_quoted(size_t node) const { return _p(node)->m_type.is_val_quoted(); } - C4_ALWAYS_INLINE bool is_quoted(size_t node) const { return _p(node)->m_type.is_quoted(); } - - C4_ALWAYS_INLINE bool parent_is_seq(size_t node) const { RYML_ASSERT(has_parent(node)); return is_seq(_p(node)->m_parent); } - C4_ALWAYS_INLINE bool parent_is_map(size_t node) const { RYML_ASSERT(has_parent(node)); return is_map(_p(node)->m_parent); } - - /** true when key and val are empty, and has no children */ - C4_ALWAYS_INLINE bool empty(size_t node) const { return ! has_children(node) && _p(node)->m_key.empty() && (( ! (_p(node)->m_type & VAL)) || _p(node)->m_val.empty()); } - /** true when the node has an anchor named a */ - C4_ALWAYS_INLINE bool has_anchor(size_t node, csubstr a) const { return _p(node)->m_key.anchor == a || _p(node)->m_val.anchor == a; } - - C4_ALWAYS_INLINE bool key_is_null(size_t node) const { RYML_ASSERT(has_key(node)); NodeData const* C4_RESTRICT n = _p(node); return !n->m_type.is_key_quoted() && _is_null(n->m_key.scalar); } - C4_ALWAYS_INLINE bool val_is_null(size_t node) const { RYML_ASSERT(has_val(node)); NodeData const* C4_RESTRICT n = _p(node); return !n->m_type.is_val_quoted() && _is_null(n->m_val.scalar); } - static bool _is_null(csubstr s) noexcept - { - return s.str == nullptr || - s == "~" || - s == "null" || - s == "Null" || - s == "NULL"; - } - - /** @} */ - -public: - - /** @name hierarchy predicates */ - /** @{ */ - - bool is_root(size_t node) const { RYML_ASSERT(_p(node)->m_parent != NONE || node == 0); return _p(node)->m_parent == NONE; } - - bool has_parent(size_t node) const { return _p(node)->m_parent != NONE; } - - /** true if @p node has a child with id @p ch */ - bool has_child(size_t node, size_t ch) const { return _p(ch)->m_parent == node; } - /** true if @p node has a child with key @p key */ - bool has_child(size_t node, csubstr key) const { return find_child(node, key) != npos; } - /** true if @p node has any children key */ - bool has_children(size_t node) const { return _p(node)->m_first_child != NONE; } - - /** true if @p node has a sibling with id @p sib */ - bool has_sibling(size_t node, size_t sib) const { return _p(node)->m_parent == _p(sib)->m_parent; } - /** true if one of the node's siblings has the given key */ - bool has_sibling(size_t node, csubstr key) const { return find_sibling(node, key) != npos; } - /** true if node is not a single child */ - bool has_other_siblings(size_t node) const - { - NodeData const *n = _p(node); - if(C4_LIKELY(n->m_parent != NONE)) - { - n = _p(n->m_parent); - return n->m_first_child != n->m_last_child; - } - return false; - } - - RYML_DEPRECATED("use has_other_siblings()") bool has_siblings(size_t /*node*/) const { return true; } - - /** @} */ - -public: - - /** @name hierarchy getters */ - /** @{ */ - - size_t parent(size_t node) const { return _p(node)->m_parent; } - - size_t prev_sibling(size_t node) const { return _p(node)->m_prev_sibling; } - size_t next_sibling(size_t node) const { return _p(node)->m_next_sibling; } - - /** O(#num_children) */ - size_t num_children(size_t node) const; - size_t child_pos(size_t node, size_t ch) const; - size_t first_child(size_t node) const { return _p(node)->m_first_child; } - size_t last_child(size_t node) const { return _p(node)->m_last_child; } - size_t child(size_t node, size_t pos) const; - size_t find_child(size_t node, csubstr const& key) const; - - /** O(#num_siblings) */ - /** counts with this */ - size_t num_siblings(size_t node) const { return is_root(node) ? 1 : num_children(_p(node)->m_parent); } - /** does not count with this */ - size_t num_other_siblings(size_t node) const { size_t ns = num_siblings(node); RYML_ASSERT(ns > 0); return ns-1; } - size_t sibling_pos(size_t node, size_t sib) const { RYML_ASSERT( ! is_root(node) || node == root_id()); return child_pos(_p(node)->m_parent, sib); } - size_t first_sibling(size_t node) const { return is_root(node) ? node : _p(_p(node)->m_parent)->m_first_child; } - size_t last_sibling(size_t node) const { return is_root(node) ? node : _p(_p(node)->m_parent)->m_last_child; } - size_t sibling(size_t node, size_t pos) const { return child(_p(node)->m_parent, pos); } - size_t find_sibling(size_t node, csubstr const& key) const { return find_child(_p(node)->m_parent, key); } - - size_t doc(size_t i) const { size_t rid = root_id(); RYML_ASSERT(is_stream(rid)); return child(rid, i); } //!< gets the @p i document node index. requires that the root node is a stream. - - /** @} */ - -public: - - /** @name node modifiers */ - /** @{ */ - - void to_keyval(size_t node, csubstr key, csubstr val, type_bits more_flags=0); - void to_map(size_t node, csubstr key, type_bits more_flags=0); - void to_seq(size_t node, csubstr key, type_bits more_flags=0); - void to_val(size_t node, csubstr val, type_bits more_flags=0); - void to_map(size_t node, type_bits more_flags=0); - void to_seq(size_t node, type_bits more_flags=0); - void to_doc(size_t node, type_bits more_flags=0); - void to_stream(size_t node, type_bits more_flags=0); - - void set_key(size_t node, csubstr key) { RYML_ASSERT(has_key(node)); _p(node)->m_key.scalar = key; } - void set_val(size_t node, csubstr val) { RYML_ASSERT(has_val(node)); _p(node)->m_val.scalar = val; } - - void set_key_tag(size_t node, csubstr tag) { RYML_ASSERT(has_key(node)); _p(node)->m_key.tag = tag; _add_flags(node, KEYTAG); } - void set_val_tag(size_t node, csubstr tag) { RYML_ASSERT(has_val(node) || is_container(node)); _p(node)->m_val.tag = tag; _add_flags(node, VALTAG); } - - void set_key_anchor(size_t node, csubstr anchor) { RYML_ASSERT( ! is_key_ref(node)); _p(node)->m_key.anchor = anchor.triml('&'); _add_flags(node, KEYANCH); } - void set_val_anchor(size_t node, csubstr anchor) { RYML_ASSERT( ! is_val_ref(node)); _p(node)->m_val.anchor = anchor.triml('&'); _add_flags(node, VALANCH); } - void set_key_ref (size_t node, csubstr ref ) { RYML_ASSERT( ! has_key_anchor(node)); NodeData* C4_RESTRICT n = _p(node); n->m_key.set_ref_maybe_replacing_scalar(ref, n->m_type.has_key()); _add_flags(node, KEY|KEYREF); } - void set_val_ref (size_t node, csubstr ref ) { RYML_ASSERT( ! has_val_anchor(node)); NodeData* C4_RESTRICT n = _p(node); n->m_val.set_ref_maybe_replacing_scalar(ref, n->m_type.has_val()); _add_flags(node, VAL|VALREF); } - - void rem_key_anchor(size_t node) { _p(node)->m_key.anchor.clear(); _rem_flags(node, KEYANCH); } - void rem_val_anchor(size_t node) { _p(node)->m_val.anchor.clear(); _rem_flags(node, VALANCH); } - void rem_key_ref (size_t node) { _p(node)->m_key.anchor.clear(); _rem_flags(node, KEYREF); } - void rem_val_ref (size_t node) { _p(node)->m_val.anchor.clear(); _rem_flags(node, VALREF); } - void rem_anchor_ref(size_t node) { _p(node)->m_key.anchor.clear(); _p(node)->m_val.anchor.clear(); _rem_flags(node, KEYANCH|VALANCH|KEYREF|VALREF); } - - /** @} */ - -public: - - /** @name tree modifiers */ - /** @{ */ - - /** reorder the tree in memory so that all the nodes are stored - * in a linear sequence when visited in depth-first order. - * This will invalidate existing ids, since the node id is its - * position in the node array. */ - void reorder(); - - /** Resolve references (aliases <- anchors) in the tree. - * - * Dereferencing is opt-in; after parsing, Tree::resolve() - * has to be called explicitly for obtaining resolved references in the - * tree. This method will resolve all references and substitute the - * anchored values in place of the reference. - * - * This method first does a full traversal of the tree to gather all - * anchors and references in a separate collection, then it goes through - * that collection to locate the names, which it does by obeying the YAML - * standard diktat that "an alias node refers to the most recent node in - * the serialization having the specified anchor" - * - * So, depending on the number of anchor/alias nodes, this is a - * potentially expensive operation, with a best-case linear complexity - * (from the initial traversal). This potential cost is the reason for - * requiring an explicit call. - */ - void resolve(); - - /** @} */ - -public: - - /** @name tag directives */ - /** @{ */ - - void resolve_tags(); - - size_t num_tag_directives() const; - size_t add_tag_directive(TagDirective const& td); - void clear_tag_directives(); - - size_t resolve_tag(substr output, csubstr tag, size_t node_id) const; - csubstr resolve_tag_sub(substr output, csubstr tag, size_t node_id) const - { - size_t needed = resolve_tag(output, tag, node_id); - return needed <= output.len ? output.first(needed) : output; - } - - using tag_directive_const_iterator = TagDirective const*; - tag_directive_const_iterator begin_tag_directives() const { return m_tag_directives; } - tag_directive_const_iterator end_tag_directives() const { return m_tag_directives + num_tag_directives(); } - - struct TagDirectiveProxy - { - tag_directive_const_iterator b, e; - tag_directive_const_iterator begin() const { return b; } - tag_directive_const_iterator end() const { return e; } - }; - - TagDirectiveProxy tag_directives() const { return TagDirectiveProxy{begin_tag_directives(), end_tag_directives()}; } - - /** @} */ - -public: - - /** @name modifying hierarchy */ - /** @{ */ - - /** create and insert a new child of @p parent. insert after the (to-be) - * sibling @p after, which must be a child of @p parent. To insert as the - * first child, set after to NONE */ - C4_ALWAYS_INLINE size_t insert_child(size_t parent, size_t after) - { - RYML_ASSERT(parent != NONE); - RYML_ASSERT(is_container(parent) || is_root(parent)); - RYML_ASSERT(after == NONE || (_p(after)->m_parent == parent)); - size_t child = _claim(); - _set_hierarchy(child, parent, after); - return child; - } - /** create and insert a node as the first child of @p parent */ - C4_ALWAYS_INLINE size_t prepend_child(size_t parent) { return insert_child(parent, NONE); } - /** create and insert a node as the last child of @p parent */ - C4_ALWAYS_INLINE size_t append_child(size_t parent) { return insert_child(parent, _p(parent)->m_last_child); } - -public: - - #if defined(__clang__) - # pragma clang diagnostic push - # pragma clang diagnostic ignored "-Wnull-dereference" - #elif defined(__GNUC__) - # pragma GCC diagnostic push - # if __GNUC__ >= 6 - # pragma GCC diagnostic ignored "-Wnull-dereference" - # endif - #endif - - //! create and insert a new sibling of n. insert after "after" - C4_ALWAYS_INLINE size_t insert_sibling(size_t node, size_t after) - { - return insert_child(_p(node)->m_parent, after); - } - /** create and insert a node as the first node of @p parent */ - C4_ALWAYS_INLINE size_t prepend_sibling(size_t node) { return prepend_child(_p(node)->m_parent); } - C4_ALWAYS_INLINE size_t append_sibling(size_t node) { return append_child(_p(node)->m_parent); } - -public: - - /** remove an entire branch at once: ie remove the children and the node itself */ - inline void remove(size_t node) - { - remove_children(node); - _release(node); - } - - /** remove all the node's children, but keep the node itself */ - void remove_children(size_t node); - - /** change the @p type of the node to one of MAP, SEQ or VAL. @p - * type must have one and only one of MAP,SEQ,VAL; @p type may - * possibly have KEY, but if it does, then the @p node must also - * have KEY. Changing to the same type is a no-op. Otherwise, - * changing to a different type will initialize the node with an - * empty value of the desired type: changing to VAL will - * initialize with a null scalar (~), changing to MAP will - * initialize with an empty map ({}), and changing to SEQ will - * initialize with an empty seq ([]). */ - bool change_type(size_t node, NodeType type); - - bool change_type(size_t node, type_bits type) - { - return change_type(node, (NodeType)type); - } - - #if defined(__clang__) - # pragma clang diagnostic pop - #elif defined(__GNUC__) - # pragma GCC diagnostic pop - #endif - -public: - - /** change the node's position in the parent */ - void move(size_t node, size_t after); - - /** change the node's parent and position */ - void move(size_t node, size_t new_parent, size_t after); - - /** change the node's parent and position to a different tree - * @return the index of the new node in the destination tree */ - size_t move(Tree * src, size_t node, size_t new_parent, size_t after); - - /** ensure the first node is a stream. Eg, change this tree - * - * DOCMAP - * MAP - * KEYVAL - * KEYVAL - * SEQ - * VAL - * - * to - * - * STREAM - * DOCMAP - * MAP - * KEYVAL - * KEYVAL - * SEQ - * VAL - * - * If the root is already a stream, this is a no-op. - */ - void set_root_as_stream(); - -public: - - /** recursively duplicate a node from this tree into a new parent, - * placing it after one of its children - * @return the index of the copy */ - size_t duplicate(size_t node, size_t new_parent, size_t after); - /** recursively duplicate a node from a different tree into a new parent, - * placing it after one of its children - * @return the index of the copy */ - size_t duplicate(Tree const* src, size_t node, size_t new_parent, size_t after); - - /** recursively duplicate the node's children (but not the node) - * @return the index of the last duplicated child */ - size_t duplicate_children(size_t node, size_t parent, size_t after); - /** recursively duplicate the node's children (but not the node), where - * the node is from a different tree - * @return the index of the last duplicated child */ - size_t duplicate_children(Tree const* src, size_t node, size_t parent, size_t after); - - void duplicate_contents(size_t node, size_t where); - void duplicate_contents(Tree const* src, size_t node, size_t where); - - /** duplicate the node's children (but not the node) in a new parent, but - * omit repetitions where a duplicated node has the same key (in maps) or - * value (in seqs). If one of the duplicated children has the same key - * (in maps) or value (in seqs) as one of the parent's children, the one - * that is placed closest to the end will prevail. */ - size_t duplicate_children_no_rep(size_t node, size_t parent, size_t after); - size_t duplicate_children_no_rep(Tree const* src, size_t node, size_t parent, size_t after); - -public: - - void merge_with(Tree const* src, size_t src_node=NONE, size_t dst_root=NONE); - - /** @} */ - -public: - - /** @name internal string arena */ - /** @{ */ - - /** get the current size of the tree's internal arena */ - RYML_DEPRECATED("use arena_size() instead") size_t arena_pos() const { return m_arena_pos; } - /** get the current size of the tree's internal arena */ - inline size_t arena_size() const { return m_arena_pos; } - /** get the current capacity of the tree's internal arena */ - inline size_t arena_capacity() const { return m_arena.len; } - /** get the current slack of the tree's internal arena */ - inline size_t arena_slack() const { RYML_ASSERT(m_arena.len >= m_arena_pos); return m_arena.len - m_arena_pos; } - - /** get the current arena */ - substr arena() const { return m_arena.first(m_arena_pos); } - - /** return true if the given substring is part of the tree's string arena */ - bool in_arena(csubstr s) const - { - return m_arena.is_super(s); - } - - /** serialize the given floating-point variable to the tree's - * arena, growing it as needed to accomodate the serialization. - * - * @note Growing the arena may cause relocation of the entire - * existing arena, and thus change the contents of individual - * nodes, and thus cost O(numnodes)+O(arenasize). To avoid this - * cost, ensure that the arena is reserved to an appropriate size - * using .reserve_arena() - * - * @see alloc_arena() */ - template - typename std::enable_if::value, csubstr>::type - to_arena(T const& C4_RESTRICT a) - { - substr rem(m_arena.sub(m_arena_pos)); - size_t num = to_chars_float(rem, a); - if(num > rem.len) - { - rem = _grow_arena(num); - num = to_chars_float(rem, a); - RYML_ASSERT(num <= rem.len); - } - rem = _request_span(num); - return rem; - } - - /** serialize the given non-floating-point variable to the tree's - * arena, growing it as needed to accomodate the serialization. - * - * @note Growing the arena may cause relocation of the entire - * existing arena, and thus change the contents of individual - * nodes, and thus cost O(numnodes)+O(arenasize). To avoid this - * cost, ensure that the arena is reserved to an appropriate size - * using .reserve_arena() - * - * @see alloc_arena() */ - template - typename std::enable_if::value, csubstr>::type - to_arena(T const& C4_RESTRICT a) - { - substr rem(m_arena.sub(m_arena_pos)); - size_t num = to_chars(rem, a); - if(num > rem.len) - { - rem = _grow_arena(num); - num = to_chars(rem, a); - RYML_ASSERT(num <= rem.len); - } - rem = _request_span(num); - return rem; - } - - /** serialize the given csubstr to the tree's arena, growing the - * arena as needed to accomodate the serialization. - * - * @note Growing the arena may cause relocation of the entire - * existing arena, and thus change the contents of individual - * nodes, and thus cost O(numnodes)+O(arenasize). To avoid this - * cost, ensure that the arena is reserved to an appropriate size - * using .reserve_arena() - * - * @see alloc_arena() */ - csubstr to_arena(csubstr a) - { - if(a.len > 0) - { - substr rem(m_arena.sub(m_arena_pos)); - size_t num = to_chars(rem, a); - if(num > rem.len) - { - rem = _grow_arena(num); - num = to_chars(rem, a); - RYML_ASSERT(num <= rem.len); - } - return _request_span(num); - } - else - { - if(a.str == nullptr) - { - return csubstr{}; - } - else if(m_arena.str == nullptr) - { - // Arena is empty and we want to store a non-null - // zero-length string. - // Even though the string has zero length, we need - // some "memory" to store a non-nullptr string - _grow_arena(1); - } - return _request_span(0); - } - } - C4_ALWAYS_INLINE csubstr to_arena(const char *s) - { - return to_arena(to_csubstr(s)); - } - C4_ALWAYS_INLINE csubstr to_arena(std::nullptr_t) - { - return csubstr{}; - } - - /** copy the given substr to the tree's arena, growing it by the - * required size - * - * @note Growing the arena may cause relocation of the entire - * existing arena, and thus change the contents of individual - * nodes, and thus cost O(numnodes)+O(arenasize). To avoid this - * cost, ensure that the arena is reserved to an appropriate size - * using .reserve_arena() - * - * @see alloc_arena() */ - substr copy_to_arena(csubstr s) - { - substr cp = alloc_arena(s.len); - RYML_ASSERT(cp.len == s.len); - RYML_ASSERT(!s.overlaps(cp)); - #if (!defined(__clang__)) && (defined(__GNUC__) && __GNUC__ >= 10) - C4_SUPPRESS_WARNING_GCC_PUSH - C4_SUPPRESS_WARNING_GCC("-Wstringop-overflow=") // no need for terminating \0 - C4_SUPPRESS_WARNING_GCC( "-Wrestrict") // there's an assert to ensure no violation of restrict behavior - #endif - if(s.len) - memcpy(cp.str, s.str, s.len); - #if (!defined(__clang__)) && (defined(__GNUC__) && __GNUC__ >= 10) - C4_SUPPRESS_WARNING_GCC_POP - #endif - return cp; - } - - /** grow the tree's string arena by the given size and return a substr - * of the added portion - * - * @note Growing the arena may cause relocation of the entire - * existing arena, and thus change the contents of individual - * nodes, and thus cost O(numnodes)+O(arenasize). To avoid this - * cost, ensure that the arena is reserved to an appropriate size - * using .reserve_arena(). - * - * @see reserve_arena() */ - substr alloc_arena(size_t sz) - { - if(sz > arena_slack()) - _grow_arena(sz - arena_slack()); - substr s = _request_span(sz); - return s; - } - - /** ensure the tree's internal string arena is at least the given capacity - * @note This operation has a potential complexity of O(numNodes)+O(arenasize). - * Growing the arena may cause relocation of the entire - * existing arena, and thus change the contents of individual nodes. */ - void reserve_arena(size_t arena_cap) - { - if(arena_cap > m_arena.len) - { - substr buf; - buf.str = (char*) m_callbacks.m_allocate(arena_cap, m_arena.str, m_callbacks.m_user_data); - buf.len = arena_cap; - if(m_arena.str) - { - RYML_ASSERT(m_arena.len >= 0); - _relocate(buf); // does a memcpy and changes nodes using the arena - m_callbacks.m_free(m_arena.str, m_arena.len, m_callbacks.m_user_data); - } - m_arena = buf; - } - } - - /** @} */ - -private: - - substr _grow_arena(size_t more) - { - size_t cap = m_arena.len + more; - cap = cap < 2 * m_arena.len ? 2 * m_arena.len : cap; - cap = cap < 64 ? 64 : cap; - reserve_arena(cap); - return m_arena.sub(m_arena_pos); - } - - substr _request_span(size_t sz) - { - substr s; - s = m_arena.sub(m_arena_pos, sz); - m_arena_pos += sz; - return s; - } - - substr _relocated(csubstr s, substr next_arena) const - { - RYML_ASSERT(m_arena.is_super(s)); - RYML_ASSERT(m_arena.sub(0, m_arena_pos).is_super(s)); - auto pos = (s.str - m_arena.str); - substr r(next_arena.str + pos, s.len); - RYML_ASSERT(r.str - next_arena.str == pos); - RYML_ASSERT(next_arena.sub(0, m_arena_pos).is_super(r)); - return r; - } - -public: - - /** @name lookup */ - /** @{ */ - - struct lookup_result - { - size_t target; - size_t closest; - size_t path_pos; - csubstr path; - - inline operator bool() const { return target != NONE; } - - lookup_result() : target(NONE), closest(NONE), path_pos(0), path() {} - lookup_result(csubstr path_, size_t start) : target(NONE), closest(start), path_pos(0), path(path_) {} - - /** get the part ot the input path that was resolved */ - csubstr resolved() const; - /** get the part ot the input path that was unresolved */ - csubstr unresolved() const; - }; - - /** for example foo.bar[0].baz */ - lookup_result lookup_path(csubstr path, size_t start=NONE) const; - - /** defaulted lookup: lookup @p path; if the lookup fails, recursively modify - * the tree so that the corresponding lookup_path() would return the - * default value. - * @see lookup_path() */ - size_t lookup_path_or_modify(csubstr default_value, csubstr path, size_t start=NONE); - - /** defaulted lookup: lookup @p path; if the lookup fails, recursively modify - * the tree so that the corresponding lookup_path() would return the - * branch @p src_node (from the tree @p src). - * @see lookup_path() */ - size_t lookup_path_or_modify(Tree const *src, size_t src_node, csubstr path, size_t start=NONE); - - /** @} */ - -private: - - struct _lookup_path_token - { - csubstr value; - NodeType type; - _lookup_path_token() : value(), type() {} - _lookup_path_token(csubstr v, NodeType t) : value(v), type(t) {} - inline operator bool() const { return type != NOTYPE; } - bool is_index() const { return value.begins_with('[') && value.ends_with(']'); } - }; - - size_t _lookup_path_or_create(csubstr path, size_t start); - - void _lookup_path (lookup_result *r) const; - void _lookup_path_modify(lookup_result *r); - - size_t _next_node (lookup_result *r, _lookup_path_token *parent) const; - size_t _next_node_modify(lookup_result *r, _lookup_path_token *parent); - - void _advance(lookup_result *r, size_t more) const; - - _lookup_path_token _next_token(lookup_result *r, _lookup_path_token const& parent) const; - -private: - - void _clear(); - void _free(); - void _copy(Tree const& that); - void _move(Tree & that); - - void _relocate(substr next_arena); - -public: - - #if ! RYML_USE_ASSERT - C4_ALWAYS_INLINE void _check_next_flags(size_t, type_bits) {} - #else - void _check_next_flags(size_t node, type_bits f) - { - auto n = _p(node); - type_bits o = n->m_type; // old - C4_UNUSED(o); - if(f & MAP) - { - RYML_ASSERT_MSG((f & SEQ) == 0, "cannot mark simultaneously as map and seq"); - RYML_ASSERT_MSG((f & VAL) == 0, "cannot mark simultaneously as map and val"); - RYML_ASSERT_MSG((o & SEQ) == 0, "cannot turn a seq into a map; clear first"); - RYML_ASSERT_MSG((o & VAL) == 0, "cannot turn a val into a map; clear first"); - } - else if(f & SEQ) - { - RYML_ASSERT_MSG((f & MAP) == 0, "cannot mark simultaneously as seq and map"); - RYML_ASSERT_MSG((f & VAL) == 0, "cannot mark simultaneously as seq and val"); - RYML_ASSERT_MSG((o & MAP) == 0, "cannot turn a map into a seq; clear first"); - RYML_ASSERT_MSG((o & VAL) == 0, "cannot turn a val into a seq; clear first"); - } - if(f & KEY) - { - RYML_ASSERT(!is_root(node)); - auto pid = parent(node); C4_UNUSED(pid); - RYML_ASSERT(is_map(pid)); - } - if((f & VAL) && !is_root(node)) - { - auto pid = parent(node); C4_UNUSED(pid); - RYML_ASSERT(is_map(pid) || is_seq(pid)); - } - } - #endif - - inline void _set_flags(size_t node, NodeType_e f) { _check_next_flags(node, f); _p(node)->m_type = f; } - inline void _set_flags(size_t node, type_bits f) { _check_next_flags(node, f); _p(node)->m_type = f; } - - inline void _add_flags(size_t node, NodeType_e f) { NodeData *d = _p(node); type_bits fb = f | d->m_type; _check_next_flags(node, fb); d->m_type = (NodeType_e) fb; } - inline void _add_flags(size_t node, type_bits f) { NodeData *d = _p(node); f |= d->m_type; _check_next_flags(node, f); d->m_type = f; } - - inline void _rem_flags(size_t node, NodeType_e f) { NodeData *d = _p(node); type_bits fb = d->m_type & ~f; _check_next_flags(node, fb); d->m_type = (NodeType_e) fb; } - inline void _rem_flags(size_t node, type_bits f) { NodeData *d = _p(node); f = d->m_type & ~f; _check_next_flags(node, f); d->m_type = f; } - - void _set_key(size_t node, csubstr key, type_bits more_flags=0) - { - _p(node)->m_key.scalar = key; - _add_flags(node, KEY|more_flags); - } - void _set_key(size_t node, NodeScalar const& key, type_bits more_flags=0) - { - _p(node)->m_key = key; - _add_flags(node, KEY|more_flags); - } - - void _set_val(size_t node, csubstr val, type_bits more_flags=0) - { - RYML_ASSERT(num_children(node) == 0); - RYML_ASSERT(!is_seq(node) && !is_map(node)); - _p(node)->m_val.scalar = val; - _add_flags(node, VAL|more_flags); - } - void _set_val(size_t node, NodeScalar const& val, type_bits more_flags=0) - { - RYML_ASSERT(num_children(node) == 0); - RYML_ASSERT( ! is_container(node)); - _p(node)->m_val = val; - _add_flags(node, VAL|more_flags); - } - - void _set(size_t node, NodeInit const& i) - { - RYML_ASSERT(i._check()); - NodeData *n = _p(node); - RYML_ASSERT(n->m_key.scalar.empty() || i.key.scalar.empty() || i.key.scalar == n->m_key.scalar); - _add_flags(node, i.type); - if(n->m_key.scalar.empty()) - { - if( ! i.key.scalar.empty()) - { - _set_key(node, i.key.scalar); - } - } - n->m_key.tag = i.key.tag; - n->m_val = i.val; - } - - void _set_parent_as_container_if_needed(size_t in) - { - NodeData const* n = _p(in); - size_t ip = parent(in); - if(ip != NONE) - { - if( ! (is_seq(ip) || is_map(ip))) - { - if((in == first_child(ip)) && (in == last_child(ip))) - { - if( ! n->m_key.empty() || has_key(in)) - { - _add_flags(ip, MAP); - } - else - { - _add_flags(ip, SEQ); - } - } - } - } - } - - void _seq2map(size_t node) - { - RYML_ASSERT(is_seq(node)); - for(size_t i = first_child(node); i != NONE; i = next_sibling(i)) - { - NodeData *C4_RESTRICT ch = _p(i); - if(ch->m_type.is_keyval()) - continue; - ch->m_type.add(KEY); - ch->m_key = ch->m_val; - } - auto *C4_RESTRICT n = _p(node); - n->m_type.rem(SEQ); - n->m_type.add(MAP); - } - - size_t _do_reorder(size_t *node, size_t count); - - void _swap(size_t n_, size_t m_); - void _swap_props(size_t n_, size_t m_); - void _swap_hierarchy(size_t n_, size_t m_); - void _copy_hierarchy(size_t dst_, size_t src_); - - inline void _copy_props(size_t dst_, size_t src_) - { - _copy_props(dst_, this, src_); - } - - inline void _copy_props_wo_key(size_t dst_, size_t src_) - { - _copy_props_wo_key(dst_, this, src_); - } - - void _copy_props(size_t dst_, Tree const* that_tree, size_t src_) - { - auto & C4_RESTRICT dst = *_p(dst_); - auto const& C4_RESTRICT src = *that_tree->_p(src_); - dst.m_type = src.m_type; - dst.m_key = src.m_key; - dst.m_val = src.m_val; - } - - void _copy_props_wo_key(size_t dst_, Tree const* that_tree, size_t src_) - { - auto & C4_RESTRICT dst = *_p(dst_); - auto const& C4_RESTRICT src = *that_tree->_p(src_); - dst.m_type = (src.m_type & ~_KEYMASK) | (dst.m_type & _KEYMASK); - dst.m_val = src.m_val; - } - - inline void _clear_type(size_t node) - { - _p(node)->m_type = NOTYPE; - } - - inline void _clear(size_t node) - { - auto *C4_RESTRICT n = _p(node); - n->m_type = NOTYPE; - n->m_key.clear(); - n->m_val.clear(); - n->m_parent = NONE; - n->m_first_child = NONE; - n->m_last_child = NONE; - } - - inline void _clear_key(size_t node) - { - _p(node)->m_key.clear(); - _rem_flags(node, KEY); - } - - inline void _clear_val(size_t node) - { - _p(node)->m_val.clear(); - _rem_flags(node, VAL); - } - -private: - - void _clear_range(size_t first, size_t num); - - size_t _claim(); - void _claim_root(); - void _release(size_t node); - void _free_list_add(size_t node); - void _free_list_rem(size_t node); - - void _set_hierarchy(size_t node, size_t parent, size_t after_sibling); - void _rem_hierarchy(size_t node); - -public: - - // members are exposed, but you should NOT access them directly - - NodeData * m_buf; - size_t m_cap; - - size_t m_size; - - size_t m_free_head; - size_t m_free_tail; - - substr m_arena; - size_t m_arena_pos; - - Callbacks m_callbacks; - - TagDirective m_tag_directives[RYML_MAX_TAG_DIRECTIVES]; - -}; - -} // namespace yml -} // namespace c4 - - -C4_SUPPRESS_WARNING_MSVC_POP -C4_SUPPRESS_WARNING_GCC_CLANG_POP - - -#endif /* _C4_YML_TREE_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/writer.hpp b/thirdparty/ryml/src/c4/yml/writer.hpp deleted file mode 100644 index 29e32d47b..000000000 --- a/thirdparty/ryml/src/c4/yml/writer.hpp +++ /dev/null @@ -1,229 +0,0 @@ -#ifndef _C4_YML_WRITER_HPP_ -#define _C4_YML_WRITER_HPP_ - -#ifndef _C4_YML_COMMON_HPP_ -#include "./common.hpp" -#endif - -#include -#include // fwrite(), fputc() -#include // memcpy() - - -namespace c4 { -namespace yml { - - -/** Repeat-Character: a character to be written a number of times. */ -struct RepC -{ - char c; - size_t num_times; -}; -inline RepC indent_to(size_t num_levels) -{ - return {' ', size_t(2) * num_levels}; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** A writer that outputs to a file. Defaults to stdout. */ -struct WriterFile -{ - FILE * m_file; - size_t m_pos; - - WriterFile(FILE *f = nullptr) : m_file(f ? f : stdout), m_pos(0) {} - - inline substr _get(bool /*error_on_excess*/) - { - substr sp; - sp.str = nullptr; - sp.len = m_pos; - return sp; - } - - template - inline void _do_write(const char (&a)[N]) - { - fwrite(a, sizeof(char), N - 1, m_file); - m_pos += N - 1; - } - - inline void _do_write(csubstr sp) - { - #if defined(__clang__) - # pragma clang diagnostic push - # pragma GCC diagnostic ignored "-Wsign-conversion" - #elif defined(__GNUC__) - # pragma GCC diagnostic push - # pragma GCC diagnostic ignored "-Wsign-conversion" - #endif - if(sp.empty()) return; - fwrite(sp.str, sizeof(csubstr::char_type), sp.len, m_file); - m_pos += sp.len; - #if defined(__clang__) - # pragma clang diagnostic pop - #elif defined(__GNUC__) - # pragma GCC diagnostic pop - #endif - } - - inline void _do_write(const char c) - { - fputc(c, m_file); - ++m_pos; - } - - inline void _do_write(RepC const rc) - { - for(size_t i = 0; i < rc.num_times; ++i) - { - fputc(rc.c, m_file); - } - m_pos += rc.num_times; - } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** A writer that outputs to an STL-like ostream. */ -template -struct WriterOStream -{ - OStream& m_stream; - size_t m_pos; - - WriterOStream(OStream &s) : m_stream(s), m_pos(0) {} - - inline substr _get(bool /*error_on_excess*/) - { - substr sp; - sp.str = nullptr; - sp.len = m_pos; - return sp; - } - - template - inline void _do_write(const char (&a)[N]) - { - m_stream.write(a, N - 1); - m_pos += N - 1; - } - - inline void _do_write(csubstr sp) - { - #if defined(__clang__) - # pragma clang diagnostic push - # pragma GCC diagnostic ignored "-Wsign-conversion" - #elif defined(__GNUC__) - # pragma GCC diagnostic push - # pragma GCC diagnostic ignored "-Wsign-conversion" - #endif - if(sp.empty()) return; - m_stream.write(sp.str, sp.len); - m_pos += sp.len; - #if defined(__clang__) - # pragma clang diagnostic pop - #elif defined(__GNUC__) - # pragma GCC diagnostic pop - #endif - } - - inline void _do_write(const char c) - { - m_stream.put(c); - ++m_pos; - } - - inline void _do_write(RepC const rc) - { - for(size_t i = 0; i < rc.num_times; ++i) - { - m_stream.put(rc.c); - } - m_pos += rc.num_times; - } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** a writer to a substr */ -struct WriterBuf -{ - substr m_buf; - size_t m_pos; - - WriterBuf(substr sp) : m_buf(sp), m_pos(0) {} - - inline substr _get(bool error_on_excess) - { - if(m_pos <= m_buf.len) - { - return m_buf.first(m_pos); - } - if(error_on_excess) - { - c4::yml::error("not enough space in the given buffer"); - } - substr sp; - sp.str = nullptr; - sp.len = m_pos; - return sp; - } - - template - inline void _do_write(const char (&a)[N]) - { - RYML_ASSERT( ! m_buf.overlaps(a)); - if(m_pos + N-1 <= m_buf.len) - { - memcpy(&(m_buf[m_pos]), a, N-1); - } - m_pos += N-1; - } - - inline void _do_write(csubstr sp) - { - if(sp.empty()) return; - RYML_ASSERT( ! sp.overlaps(m_buf)); - if(m_pos + sp.len <= m_buf.len) - { - memcpy(&(m_buf[m_pos]), sp.str, sp.len); - } - m_pos += sp.len; - } - - inline void _do_write(const char c) - { - if(m_pos + 1 <= m_buf.len) - { - m_buf[m_pos] = c; - } - ++m_pos; - } - - inline void _do_write(RepC const rc) - { - if(m_pos + rc.num_times <= m_buf.len) - { - for(size_t i = 0; i < rc.num_times; ++i) - { - m_buf[m_pos + i] = rc.c; - } - } - m_pos += rc.num_times; - } -}; - - -} // namespace yml -} // namespace c4 - -#endif /* _C4_YML_WRITER_HPP_ */ diff --git a/thirdparty/ryml/src/c4/yml/yml.hpp b/thirdparty/ryml/src/c4/yml/yml.hpp deleted file mode 100644 index 36f78fe82..000000000 --- a/thirdparty/ryml/src/c4/yml/yml.hpp +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef _C4_YML_YML_HPP_ -#define _C4_YML_YML_HPP_ - -#include "c4/yml/tree.hpp" -#include "c4/yml/node.hpp" -#include "c4/yml/emit.hpp" -#include "c4/yml/parse.hpp" -#include "c4/yml/preprocess.hpp" - -#endif // _C4_YML_YML_HPP_ diff --git a/thirdparty/ryml/src/ryml-gdbtypes.py b/thirdparty/ryml/src/ryml-gdbtypes.py deleted file mode 100644 index 2625a6ccd..000000000 --- a/thirdparty/ryml/src/ryml-gdbtypes.py +++ /dev/null @@ -1,391 +0,0 @@ -# To make this file known to Qt Creator using: -# Tools > Options > Debugger > Locals & Expressions > Extra Debugging Helpers -# Any contents here will be picked up by GDB, LLDB, and CDB based -# debugging in Qt Creator automatically. - - -# Example to display a simple type -# template struct MapNode -# { -# U key; -# V data; -# } -# -# def qdump__MapNode(d, value): -# d.putValue("This is the value column contents") -# d.putExpandable() -# if d.isExpanded(): -# with Children(d): -# # Compact simple case. -# d.putSubItem("key", value["key"]) -# # Same effect, with more customization possibilities. -# with SubItem(d, "data") -# d.putItem("data", value["data"]) - -# Check http://doc.qt.io/qtcreator/creator-debugging-helpers.html -# for more details or look at qttypes.py, stdtypes.py, boosttypes.py -# for more complex examples. - -# to try parsing: -# env PYTHONPATH=/usr/share/qtcreator/debugger/ python src/ryml-gdbtypes.py - - -import dumper -#from dumper import Dumper, Value, Children, SubItem -#from dumper import SubItem, Children -from dumper import * - -import sys -import os - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - -# QtCreator makes it really hard to figure out problems in this code. -# So here are some debugging utilities. - - -# FIXME. this decorator is not working; find out why. -def dbglog(func): - """a decorator that logs entry and exit of functions""" - if not _DBG: - return func - def func_wrapper(*args, **kwargs): - _dbg_enter(func.__name__) - ret = func(*args, **kwargs) - _dbg_exit(func.__name__) - return ret - return func_wrapper - - -_DBG = False -_dbg_log = None -_dbg_stack = 0 -def _dbg(*args, **kwargs): - global _dbg_log, _dbg_stack - if not _DBG: - return - if _dbg_log is None: - filename = os.path.join(os.path.dirname(__file__), "dbg.txt") - _dbg_log = open(filename, "w") - kwargs['file'] = _dbg_log - kwargs['flush'] = True - print(" " * _dbg_stack, *args, **kwargs) - - -def _dbg_enter(name): - global _dbg_stack - _dbg(name, "- enter") - _dbg_stack += 1 - - -def _dbg_exit(name): - global _dbg_stack - _dbg_stack -= 1 - _dbg(name, "- exit!") - - - -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- -# ----------------------------------------------------------------------------- - - -NPOS = 18446744073709551615 -MAX_SUBSTR_LEN_DISPLAY = 80 -MAX_SUBSTR_LEN_EXPAND = 1000 - - -def get_str_value(d, value, limit=0): - # adapted from dumper.py::Dumper::putCharArrayValue() - m_str = value["str"].pointer() - m_len = value["len"].integer() - if m_len == NPOS: - _dbg("getstr... 1", m_len) - m_str = "!!!!!!!!!!" - m_len = len(m_str) - return m_str, m_len - if limit == 0: - limit = d.displayStringLimit - elided, shown = d.computeLimit(m_len, limit) - mem = bytes(d.readRawMemory(m_str, shown)) - mem = mem.decode('utf8') - return mem, m_len - - -def __display_csubstr(d, value, limit=0): - m_str, m_len = get_str_value(d, value) - safe_len = min(m_len, MAX_SUBSTR_LEN_DISPLAY) - disp = m_str[0:safe_len] - # ensure the string escapes characters like \n\r\t etc - disp = disp.encode('unicode_escape').decode('utf8') - # WATCHOUT. quotes in the string will make qtcreator hang!!! - disp = disp.replace('"', '\\"') - disp = disp.replace('\'', '\\') - if m_len <= MAX_SUBSTR_LEN_DISPLAY: - d.putValue(f"[{m_len}] '{disp}'") - else: - d.putValue(f"[{m_len}] '{disp}'...") - return m_str, m_len - - -def qdump__c4__csubstr(d, value): - m_str, m_len = __display_csubstr(d, value) - d.putExpandable() - if d.isExpanded(): - with Children(d): - safe_len = min(m_len, MAX_SUBSTR_LEN_EXPAND) - for i in range(safe_len): - ct = d.createType('char') - d.putSubItem(safe_len, d.createValue(value["str"].pointer() + i, ct)) - d.putSubItem("len", value["len"]) - d.putPtrItem("str", value["str"].pointer()) - - -def qdump__c4__substr(d, value): - qdump__c4__csubstr(d, value) - - -def qdump__c4__basic_substring(d, value): - qdump__c4__csubstr(d, value) - - -def qdump__c4__yml__NodeScalar(d, value): - alen = value["anchor"]["len"].integer() - tlen = value["tag" ]["len"].integer() - m_str, m_len = get_str_value(d, value["scalar"]) - if alen == 0 and tlen == 0: - d.putValue(f'\'{m_str}\'') - elif alen == 0 and tlen > 0: - d.putValue(f'\'{m_str}\' [Ta]') - elif alen > 0 and tlen == 0: - d.putValue(f'\'{m_str}\' [tA]') - elif alen > 0 and tlen > 0: - d.putValue(f'\'{m_str}\' [TA]') - d.putExpandable() - if d.isExpanded(): - with Children(d): - d.putSubItem("[scalar]", value["scalar"]) - if tlen > 0: - d.putSubItem("[tag]", value["tag"]) - if alen > 0: - d.putSubItem("[anchor or ref]", value["anchor"]) - - -def _format_enum_value(int_value, enum_map): - str_value = enum_map.get(int_value, None) - display = f'{int_value}' if str_value is None else f'{str_value} ({int_value})' - return display - - -def _format_bitmask_value(int_value, enum_map): - str_value = enum_map.get(int_value, None) - if str_value: - return f'{str_value} ({int_value})' - else: - out = "" - orig = int_value - # do in reverse to get compound flags first - for k, v in reversed(enum_map.items()): - if (k != 0): - if (int_value & k) == k: - if len(out) > 0: - out += '|' - out += v - int_value &= ~k - else: - if len(out) == 0 and int_value == 0: - return v - if out == "": - return f'{int_value}' - return f"{out} ({orig})" - - -def _c4bit(*ints): - ret = 0 - for i in ints: - ret |= 1 << i - return ret - - -node_types = { - 0: "NOTYPE", - _c4bit(0): "VAL" , - _c4bit(1): "KEY" , - _c4bit(2): "MAP" , - _c4bit(3): "SEQ" , - _c4bit(4): "DOC" , - _c4bit(5,3): "STREAM", - _c4bit(6): "KEYREF" , - _c4bit(7): "VALREF" , - _c4bit(8): "KEYANCH" , - _c4bit(9): "VALANCH" , - _c4bit(10): "KEYTAG" , - _c4bit(11): "VALTAG" , - _c4bit(12): "VALQUO" , - _c4bit(13): "KEYQUO" , - _c4bit(1,0): "KEYVAL", - _c4bit(1,3): "KEYSEQ", - _c4bit(1,2): "KEYMAP", - _c4bit(4,2): "DOCMAP", - _c4bit(4,3): "DOCSEQ", - _c4bit(4,0): "DOCVAL", - # - _c4bit(14): "STYLE_FLOW_SL", - _c4bit(15): "STYLE_FLOW_ML", - _c4bit(16): "STYLE_BLOCK", - # - _c4bit(17): "KEY_LITERAL", - _c4bit(18): "VAL_LITERAL", - _c4bit(19): "KEY_FOLDED", - _c4bit(20): "VAL_FOLDED", - _c4bit(21): "KEY_SQUO", - _c4bit(22): "VAL_SQUO", - _c4bit(23): "KEY_DQUO", - _c4bit(24): "VAL_DQUO", - _c4bit(25): "KEY_PLAIN", - _c4bit(26): "VAL_PLAIN", -} -node_types_rev = {v: k for k, v in node_types.items()} - - -def _node_type_has_all(node_type_value, type_name): - exp = node_types_rev[type_name] - return (node_type_value & exp) == exp - - -def _node_type_has_any(node_type_value, type_name): - exp = node_types_rev[type_name] - return (node_type_value & exp) != 0 - - -def qdump__c4__yml__NodeType_e(d, value): - v = _format_bitmask_value(value.integer(), node_types) - d.putValue(v) - - -def qdump__c4__yml__NodeType(d, value): - qdump__c4__yml__NodeType_e(d, value["type"]) - - -def qdump__c4__yml__NodeData(d, value): - d.putValue("wtf") - ty = _format_bitmask_value(value.integer(), node_types) - t = value["m_type"]["type"].integer() - k = value["m_key"]["scalar"] - v = value["m_val"]["scalar"] - sk, lk = get_str_value(d, k) - sv, lv = get_str_value(d, v) - if _node_type_has_all(t, "KEYVAL"): - d.putValue(f"'{sk}': '{sv}' {ty}") - elif _node_type_has_any(t, "KEY"): - d.putValue(f"'{sk}': {ty}") - elif _node_type_has_any(t, "VAL"): - d.putValue(f"'{sv}' {ty}") - else: - d.putValue(f"{ty}") - d.putExpandable() - if d.isExpanded(): - with Children(d): - d.putSubItem("m_type", value["m_type"]) - # key - if _node_type_has_any(t, "KEY"): - d.putSubItem("m_key", value["m_key"]) - if _node_type_has_any(t, "KEYREF"): - with SubItem(d, "m_key.ref"): - s_, _ = get_str_value(d, value["m_key"]["anchor"]) - d.putValue(f"'{s_}'") - if _node_type_has_any(t, "KEYANCH"): - with SubItem(d, "m_key.anchor"): - s_, _ = get_str_value(d, value["m_key"]["anchor"]) - d.putValue(f"'{s_}'") - if _node_type_has_any(t, "KEYTAG"): - with SubItem(d, "m_key.tag"): - s_, _ = get_str_value(d, value["m_key"]["tag"]) - d.putValue(f"'{s_}'") - # val - if _node_type_has_any(t, "VAL"): - d.putSubItem("m_val", value["m_val"]) - if _node_type_has_any(t, "VALREF"): - with SubItem(d, "m_val.ref"): - s_, _ = get_str_value(d, value["m_val"]["anchor"]) - d.putValue(f"'{s_}'") - if _node_type_has_any(t, "VALANCH"): - with SubItem(d, "m_val.anchor"): - s_, _ = get_str_value(d, value["m_val"]["anchor"]) - d.putValue(f"'{s_}'") - if _node_type_has_any(t, "VALTAG"): - with SubItem(d, "m_val.tag"): - s_, _ = get_str_value(d, value["m_val"]["tag"]) - d.putValue(f"'{s_}'") - # hierarchy - _dump_node_index(d, "m_parent", value) - _dump_node_index(d, "m_first_child", value) - _dump_node_index(d, "m_last_child", value) - _dump_node_index(d, "m_next_sibling", value) - _dump_node_index(d, "m_prev_sibling", value) - - -def _dump_node_index(d, name, value): - if int(value[name].integer()) == NPOS: - pass - #with SubItem(d, name): - # d.putValue("-") - else: - d.putSubItem(name, value[name]) - - -# c4::yml::Tree -def qdump__c4__yml__Tree(d, value): - m_size = value["m_size"].integer() - m_cap = value["m_cap"].integer() - d.putExpandable() - if d.isExpanded(): - #d.putArrayData(value["m_buf"], m_size, value["m_buf"].dereference()) - with Children(d): - with SubItem(d, f"[nodes]"): - d.putItemCount(m_size) - d.putArrayData(value["m_buf"].pointer(), m_size, value["m_buf"].type.dereference()) - d.putPtrItem("m_buf", value["m_buf"].pointer()) - d.putIntItem("m_size", value["m_size"]) - d.putIntItem("m_cap (capacity)", value["m_cap"]) - d.putIntItem("[slack]", m_cap - m_size) - d.putIntItem("m_free_head", value["m_free_head"]) - d.putIntItem("m_free_tail", value["m_free_tail"]) - d.putSubItem("m_arena", value["m_arena"]) - - -def qdump__c4__yml__detail__stack(d, value): - T = value.type[0] - N = value.type[0] - m_size = value["m_size"].integer() - m_capacity = value["m_capacity"].integer() - d.putItemCount(m_size) - if d.isExpanded(): - with Children(d): - with SubItem(d, f"[nodes]"): - d.putItemCount(m_size) - d.putArrayData(value["m_stack"].pointer(), m_size, T) - d.putIntItem("m_size", value["m_size"]) - d.putIntItem("m_capacity", value["m_capacity"]) - #d.putIntItem("[small capacity]", N) - d.putIntItem("[is large]", value["m_buf"].address() == value["m_stack"].pointer()) - d.putPtrItem("m_stack", value["m_stack"].pointer()) - d.putPtrItem("m_buf", value["m_buf"].address()) - - -def qdump__c4__yml__detail__ReferenceResolver__refdata(d, value): - node = value["node"].integer() - ty = _format_bitmask_value(value["type"].integer(), node_types) - d.putValue(f'{node} {ty}') - d.putExpandable() - if d.isExpanded(): - with Children(d): - d.putSubItem("type", value["type"]) - d.putSubItem("node", value["node"]) - _dump_node_index(d, "prev_anchor", value) - _dump_node_index(d, "target", value) - _dump_node_index(d, "parent_ref", value) - _dump_node_index(d, "parent_ref_sibling", value) diff --git a/thirdparty/ryml/src/ryml.hpp b/thirdparty/ryml/src/ryml.hpp deleted file mode 100644 index c2b3e74b2..000000000 --- a/thirdparty/ryml/src/ryml.hpp +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef _RYML_HPP_ -#define _RYML_HPP_ - -#include "c4/yml/yml.hpp" - -namespace ryml { -using namespace c4::yml; -using namespace c4; -} - -#endif /* _RYML_HPP_ */ diff --git a/thirdparty/ryml/src/ryml.natvis b/thirdparty/ryml/src/ryml.natvis deleted file mode 100644 index 5e43b1a5c..000000000 --- a/thirdparty/ryml/src/ryml.natvis +++ /dev/null @@ -1,194 +0,0 @@ - - - - - - - - {scalar.str,[scalar.len]} - {scalar.str,[scalar.len]} [T] - {scalar.str,[scalar.len]} [A] - {scalar.str,[scalar.len]} [T][A] - - scalar - tag - anchor - - - - - {type} - - - - c4::yml::VAL - c4::yml::KEY - c4::yml::MAP - c4::yml::SEQ - c4::yml::DOC - c4::yml::STREAM - c4::yml::KEYREF - c4::yml::VALREF - c4::yml::KEYANCH - c4::yml::VALANCH - c4::yml::KEYTAG - c4::yml::VALTAG - c4::yml::VALQUO - c4::yml::KEYQUO - - - - - - - [KEYVAL] {m_key.scalar.str,[m_key.scalar.len]}: {m_val.scalar.str,[m_val.scalar.len]} - [KEYSEQ] {m_key.scalar.str,[m_key.scalar.len]} - [KEYMAP] {m_key.scalar.str,[m_key.scalar.len]} - [DOCSEQ] - [DOCMAP] - [VAL] {m_val.scalar.str,[m_val.scalar.len]} - [KEY] {m_key.scalar.str,[m_key.scalar.len]} - [SEQ] - [MAP] - [DOC] - [STREAM] - [NOTYPE] - - m_type - m_key - m_val - c4::yml::KEYQUO - c4::yml::VALQUO - m_key.anchor - m_val.anchor - m_key.anchor - m_val.anchor - m_parent - m_first_child - m_last_child - m_prev_sibling - m_next_sibling - - - - - sz={m_size}, cap={m_cap} - - m_size - m_cap - - - - m_cap - m_buf - - - - m_free_head - m_arena - - - - - {value} ({type}) - - value - type - - - - - {path} -- target={target} closest={closest} - - target - closest - path_pos - path - - {path.str,[path_pos]} - - - {path.str+path_pos,[path.len-path_pos]} - - - - - - (void) - [INDEX SEED for] {*(m_tree->m_buf + m_id)} - [NAMED SEED for] {*(m_tree->m_buf + m_id)} - {*(m_tree->m_buf + m_id)} - - m_id - *(m_tree->m_buf + m_id) - m_tree - - - - - - - - buf + curr - curr = (buf + curr)->m_next_sibling - - - - - - - - - - #refs={refs.m_size} #nodes={t->m_size} - - - - - - - t->m_buf + (refs.m_stack + curr)->node - curr = curr+1 - - - - - - - - - refs.m_size - refs.m_stack - - - - t - - - - - sz={m_size} cap={m_capacity} - - m_size - m_capacity - m_buf == m_stack - - - - m_size - m_stack - - - - - - - diff --git a/thirdparty/ryml/src/ryml_std.hpp b/thirdparty/ryml/src/ryml_std.hpp deleted file mode 100644 index 5e81439ac..000000000 --- a/thirdparty/ryml/src/ryml_std.hpp +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _RYML_STD_HPP_ -#define _RYML_STD_HPP_ - -#include "./c4/yml/std/std.hpp" - -#endif /* _RYML_STD_HPP_ */ diff --git a/thirdparty/ryml/tbump.toml b/thirdparty/ryml/tbump.toml deleted file mode 100644 index a14130303..000000000 --- a/thirdparty/ryml/tbump.toml +++ /dev/null @@ -1,56 +0,0 @@ -# https://github.com/TankerHQ/tbump -# https://hackernoon.com/lets-automate-version-number-updates-not-a91q3x7n - -# Uncomment this if your project is hosted on GitHub: -github_url = "https://github.com/biojppm/rapidyaml/" - -[version] -current = "0.5.0" - -# Example of a semver regexp. -# Make sure this matches current_version before -# using tbump -regex = ''' - (?P\d+) - \. - (?P\d+) - \. - (?P\d+) - (-(?P[a-z]+)(?P\d+))? - ''' - -[git] -message_template = "chg: pkg: version {new_version}" -tag_template = "v{new_version}" - -# For each file to patch, add a [[file]] config section containing -# the path of the file, relative to the tbump.toml location. -[[file]] -src = "CMakeLists.txt" -search = "c4_project\\(VERSION {current_version}" -[[file]] -src = "test/test_install/CMakeLists.txt" -search = "c4_project\\(VERSION {current_version}" -[[file]] -src = "test/test_singleheader/CMakeLists.txt" -search = "c4_project\\(VERSION {current_version}" - -# You can specify a list of commands to -# run after the files have been patched -# and before the git commit is made - -# [[before_commit]] -# name = "check changelog" -# cmd = "grep -q {new_version} Changelog.rst" - -# TODO: add version header, containing commit hash -# TODO: consolidate changelog from the git logs: -# https://pypi.org/project/gitchangelog/ -# https://blogs.sap.com/2018/06/22/generating-release-notes-from-git-commit-messages-using-basic-shell-commands-gitgrep/ -# https://medium.com/better-programming/create-your-own-changelog-generator-with-git-aefda291ea93 - -# Or run some commands after the git tag and the branch -# have been pushed: -# [[after_push]] -# name = "publish" -# cmd = "./publish.sh" diff --git a/thirdparty/ryml/test/callbacks_tester.hpp b/thirdparty/ryml/test/callbacks_tester.hpp deleted file mode 100644 index 5286e0c34..000000000 --- a/thirdparty/ryml/test/callbacks_tester.hpp +++ /dev/null @@ -1,77 +0,0 @@ -#ifndef C4_TEST_CALLBACKS_TESTER_HPP_ -#define C4_TEST_CALLBACKS_TESTER_HPP_ - -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/common.hpp" -#endif -#include -#include - -namespace c4 { -namespace yml { - -struct CallbacksTester -{ - std::vector memory_pool; - const char *id; - size_t num_allocs, alloc_size; - size_t num_deallocs, dealloc_size; - - CallbacksTester(const char *id_="notset", size_t sz=10u * 1024u) // 10KB - : memory_pool(sz) - , id(id_) - , num_allocs() - , alloc_size() - , num_deallocs() - , dealloc_size() - { - } - - // checking - ~CallbacksTester() - { - check(); - } - - void check() - { - std::cout << "size: alloc=" << alloc_size << " dealloc=" << dealloc_size << std::endl; - std::cout << "count: #allocs=" << num_allocs << " #deallocs=" << num_deallocs << std::endl; - RYML_CHECK(num_allocs == num_deallocs); - RYML_CHECK(alloc_size == dealloc_size); - } - - Callbacks callbacks() const - { - Callbacks cb = get_callbacks(); - cb.m_user_data = (void*) this; - cb.m_allocate = [](size_t len, void *, void *data){ return ((CallbacksTester*) data)->allocate(len); }; - cb.m_free = [](void *mem, size_t len, void *data){ return ((CallbacksTester*) data)->free(mem, len); }; - return cb; - } - - void *allocate(size_t len) - { - std::cout << "alloc[" << num_allocs << "]=" << len << "B\n"; - void *ptr = &memory_pool[alloc_size]; - alloc_size += len; - ++num_allocs; - RYML_CHECK(alloc_size < memory_pool.size()); - return ptr; - } - - void free(void *mem, size_t len) - { - RYML_CHECK((char*)mem >= &memory_pool.front() && (char*)mem < &memory_pool.back()); - RYML_CHECK((char*)mem+len >= &memory_pool.front() && (char*)mem+len <= &memory_pool.back()); - std::cout << "free[" << num_deallocs << "]=" << len << "B\n"; - dealloc_size += len; - ++num_deallocs; - // no need to free here - } -}; - -} // namespace yml -} // namespace c4 - -#endif /* C4_TEST_CALLBACKS_TESTER_HPP_ */ diff --git a/thirdparty/ryml/test/test_basic.cpp b/thirdparty/ryml/test/test_basic.cpp deleted file mode 100644 index 7a10c073b..000000000 --- a/thirdparty/ryml/test/test_basic.cpp +++ /dev/null @@ -1,304 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/std/std.hpp" -#include "c4/yml/parse.hpp" -#include "c4/yml/emit.hpp" -#include -#include -#include -#endif - -#include "./test_case.hpp" - -#include - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4389) // signed/unsigned mismatch -#elif defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wdollar-in-identifier-extension" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -#endif - -namespace c4 { -namespace yml { - -TEST(general, parsing) -{ - auto tree = parse_in_arena("{foo: 1}"); - - char cmpbuf[128] = {0}; - substr cmp(cmpbuf); - size_t ret; - - ret = cat(cmp, tree["foo"].val()); - EXPECT_EQ(cmp.first(ret), "1"); - - ret = cat(cmp, tree["foo"].key()); - EXPECT_EQ(cmp.first(ret), "foo"); -} - -TEST(general, emitting) -{ - std::string cmpbuf; - - Tree tree; - auto r = tree.rootref(); - - r |= MAP; // this is needed to make the root a map - - r["foo"] = "1"; // ryml works only with strings. - // Note that the tree will be __pointing__ at the - // strings "foo" and "1" used here. You need - // to make sure they have at least the same - // lifetime as the tree. - - auto s = r["seq"]; // does not change the tree until s is written to. - s |= SEQ; - r["seq"].append_child() = "bar0"; // value of this child is now __pointing__ at "bar0" - r["seq"].append_child() = "bar1"; - r["seq"].append_child() = "bar2"; - - //print_tree(tree); - - // emit to stdout (can also emit to FILE* or ryml::span) - emitrs_yaml(tree, &cmpbuf); - const char* exp = R"(foo: 1 -seq: - - bar0 - - bar1 - - bar2 -)"; - EXPECT_EQ(cmpbuf, exp); - - // serializing: using operator<< instead of operator= - // will make the tree serialize the value into a char - // arena inside the tree. This arena can be reserved at will. - int ch3 = 33, ch4 = 44; - s.append_child() << ch3; - s.append_child() << ch4; - - { - std::string tmp = "child5"; - s.append_child() << tmp; // requires #include - // now tmp can go safely out of scope, as it was - // serialized to the tree's internal string arena - // Note the include highlighted above is required so that ryml - // knows how to turn an std::string into a c4::csubstr/c4::substr. - } - - emitrs_yaml(tree, &cmpbuf); - exp = R"(foo: 1 -seq: - - bar0 - - bar1 - - bar2 - - 33 - - 44 - - child5 -)"; - EXPECT_EQ(cmpbuf, exp); - - // to serialize keys: - int k=66; - r.append_child() << key(k) << 7; - - emitrs_yaml(tree, &cmpbuf); - exp = R"(foo: 1 -seq: - - bar0 - - bar1 - - bar2 - - 33 - - 44 - - child5 -66: 7 -)"; - EXPECT_EQ(cmpbuf, exp); -} - -TEST(general, map_to_root) -{ - std::string cmpbuf; const char *exp; - std::map m({{"bar", 2}, {"foo", 1}}); - Tree t; - t.rootref() << m; - - emitrs_yaml(t, &cmpbuf); - exp = R"(bar: 2 -foo: 1 -)"; - EXPECT_EQ(cmpbuf, exp); - - t["foo"] << 10; - t["bar"] << 20; - - m.clear(); - t.rootref() >> m; - - EXPECT_EQ(m["foo"], 10); - EXPECT_EQ(m["bar"], 20); -} - -TEST(general, print_tree) -{ - const char yaml[] = R"( -a: - b: bval - c: - d: - - e - - d - - f: fval - g: gval - h: - - - x: a - y: b - - - z: c - u: -)"; - Tree t = parse_in_arena(yaml); - print_tree(t); // to make sure this is covered too -} - -TEST(general, numbers) -{ - const char yaml[] = R"(- -1 -- -1.0 -- +1.0 -- 1e-2 -- 1e+2 -)"; - Tree t = parse_in_arena(yaml); - auto s = emitrs_yaml(t); - EXPECT_EQ(s, std::string(yaml)); -} - -// github issue 29: https://github.com/biojppm/rapidyaml/issues/29 -TEST(general, newlines_on_maps_nested_in_seqs) -{ - const char yaml[] = R"(enemy: -- actors: - - {name: Enemy_Bokoblin_Junior, value: 4.0} - - {name: Enemy_Bokoblin_Middle, value: 16.0} - - {name: Enemy_Bokoblin_Senior, value: 32.0} - - {name: Enemy_Bokoblin_Dark, value: 48.0} - species: BokoblinSeries -)"; - std::string expected = R"(enemy: - - actors: - - name: Enemy_Bokoblin_Junior - value: 4.0 - - name: Enemy_Bokoblin_Middle - value: 16.0 - - name: Enemy_Bokoblin_Senior - value: 32.0 - - name: Enemy_Bokoblin_Dark - value: 48.0 - species: BokoblinSeries -)"; - Tree t = parse_in_arena(yaml); - auto s = emitrs_yaml(t); - EXPECT_EQ(expected, s); -} - - -TEST(general, test_suite_RZT7) -{ - csubstr yaml = R"( ---- -Time: 2001-11-23 15:01:42 -5 -User: ed -Warning: - This is an error message - for the log file ---- -Time: 2001-11-23 15:02:31 -5 -User: ed -Warning: - A slightly different error - message. ---- -Date: 2001-11-23 15:03:17 -5 -User: ed -Fatal: - Unknown variable "bar" -Stack: - - file: TopClass.py - line: 23 - code: | - x = MoreObject("345\n") - - file: MoreClass.py - line: 58 - code: |- - foo = bar -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_stream()); - ConstNodeRef doc0 = t.rootref()[0]; - EXPECT_EQ(doc0["Time"].val(), csubstr("2001-11-23 15:01:42 -5")); - EXPECT_EQ(doc0["User"].val(), csubstr("ed")); - EXPECT_EQ(doc0["Warning"].val(), csubstr("This is an error message for the log file")); - ConstNodeRef doc1 = t.rootref()[1]; - EXPECT_EQ(doc1["Time"].val(), csubstr("2001-11-23 15:02:31 -5")); - EXPECT_EQ(doc1["User"].val(), csubstr("ed")); - EXPECT_EQ(doc1["Warning"].val(), csubstr("A slightly different error message.")); - ConstNodeRef doc2 = t.rootref()[2]; - EXPECT_EQ(doc2["Date"].val(), csubstr("2001-11-23 15:03:17 -5")); - EXPECT_EQ(doc2["User"].val(), csubstr("ed")); - EXPECT_EQ(doc2["Fatal"].val(), csubstr("Unknown variable \"bar\"")); - EXPECT_EQ(doc2["Stack"][0]["file"].val(), csubstr("TopClass.py")); - EXPECT_EQ(doc2["Stack"][0]["line"].val(), csubstr("23")); - EXPECT_EQ(doc2["Stack"][0]["code"].val(), csubstr("x = MoreObject(\"345\\n\")\n")); - EXPECT_EQ(doc2["Stack"][1]["file"].val(), csubstr("MoreClass.py")); - EXPECT_EQ(doc2["Stack"][1]["line"].val(), csubstr("58")); - EXPECT_EQ(doc2["Stack"][1]["code"].val(), csubstr("foo = bar")); - }); -} - - -TEST(general, github_issue_124) -{ - // All these inputs are basically the same. - // However, the comment was found to confuse the parser in #124. - csubstr yaml[] = { - "a:\n - b\nc: d", - "a:\n - b\n\n# ignore me:\nc: d", - "a:\n - b\n\n # ignore me:\nc: d", - "a:\n - b\n\n # ignore me:\nc: d", - "a:\n - b\n\n#:\nc: d", // also try with just a ':' in the comment - "a:\n - b\n\n# :\nc: d", - "a:\n - b\n\n#\nc: d", // also try with empty comment - }; - for(csubstr inp : yaml) - { - SCOPED_TRACE(inp); - Tree t = parse_in_arena(inp); - std::string s = emitrs_yaml(t); - // The re-emitted output should not contain the comment. - EXPECT_EQ(c4::to_csubstr(s), "a:\n - b\nc: d\n"); - } -} - - - -//------------------------------------------- -// this is needed to use the test case library -Case const* get_case(csubstr /*name*/) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 - -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/test/test_block_folded.cpp b/thirdparty/ryml/test/test_block_folded.cpp deleted file mode 100644 index 9d579c5a1..000000000 --- a/thirdparty/ryml/test/test_block_folded.cpp +++ /dev/null @@ -1,1574 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -TEST(block_folded, basic) -{ - { - Tree t = parse_in_arena(R"(> -hello -there - -got it - - -really -)"); - EXPECT_EQ(t.rootref().val(), csubstr("hello there\ngot it\n\nreally\n")); - } -} - -TEST(block_folded, empty_block) -{ - { - Tree t = parse_in_arena(R"(- > -)"); - EXPECT_EQ(t[0].val(), csubstr("")); - } - { - Tree t = parse_in_arena(R"(- >- -)"); - EXPECT_EQ(t[0].val(), csubstr("")); - } - { - Tree t = parse_in_arena(R"(- >+ -)"); - EXPECT_EQ(t[0].val(), csubstr("")); - } - { - Tree t = parse_in_arena(R"( -- > - -- >- - -- >+ - -)"); - EXPECT_FALSE(t.empty()); - EXPECT_EQ(t[0].val(), csubstr("")); - EXPECT_EQ(t[1].val(), csubstr("")); - EXPECT_EQ(t[2].val(), csubstr("\n")); - } - { - Tree t = parse_in_arena(R"( -- > - -- >- - -- >+ - -)"); - EXPECT_FALSE(t.empty()); - EXPECT_EQ(t[0].val(), csubstr("")); - EXPECT_EQ(t[1].val(), csubstr("")); - EXPECT_EQ(t[2].val(), csubstr("\n")); - } - { - Tree t = parse_in_arena(R"( -- > -- >- -- >+ -)"); - EXPECT_FALSE(t.empty()); - EXPECT_EQ(t[0].val(), csubstr("")); - EXPECT_EQ(t[1].val(), csubstr("")); - EXPECT_EQ(t[2].val(), csubstr("")); - } -} - -TEST(block_folded, empty_block0) -{ - Tree t = parse_in_arena(R"(- > -)"); - EXPECT_EQ(t[0].val(), csubstr("")); - t = parse_in_arena(R"(- >- -)"); - EXPECT_EQ(t[0].val(), csubstr("")); - t = parse_in_arena(R"(- >+ -)"); - EXPECT_EQ(t[0].val(), csubstr("")); -} - -TEST(block_folded, empty_block1) -{ - const Tree t = parse_in_arena(R"( -- >- - a -- >- - -- >- - -- >- - - - -- >- - - - -)"); - EXPECT_EQ(t[0].val(), csubstr("a")); - EXPECT_EQ(t[1].val(), csubstr("")); - EXPECT_EQ(t[2].val(), csubstr("")); - EXPECT_EQ(t[3].val(), csubstr("")); - EXPECT_EQ(t[4].val(), csubstr("")); -} - -TEST(block_folded, empty_block_as_container_member) -{ - // this was ok - test_check_emit_check(R"( -map: - a: "" - b: '' - d: | - c: > - e: -)", [](Tree const &t){ - EXPECT_TRUE(t["map"].has_key()); - EXPECT_TRUE(t["map"].is_map()); - EXPECT_EQ(t["map"].num_children(), 5u); - for(const auto &child : t["map"].children()) - { - EXPECT_EQ(child.val(), ""); - if(child.key() != "e") - { - EXPECT_TRUE(child.type().is_val_quoted()); - EXPECT_FALSE(child.val_is_null()); - } - } - }); - // this was ok - test_check_emit_check(R"( -map: - a: "" - b: '' - d: | - c: > -)", [](Tree const &t){ - EXPECT_TRUE(t["map"].has_key()); - EXPECT_TRUE(t["map"].is_map()); - EXPECT_TRUE(t["map"].is_map()); - EXPECT_EQ(t["map"].num_children(), 4u); - for(const auto &child : t["map"].children()) - { - EXPECT_EQ(child.val(), ""); - EXPECT_TRUE(child.type().is_val_quoted()); - EXPECT_FALSE(child.val_is_null()); - } - }); - // this was not ok! the block literal before next is extended: to - // include the YAML for next! - test_check_emit_check(R"( -map: - a: "" - b: '' - d: | - c: > -next: - a: "" - b: '' - d: | - c: > -)", [](Tree const &t){ - for(const char *name : {"map", "next"}) - { - ASSERT_TRUE(t.rootref().has_child(to_csubstr(name))) << "name=" << name; - ConstNodeRef node = t[to_csubstr(name)]; - EXPECT_TRUE(node.has_key()); - EXPECT_TRUE(node.is_map()); - EXPECT_TRUE(node.is_map()); - ASSERT_EQ(node.num_children(), 4u); - for(const auto &child : node.children()) - { - EXPECT_EQ(child.val(), ""); - EXPECT_TRUE(child.type().is_val_quoted()); - EXPECT_FALSE(child.val_is_null()); - } - } - }); - test_check_emit_check(R"( -seq: - - "" - - '' - - | - - > -next: - - "" - - '' - - | - - > -)", [](Tree const &t){ - for(const char *name : {"seq", "next"}) - { - ASSERT_TRUE(t.rootref().has_child(to_csubstr(name))) << "name=" << name; - ConstNodeRef node = t[to_csubstr(name)]; - EXPECT_TRUE(node.has_key()); - EXPECT_TRUE(node.is_seq()); - ASSERT_EQ(node.num_children(), 4u); - for(const auto &child : node.children()) - { - EXPECT_EQ(child.val(), ""); - EXPECT_TRUE(child.type().is_val_quoted()); - EXPECT_FALSE(child.val_is_null()); - } - } - }); -} - -TEST(block_folded, issue152_not_indented) -{ - const Tree t = parse_in_arena(R"( -ok: - - | - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - - parses - yes -ok_parses: yes -err: - - | - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 -err_parses: no -err2: - - > - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 -err2_parses: no -err3: - - >- - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 -err3_parses: no -)"); - EXPECT_EQ(t["ok" ][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(t["err" ][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(t["err2"][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(t["err3"][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432")); -} - -TEST(block_folded, issue152_indented_once) -{ - const Tree t = parse_in_arena(R"( -indented_once: - ok: - - | - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - - parses - yes - ok_parses: yes - err: - - | - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - err_parses: no - err2: - - > - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - err2_parses: no - err3: - - >- - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - err3_parses: no -)"); - ConstNodeRef n = t["indented_once"]; - EXPECT_EQ(n["ok" ][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(n["err" ][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(n["err2"][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(n["err3"][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432")); -} - -TEST(block_folded, issue152_indented_twice) -{ - const Tree t = parse_in_arena(R"( -indented_once: - indented_twice: - ok: - - | - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - - parses - yes - ok_parses: yes - err: - - | - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - err_parses: no - err2: - - > - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - err2_parses: no - err3: - - >- - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - err3_parses: no -)"); - ConstNodeRef n = t["indented_once"]["indented_twice"]; - EXPECT_EQ(n["ok" ][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(n["err" ][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(n["err2"][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(n["err3"][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432")); -} - -TEST(block_folded, issue152_indented_thrice) -{ - const Tree t = parse_in_arena(R"( -indented_once: - indented_twice: - indented_thrice: - ok: - - | - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - - parses - yes - ok_parses: yes - err: - - | - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - err_parses: no - err2: - - > - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - err2_parses: no - err3: - - >- - exec pg_isready -U "dog" -d "dbname=dog" -h 127.0.0.1 -p 5432 - err3_parses: no -)"); - ConstNodeRef n = t["indented_once"]["indented_twice"]["indented_thrice"]; - EXPECT_EQ(n["ok" ][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(n["err" ][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(n["err2"][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432\n")); - EXPECT_EQ(n["err3"][0].val(), csubstr("exec pg_isready -U \"dog\" -d \"dbname=dog\" -h 127.0.0.1 -p 5432")); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(block_folded, test_suite_4QFQ) -{ - csubstr yaml = R"( -- |1 - child2 -- |3 - child2 -- ' child2 - -' -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t[0].val(), csubstr(" child2\n")); - EXPECT_EQ(t[1].val(), csubstr(" child2\n")); - EXPECT_EQ(t[2].val(), csubstr(" child2\n")); - }); -} - -TEST(block_folded, test_suite_4QFQ_pt2) -{ - csubstr yaml = R"(--- -- | - child0 -- > - - - # child1 -- |1 - child2 -- > - child3 ---- -foo: - - | - child0 - - > - - - # child1 - - |2 - child2 - - > - child3 -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_stream()); - ConstNodeRef doc = t.rootref().child(0); - ASSERT_TRUE(doc.is_seq()); - ASSERT_EQ(doc.num_children(), 4u); - EXPECT_EQ(doc[0].val(), csubstr("child0\n")); - EXPECT_EQ(doc[1].val(), csubstr("\n\n# child1\n")); - EXPECT_EQ(doc[2].val(), csubstr(" child2\n")); - EXPECT_EQ(doc[3].val(), csubstr("child3\n")); - doc = t.rootref().child(1); - ASSERT_TRUE(doc.is_map()); - ASSERT_EQ(doc["foo"].num_children(), 4u); - EXPECT_EQ(doc["foo"][0].val(), csubstr("child0\n")); - EXPECT_EQ(doc["foo"][1].val(), csubstr("\n\n# child1\n")); - EXPECT_EQ(doc["foo"][2].val(), csubstr(" child2\n")); - EXPECT_EQ(doc["foo"][3].val(), csubstr("child3\n")); - }); -} - -TEST(block_folded, test_suite_6VJK) -{ - csubstr yaml = R"(- > - Sammy Sosa completed another - fine season with great stats. - - 63 Home Runs - 0.288 Batting Average - - What a year! -- > - Sammy Sosa completed another - fine season with great stats. - 63 Home Runs - 0.288 Batting Average - What a year! -- > - Sammy Sosa completed another - fine season with great stats. - - 63 Home Runs - 0.288 Batting Average - - What a year! -- > - Sammy Sosa completed another - fine season with great stats. - - - 63 Home Runs - 0.288 Batting Average - - - What a year! -- > - Sammy Sosa completed another - fine season with great stats. - - - - 63 Home Runs - 0.288 Batting Average - - - - What a year! -- >- - No folding needed -- > - No folding needed)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t[0].val(), csubstr("Sammy Sosa completed another fine season with great stats.\n63 Home Runs 0.288 Batting Average\nWhat a year!\n")); - EXPECT_EQ(t[1].val(), csubstr("Sammy Sosa completed another fine season with great stats.\n 63 Home Runs\n 0.288 Batting Average\nWhat a year!\n")); - EXPECT_EQ(t[2].val(), csubstr("Sammy Sosa completed another fine season with great stats.\n\n 63 Home Runs\n 0.288 Batting Average\n\nWhat a year!\n")); - EXPECT_EQ(t[3].val(), csubstr("Sammy Sosa completed another fine season with great stats.\n\n\n 63 Home Runs\n 0.288 Batting Average\n\n\nWhat a year!\n")); - EXPECT_EQ(t[4].val(), csubstr("Sammy Sosa completed another fine season with great stats.\n\n\n\n 63 Home Runs\n 0.288 Batting Average\n\n\n\nWhat a year!\n")); - EXPECT_EQ(t[5].val(), csubstr("No folding needed")); - EXPECT_EQ(t[6].val(), csubstr("No folding needed\n")); - }); -} - -TEST(block_folded, test_suite_7T8X) -{ - csubstr yaml = R"(> - - folded - line - - next - line - * bullet - - * list - * lines - - last - line - -# Comment -)"; - Tree t = parse_in_arena(yaml); - EXPECT_EQ(t.rootref().val(), "\nfolded line\nnext line\n * bullet\n\n * list\n * lines\n\nlast line\n"); -} - -TEST(block_folded, test_suite_A6F9) -{ - csubstr yaml = R"( -strip: |- - text -clip: | - text -keep: |+ - text -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t["strip"].val(), "text"); - EXPECT_EQ(t["clip"].val(), "text\n"); - EXPECT_EQ(t["keep"].val(), "text\n"); - }); -} - -TEST(block_folded, test_suite_B3HG) -{ - csubstr yaml = R"( ---- > - folded - text - - ---- > - folded - text ---- > - folded text -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t.docref(0).val(), csubstr("folded text\n")); - EXPECT_EQ(t.docref(1).val(), csubstr("folded text\n")); - EXPECT_EQ(t.docref(2).val(), csubstr("folded text\n")); - }); -} - -TEST(block_folded, test_suite_D83L) -{ - csubstr yaml = R"( -- |2- - explicit indent and chomp -- |-2 - chomp and explicit indent -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_TRUE(t.rootref().is_seq()); - EXPECT_EQ(t[0].val(), csubstr("explicit indent and chomp")); - EXPECT_EQ(t[1].val(), csubstr("chomp and explicit indent")); - }); -} - -TEST(block_folded, test_suite_DWX9) -{ - csubstr yaml = R"( -| - - - literal - - - text - - # Comment -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t.rootref().val(), csubstr("\n\nliteral\n \n\ntext\n")); - }); -} - -TEST(block_folded, test_suite_F6MC) -{ - csubstr yaml = R"( -a: >2 - more indented - regular -b: >2 - - - more indented - regular -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t["a"].val(), csubstr(" more indented\nregular\n")); - EXPECT_EQ(t["b"].val(), csubstr("\n\n more indented\nregular\n")); - }); -} - -TEST(block_folded, test_suite_K858) -{ - csubstr yaml = R"(--- -# strip -- >- - -# clip -- > - -# keep -- |+ - ---- -strip: >- - -clip: > - -keep: |+ - -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_EQ(t.docref(0).num_children(), 3u); - EXPECT_EQ(t.docref(0)[0].val(), csubstr{}); - EXPECT_EQ(t.docref(0)[1].val(), csubstr{}); - EXPECT_EQ(t.docref(0)[2].val(), csubstr("\n")); - ASSERT_TRUE(t.docref(1).has_child("strip")); - ASSERT_TRUE(t.docref(1).has_child("keep")); - ASSERT_TRUE(t.docref(1).has_child("clip")); - EXPECT_EQ(t.docref(1)["strip"].val(), csubstr{}); - EXPECT_EQ(t.docref(1)["clip"].val(), csubstr{}); - EXPECT_EQ(t.docref(1)["keep"].val(), csubstr("\n")); - }); -} - - -TEST(block_folded, test_suite_MJS9) -{ - csubstr yaml = R"( -- > - foo - - bar - - baz -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t[0].val(), csubstr("foo \n\n\t bar\n\nbaz\n")); // "foo \n\n \t bar\n\nbaz\n" - }); -} - -TEST(block_folded, test_suite_P2AD) -{ - csubstr yaml = R"( -- | # Empty header↓ - literal -- >1 # Indentation indicator↓ - folded -- |+ # Chomping indicator↓ - keep - -- >1- # Both indicators↓ - strip -- >-1 # Both indicators↓ - strip -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_seq()); - ASSERT_EQ(t.rootref().num_children(), 5u); - EXPECT_EQ(t[0].val(), csubstr("literal\n")); - EXPECT_EQ(t[1].val(), csubstr(" folded\n")); - EXPECT_EQ(t[2].val(), csubstr("keep\n\n")); - EXPECT_EQ(t[3].val(), csubstr(" strip")); - EXPECT_EQ(t[4].val(), csubstr(" strip")); - }); -} - - -TEST(block_folded, test_suite_R4YG) -{ - csubstr yaml = R"( -- | - detected0 -- > - - - # detected1 -- |1 - explicit2 -- > - - detected3 -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_seq()); - ASSERT_EQ(t.rootref().num_children(), 4u); - EXPECT_EQ(t[0].val(), csubstr("detected0\n")); - EXPECT_EQ(t[1].val(), csubstr("\n\n# detected1\n")); - EXPECT_EQ(t[2].val(), csubstr(" explicit2\n")); - EXPECT_EQ(t[3].val(), csubstr("\t\ndetected3\n")); - }); -} - - -TEST(block_folded, test_suite_T26H) -{ - csubstr yaml = R"( ---- | - - - literal - - - text - - # Comment -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_stream()); - ASSERT_TRUE(t.rootref().first_child().is_doc()); - EXPECT_EQ(t.rootref().first_child().val(), csubstr("\n\nliteral\n \n\ntext\n")); - }); -} - - -TEST(block_folded, test_suite_T5N4) -{ - csubstr yaml = R"( ---- | - literal - text - - -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_stream()); - ASSERT_TRUE(t.rootref().first_child().is_doc()); - EXPECT_EQ(t.rootref().first_child().val(), csubstr("literal\n\ttext\n")); - }); -} - - -TEST(block_folded, test_suite_W4TN) -{ - csubstr yaml = R"( ---- | - %!PS-Adobe-2.0 -... ---- > - %!PS-Adobe-2.0 -... ---- | -%!PS-Adobe-2.0 -... ---- > -%!PS-Adobe-2.0 -... ---- -# Empty -... ---- | - %!PS-Adobe-2.0 ---- > - %!PS-Adobe-2.0 ---- | -%!PS-Adobe-2.0 ---- > -%!PS-Adobe-2.0 ---- -# empty -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ConstNodeRef r = t.rootref(); - ASSERT_TRUE(r.is_stream()); - ASSERT_EQ(r.num_children(), 10u); - ASSERT_TRUE(r.doc(0).is_doc()); - ASSERT_TRUE(r.doc(0).is_val()); - EXPECT_EQ(r.doc(0).val(), csubstr("%!PS-Adobe-2.0\n")); - ASSERT_TRUE(r.doc(1).is_doc()); - ASSERT_TRUE(r.doc(1).is_val()); - EXPECT_EQ(r.doc(1).val(), csubstr("%!PS-Adobe-2.0\n")); - ASSERT_TRUE(r.doc(2).is_doc()); - ASSERT_TRUE(r.doc(2).is_val()); - EXPECT_EQ(r.doc(2).val(), csubstr("%!PS-Adobe-2.0\n")); - ASSERT_TRUE(r.doc(3).is_doc()); - ASSERT_TRUE(r.doc(3).is_val()); - EXPECT_EQ(r.doc(3).val(), csubstr("%!PS-Adobe-2.0\n")); - ASSERT_TRUE(r.doc(4).is_doc()); - ASSERT_TRUE(r.doc(4).is_val()); - EXPECT_EQ(r.doc(4).val(), csubstr{}); - ASSERT_TRUE(r.doc(5).is_doc()); - ASSERT_TRUE(r.doc(5).is_val()); - EXPECT_EQ(r.doc(5).val(), csubstr("%!PS-Adobe-2.0\n")); - ASSERT_TRUE(r.doc(6).is_doc()); - ASSERT_TRUE(r.doc(6).is_val()); - EXPECT_EQ(r.doc(6).val(), csubstr("%!PS-Adobe-2.0\n")); - ASSERT_TRUE(r.doc(7).is_doc()); - ASSERT_TRUE(r.doc(7).is_val()); - EXPECT_EQ(r.doc(7).val(), csubstr("%!PS-Adobe-2.0\n")); - ASSERT_TRUE(r.doc(8).is_doc()); - ASSERT_TRUE(r.doc(8).is_val()); - EXPECT_EQ(r.doc(8).val(), csubstr("%!PS-Adobe-2.0\n")); - ASSERT_TRUE(r.doc(4).is_doc()); - ASSERT_TRUE(r.doc(4).is_val()); - EXPECT_EQ(r.doc(4).val(), csubstr{}); - }); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -CASE_GROUP(BLOCK_FOLDED) -{ -// -ADD_CASE_TO_GROUP("indentation requirements", -R"(--- -> -hello -there ---- -> - hello - there ---- -> - hello - there ---- -> -ciao -qua ---- -> - ciao - qua ---- -> - ciao - qua ---- -- > - hello - there -- > - ciao - qua ---- -foo: > - hello - there -bar: > - ciao - qua -)", -N(STREAM, L{ - N(DOCVAL|QV, "hello there\n"), - N(DOCVAL|QV, "hello there\n"), - N(DOCVAL|QV, "hello there\n"), - N(DOCVAL|QV, "ciao qua\n"), - N(DOCVAL|QV, "ciao qua\n"), - N(DOCVAL|QV, "ciao qua\n"), - N(SEQ|DOC, L{N(QV, "hello there\n"), N(QV, "ciao qua\n")}), - N(MAP|DOC, L{N(QV, "foo", "hello there\n"), N(QV, "bar", "ciao qua\n")}), - })); - -ADD_CASE_TO_GROUP("indentation requirements err seq", EXPECT_PARSE_ERROR, -R"(- > -hello -there -- > -ciao -qua -)", -N(L{N(QV, "hello there"), N(QV, "ciao qua\n")})); - -ADD_CASE_TO_GROUP("indentation requirements err map", EXPECT_PARSE_ERROR, -R"(foo: > -hello -there -bar: > -ciao -qua -)", -N(L{N(QV, "foo", "hello there\n"), N(QV, "bar" "ciao qua\n")})); - -ADD_CASE_TO_GROUP("indentation requirements err level", EXPECT_PARSE_ERROR, -R"(--- >2 - hello - there -)", -N(NOTYPE)); - -ADD_CASE_TO_GROUP("foo without space after", -R"(> - foo -)", -N(DOCVAL|QV, "foo\n")); - -ADD_CASE_TO_GROUP("foo with space after", -R"(> - foo - -)", -N(DOCVAL|QV, "foo\n")); - -ADD_CASE_TO_GROUP("simple with indents", -R"(> - foo - - bar -)", -N(DOCVAL|QV, "foo\n \n bar\n")); - - -ADD_CASE_TO_GROUP("7T8X", -R"(- > - - folded - line - - next - line - * bullet - - * list - * lines - - last - line - -# Comment - -##### this is the original scalar: -- > - - folded - line - - next - line - * bullet - - * list - * lines - - last - line - -# Comment - -##### without any indentation -- > - - folded - line - - next - line - * bullet - - * list - * lines - - last - line - -# Comment -)", - L{ - N(QV, "\nfolded line\nnext line\n * bullet\n\n * list\n * lines\n\nlast line\n"), - N(QV, "\nfolded line\nnext line\n * bullet\n\n * list\n * lines\n\nlast line\n"), - N(QV, "\nfolded line\nnext line\n * bullet\n\n * list\n * lines\n\nlast line\n"), - } -); - - -ADD_CASE_TO_GROUP("block folded as seq val, implicit indentation 2", -R"( -- > - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -- another val -)", - L{ - N(QV, "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, implicit indentation 2", -R"( -example: > - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, implicit indentation 2, chomp=keep", -R"( -example: >+ - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n\n\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, implicit indentation 2, chomp=strip", -R"( -example: >- - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end."), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, explicit indentation 2", -R"( -example: >2 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, explicit indentation 2, chomp=keep", -R"( -example: >+2 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -example2: >2+ - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n\n\n"), - N(QV, "example2", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n\n\n"), - } -); - -ADD_CASE_TO_GROUP("block folded as map val, explicit indentation 2, chomp=strip", -R"( -example: >-2 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -example2: >2- - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end."), - N(QV, "example2", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end."), - } -); - -ADD_CASE_TO_GROUP("block folded as map val, implicit indentation 3", -R"( -example: > - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, explicit indentation 3", -R"( -example: >3 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, implicit indentation 4", -R"( -example: > - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, explicit indentation 4", -R"( -example: >4 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, implicit indentation 9", -R"( -example: > - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block folded as map val, explicit indentation 9", -R"( -example: >9 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another", "val") - } -); - - -ADD_CASE_TO_GROUP("block folded as map entry", -R"( -data: > - Wrapped text - will be folded - into a single - paragraph - - Blank lines denote - paragraph breaks -)", - N(L{N(KEYVAL|VALQUO, "data", "Wrapped text will be folded into a single paragraph\nBlank lines denote paragraph breaks\n")}) -); - -ADD_CASE_TO_GROUP("block folded, no chomp, no indentation", -R"(example: > - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - -another: text -)", - N(L{ - N(KEYVAL|VALQUO, "example", "Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\nplus another line at the end.\n"), - N("another", "text"), - }) -); - -ADD_CASE_TO_GROUP("block folded with tab and spaces", -R"(> - )", - N(DOCVAL|VALQUO, "\t \n") - ); - - -ADD_CASE_TO_GROUP("block folded with empty docval 1", -R"(>)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 2", -R"(> -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 3", -R"(> - )", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 4", -R"(> - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 5", -R"(> - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 8", -R"(> - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 9", -R"(> - - - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 10", -R"(> - - - - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 11", -R"(> - - - - )", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 12", -R"(> - - - - - - - - - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with empty docval 13", -R"(> - - - - - - - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 0", -R"(> - asd)", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 1", -R"(> - asd -)", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 2", -R"(> - asd - -)", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 3", -R"(> - asd - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 4", -R"(> - asd - - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 5", -R"(> - asd - - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 5.1", -R"(> - asd - - - - - - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 5.2", -R"(> - asd - - - - - - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 5.3", -R"(> - asd - - - - - - )", - N(DOCVAL|VALQUO, "asd\n\n\n \n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 6", -R"(> - asd - )", - N(DOCVAL|VALQUO, "asd\n \n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 7", -R"(> - asd - -)", - N(DOCVAL|VALQUO, "asd\n \n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 8", -R"(> - asd - )", - N(DOCVAL|VALQUO, "asd\n \n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 9", -R"(> - asd - -)", - N(DOCVAL|VALQUO, "asd\n \n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 10", -R"(> - asd - )", - N(DOCVAL|VALQUO, "asd\n\t \n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 11", -R"(> - asd - )", - N(DOCVAL|VALQUO, "asd\n \t \n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 12", -R"(> - asd - -)", - N(DOCVAL|VALQUO, "asd\n\t \n") - ); - -ADD_CASE_TO_GROUP("block folded with docval no newlines at end 13", -R"(> - asd - -)", - N(DOCVAL|VALQUO, "asd\n \t \n") - ); - - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 0", -R"(>+)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 1", -R"(>+ -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 1.1", -R"(>+ - )", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 1.2", -R"(>+ - asd)", - N(DOCVAL|VALQUO, "asd") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 1.3", -R"(>+ - asd -)", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 1.4", -R"(>+ - asd - -)", - N(DOCVAL|VALQUO, "asd\n\n") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 2", -R"(>+ - -)", - N(DOCVAL|VALQUO, "\n") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 2.1", -R"(>+ - - )", - N(DOCVAL|VALQUO, "\n") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 3", -R"(>+ - - -)", - N(DOCVAL|VALQUO, "\n\n") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 4", -R"(>+ - - - -)", - N(DOCVAL|VALQUO, "\n\n\n") - ); - -ADD_CASE_TO_GROUP("block folded, keep, empty docval trailing 5", -R"(>+ - - - - -)", - N(DOCVAL|VALQUO, "\n\n\n\n") - ); - -ADD_CASE_TO_GROUP("block folded, empty block vals in seq 0", -R"(- >+ - -- >+ - )", -N(L{N(QV, "\n"), N(QV, ""),})); - -ADD_CASE_TO_GROUP("block folded, empty block vals in seq 1", -R"(- >+ - -- >+ - -)", -N(L{N(QV, "\n"), N(QV, "\n"),})); - -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_block_literal.cpp b/thirdparty/ryml/test/test_block_literal.cpp deleted file mode 100644 index fe8e352be..000000000 --- a/thirdparty/ryml/test/test_block_literal.cpp +++ /dev/null @@ -1,1261 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -TEST(block_literal, empty_block) -{ - { - Tree t = parse_in_arena(R"(- | -)"); - EXPECT_EQ(t[0].val(), csubstr("")); - } - { - Tree t = parse_in_arena(R"(- |- -)"); - EXPECT_EQ(t[0].val(), csubstr("")); - } - { - Tree t = parse_in_arena(R"(- |+ -)"); - EXPECT_EQ(t[0].val(), csubstr("")); - } - { - Tree t = parse_in_arena(R"(# no indentation: fails! -- | - -- |- - -- |+ - -)"); - EXPECT_FALSE(t.empty()); - EXPECT_EQ(t[0].val(), csubstr("")); - EXPECT_EQ(t[1].val(), csubstr("")); - EXPECT_EQ(t[2].val(), csubstr("\n")); - } - { - Tree t = parse_in_arena(R"( -- | - -- |- - -- |+ - -)"); - EXPECT_FALSE(t.empty()); - EXPECT_EQ(t[0].val(), csubstr("")); - EXPECT_EQ(t[1].val(), csubstr("")); - EXPECT_EQ(t[2].val(), csubstr("\n")); - } - { - Tree t = parse_in_arena(R"( -- | -- |- -- |+ -)"); - EXPECT_FALSE(t.empty()); - EXPECT_EQ(t[0].val(), csubstr("")); - EXPECT_EQ(t[1].val(), csubstr("")); - EXPECT_EQ(t[2].val(), csubstr("")); - } -} - - -TEST(block_literal, empty_block_as_container_member) -{ - // this was ok - test_check_emit_check(R"( -map: - a: "" - b: '' - c: > - d: | - e: -)", [](Tree const &t){ - EXPECT_TRUE(t["map"].has_key()); - EXPECT_TRUE(t["map"].is_map()); - EXPECT_EQ(t["map"].num_children(), 5u); - for(const auto &child : t["map"].children()) - { - EXPECT_EQ(child.val(), ""); - if(child.key() != "e") - { - EXPECT_TRUE(child.type().is_val_quoted()); - EXPECT_FALSE(child.val_is_null()); - } - } - }); - // this was ok - test_check_emit_check(R"( -map: - a: "" - b: '' - c: > - d: | -)", [](Tree const &t){ - EXPECT_TRUE(t["map"].has_key()); - EXPECT_TRUE(t["map"].is_map()); - EXPECT_TRUE(t["map"].is_map()); - EXPECT_EQ(t["map"].num_children(), 4u); - for(const auto &child : t["map"].children()) - { - EXPECT_EQ(child.val(), ""); - EXPECT_TRUE(child.type().is_val_quoted()); - EXPECT_FALSE(child.val_is_null()); - } - }); - // this was not ok! the block literal before next is extended: to - // include the YAML for next! - test_check_emit_check(R"( -map: - a: "" - b: '' - c: > - d: | -next: - a: "" - b: '' - c: > - d: | -)", [](Tree const &t){ - for(const char *name : {"map", "next"}) - { - ASSERT_TRUE(t.rootref().has_child(to_csubstr(name))) << "name=" << name; - ConstNodeRef node = t[to_csubstr(name)]; - EXPECT_TRUE(node.has_key()); - EXPECT_TRUE(node.is_map()); - ASSERT_EQ(node.num_children(), 4u); - for(const auto &child : node.children()) - { - EXPECT_EQ(child.val(), ""); - EXPECT_TRUE(child.type().is_val_quoted()); - EXPECT_FALSE(child.val_is_null()); - } - } - }); - test_check_emit_check(R"( -seq: - - "" - - '' - - > - - | -next: - - "" - - '' - - > - - | -)", [](Tree const &t){ - for(const char *name : {"seq", "next"}) - { - ASSERT_TRUE(t.rootref().has_child(to_csubstr(name))) << "name=" << name; - ConstNodeRef node = t[to_csubstr(name)]; - EXPECT_TRUE(node.has_key()); - EXPECT_TRUE(node.is_seq()); - ASSERT_EQ(node.num_children(), 4u); - for(const auto &child : node.children()) - { - EXPECT_EQ(child.val(), ""); - EXPECT_TRUE(child.type().is_val_quoted()); - EXPECT_FALSE(child.val_is_null()); - } - } - }); -} - - -TEST(block_literal, emit_does_not_add_lines_to_multi_at_end_1) -{ - Tree t = parse_in_arena("[]"); - NodeRef r = t.rootref(); - r.append_child() = "\n\n"; - r.append_child() = "\n\n"; - r.append_child() = "last"; - std::string out = emitrs_yaml(t); - t.clear(); - t = parse_in_arena(to_csubstr(out)); - EXPECT_EQ(t[0].val(), csubstr("\n\n")); - EXPECT_EQ(t[1].val(), csubstr("\n\n")); - EXPECT_EQ(t[2].val(), csubstr("last")); - out = emitrs_yaml(t); - t.clear(); - t = parse_in_arena(to_csubstr(out)); - EXPECT_EQ(t[0].val(), csubstr("\n\n")); - EXPECT_EQ(t[1].val(), csubstr("\n\n")); - EXPECT_EQ(t[2].val(), csubstr("last")); - out = emitrs_yaml(t); - t.clear(); - t = parse_in_arena(to_csubstr(out)); - EXPECT_EQ(t[0].val(), csubstr("\n\n")); - EXPECT_EQ(t[1].val(), csubstr("\n\n")); - EXPECT_EQ(t[2].val(), csubstr("last")); - EXPECT_EQ(csubstr("ab\n\n \n").trimr(" \t\n"), csubstr("ab")); -} - -TEST(block_literal, emit_does_not_add_lines_to_multi_at_end_2) -{ - Tree t = parse_in_arena(R"(--- |+ - ab - - -)"); - EXPECT_EQ(t.docref(0).val(), csubstr("ab\n\n \n")); - std::string expected = R"(--- | - ab - - - -)"; - std::string out = emitrs_yaml(t); - EXPECT_EQ(out, expected); - t = parse_in_arena(to_csubstr(out)); - EXPECT_EQ(t.docref(0).val(), csubstr("ab\n\n \n")); - out = emitrs_yaml(t); - EXPECT_EQ(out, expected); - t = parse_in_arena(to_csubstr(out)); - EXPECT_EQ(t.docref(0).val(), csubstr("ab\n\n \n")); - out = emitrs_yaml(t); - EXPECT_EQ(out, expected); - t = parse_in_arena(to_csubstr(out)); - EXPECT_EQ(t.docref(0).val(), csubstr("ab\n\n \n")); -} - -TEST(block_literal, emit_does_not_add_lines_to_multi_at_end_3) -{ - std::string yaml = R"( -- | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - - -- |+ - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - -- last -)"; - std::string expected = R"(- | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - -- |+ - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - -- last -)"; - Tree t = parse_in_arena(to_csubstr(yaml)); - EXPECT_EQ(t[0].val(), "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"); - EXPECT_EQ(t[1].val(), "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n\n"); - std::string out = emitrs_yaml(t); - EXPECT_EQ(out, expected); - t = parse_in_arena(to_csubstr(out)); - EXPECT_EQ(t[0].val(), "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"); - EXPECT_EQ(t[1].val(), "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n\n"); - out = emitrs_yaml(t); - EXPECT_EQ(out, expected); -} - -TEST(block_literal, carriage_return) -{ - std::string yaml = "with: |\r\n" -" text\r\n" -" lines\r\n" -"without: |\n" -" text\n" -" lines\n"; - Tree t = parse_in_arena(to_csubstr(yaml)); - EXPECT_EQ(t["with"].val(), "text\n \tlines\n"); - EXPECT_EQ(t["without"].val(), "text\n \tlines\n"); - auto emitted = emitrs_yaml(t); - #ifdef RYML_DBG - __c4presc(emitted.data(), emitted.size()); - #endif - Tree r = parse_in_arena(to_csubstr(emitted)); - EXPECT_EQ(t["with"].val(), "text\n \tlines\n"); - EXPECT_EQ(t["without"].val(), "text\n \tlines\n"); -} - -#ifdef JAVAI -TEST(block_literal, errors_on_tab_indents) -{ - Tree tree; - ExpectError::do_check(&tree, [&]{ - parse_in_arena("foo: |4\n this is foo\n now with tab-\n \t \tmust not work\n", &tree); - }); -} -#endif - -TEST(block_literal, test_suite_L24T_00) -{ - // this is double quoted, but will be emitted as a block literal - csubstr yaml = R"(foo: "x\n \n" -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t["foo"].val(), csubstr("x\n \n")); - }); -} - -TEST(block_literal, error_on_bad_spec) -{ - Tree t; - ExpectError::do_check(&t, [&t]{ - t = parse_in_arena("- |012abc\n must have errors above\n"); - }); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -CASE_GROUP(BLOCK_LITERAL) -{ -// -ADD_CASE_TO_GROUP("indentation requirements", -R"(--- -| -hello -there ---- -| - hello - there ---- -| - hello - there ---- -| -ciao -qua ---- -| - ciao - qua ---- -| - ciao - qua ---- -- | - hello - there -- | - ciao - qua ---- -foo: | - hello - there -bar: | - ciao - qua -)", -N(STREAM, L{ - N(DOCVAL|QV, "hello\nthere\n"), - N(DOCVAL|QV, "hello\nthere\n"), - N(DOCVAL|QV, "hello\nthere\n"), - N(DOCVAL|QV, "ciao\nqua\n"), - N(DOCVAL|QV, "ciao\nqua\n"), - N(DOCVAL|QV, "ciao\nqua\n"), - N(SEQ|DOC, L{N(QV, "hello\nthere\n"), N(QV, "ciao\nqua\n")}), - N(MAP|DOC, L{N(QV, "foo", "hello\nthere\n"), N(QV, "bar", "ciao\nqua\n")}), - })); - -ADD_CASE_TO_GROUP("indentation requirements err seq", EXPECT_PARSE_ERROR, -R"(- | -hello -there -- | -ciao -qua -)", -N(L{N(QV, "hello\nthere\n"), N(QV, "ciao\nqua\n")})); - -ADD_CASE_TO_GROUP("indentation requirements err map", EXPECT_PARSE_ERROR, -R"(foo: | -hello -there -bar: | -ciao -qua -)", -N(L{N(QV, "foo", "hello\nthere\n"), N(QV, "bar" "ciao\nqua\n")})); - -ADD_CASE_TO_GROUP("indentation requirements err level", EXPECT_PARSE_ERROR, -R"(--- |2 - hello - there -)", -N(NOTYPE)); - -ADD_CASE_TO_GROUP("empty, specs only 2G84_02", -"--- |1-", -N(STREAM, L{N(DOCVAL|VALQUO, {})})); - -ADD_CASE_TO_GROUP("empty, specs only 2G84_03", -"--- |1+", -N(STREAM, L{N(DOCVAL|VALQUO, {})})); - -ADD_CASE_TO_GROUP("empty, specs only 2G84_xx", -"--- |+", -N(STREAM, L{N(DOCVAL|VALQUO, {})})); - -ADD_CASE_TO_GROUP("empty, specs only 2G84_02_1", -"|1-", -N(DOCVAL|VALQUO, {})); - -ADD_CASE_TO_GROUP("empty, specs only 2G84_03_1", -"|1+", -N(DOCVAL|VALQUO, {})); - -ADD_CASE_TO_GROUP("empty, specs only 2G84_xx_1", -"|+", -N(DOCVAL|VALQUO, {})); - -ADD_CASE_TO_GROUP("block literal as map entry", -R"( -data: | - There once was a short man from Ealing - Who got on a bus to Darjeeling - It said on the door - "Please don't spit on the floor" - So he carefully spat on the ceiling -)", - N(MAP, { - N(KEYVAL|VALQUO, "data", "There once was a short man from Ealing\nWho got on a bus to Darjeeling\n It said on the door\n \"Please don't spit on the floor\"\nSo he carefully spat on the ceiling\n") - }) -); - -ADD_CASE_TO_GROUP("block literal and two scalars", -R"( -example: > - HTML goes into YAML without modification -message: | -
      -

      "Three is always greater than two, - even for large values of two"

      -

      --Author Unknown

      -
      -date: 2007-06-01 -)", - N(MAP, L{ - N(KEYVAL|VALQUO, "example", "HTML goes into YAML without modification\n"), - N(KEYVAL|VALQUO, "message", R"(
      -

      "Three is always greater than two, - even for large values of two"

      -

      --Author Unknown

      -
      -)"), - N("date", "2007-06-01"), - }) -); - -ADD_CASE_TO_GROUP("block literal no chomp, no indentation", -R"(example: | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - -another: text -)", - N(MAP, L{ - N(KEYVAL|VALQUO, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another", "text"), - }) -); - -ADD_CASE_TO_GROUP("block literal as seq val, implicit indentation 2", -R"( -- | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -- another val -)", - L{ - N(QV, "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another val") - } -); - -ADD_CASE_TO_GROUP("block literal as seq val, implicit indentation 2, chomp=keep", -R"( -- |+ - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -- another val -)", - L{ - N(QV, "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n\n\n"), - N("another val") - } -); - -ADD_CASE_TO_GROUP("block literal as seq val, implicit indentation 2, chomp=strip", -R"( -- |- - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -- another val -)", - L{ - N(QV, "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end."), - N("another val") - } -); - -ADD_CASE_TO_GROUP("block literal as seq val at eof, implicit indentation 2", -R"( -- | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -)", - L{ - N(QV, "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - } -); - -ADD_CASE_TO_GROUP("block literal as seq val at eof, implicit indentation 4", -R"( -- | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -)", - L{ - N(QV, "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - } -); - -ADD_CASE_TO_GROUP("block literal as map val, implicit indentation 2", -R"( -example: | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal as map val, explicit indentation 2", -R"( -example: |2 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal as map val, explicit indentation 2, chomp=keep", -R"( -example: |+2 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n\n\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal as map val, explicit indentation 2, chomp=strip", -R"( -example: |-2 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end."), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal as map val, implicit indentation 3", -R"( -example: | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal as map val, explicit indentation 3", -R"( -example: |3 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal as map val, implicit indentation 4", -R"( -example: | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal as map val, explicit indentation 4", -R"( -example: |4 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal as map val at eof, implicit indentation 2", -R"( -example: | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - } -); - -ADD_CASE_TO_GROUP("block literal as map val at eof, implicit indentation 4", -R"( -example: | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - } -); - -ADD_CASE_TO_GROUP("block literal as map val, implicit indentation 9", -R"( -example: | - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal as map val, explicit indentation 9", -R"( -example: |9 - Several lines of text, - with some "quotes" of various 'types', - and also a blank line: - - plus another line at the end. - - -another: val -)", - L{ - N(QV, "example", "Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nplus another line at the end.\n"), - N("another", "val") - } -); - -ADD_CASE_TO_GROUP("block literal with empty unindented lines, without quotes", - R"(tpl: - src: | - #include <{{hdr.filename}}> - - {{src.gencode}} -)", - L{ - N("tpl", L{N(QV, "src", "#include <{{hdr.filename}}>\n\n{{src.gencode}}\n")}) - } -); - -ADD_CASE_TO_GROUP("block literal with empty unindented lines, with double quotes", - R"(tpl: - src: | - #include "{{hdr.filename}}" - - {{src.gencode}} -)", - L{ - N("tpl", L{N(QV, "src", "#include \"{{hdr.filename}}\"\n\n{{src.gencode}}\n")}) - } -); - -ADD_CASE_TO_GROUP("block literal with empty unindented lines, with single quotes", - R"(tpl: - src: | - #include '{{hdr.filename}}' - - {{src.gencode}} -)", - L{ - N("tpl", L{N(QV, "src", "#include '{{hdr.filename}}'\n\n{{src.gencode}}\n")}) - } -); - -ADD_CASE_TO_GROUP("block literal with same indentation level 0", -R"( -aaa: |2 - xxx -bbb: | - yyy -)", - L{N(QV, "aaa", "xxx\n"), N(QV, "bbb", "yyy\n")} - ); - -ADD_CASE_TO_GROUP("block literal with same indentation level 1", -R"( -- aaa: |2 - xxx - bbb: | - yyy -)", - L{N(L{N(QV, "aaa", "xxx\n"), N(QV, "bbb", "yyy\n")})} - ); - -ADD_CASE_TO_GROUP("block literal with tab and spaces", -R"(| - )", - N(DOCVAL|VALQUO, "\t \n") - ); - - -ADD_CASE_TO_GROUP("block literal with empty docval 1", -R"(|)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 2", -R"(| -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 3", -R"(| - )", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 4", -R"(| - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 5", -R"(| - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 8", -R"(| - - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 9", -R"(| - - - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 10", -R"(| - - - - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 11", -R"(| - - - - )", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 12", -R"(| - - - - - - - - - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 13", -R"(| - - - - - - - -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 14.0", -R"(- |+ -)", - N(SEQ, L{N(VALQUO, "")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 14.0.1", -R"(- |+ - )", - N(SEQ, L{N(VALQUO, "\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 14.0.2", -R"(- |+ - )", - N(SEQ, L{N(VALQUO, "\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 14.1", -R"(foo: |+ -)", - N(MAP, L{N(VALQUO, "foo", "")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 14.1.1", -R"(foo: |+ - )", - N(MAP, L{N(VALQUO, "foo", "\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 14.1.2", -R"(foo: |+ - )", - N(MAP, L{N(VALQUO, "foo", "\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 14.2", -R"(|+ -)", - N(DOCVAL|VALQUO, "") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 14.2.1", -R"(|+ - )", - N(DOCVAL|VALQUO, "\n") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 14.2.2", -R"(|+ - )", - N(DOCVAL|VALQUO, "\n") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 15.0", -R"(- |+ - -)", - N(SEQ, L{N(VALQUO, "\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 15.0.1", -R"(- |+ - - )", - N(SEQ, L{N(VALQUO, "\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 15.1", -R"(foo: |+ - -)", - N(MAP, L{N(VALQUO, "foo", "\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 15.1.1", -R"(foo: |+ - - )", - N(MAP, L{N(VALQUO, "foo", "\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 15.2", -R"(|+ - -)", - N(DOCVAL|VALQUO, "\n") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 15.2.1", -R"(|+ - - )", - N(DOCVAL|VALQUO, "\n") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 16", -R"(|+ - - -)", - N(DOCVAL|VALQUO, "\n\n") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 16.1", -R"(foo: |+ - - -)", - N(MAP, L{N(VALQUO, "foo", "\n\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 16.2", -R"(- |+ - - -)", - N(SEQ, L{N(VALQUO, "\n\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 17", -R"(|+ - - - -)", - N(DOCVAL|VALQUO, "\n\n\n") - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 17.1", -R"(foo: |+ - - - -)", - N(MAP, L{N(VALQUO, "foo", "\n\n\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with empty docval 17.2", -R"(- |+ - - - -)", - N(SEQ, L{N(VALQUO, "\n\n\n")}) - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 0", -R"(| - asd)", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 1", -R"(| - asd -)", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 1.1", -R"(| - asd - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 1.2", -R"(|+ - asd - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 2", -R"(| - asd - -)", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 3", -R"(| - asd - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 4", -R"(| - asd - - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 5", -R"(| - asd - - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 5.1", -R"(| - asd - - - - - - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 5.2", -R"(| - asd - - - - - - )", - N(DOCVAL|VALQUO, "asd\n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 5.3", -R"(| - asd - - - - - - )", - N(DOCVAL|VALQUO, "asd\n\n\n \n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 6", -R"(| - asd - )", - N(DOCVAL|VALQUO, "asd\n \n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 7", -R"(| - asd - -)", - N(DOCVAL|VALQUO, "asd\n \n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 8", -R"(| - asd - )", - N(DOCVAL|VALQUO, "asd\n \n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 9", -R"(| - asd - -)", - N(DOCVAL|VALQUO, "asd\n \n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 10", -R"(| - asd - )", - N(DOCVAL|VALQUO, "asd\n\t \n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 11", -R"(| - asd - )", - N(DOCVAL|VALQUO, "asd\n \t \n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 12", -R"(| - asd - -)", - N(DOCVAL|VALQUO, "asd\n\t \n") - ); - -ADD_CASE_TO_GROUP("block literal with docval no newlines at end 13", -R"(| - asd - -)", - N(DOCVAL|VALQUO, "asd\n \t \n") - ); - -ADD_CASE_TO_GROUP("block literal, empty block vals in seq 0", -R"(- |+ - -- |+ - )", -N(L{N(QV, "\n"), N(QV, "\n"),})); - -ADD_CASE_TO_GROUP("block literal, empty block vals in seq 1", -R"(- |+ - -- |+ - -)", -N(L{N(QV, "\n"), N(QV, "\n"),})); - -} - - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_callbacks.cpp b/thirdparty/ryml/test/test_callbacks.cpp deleted file mode 100644 index 6f4bbf0ed..000000000 --- a/thirdparty/ryml/test/test_callbacks.cpp +++ /dev/null @@ -1,356 +0,0 @@ -#include "./test_case.hpp" -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/common.hpp" -#endif -#include - - -namespace c4 { -namespace yml { - -static_assert(std::is_same::type, size_t>::value, "invalid type"); -static_assert(std::is_same::type, size_t>::value, "invalid type"); -static_assert(size_t(c4::yml::npos) == ((size_t)-1), "invalid value"); // some debuggers show the wrong value... -static_assert(size_t(c4::yml::NONE) == ((size_t)-1), "invalid value"); // some debuggers show the wrong value... - -std::string stored_msg; -Location stored_location; -void * stored_mem; -size_t stored_length; - -void test_error_impl(const char* msg, size_t length, Location loc, void * /*user_data*/) -{ - stored_msg = std::string(msg, length); - stored_location = loc; -} - -void* test_allocate_impl(size_t length, void * /*hint*/, void * /*user_data*/) -{ - void *mem = ::malloc(length); - stored_length = length; - stored_mem = mem; - if(mem == nullptr) - { - const char msg[] = "could not allocate memory"; - test_error_impl(msg, sizeof(msg)-1, {}, nullptr); - } - return mem; -} - -void test_free_impl(void *mem, size_t length, void * /*user_data*/) -{ - stored_mem = mem; - stored_length = length; - ::free(mem); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -TEST(Callbacks, ctor) -{ - { - Callbacks cb; - EXPECT_NE(cb.m_allocate, &test_allocate_impl); - EXPECT_NE(cb.m_free, &test_free_impl); - EXPECT_NE(cb.m_error, &test_error_impl); - #ifndef RYML_NO_DEFAULT_CALLBACKS - EXPECT_EQ(cb.m_user_data, nullptr); - EXPECT_NE(cb.m_allocate, nullptr); - EXPECT_NE(cb.m_free, nullptr); - EXPECT_NE(cb.m_error, nullptr); - #else - EXPECT_EQ(cb.m_user_data, nullptr); - EXPECT_EQ(cb.m_allocate, nullptr); - EXPECT_EQ(cb.m_free, nullptr); - EXPECT_EQ(cb.m_error, nullptr); - #endif - } - { - Callbacks cb((void*)0xff, nullptr, nullptr, nullptr); - EXPECT_NE(cb.m_allocate, &test_allocate_impl); - EXPECT_NE(cb.m_free, &test_free_impl); - EXPECT_NE(cb.m_error, &test_error_impl); - #ifndef RYML_NO_DEFAULT_CALLBACKS - EXPECT_EQ(cb.m_user_data, (void*)0xff); - EXPECT_NE(cb.m_allocate, nullptr); - EXPECT_NE(cb.m_free, nullptr); - EXPECT_NE(cb.m_error, nullptr); - #else - EXPECT_EQ(cb.m_user_data, (void*)0xff); - EXPECT_EQ(cb.m_allocate, nullptr); - EXPECT_EQ(cb.m_free, nullptr); - EXPECT_EQ(cb.m_error, nullptr); - #endif - } - { - Callbacks cb((void*)0xff, &test_allocate_impl, nullptr, nullptr); - #ifndef RYML_NO_DEFAULT_CALLBACKS - EXPECT_EQ(cb.m_user_data, (void*)0xff); - EXPECT_EQ(cb.m_allocate, &test_allocate_impl); - EXPECT_NE(cb.m_free, nullptr); - EXPECT_NE(cb.m_error, nullptr); - #else - EXPECT_EQ(cb.m_user_data, (void*)0xff); - EXPECT_EQ(cb.m_allocate, nullptr); - EXPECT_EQ(cb.m_free, nullptr); - EXPECT_EQ(cb.m_error, nullptr); - #endif - } - { - Callbacks cb((void*)0xff, nullptr, &test_free_impl, nullptr); - #ifndef RYML_NO_DEFAULT_CALLBACKS - EXPECT_EQ(cb.m_user_data, (void*)0xff); - EXPECT_NE(cb.m_allocate, nullptr); - EXPECT_EQ(cb.m_free, &test_free_impl); - EXPECT_NE(cb.m_error, nullptr); - #else - EXPECT_EQ(cb.m_user_data, (void*)0xff); - EXPECT_EQ(cb.m_allocate, nullptr); - EXPECT_EQ(cb.m_free, &test_free_impl); - EXPECT_EQ(cb.m_error, nullptr); - #endif - } - { - Callbacks cb((void*)0xff, nullptr, nullptr, &test_error_impl); - #ifndef RYML_NO_DEFAULT_CALLBACKS - EXPECT_EQ(cb.m_user_data, (void*)0xff); - EXPECT_NE(cb.m_allocate, nullptr); - EXPECT_NE(cb.m_free, nullptr); - EXPECT_EQ(cb.m_error, &test_error_impl); - #else - EXPECT_EQ(cb.m_user_data, (void*)0xff); - EXPECT_EQ(cb.m_allocate, nullptr); - EXPECT_EQ(cb.m_free, nullptr); - EXPECT_EQ(cb.m_error, test_error_impl); - #endif - } -} - -TEST(Callbacks, get) -{ - Callbacks cb = get_callbacks(); - EXPECT_NE(cb.m_allocate, &test_allocate_impl); - EXPECT_NE(cb.m_free, &test_free_impl); - EXPECT_NE(cb.m_error, &test_error_impl); - #ifndef RYML_NO_DEFAULT_CALLBACKS - EXPECT_EQ(cb.m_user_data, nullptr); - EXPECT_NE(cb.m_allocate, nullptr); - EXPECT_NE(cb.m_free, nullptr); - EXPECT_NE(cb.m_error, nullptr); - #else - EXPECT_EQ(cb.m_user_data, nullptr); - EXPECT_EQ(cb.m_allocate, nullptr); - EXPECT_EQ(cb.m_free, nullptr); - EXPECT_EQ(cb.m_error, nullptr); - #endif -} - -TEST(Callbacks, set) -{ - Callbacks before = get_callbacks(); - Callbacks cb((void*)0xff, &test_allocate_impl, &test_free_impl, &test_error_impl); - - set_callbacks(cb); - Callbacks after = get_callbacks(); - EXPECT_EQ(cb.m_user_data, after.m_user_data); - EXPECT_EQ(cb.m_allocate, after.m_allocate); - EXPECT_EQ(cb.m_free, after.m_free); - EXPECT_EQ(cb.m_error, after.m_error); - - set_callbacks(before); - after = get_callbacks(); - EXPECT_EQ(before.m_user_data, after.m_user_data); - EXPECT_EQ(before.m_allocate, after.m_allocate); - EXPECT_EQ(before.m_free, after.m_free); - EXPECT_EQ(before.m_error, after.m_error); - - set_callbacks(cb); - after = get_callbacks(); - EXPECT_EQ(cb.m_user_data, after.m_user_data); - EXPECT_EQ(cb.m_allocate, after.m_allocate); - EXPECT_EQ(cb.m_free, after.m_free); - EXPECT_EQ(cb.m_error, after.m_error); -} - -TEST(Callbacks, reset) -{ - Callbacks before = get_callbacks(); - Callbacks cb((void*)0xff, &test_allocate_impl, &test_free_impl, &test_error_impl); - - set_callbacks(cb); - Callbacks after = get_callbacks(); - EXPECT_EQ(cb.m_user_data, after.m_user_data); - EXPECT_EQ(cb.m_allocate, after.m_allocate); - EXPECT_EQ(cb.m_free, after.m_free); - EXPECT_EQ(cb.m_error, after.m_error); - - reset_callbacks(); - EXPECT_EQ(before.m_user_data, after.m_user_data); - EXPECT_EQ(before.m_allocate, after.m_allocate); - EXPECT_EQ(before.m_free, after.m_free); - EXPECT_EQ(before.m_error, after.m_error); -} - -TEST(Callbacks, eq) -{ - Callbacks before = get_callbacks(); - Callbacks bf2 = get_callbacks(); - Callbacks cb((void*)0xff, &test_allocate_impl, &test_free_impl, &test_error_impl); - - EXPECT_EQ(bf2, before); - EXPECT_TRUE(bf2 == before); - EXPECT_FALSE(!(bf2 == before)); - EXPECT_TRUE(!(cb == before)); -} - -TEST(Callbacks, ne) -{ - Callbacks before = get_callbacks(); - Callbacks bf2 = get_callbacks(); - Callbacks cb((void*)0xff, &test_allocate_impl, &test_free_impl, &test_error_impl); - - EXPECT_NE(cb, before); - EXPECT_TRUE(cb != before); - EXPECT_TRUE(!(bf2 != before)); - EXPECT_FALSE(!(cb != before)); -} - -TEST(Callbacks, cmp_user_data) -{ - Callbacks before = get_callbacks(); - Callbacks cp = before; - EXPECT_EQ(cp, before); - cp.m_user_data = (void*)(((char*)before.m_user_data) + 100u); - EXPECT_NE(cp, before); -} - -TEST(Callbacks, cmp_allocate) -{ - Callbacks before = get_callbacks(); - Callbacks cp = before; - EXPECT_NE(cp.m_allocate, nullptr); - EXPECT_EQ(cp, before); - cp.m_allocate = nullptr; - EXPECT_NE(cp, before); -} - -TEST(Callbacks, cmp_free) -{ - Callbacks before = get_callbacks(); - Callbacks cp = before; - EXPECT_NE(cp.m_free, nullptr); - EXPECT_EQ(cp, before); - cp.m_free = nullptr; - EXPECT_NE(cp, before); -} - -TEST(Callbacks, cmp_error) -{ - Callbacks before = get_callbacks(); - Callbacks cp = before; - EXPECT_NE(cp.m_error, nullptr); - EXPECT_EQ(cp, before); - cp.m_error = nullptr; - EXPECT_NE(cp, before); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(allocate_and_free, basic) -{ - EXPECT_NE(get_callbacks().m_allocate, &test_allocate_impl); - EXPECT_NE(get_callbacks().m_error, &test_error_impl); - Callbacks cb(nullptr, &test_allocate_impl, &test_free_impl, nullptr); - set_callbacks(cb); - void *mem = get_callbacks().m_allocate(32, /*hint*/0, get_callbacks().m_user_data); - EXPECT_EQ(stored_mem, mem); - EXPECT_EQ(stored_length, 32u); - stored_mem = nullptr; - stored_length = 0; - get_callbacks().m_free(mem, 32u, get_callbacks().m_user_data); - EXPECT_EQ(stored_mem, mem); - EXPECT_EQ(stored_length, 32u); -} - -TEST(error, basic) -{ - EXPECT_NE(get_callbacks().m_error, &test_error_impl); - Callbacks cb(nullptr, nullptr, nullptr, &test_error_impl); - set_callbacks(cb); - // message - EXPECT_EQ(get_callbacks().m_error, &test_error_impl); - c4::yml::error("some message 123"); // calls test_error_impl, which sets stored_msg and stored_location - EXPECT_EQ(stored_msg, "some message 123"); - EXPECT_EQ(stored_location.name, ""); - EXPECT_EQ(stored_location.offset, 0u); - EXPECT_EQ(stored_location.line, 0u); - EXPECT_EQ(stored_location.col, 0u); - // location - c4::yml::error("some message 456", Location("file.yml", 433u, 123u, 4u)); - EXPECT_EQ(stored_msg, "some message 456"); - EXPECT_EQ(stored_location.name, "file.yml"); - EXPECT_EQ(stored_location.offset, 433u); - EXPECT_EQ(stored_location.line, 123u); - EXPECT_EQ(stored_location.col, 4u); - reset_callbacks(); - EXPECT_NE(get_callbacks().m_error, &test_error_impl); -} - -TEST(RYML_CHECK, basic) -{ - EXPECT_NE(get_callbacks().m_error, &test_error_impl); - Callbacks cb(nullptr, nullptr, nullptr, &test_error_impl); - set_callbacks(cb); - size_t the_line = __LINE__; RYML_CHECK(false); // keep both statements in the same line - EXPECT_EQ(stored_msg, "check failed: false"); - EXPECT_EQ(stored_location.name, __FILE__); - EXPECT_EQ(stored_location.offset, 0u); - EXPECT_EQ(stored_location.line, the_line); - EXPECT_EQ(stored_location.col, 0u); - reset_callbacks(); - EXPECT_NE(get_callbacks().m_error, &test_error_impl); -} - - -TEST(RYML_ASSERT, basic) -{ - EXPECT_NE(get_callbacks().m_error, &test_error_impl); - Callbacks cb(nullptr, nullptr, nullptr, &test_error_impl); - set_callbacks(cb); - stored_msg = ""; - stored_location = {}; - size_t the_line = __LINE__; RYML_ASSERT(false); // keep both statements in the same line - #if RYML_USE_ASSERT - EXPECT_EQ(stored_msg, "check failed: false"); - EXPECT_EQ(stored_location.name, __FILE__); - EXPECT_EQ(stored_location.offset, 0u); - EXPECT_EQ(stored_location.line, the_line); - EXPECT_EQ(stored_location.col, 0u); - #else - C4_UNUSED(the_line); - EXPECT_EQ(stored_msg, ""); - EXPECT_EQ(stored_location.name, nullptr); - EXPECT_EQ(stored_location.offset, 0u); - EXPECT_EQ(stored_location.line, 0u); - EXPECT_EQ(stored_location.col, 0u); - #endif - reset_callbacks(); - EXPECT_NE(get_callbacks().m_error, &test_error_impl); -} - - -// FIXME this is here merely to avoid a linker error -Case const* get_case(csubstr) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_case.cpp b/thirdparty/ryml/test/test_case.cpp deleted file mode 100644 index a850d242b..000000000 --- a/thirdparty/ryml/test/test_case.cpp +++ /dev/null @@ -1,898 +0,0 @@ -#include "./test_case.hpp" -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/common.hpp" -#include "c4/format.hpp" -#include "c4/span.hpp" -#include "c4/yml/std/std.hpp" -#include "c4/yml/detail/print.hpp" -#include "c4/yml/detail/checks.hpp" -#endif - -#include - -#if defined(_MSC_VER) -# pragma warning(push) -#elif defined(__clang__) -# pragma clang diagnostic push -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wuseless-cast" -# if __GNUC__ >= 6 -# pragma GCC diagnostic ignored "-Wnull-dereference" -# endif -#endif - -namespace c4 { -namespace yml { - - -size_t _num_leaves(Tree const& t, size_t node) -{ - size_t count = 0; - for(size_t ch = t.first_child(node); ch != NONE; ch = t.next_sibling(ch)) - count += _num_leaves(t, ch); - return count; -} - - -void test_compare(Tree const& actual, Tree const& expected) -{ - ASSERT_EQ(actual.empty(), expected.empty()); - if(actual.empty() || expected.empty()) - return; - EXPECT_EQ(actual.size(), expected.size()); - EXPECT_EQ(_num_leaves(actual, actual.root_id()), _num_leaves(expected, expected.root_id())); - test_compare(actual, actual.root_id(), expected, expected.root_id(), 0); -} - - -void test_compare(Tree const& actual, size_t node_actual, - Tree const& expected, size_t node_expected, - size_t level) -{ - #define _MORE_INFO "actual=" << node_actual << " vs expected=" << node_expected - - ASSERT_NE(node_actual, (size_t)NONE); - ASSERT_NE(node_expected, (size_t)NONE); - ASSERT_LT(node_actual, actual.capacity()); - ASSERT_LT(node_expected, expected.capacity()); - - EXPECT_EQ((type_bits)(actual.type(node_actual)&_TYMASK), (type_bits)(expected.type(node_expected)&_TYMASK)) << _MORE_INFO; - - EXPECT_EQ(actual.has_key(node_actual), expected.has_key(node_expected)) << _MORE_INFO; - if(actual.has_key(node_actual) && expected.has_key(node_expected)) - { - EXPECT_EQ(actual.key(node_actual), expected.key(node_expected)) << _MORE_INFO; - } - - EXPECT_EQ(actual.has_val(node_actual), expected.has_val(node_expected)) << _MORE_INFO; - if(actual.has_val(node_actual) && expected.has_val(node_expected)) - { - EXPECT_EQ(actual.val(node_actual), expected.val(node_expected)) << _MORE_INFO; - } - - EXPECT_EQ(actual.has_key_tag(node_actual), expected.has_key_tag(node_expected)) << _MORE_INFO; - if(actual.has_key_tag(node_actual) && expected.has_key_tag(node_expected)) - { - EXPECT_EQ(actual.key_tag(node_actual), expected.key_tag(node_expected)) << _MORE_INFO; - } - - EXPECT_EQ(actual.has_val_tag(node_actual), expected.has_val_tag(node_expected)) << _MORE_INFO; - if(actual.has_val_tag(node_actual) && expected.has_val_tag(node_expected)) - { - auto filtered = [](csubstr tag) { - if(tag.begins_with("!')) - return tag.offs(3, 1); - return tag; - }; - csubstr actual_tag = filtered(actual.val_tag(node_actual)); - csubstr expected_tag = filtered(actual.val_tag(node_expected)); - EXPECT_EQ(actual_tag, expected_tag) << _MORE_INFO; - } - - EXPECT_EQ(actual.has_key_anchor(node_actual), expected.has_key_anchor(node_expected)) << _MORE_INFO; - if(actual.has_key_anchor(node_actual) && expected.has_key_anchor(node_expected)) - { - EXPECT_EQ(actual.key_anchor(node_actual), expected.key_anchor(node_expected)) << _MORE_INFO; - } - - EXPECT_EQ(actual.has_val_anchor(node_actual), expected.has_val_anchor(node_expected)) << _MORE_INFO; - if(actual.has_val_anchor(node_actual) && expected.has_val_anchor(node_expected)) - { - EXPECT_EQ(actual.val_anchor(node_actual), expected.val_anchor(node_expected)) << _MORE_INFO; - } - - EXPECT_EQ(actual.num_children(node_actual), expected.num_children(node_expected)) << _MORE_INFO; - for(size_t ia = actual.first_child(node_actual), ib = expected.first_child(node_expected); - ia != NONE && ib != NONE; - ia = actual.next_sibling(ia), ib = expected.next_sibling(ib)) - { - test_compare(actual, ia, expected, ib, level+1); - } - - #undef _MORE_INFO -} - -void test_arena_not_shared(Tree const& a, Tree const& b) -{ - for(NodeData const* n = a.m_buf, *e = a.m_buf + a.m_cap; n != e; ++n) - { - EXPECT_FALSE(b.in_arena(n->m_key.scalar)) << n - a.m_buf; - EXPECT_FALSE(b.in_arena(n->m_key.tag )) << n - a.m_buf; - EXPECT_FALSE(b.in_arena(n->m_key.anchor)) << n - a.m_buf; - EXPECT_FALSE(b.in_arena(n->m_val.scalar)) << n - a.m_buf; - EXPECT_FALSE(b.in_arena(n->m_val.tag )) << n - a.m_buf; - EXPECT_FALSE(b.in_arena(n->m_val.anchor)) << n - a.m_buf; - } - for(NodeData const* n = b.m_buf, *e = b.m_buf + b.m_cap; n != e; ++n) - { - EXPECT_FALSE(a.in_arena(n->m_key.scalar)) << n - b.m_buf; - EXPECT_FALSE(a.in_arena(n->m_key.tag )) << n - b.m_buf; - EXPECT_FALSE(a.in_arena(n->m_key.anchor)) << n - b.m_buf; - EXPECT_FALSE(a.in_arena(n->m_val.scalar)) << n - b.m_buf; - EXPECT_FALSE(a.in_arena(n->m_val.tag )) << n - b.m_buf; - EXPECT_FALSE(a.in_arena(n->m_val.anchor)) << n - b.m_buf; - } - for(TagDirective const& td : a.m_tag_directives) - { - EXPECT_FALSE(b.in_arena(td.handle)); - EXPECT_FALSE(b.in_arena(td.prefix)); - } - for(TagDirective const& td : b.m_tag_directives) - { - EXPECT_FALSE(a.in_arena(td.handle)); - EXPECT_FALSE(a.in_arena(td.prefix)); - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// ensure coverage of the default callback report -#ifndef RYML_NO_DEFAULT_CALLBACKS -extern void report_error_impl(const char* msg, size_t len, Location loc, FILE *file); -#endif - -std::string format_error(const char* msg, size_t len, Location loc) -{ - // ensure coverage of the default callback report - #ifndef RYML_NO_DEFAULT_CALLBACKS - report_error_impl(msg, len, loc, nullptr); - #endif - if(!loc) return msg; - std::string out; - if(!loc.name.empty()) c4::formatrs(append, &out, "{}:", loc.name); - c4::formatrs(append, &out, "{}:{}:", loc.line, loc.col); - if(loc.offset) c4::formatrs(append, &out, " (@{}B):", loc.offset); - c4::formatrs(append, &out, "{}:", csubstr(msg, len)); - return out; -} - -struct ExpectedError : public std::runtime_error -{ - Location error_location; - ExpectedError(const char* msg, size_t len, Location loc) - : std::runtime_error(format_error(msg, len, loc)) - , error_location(loc) - { - } -}; - - -//----------------------------------------------------------------------------- - -ExpectError::ExpectError(Tree *tree, Location loc) - : m_got_an_error(false) - , m_tree(tree) - , m_glob_prev(get_callbacks()) - , m_tree_prev(tree ? tree->callbacks() : Callbacks{}) - , expected_location(loc) -{ - auto err = [](const char* msg, size_t len, Location errloc, void *this_) { - ((ExpectError*)this_)->m_got_an_error = true; - throw ExpectedError(msg, len, errloc); - }; - #ifdef RYML_NO_DEFAULT_CALLBACKS - c4::yml::Callbacks tcb((void*)this, nullptr, nullptr, err); - c4::yml::Callbacks gcb((void*)this, nullptr, nullptr, err); - #else - c4::yml::Callbacks tcb((void*)this, tree ? m_tree_prev.m_allocate : nullptr, tree ? m_tree_prev.m_free : nullptr, err); - c4::yml::Callbacks gcb((void*)this, m_glob_prev.m_allocate, m_glob_prev.m_free, err); - #endif - if(tree) - tree->callbacks(tcb); - set_callbacks(gcb); -} - -ExpectError::~ExpectError() -{ - if(m_tree) - m_tree->callbacks(m_tree_prev); - set_callbacks(m_tree_prev); -} - -void ExpectError::do_check(Tree *tree, std::function fn, Location expected_location) -{ - auto context = ExpectError(tree, expected_location); - try - { - fn(); - } - catch(ExpectedError const& e) - { - #if defined(RYML_DBG) - std::cout << "---------------\n"; - std::cout << "got an expected error:\n" << e.what() << "\n"; - std::cout << "---------------\n"; - #endif - if(context.expected_location) - { - EXPECT_EQ(static_cast(context.expected_location), - static_cast(e.error_location)); - EXPECT_EQ(e.error_location.line, context.expected_location.line); - EXPECT_EQ(e.error_location.col, context.expected_location.col); - if(context.expected_location.offset) - { - EXPECT_EQ(e.error_location.offset, context.expected_location.offset); - } - } - }; - EXPECT_TRUE(context.m_got_an_error); -} - -void ExpectError::check_assertion(Tree *tree, std::function fn, Location expected_location) -{ - #if RYML_USE_ASSERT - ExpectError::do_check(tree, fn, expected_location); - #else - C4_UNUSED(tree); - C4_UNUSED(fn); - C4_UNUSED(expected_location); - #endif -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -using N = CaseNode; -using L = CaseNode::iseqmap; - -TEST(CaseNode, setting_up) -{ - L tl1 = {DOC, DOC}; - L tl2 = {(DOC), (DOC)}; - - ASSERT_EQ(tl1.size(), tl2.size()); - N const& d1 = *tl1.begin(); - N const& d2 = *(tl1.begin() + 1); - ASSERT_EQ(d1.reccount(), d2.reccount()); - ASSERT_EQ((type_bits)d1.type, (type_bits)DOC); - ASSERT_EQ((type_bits)d2.type, (type_bits)DOC); - - N n1(tl1); - N n2(tl2); - ASSERT_EQ(n1.reccount(), n2.reccount()); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -NodeType_e CaseNode::_guess() const -{ - NodeType t; - C4_ASSERT(!val.empty() != !children.empty() || (val.empty() && children.empty())); - if(children.empty()) - { - C4_ASSERT(parent); - if(key.empty()) - { - t = VAL; - } - else - { - t = KEYVAL; - } - } - else - { - NodeType_e has_key = key.empty() ? NOTYPE : KEY; - auto const& ch = children.front(); - if(ch.key.empty()) - { - t = (has_key|SEQ); - } - else - { - t = (has_key|MAP); - } - } - if( ! key_tag.empty()) - { - C4_ASSERT( ! key.empty()); - t.add(KEYTAG); - } - if( ! val_tag.empty()) - { - C4_ASSERT( ! val.empty() || ! children.empty()); - t.add(VALTAG); - } - if( ! key_anchor.str.empty()) - { - t.add(key_anchor.type); - } - if( ! val_anchor.str.empty()) - { - t.add(val_anchor.type); - } - return t; -} - - -//----------------------------------------------------------------------------- -void CaseNode::compare_child(yml::ConstNodeRef const& n, size_t pos) const -{ - EXPECT_TRUE(pos < n.num_children()); - EXPECT_TRUE(pos < children.size()); - - if(pos >= n.num_children() || pos >= children.size()) return; - - ASSERT_GT(n.num_children(), pos); - auto const& expectedch = children[pos]; - - if(type & MAP) - { - auto actualch = n.find_child(expectedch.key); - if(actualch != nullptr) - { - // there may be duplicate keys. - if(actualch.id() != n[pos].id()) - actualch = n[pos]; - //EXPECT_EQ(fch, n[ch.key]); - EXPECT_EQ(actualch.get(), n[pos].get()); - //EXPECT_EQ(n[pos], n[ch.key]); - EXPECT_EQ(n[expectedch.key].key(), expectedch.key); - } - else - { - printf("error: node should have child %.*s: ", (int)expectedch.key.len, expectedch.key.str); - print_path(n); - printf("\n"); - print_node(n); - GTEST_FAIL(); - } - } - - if(type & SEQ) - { - EXPECT_FALSE(n[pos].has_key()); - EXPECT_EQ(n[pos].get()->m_key.scalar, children[pos].key); - auto actualch = n.child(pos); - EXPECT_EQ(actualch.get(), n[pos].get()); - } - - if(expectedch.type & KEY) - { - auto actualfch = n[pos]; - EXPECT_TRUE(actualfch.has_key()) << "id=" << actualfch.id(); - if(actualfch.has_key()) - { - EXPECT_EQ(actualfch.key(), expectedch.key) << "id=" << actualfch.id(); - } - - if( ! expectedch.key_tag.empty()) - { - EXPECT_TRUE(actualfch.has_key_tag()) << "id=" << actualfch.id(); - if(actualfch.has_key_tag()) - { - EXPECT_EQ(actualfch.key_tag(), expectedch.key_tag) << "id=" << actualfch.id(); - } - } - } - - if(expectedch.type & VAL) - { - auto actualch = n[pos]; - EXPECT_TRUE(actualch.has_val()) << "id=" << actualch.id(); - if(actualch.has_val()) - { - EXPECT_EQ(actualch.val(), expectedch.val) << "id=" << actualch.id(); - } - - if( ! expectedch.val_tag.empty()) - { - EXPECT_TRUE(actualch.has_val_tag()) << "id=" << actualch.id(); - if(actualch.has_val_tag()) - { - EXPECT_EQ(actualch.val_tag(), expectedch.val_tag) << "id=" << actualch.id(); - } - } - } -} - -void CaseNode::compare(yml::ConstNodeRef const& actual, bool ignore_quote) const -{ - if(ignore_quote) - { - const auto actual_type = actual.get()->m_type & ~(VALQUO | KEYQUO); - const auto expected_type = type & ~(VALQUO | KEYQUO); - EXPECT_EQ(expected_type, actual_type) << "id=" << actual.id(); - } - else - { - EXPECT_EQ((int)actual.get()->m_type, (int)type) << "id=" << actual.id(); // the type() method masks the type, and thus tag flags are omitted on its return value - } - - EXPECT_EQ(actual.num_children(), children.size()) << "id=" << actual.id(); - - if(actual.has_key()) - { - EXPECT_EQ(actual.key(), key) << "id=" << actual.id(); - } - - if(actual.has_val()) - { - EXPECT_EQ(actual.val(), val) << "id=" << actual.id(); - } - - // check that the children are in the same order - { - EXPECT_EQ(children.size(), actual.num_children()) << "id=" << actual.id(); - - size_t ic = 0; - for(auto const &expectedch : children) - { - SCOPED_TRACE("comparing: iteration based on the ref children"); - (void)expectedch; // unused - compare_child(actual, ic++); - } - - ic = 0; - for(auto const actualch : actual.children()) - { - SCOPED_TRACE("comparing: iteration based on the yml::Node children"); - (void)actualch; // unused - compare_child(actual, ic++); - } - - if(actual.first_child() != nullptr) - { - ic = 0; - for(auto const ch : actual.first_child().siblings()) - { - SCOPED_TRACE("comparing: iteration based on the yml::Node siblings"); - (void)ch; // unused - compare_child(actual, ic++); - } - } - } - - for(size_t i = 0, ei = actual.num_children(), j = 0, ej = children.size(); i < ei && j < ej; ++i, ++j) - { - children[j].compare(actual[i], ignore_quote); - } -} - -void CaseNode::recreate(yml::NodeRef *n) const -{ - C4_ASSERT( ! n->has_children()); - auto *nd = n->get(); - nd->m_type = type|key_anchor.type|val_anchor.type; - nd->m_key.scalar = key; - nd->m_key.tag = (key_tag); - nd->m_key.anchor = key_anchor.str; - nd->m_val.scalar = val; - nd->m_val.tag = (val_tag); - nd->m_val.anchor = val_anchor.str; - auto &tree = *n->tree(); - size_t nid = n->id(); // don't use node from now on - for(auto const& ch : children) - { - size_t id = tree.append_child(nid); - NodeRef chn(n->tree(), id); - ch.recreate(&chn); - } -} - - -//----------------------------------------------------------------------------- - -void print_path(ConstNodeRef const& n) -{ - size_t len = 0; - char buf[1024]; - ConstNodeRef p = n; - while(p != nullptr) - { - if(p.has_key()) - { - len += 1 + p.key().len; - } - else - { - int ret = snprintf(buf, sizeof(buf), "/%zd", p.has_parent() ? p.parent().child_pos(p) : 0); - RYML_ASSERT(ret >= 0); - len += static_cast(ret); - } - p = p.parent(); - }; - C4_ASSERT(len < sizeof(buf)); - size_t pos = len; - p = n; - while(p.valid() && p != nullptr) - { - if(p.has_key()) - { - size_t tl = p.key().len; - int ret = snprintf(buf + pos - tl, tl, "%.*s", (int)tl, p.key().str); - RYML_ASSERT(ret >= 0); - pos -= static_cast(ret); - } - else if(p.has_parent()) - { - pos = p.parent().child_pos(p); - int ret = snprintf(buf, 0, "/%zd", pos); - RYML_ASSERT(ret >= 0); - size_t tl = static_cast(ret); - RYML_ASSERT(pos >= tl); - ret = snprintf(buf + static_cast(pos - tl), tl, "/%zd", pos); - RYML_ASSERT(ret >= 0); - pos -= static_cast(ret); - } - p = p.parent(); - }; - printf("%.*s", (int)len, buf); -} - - - -void print_node(CaseNode const& p, int level) -{ - printf("%*s%p", (2*level), "", (void*)&p); - if( ! p.parent) - { - printf(" [ROOT]"); - } - printf(" %s:", NodeType::type_str(p.type)); - if(p.has_key()) - { - if(p.has_key_anchor()) - { - csubstr ka = p.key_anchor.str; - printf(" &%.*s", (int)ka.len, ka.str); - } - if(p.key_tag.empty()) - { - csubstr v = p.key; - printf(" '%.*s'", (int)v.len, v.str); - } - else - { - csubstr vt = p.key_tag; - csubstr v = p.key; - printf(" '%.*s %.*s'", (int)vt.len, vt.str, (int)v.len, v.str); - } - } - if(p.has_val()) - { - if(p.val_tag.empty()) - { - csubstr v = p.val; - printf(" '%.*s'", (int)v.len, v.str); - } - else - { - csubstr vt = p.val_tag; - csubstr v = p.val; - printf(" '%.*s %.*s'", (int)vt.len, vt.str, (int)v.len, v.str); - } - } - else - { - if( ! p.val_tag.empty()) - { - csubstr vt = p.val_tag; - printf(" %.*s", (int)vt.len, vt.str); - } - } - if(p.has_val_anchor()) - { - auto &a = p.val_anchor.str; - printf(" valanchor='&%.*s'", (int)a.len, a.str); - } - printf(" (%zd sibs)", p.parent ? p.parent->children.size() : 0); - if(p.is_container()) - { - printf(" %zd children:", p.children.size()); - } - printf("\n"); -} - - -void print_tree(ConstNodeRef const& p, int level) -{ - print_node(p, level); - for(ConstNodeRef ch : p.children()) - { - print_tree(ch, level+1); - } -} - -void print_tree(CaseNode const& p, int level) -{ - print_node(p, level); - for(auto const& ch : p.children) - print_tree(ch, level+1); -} - -void print_tree(CaseNode const& t) -{ - printf("--------------------------------------\n"); - print_tree(t, 0); - printf("#nodes: %zd\n", t.reccount()); - printf("--------------------------------------\n"); -} - -void test_invariants(ConstNodeRef const& n) -{ - #define _MORE_INFO << "id=" << n.id() - - if(n.is_root()) - { - EXPECT_FALSE(n.has_other_siblings()) _MORE_INFO; - } - // keys or vals cannot be root - if(n.has_key() || n.is_val() || n.is_keyval()) - { - EXPECT_TRUE(!n.is_root() || (n.is_doc() && !n.has_key())) _MORE_INFO; - } - // vals cannot be containers - if( ! n.empty() && ! n.is_doc()) - { - EXPECT_NE(n.has_val(), n.is_container()) _MORE_INFO; - } - if(n.has_children()) - { - EXPECT_TRUE(n.is_container()) _MORE_INFO; - EXPECT_FALSE(n.is_val()) _MORE_INFO; - } - // check parent & sibling reciprocity - for(ConstNodeRef s : n.siblings()) - { - EXPECT_TRUE(n.has_sibling(s)) _MORE_INFO; - EXPECT_TRUE(s.has_sibling(n)) _MORE_INFO; - EXPECT_EQ(s.parent().get(), n.parent().get()) _MORE_INFO; - } - if(n.parent() != nullptr) - { - EXPECT_EQ(n.parent().num_children() > 1, n.has_other_siblings()) _MORE_INFO; - EXPECT_TRUE(n.parent().has_child(n)) _MORE_INFO; - EXPECT_EQ(n.parent().num_children(), n.num_siblings()) _MORE_INFO; - // doc parent must be a seq and a stream - if(n.is_doc()) - { - EXPECT_TRUE(n.parent().is_seq()) _MORE_INFO; - EXPECT_TRUE(n.parent().is_stream()) _MORE_INFO; - } - } - else - { - EXPECT_TRUE(n.is_root()) _MORE_INFO; - } - if(n.is_seq()) - { - EXPECT_TRUE(n.is_container()) _MORE_INFO; - EXPECT_FALSE(n.is_map()) _MORE_INFO; - for(ConstNodeRef ch : n.children()) - { - EXPECT_FALSE(ch.is_keyval()) _MORE_INFO; - EXPECT_FALSE(ch.has_key()) _MORE_INFO; - } - } - if(n.is_map()) - { - EXPECT_TRUE(n.is_container()) _MORE_INFO; - EXPECT_FALSE(n.is_seq()) _MORE_INFO; - for(ConstNodeRef ch : n.children()) - { - EXPECT_TRUE(ch.has_key()) _MORE_INFO; - } - } - if(n.has_key_anchor()) - { - EXPECT_FALSE(n.key_anchor().empty()) _MORE_INFO; - EXPECT_FALSE(n.is_key_ref()) _MORE_INFO; - } - if(n.has_val_anchor()) - { - EXPECT_FALSE(n.val_anchor().empty()) _MORE_INFO; - EXPECT_FALSE(n.is_val_ref()) _MORE_INFO; - } - if(n.is_key_ref()) - { - EXPECT_FALSE(n.key_ref().empty()) _MORE_INFO; - EXPECT_FALSE(n.has_key_anchor()) _MORE_INFO; - } - if(n.is_val_ref()) - { - EXPECT_FALSE(n.val_ref().empty()) _MORE_INFO; - EXPECT_FALSE(n.has_val_anchor()) _MORE_INFO; - } - // ... add more tests here - - // now recurse into the children - for(ConstNodeRef ch : n.children()) - { - test_invariants(ch); - } - - #undef _MORE_INFO -} - -size_t test_tree_invariants(ConstNodeRef const& n) -{ - auto parent = n.parent(); - - if(n.get()->m_prev_sibling == NONE) - { - if(parent != nullptr) - { - EXPECT_EQ(parent.first_child().get(), n.get()); - EXPECT_EQ(parent.first_child().id(), n.id()); - } - } - - if(n.get()->m_next_sibling == NONE) - { - if(parent != nullptr) - { - EXPECT_EQ(parent.last_child().get(), n.get()); - EXPECT_EQ(parent.last_child().id(), n.id()); - } - } - - if(parent == nullptr) - { - EXPECT_TRUE(n.is_root()); - EXPECT_EQ(n.prev_sibling().get(), nullptr); - EXPECT_EQ(n.next_sibling().get(), nullptr); - } - - size_t count = 1, num = 0; - for(ConstNodeRef ch : n.children()) - { - EXPECT_NE(ch.id(), n.id()); - count += test_tree_invariants(ch); - ++num; - } - - EXPECT_EQ(num, n.num_children()); - - return count; -} - -void test_invariants(Tree const& t) -{ - - ASSERT_LE(t.size(), t.capacity()); - EXPECT_EQ(t.size() + t.slack(), t.capacity()); - - ASSERT_LE(t.arena_size(), t.arena_capacity()); - ASSERT_LE(t.arena_slack(), t.arena_capacity()); - EXPECT_EQ(t.arena_size() + t.arena_slack(), t.arena_capacity()); - - if(t.empty()) - return; - - size_t count = test_tree_invariants(t.rootref()); - EXPECT_EQ(count, t.size()); - - check_invariants(t); - test_invariants(t.rootref()); - - if(!testing::UnitTest::GetInstance()->current_test_info()->result()->Passed()) - { - print_tree(t); - } - - return; -#if 0 == 1 - for(size_t i = 0; i < t.m_size; ++i) - { - auto n = t.get(i); - if(n->m_prev_sibling == NONE) - { - EXPECT_TRUE(i == t.m_head || i == t.m_free_head); - } - if(n->m_next_sibling == NONE) - { - EXPECT_TRUE(i == t.m_tail || i == t.m_free_tail); - } - } - - std::vector touched(t.capacity()); - - for(size_t i = t.m_head; i != NONE; i = t.get(i)->m_next_sibling) - touched[i] = true; - - size_t size = 0; - for(bool v : touched) - size += v; - - EXPECT_EQ(size, t.size()); - - touched.clear(); - touched.resize(t.capacity()); - - for(size_t i = t.m_free_head; i != NONE; i = t.get(i)->m_next_sibling) - { - touched[i] = true; - } - - size_t slack = 0; - for(auto v : touched) - { - slack += v; - } - - EXPECT_EQ(slack, t.slack()); - EXPECT_EQ(size+slack, t.capacity()); - - // there are more checks to be done -#endif -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -CaseData* get_data(csubstr name) -{ - static std::map m; - - auto it = m.find(name); - CaseData *cd; - if(it == m.end()) - { - cd = &m[name]; - Case const* c = get_case(name); - RYML_CHECK(c->src.find("\n\r") == csubstr::npos); - { - std::string tmp; - replace_all("\r", "", c->src, &tmp); - cd->unix_style.src_buf.assign(tmp.begin(), tmp.end()); - cd->unix_style.src = to_substr(cd->unix_style.src_buf); - cd->unix_style_json.src_buf.assign(tmp.begin(), tmp.end()); - cd->unix_style_json.src = to_substr(cd->unix_style.src_buf); - } - { - std::string tmp; - replace_all("\n", "\r\n", cd->unix_style.src, &tmp); - cd->windows_style.src_buf.assign(tmp.begin(), tmp.end()); - cd->windows_style.src = to_substr(cd->windows_style.src_buf); - cd->windows_style_json.src_buf.assign(tmp.begin(), tmp.end()); - cd->windows_style_json.src = to_substr(cd->windows_style.src_buf); - } - } - else - { - cd = &it->second; - } - return cd; -} - -} // namespace yml -} // namespace c4 - -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/test/test_case.hpp b/thirdparty/ryml/test/test_case.hpp deleted file mode 100644 index 7ddc0a96d..000000000 --- a/thirdparty/ryml/test/test_case.hpp +++ /dev/null @@ -1,533 +0,0 @@ -#ifndef _TEST_CASE_HPP_ -#define _TEST_CASE_HPP_ - -#ifdef RYML_SINGLE_HEADER -#include -#else -#include "c4/std/vector.hpp" -#include "c4/std/string.hpp" -#include "c4/format.hpp" -#include -#include -#endif - -#include -#include - -#ifdef __GNUC__ -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wtype-limits" -#endif - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4296/*expression is always 'boolean_value'*/) -# pragma warning(disable: 4389/*'==': signed/unsigned mismatch*/) -# if C4_MSVC_VERSION != C4_MSVC_VERSION_2017 -# pragma warning(disable: 4800/*'int': forcing value to bool 'true' or 'false' (performance warning)*/) -# endif -#endif - -#ifdef RYML_DBG -# include -#endif - -namespace c4 { - -inline void PrintTo(substr s, ::std::ostream* os) { os->write(s.str, (std::streamsize)s.len); } -inline void PrintTo(csubstr s, ::std::ostream* os) { os->write(s.str, (std::streamsize)s.len); } - -namespace yml { - -inline void PrintTo(NodeType ty, ::std::ostream* os) -{ - *os << ty.type_str(); -} -inline void PrintTo(NodeType_e ty, ::std::ostream* os) -{ - *os << NodeType::type_str(ty); -} - -inline void PrintTo(Callbacks const& cb, ::std::ostream* os) -{ -#ifdef __GNUC__ -#define RYML_GNUC_EXTENSION __extension__ -#else -#define RYML_GNUC_EXTENSION -#endif - *os << '{' - << "userdata." << (void*)cb.m_user_data << ',' - << "allocate." << RYML_GNUC_EXTENSION (void*)cb.m_allocate << ',' - << "free." << RYML_GNUC_EXTENSION (void*)cb.m_free << ',' - << "error." << RYML_GNUC_EXTENSION (void*)cb.m_error << '}'; -#undef RYML_GNUC_EXTENSION -} - -struct Case; -struct CaseNode; -struct CaseData; - -Case const* get_case(csubstr name); -CaseData* get_data(csubstr name); - -void test_compare(Tree const& actual, Tree const& expected); -void test_compare(Tree const& actual, size_t node_actual, - Tree const& expected, size_t node_expected, - size_t level=0); - -void test_arena_not_shared(Tree const& a, Tree const& b); - -void test_invariants(Tree const& t); -void test_invariants(ConstNodeRef const& n); - -void print_node(CaseNode const& t, int level=0); -void print_tree(CaseNode const& p, int level=0); -void print_path(ConstNodeRef const& p); - - - -template -void test_check_emit_check(Tree const& t, CheckFn check_fn) -{ - #ifdef RYML_DBG - print_tree(t); - #endif - { - SCOPED_TRACE("original yaml"); - test_invariants(t); - check_fn(t); - } - auto emit_and_parse = [&check_fn](Tree const& tp, const char* identifier){ - SCOPED_TRACE(identifier); - std::string emitted = emitrs_yaml(tp); - #ifdef RYML_DBG - printf("~~~%s~~~\n%.*s", identifier, (int)emitted.size(), emitted.data()); - #endif - Tree cp = parse_in_arena(to_csubstr(emitted)); - #ifdef RYML_DBG - print_tree(cp); - #endif - test_invariants(cp); - check_fn(cp); - return cp; - }; - Tree cp = emit_and_parse(t, "emitted 1"); - cp = emit_and_parse(cp, "emitted 2"); - cp = emit_and_parse(cp, "emitted 3"); -} - -template -void test_check_emit_check(csubstr yaml, CheckFn check_fn) -{ - Tree t = parse_in_arena(yaml); - test_check_emit_check(t, check_fn); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -inline c4::substr replace_all(c4::csubstr pattern, c4::csubstr repl, c4::csubstr subject, std::string *dst) -{ - RYML_CHECK(!subject.overlaps(to_csubstr(*dst))); - size_t ret = subject.replace_all(to_substr(*dst), pattern, repl); - if(ret != dst->size()) - { - dst->resize(ret); - ret = subject.replace_all(to_substr(*dst), pattern, repl); - } - RYML_CHECK(ret == dst->size()); - return c4::to_substr(*dst); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -struct ExpectError -{ - bool m_got_an_error; - Tree *m_tree; - c4::yml::Callbacks m_glob_prev; - c4::yml::Callbacks m_tree_prev; - Location expected_location; - - ExpectError(Location loc={}) : ExpectError(nullptr, loc) {} - ExpectError(Tree *tree, Location loc={}); - ~ExpectError(); - - static void do_check( std::function fn, Location expected={}) { do_check(nullptr, fn, expected); } - static void do_check(Tree *tree, std::function fn, Location expected={}); - static void check_assertion( std::function fn, Location expected={}) { check_assertion(nullptr, fn, expected); } - static void check_assertion(Tree *tree, std::function fn, Location expected={}); -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -struct TaggedScalar -{ - csubstr tag; - csubstr scalar; - template - TaggedScalar(const char (&t)[N], const char (&s)[M]) : tag(t), scalar(s) {} - template - TaggedScalar(const char (&t)[N], std::nullptr_t) : tag(t), scalar() {} -}; - -struct AnchorRef -{ - NodeType_e type; - csubstr str; - AnchorRef() : type(NOTYPE), str() {} - AnchorRef(NodeType_e t) : type(t), str() {} - AnchorRef(NodeType_e t, csubstr v) : type(t), str(v) {} -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -/** a node class against which ryml structures are tested. Uses initializer - * lists to facilitate minimal specification. */ -struct CaseNode -{ -public: - - using seqmap = std::vector; - using iseqmap = std::initializer_list; - - struct TaggedList - { - csubstr tag; - iseqmap ilist; - template TaggedList(const char (&t)[N], iseqmap l) : tag(t), ilist(l) {} - }; - -public: - - NodeType type; - csubstr key, key_tag; AnchorRef key_anchor; - csubstr val, val_tag; AnchorRef val_anchor; - seqmap children; - CaseNode * parent; - -public: - - CaseNode(CaseNode && that) noexcept { _move(std::move(that)); } - CaseNode(CaseNode const& that) noexcept { _copy(that); } - - CaseNode& operator= (CaseNode && that) noexcept { _move(std::move(that)); return *this; } - CaseNode& operator= (CaseNode const& that) noexcept { _copy(that); return *this; } - - ~CaseNode() = default; - -public: - - // brace yourself: what you are about to see is ... crazy. - - CaseNode() : CaseNode(NOTYPE) {} - CaseNode(NodeType_e t) : type(t), key(), key_tag(), key_anchor(), val(), val_tag(), val_anchor(), children(), parent(nullptr) { _set_parent(); } - - // val - template explicit CaseNode(const char (&v)[N] ) : type((VAL )), key(), key_tag(), key_anchor(), val(v ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(TaggedScalar const& v) : type((VAL|VALTAG)), key(), key_tag(), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(std::nullptr_t ) : type((VAL )), key(), key_tag(), key_anchor(), val( ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - // val, with anchor/ref - template explicit CaseNode(const char (&v)[N] , AnchorRef const& arv) : type((arv.type|VAL )), key(), key_tag(), key_anchor(), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(TaggedScalar const& v, AnchorRef const& arv) : type((arv.type|VAL|VALTAG)), key(), key_tag(), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(std::nullptr_t , AnchorRef const& arv) : type((arv.type|VAL )), key(), key_tag(), key_anchor(), val( ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode( AnchorRef const& arv) : type((arv.type|VAL )), key(), key_tag(), key_anchor(), val(arv.str ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); RYML_ASSERT(arv.type == VALREF); } - - - // val, explicit type - template explicit CaseNode(NodeType t, const char (&v)[N] ) : type((VAL|t )), key(), key_tag(), key_anchor(), val(v ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& v) : type((VAL|VALTAG|t)), key(), key_tag(), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, std::nullptr_t ) : type((VAL |t)), key(), key_tag(), key_anchor(), val( ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - // val, explicit type, with val anchor/ref - template explicit CaseNode(NodeType t, const char (&v)[N] , AnchorRef const& arv) : type((arv.type|VAL|t )), key(), key_tag(), key_anchor(), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& v, AnchorRef const& arv) : type((arv.type|VAL|VALTAG|t)), key(), key_tag(), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, std::nullptr_t , AnchorRef const& arv) : type((arv.type|VAL |t)), key(), key_tag(), key_anchor(), val( ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - - - // keyval - template explicit CaseNode(const char (&k)[N] , const char (&v)[M] ) : type((KEYVAL )), key(k ), key_tag( ), key_anchor( ), val(v ), val_tag( ), val_anchor( ), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(std::nullptr_t , const char (&v)[M] ) : type((KEYVAL )), key( ), key_tag( ), key_anchor( ), val(v ), val_tag( ), val_anchor( ), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(const char (&k)[N] , std::nullptr_t ) : type((KEYVAL )), key(k ), key_tag( ), key_anchor( ), val( ), val_tag( ), val_anchor( ), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(const char (&k)[N] , TaggedScalar const& v) : type((KEYVAL|VALTAG )), key(k ), key_tag( ), key_anchor( ), val(v.scalar), val_tag(v.tag), val_anchor( ), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(TaggedScalar const& k, const char (&v)[M] ) : type((KEYVAL|KEYTAG )), key(k.scalar), key_tag(k.tag), key_anchor( ), val(v ), val_tag( ), val_anchor( ), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(TaggedScalar const& k, TaggedScalar const& v) : type((KEYVAL|KEYTAG|VALTAG )), key(k.scalar), key_tag(k.tag), key_anchor( ), val(v.scalar), val_tag(v.tag), val_anchor( ), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(std::nullptr_t , TaggedScalar const& v) : type((KEYVAL |VALTAG )), key( ), key_tag( ), key_anchor( ), val(v.scalar), val_tag(v.tag), val_anchor( ), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(TaggedScalar const& k, std::nullptr_t ) : type((KEYVAL|KEYTAG )), key(k.scalar), key_tag(k.tag), key_anchor( ), val( ), val_tag( ), val_anchor( ), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(std::nullptr_t , std::nullptr_t ) : type((KEYVAL )), key( ), key_tag( ), key_anchor( ), val( ), val_tag( ), val_anchor( ), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(AnchorRef const& ark, AnchorRef const& arv) : type((KEYVAL|ark.type|arv.type)), key(ark.str ), key_tag( ), key_anchor(ark), val(arv.str ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); RYML_ASSERT(ark.type == KEYREF); RYML_ASSERT(arv.type == VALREF); } - // keyval, with val anchor/ref - template explicit CaseNode(const char (&k)[N] , const char (&v)[M] , AnchorRef const& arv) : type((arv.type|KEYVAL )), key(k ), key_tag( ), key_anchor(), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(const char (&k)[N] , TaggedScalar const& v, AnchorRef const& arv) : type((arv.type|KEYVAL|VALTAG )), key(k ), key_tag( ), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(TaggedScalar const& k, const char (&v)[M] , AnchorRef const& arv) : type((arv.type|KEYVAL|KEYTAG )), key(k.scalar), key_tag(k.tag), key_anchor(), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(TaggedScalar const& k, TaggedScalar const& v, AnchorRef const& arv) : type((arv.type|KEYVAL|KEYTAG|VALTAG)), key(k.scalar), key_tag(k.tag), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - // keyval, with key anchor/ref - template explicit CaseNode(const char (&k)[N] , AnchorRef const& ark, const char (&v)[M] ) : type((ark.type|KEYVAL )), key(k ), key_tag( ), key_anchor(ark), val(v ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(const char (&k)[N] , AnchorRef const& ark, TaggedScalar const& v) : type((ark.type|KEYVAL|VALTAG )), key(k ), key_tag( ), key_anchor(ark), val(v.scalar), val_tag(v.tag), val_anchor(), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(TaggedScalar const& k, AnchorRef const& ark, const char (&v)[M] ) : type((ark.type|KEYVAL|KEYTAG )), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(v ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(TaggedScalar const& k, AnchorRef const& ark, TaggedScalar const& v) : type((ark.type|KEYVAL|KEYTAG|VALTAG)), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(v.scalar), val_tag(v.tag), val_anchor(), children(), parent(nullptr) { _set_parent(); } - // keyval, with key anchor/ref + val anchor/ref - template explicit CaseNode(const char (&k)[N] , AnchorRef const& ark, const char (&v)[M] , AnchorRef const& arv) : type((ark.type|arv.type|KEYVAL )), key(k ), key_tag( ), key_anchor(ark), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(const char (&k)[N] , AnchorRef const& ark, TaggedScalar const& v, AnchorRef const& arv) : type((ark.type|arv.type|KEYVAL|VALTAG )), key(k ), key_tag( ), key_anchor(ark), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(TaggedScalar const& k, AnchorRef const& ark, const char (&v)[M] , AnchorRef const& arv) : type((ark.type|arv.type|KEYVAL|KEYTAG )), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(TaggedScalar const& k, AnchorRef const& ark, TaggedScalar const& v, AnchorRef const& arv) : type((ark.type|arv.type|KEYVAL|KEYTAG|VALTAG)), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - - - // keyval, explicit type - template explicit CaseNode(NodeType t, const char (&k)[N] , const char (&v)[M] ) : type((KEYVAL|t )), key(k ), key_tag( ), key_anchor(), val(v ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, const char (&k)[N] , std::nullptr_t ) : type((KEYVAL|t )), key(k ), key_tag( ), key_anchor(), val( ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, std::nullptr_t , const char (&v)[M] ) : type((KEYVAL|t )), key( ), key_tag( ), key_anchor(), val(v ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, const char (&k)[N] , TaggedScalar const& v) : type((KEYVAL|VALTAG|t )), key(k ), key_tag( ), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, TaggedScalar const& k, const char (&v)[M] ) : type((KEYVAL|KEYTAG|t )), key(k.scalar), key_tag(k.tag), key_anchor(), val(v ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& k, TaggedScalar const& v) : type((KEYVAL|KEYTAG|VALTAG|t)), key(k.scalar), key_tag(k.tag), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& k, std::nullptr_t ) : type((KEYVAL|KEYTAG |t)), key(k.scalar), key_tag(k.tag), key_anchor(), val( ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, std::nullptr_t , TaggedScalar const& v) : type((KEYVAL |VALTAG|t)), key( ), key_tag( ), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, std::nullptr_t , std::nullptr_t ) : type((KEYVAL |t)), key( ), key_tag( ), key_anchor(), val( ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - // keyval, explicit type, with val anchor/ref - template explicit CaseNode(NodeType t, const char (&k)[N] , const char (&v)[M] , AnchorRef const& arv) : type((arv.type|KEYVAL|t )), key(k ), key_tag( ), key_anchor(), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, const char (&k)[N] , TaggedScalar const& v, AnchorRef const& arv) : type((arv.type|KEYVAL|VALTAG|t )), key(k ), key_tag( ), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, TaggedScalar const& k, const char (&v)[M] , AnchorRef const& arv) : type((arv.type|KEYVAL|KEYTAG|t )), key(k.scalar), key_tag(k.tag), key_anchor(), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& k, TaggedScalar const& v, AnchorRef const& arv) : type((arv.type|KEYVAL|KEYTAG|VALTAG|t)), key(k.scalar), key_tag(k.tag), key_anchor(), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - // keyval, explicit type, with key anchor/ref - template explicit CaseNode(NodeType t, const char (&k)[N] , AnchorRef const& ark, const char (&v)[M] ) : type((ark.type|KEYVAL|t )), key(k ), key_tag( ), key_anchor(ark), val(v ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, const char (&k)[N] , AnchorRef const& ark, TaggedScalar const& v) : type((ark.type|KEYVAL|VALTAG|t )), key(k ), key_tag( ), key_anchor(ark), val(v.scalar), val_tag(v.tag), val_anchor(), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, TaggedScalar const& k, AnchorRef const& ark, const char (&v)[M] ) : type((ark.type|KEYVAL|KEYTAG|t )), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(v ), val_tag( ), val_anchor(), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& k, AnchorRef const& ark, TaggedScalar const& v) : type((ark.type|KEYVAL|KEYTAG|VALTAG|t)), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(v.scalar), val_tag(v.tag), val_anchor(), children(), parent(nullptr) { _set_parent(); } - // keyval, explicit type, with key anchor/ref + val anchor/ref - template explicit CaseNode(NodeType t, const char (&k)[N] , AnchorRef const& ark, const char (&v)[M] , AnchorRef const& arv) : type((ark.type|arv.type|KEYVAL|t )), key(k ), key_tag( ), key_anchor(ark), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, const char (&k)[N] , AnchorRef const& ark, TaggedScalar const& v, AnchorRef const& arv) : type((ark.type|arv.type|KEYVAL|VALTAG|t )), key(k ), key_tag( ), key_anchor(ark), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, TaggedScalar const& k, AnchorRef const& ark, const char (&v)[M] , AnchorRef const& arv) : type((ark.type|arv.type|KEYVAL|KEYTAG|t )), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(v ), val_tag( ), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& k, AnchorRef const& ark, TaggedScalar const& v, AnchorRef const& arv) : type((ark.type|arv.type|KEYVAL|KEYTAG|VALTAG|t)), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(v.scalar), val_tag(v.tag), val_anchor(arv), children(), parent(nullptr) { _set_parent(); } - - - // container - template explicit CaseNode(const char (&k)[N] , iseqmap s) : type(), key(k ), key_tag( ), key_anchor(), val(), val_tag( ), val_anchor(), children(s ), parent(nullptr) { _set_parent(); type = _guess(); } - template explicit CaseNode(const char (&k)[N] , TaggedList s) : type(), key(k ), key_tag( ), key_anchor(), val(), val_tag(s.tag), val_anchor(), children(s.ilist), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode(TaggedScalar const& k, iseqmap s) : type(), key(k.scalar), key_tag(k.tag), key_anchor(), val(), val_tag( ), val_anchor(), children(s ), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode(TaggedScalar const& k, TaggedList s) : type(), key(k.scalar), key_tag(k.tag), key_anchor(), val(), val_tag(s.tag), val_anchor(), children(s.ilist), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode( iseqmap m) : CaseNode("", m) {} - explicit CaseNode( TaggedList m) : CaseNode("", m) {} - // container, with val anchor/ref - template explicit CaseNode(const char (&k)[N] , iseqmap s, AnchorRef const& arv) : type(), key(k ), key_tag( ), key_anchor(), val(), val_tag( ), val_anchor(arv), children(s ), parent(nullptr) { _set_parent(); type = _guess(); } - template explicit CaseNode(const char (&k)[N] , TaggedList s, AnchorRef const& arv) : type(), key(k ), key_tag( ), key_anchor(), val(), val_tag(s.tag), val_anchor(arv), children(s.ilist), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode(TaggedScalar const& k, iseqmap s, AnchorRef const& arv) : type(), key(k.scalar), key_tag(k.tag), key_anchor(), val(), val_tag( ), val_anchor(arv), children(s ), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode(TaggedScalar const& k, TaggedList s, AnchorRef const& arv) : type(), key(k.scalar), key_tag(k.tag), key_anchor(), val(), val_tag(s.tag), val_anchor(arv), children(s.ilist), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode( iseqmap m, AnchorRef const& arv) : CaseNode("", m, arv) {} - explicit CaseNode( TaggedList m, AnchorRef const& arv) : CaseNode("", m, arv) {} - // container, with key anchor/ref - template explicit CaseNode(const char (&k)[N] , AnchorRef const& ark, iseqmap s) : type(), key(k ), key_tag( ), key_anchor(ark), val(), val_tag( ), val_anchor(), children(s ), parent(nullptr) { _set_parent(); type = _guess(); } - template explicit CaseNode(const char (&k)[N] , AnchorRef const& ark, TaggedList s) : type(), key(k ), key_tag( ), key_anchor(ark), val(), val_tag(s.tag), val_anchor(), children(s.ilist), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode(TaggedScalar const& k, AnchorRef const& ark, iseqmap s) : type(), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(), val_tag( ), val_anchor(), children(s ), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode(TaggedScalar const& k, AnchorRef const& ark, TaggedList s) : type(), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(), val_tag(s.tag), val_anchor(), children(s.ilist), parent(nullptr) { _set_parent(); type = _guess(); } - // container, with key anchor/ref + val anchor/ref - template explicit CaseNode(const char (&k)[N] , AnchorRef const& ark, iseqmap s, AnchorRef const& arv) : type(), key(k ), key_tag( ), key_anchor(ark), val(), val_tag( ), val_anchor(arv), children(s ), parent(nullptr) { _set_parent(); type = _guess(); } - template explicit CaseNode(const char (&k)[N] , AnchorRef const& ark, TaggedList s, AnchorRef const& arv) : type(), key(k ), key_tag( ), key_anchor(ark), val(), val_tag(s.tag), val_anchor(arv), children(s.ilist), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode(TaggedScalar const& k, AnchorRef const& ark, iseqmap s, AnchorRef const& arv) : type(), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(), val_tag( ), val_anchor(arv), children(s ), parent(nullptr) { _set_parent(); type = _guess(); } - explicit CaseNode(TaggedScalar const& k, AnchorRef const& ark, TaggedList s, AnchorRef const& arv) : type(), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(), val_tag(s.tag), val_anchor(arv), children(s.ilist), parent(nullptr) { _set_parent(); type = _guess(); } - - - // container, explicit type - template explicit CaseNode(NodeType t, const char (&k)[N] , iseqmap s) : type((t )), key(k ), key_tag( ), key_anchor(), val(), val_tag( ), val_anchor(), children(s ), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, const char (&k)[N] , TaggedList s) : type((t|VALTAG)), key(k ), key_tag( ), key_anchor(), val(), val_tag(s.tag), val_anchor(), children(s.ilist), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& k, iseqmap s) : type((t|KEYTAG)), key(k.scalar), key_tag(k.tag), key_anchor(), val(), val_tag( ), val_anchor(), children(s ), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, iseqmap s) : type((t )), key( ), key_tag( ), key_anchor(), val(), val_tag( ), val_anchor(), children(s ), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedList s) : type((t|VALTAG)), key( ), key_tag( ), key_anchor(), val(), val_tag(s.tag), val_anchor(), children(s.ilist), parent(nullptr) { _set_parent(); } - // container, explicit type, with val anchor/ref - template explicit CaseNode(NodeType t, const char (&k)[N] , iseqmap s, AnchorRef const& arv) : type((t |VALANCH)), key(k ), key_tag( ), key_anchor(), val(), val_tag( ), val_anchor(arv), children(s ), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, const char (&k)[N] , TaggedList s, AnchorRef const& arv) : type((t|VALTAG|VALANCH)), key(k ), key_tag( ), key_anchor(), val(), val_tag(s.tag), val_anchor(arv), children(s.ilist), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& k, iseqmap s, AnchorRef const& arv) : type((t|KEYTAG|VALANCH)), key(k.scalar), key_tag(k.tag), key_anchor(), val(), val_tag( ), val_anchor(arv), children(s ), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, iseqmap s, AnchorRef const& arv) : type((t |VALANCH)), key( ), key_tag( ), key_anchor(), val(), val_tag( ), val_anchor(arv), children(s ), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedList s, AnchorRef const& arv) : type((t|VALTAG|VALANCH)), key( ), key_tag( ), key_anchor(), val(), val_tag(s.tag), val_anchor(arv), children(s.ilist), parent(nullptr) { _set_parent(); } - // container, explicit type, with key anchor/ref - template explicit CaseNode(NodeType t, const char (&k)[N] , AnchorRef const& ark, iseqmap s) : type((t |KEYANCH)), key(k ), key_tag( ), key_anchor(ark), val(), val_tag( ), val_anchor(), children(s ), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, const char (&k)[N] , AnchorRef const& ark, TaggedList s) : type((t|VALTAG|KEYANCH)), key(k ), key_tag( ), key_anchor(ark), val(), val_tag(s.tag), val_anchor(), children(s.ilist), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& k, AnchorRef const& ark, iseqmap s) : type((t|KEYTAG|KEYANCH)), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(), val_tag( ), val_anchor(), children(s ), parent(nullptr) { _set_parent(); } - // container, explicit type, with key anchor/ref + val anchor/ref - template explicit CaseNode(NodeType t, const char (&k)[N] , AnchorRef const& ark, iseqmap s, AnchorRef const& arv) : type((t |KEYANCH|VALANCH)), key(k ), key_tag( ), key_anchor(ark), val(), val_tag( ), val_anchor(arv), children(s ), parent(nullptr) { _set_parent(); } - template explicit CaseNode(NodeType t, const char (&k)[N] , AnchorRef const& ark, TaggedList s, AnchorRef const& arv) : type((t|VALTAG|KEYANCH|VALANCH)), key(k ), key_tag( ), key_anchor(ark), val(), val_tag(s.tag), val_anchor(arv), children(s.ilist), parent(nullptr) { _set_parent(); } - explicit CaseNode(NodeType t, TaggedScalar const& k, AnchorRef const& ark, iseqmap s, AnchorRef const& arv) : type((t|KEYTAG|KEYANCH|VALANCH)), key(k.scalar), key_tag(k.tag), key_anchor(ark), val(), val_tag( ), val_anchor(arv), children(s ), parent(nullptr) { _set_parent(); } - - -public: - - void _move(CaseNode&& that) - { - type = that.type; - key = that.key; - key_tag = that.key_tag; - key_anchor = that.key_anchor; - val = that.val; - val_tag = that.val_tag; - val_anchor = that.val_anchor; - children = std::move(that.children); - parent = nullptr; - _set_parent(); - } - void _copy(CaseNode const& that) - { - type = that.type; - key = that.key; - key_tag = that.key_tag; - key_anchor = that.key_anchor; - val = that.val; - val_tag = that.val_tag; - val_anchor = that.val_anchor; - children = that.children; - parent = nullptr; - _set_parent(); - } - - void _set_parent() - { - for(auto &ch : children) - { - ch.parent = this; - } - } - - NodeType_e _guess() const; - - bool is_root() const { return parent; } - bool is_doc() const { return type & DOC; } - bool is_map() const { return type & MAP; } - bool is_seq() const { return type & SEQ; } - bool has_val() const { return type & VAL; } - bool has_key() const { return type & KEY; } - bool is_container() const { return type & (SEQ|MAP); } - bool has_key_anchor() const { return type & KEYANCH; } - bool has_val_anchor() const { return type & VALANCH; } - -public: - - CaseNode const& operator[] (size_t i) const - { - C4_ASSERT(i >= 0 && i < children.size()); - return children[i]; - } - - CaseNode const& operator[] (csubstr const& name) const - { - auto ch = lookup(name); - C4_ASSERT(ch != nullptr); - return *ch; - } - - CaseNode const* lookup(csubstr const& name) const - { - C4_ASSERT( ! children.empty()); - for(auto const& ch : children) - if(ch.key == name) - return &ch; - return nullptr; - } - -public: - - void compare(yml::ConstNodeRef const& n, bool ignore_quote=false) const; - void compare_child(yml::ConstNodeRef const& n, size_t pos) const; - - size_t reccount() const - { - size_t c = 1; - for(auto const& ch : children) - c += ch.reccount(); - return c; - } - - void recreate(yml::NodeRef *n) const; - -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -typedef enum { - EXPECT_PARSE_ERROR = (1<<0), - RESOLVE_REFS = (1<<1), - JSON_ALSO = (1<<2), // TODO: make it the opposite: opt-out instead of opt-in -} TestCaseFlags_e; - - -struct Case -{ - std::string filelinebuf; - csubstr fileline; - csubstr name; - csubstr src; - CaseNode root; - TestCaseFlags_e flags; - Location expected_location; - - //! create a standard test case: name, source and expected CaseNode structure - template Case(csubstr file, int line, const char *name_, const char *src_, Args&& ...args) : filelinebuf(catrs(file, ':', line)), fileline(to_csubstr(filelinebuf)), name(to_csubstr(name_)), src(to_csubstr(src_)), root(std::forward(args)...), flags(), expected_location() {} - //! create a test case with explicit flags: name, source flags, and expected CaseNode structure - template Case(csubstr file, int line, const char *name_, int f_, const char *src_, Args&& ...args) : filelinebuf(catrs(file, ':', line)), fileline(to_csubstr(filelinebuf)), name(to_csubstr(name_)), src(to_csubstr(src_)), root(std::forward(args)...), flags((TestCaseFlags_e)f_), expected_location() {} - //! create a test case with an error on an expected location - Case(csubstr file, int line, const char *name_, int f_, const char *src_, LineCol loc) : filelinebuf(catrs(file, ':', line)), fileline(to_csubstr(filelinebuf)), name(to_csubstr(name_)), src(to_csubstr(src_)), root(), flags((TestCaseFlags_e)f_), expected_location(name, loc.line, loc.col) {} -}; - -//----------------------------------------------------------------------------- - -// a persistent data store to avoid repeating operations on every test -struct CaseDataLineEndings -{ - std::vector src_buf; - substr src; - - Tree parsed_tree; - - size_t numbytes_stdout; - size_t numbytes_stdout_json; - - std::string emit_buf; - csubstr emitted_yml; - - std::string emitjson_buf; - csubstr emitted_json; - - std::string parse_buf; - substr parsed_yml; - - std::string parse_buf_json; - substr parsed_json; - - Tree emitted_tree; - Tree emitted_tree_json; - - Tree recreated; -}; - - -struct CaseData -{ - CaseDataLineEndings unix_style; - CaseDataLineEndings unix_style_json; - CaseDataLineEndings windows_style; - CaseDataLineEndings windows_style_json; -}; - - -} // namespace yml -} // namespace c4 - -#if defined(_MSC_VER) -# pragma warning(pop) -#endif - -#ifdef __GNUC__ -# pragma GCC diagnostic pop -#endif - -#endif /* _TEST_CASE_HPP_ */ diff --git a/thirdparty/ryml/test/test_double_quoted.cpp b/thirdparty/ryml/test/test_double_quoted.cpp deleted file mode 100644 index 6c3915873..000000000 --- a/thirdparty/ryml/test/test_double_quoted.cpp +++ /dev/null @@ -1,610 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -TEST(double_quoted, escaped_chars) -{ - csubstr yaml = R"("\\\"\n\r\t\ \/\ \0\b\f\a\v\e\_\N\L\P")"; - // build the string like this because some of the characters are - // filtered out under the double quotes - std::string expected; - expected += '\\'; - expected += '"'; - expected += '\n'; - expected += '\r'; - expected += '\t'; - expected += '\t'; - expected += '/'; - expected += ' '; - expected += '\0'; - expected += '\b'; - expected += '\f'; - expected += '\a'; - expected += '\v'; - expected += INT8_C(0x1b); // \e - // - // wrap explicitly to avoid overflow - expected += _RYML_CHCONST(-0x3e, 0xc2); // \_ (1) - expected += _RYML_CHCONST(-0x60, 0xa0); // \_ (2) - // - expected += _RYML_CHCONST(-0x3e, 0xc2); // \N (1) - expected += _RYML_CHCONST(-0x7b, 0x85); // \N (2) - // - expected += _RYML_CHCONST(-0x1e, 0xe2); // \L (1) - expected += _RYML_CHCONST(-0x80, 0x80); // \L (2) - expected += _RYML_CHCONST(-0x58, 0xa8); // \L (3) - // - expected += _RYML_CHCONST(-0x1e, 0xe2); // \P (1) - expected += _RYML_CHCONST(-0x80, 0x80); // \P (2) - expected += _RYML_CHCONST(-0x57, 0xa9); // \P (3) - // - Tree t = parse_in_arena(yaml); - csubstr v = t.rootref().val(); - std::string actual = {v.str, v.len}; - EXPECT_EQ(actual, expected); -} - -TEST(double_quoted, test_suite_3RLN) -{ - csubstr yaml = R"(--- -"1 leading - \ttab" ---- -"2 leading - \ tab" ---- -"3 leading - tab" ---- -"4 leading - \t tab" ---- -"5 leading - \ tab" ---- -"6 leading - tab" -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t.docref(0).val(), "1 leading \ttab"); - EXPECT_EQ(t.docref(1).val(), "2 leading \ttab"); - EXPECT_EQ(t.docref(2).val(), "3 leading tab"); - EXPECT_EQ(t.docref(3).val(), "4 leading \t tab"); - EXPECT_EQ(t.docref(4).val(), "5 leading \t tab"); - EXPECT_EQ(t.docref(5).val(), "6 leading tab"); - }); -} - -TEST(double_quoted, test_suite_5GBF) -{ - csubstr yaml = R"( -Folding: - "Empty line - - as a line feed" -Folding2: - "Empty line - - as a line feed" -Folding3: - "Empty line - - as a line feed" -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - EXPECT_EQ(t["Folding"].val(), csubstr("Empty line\nas a line feed")); - EXPECT_EQ(t["Folding2"].val(), csubstr("Empty line\nas a line feed")); - EXPECT_EQ(t["Folding3"].val(), csubstr("Empty line\nas a line feed")); - }); -} - -TEST(double_quoted, test_suite_6SLA) -{ - csubstr yaml = R"( -"foo\nbar:baz\tx \\$%^&*()x": 23 -'x\ny:z\tx $%^&*()x': 24 -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_TRUE(t.rootref().has_child("foo\nbar:baz\tx \\$%^&*()x")); - ASSERT_TRUE(t.rootref().has_child("x\\ny:z\\tx $%^&*()x")); - ASSERT_EQ(t["foo\nbar:baz\tx \\$%^&*()x"].val(), csubstr("23")); - ASSERT_EQ(t["x\\ny:z\\tx $%^&*()x"].val(), csubstr("24")); - }); -} - -TEST(double_quoted, test_suite_6WPF) -{ - csubstr yaml = R"( -" - foo - - bar - - baz -" -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_val()); - EXPECT_EQ(t.rootref().val(), csubstr(" foo\nbar\nbaz ")); - }); -} - -TEST(double_quoted, test_suite_9TFX) -{ - csubstr yaml = R"( -" 1st non-empty - - 2nd non-empty - 3rd non-empty " -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_val()); - EXPECT_EQ(t.rootref().val(), csubstr(" 1st non-empty\n2nd non-empty 3rd non-empty ")); - }); -} - -TEST(double_quoted, test_suite_G4RS) -{ - csubstr yaml = R"(--- -unicode: "\u263A\u2705\U0001D11E" -control: "\b1998\t1999\t2000\n" -#hex esc: "\x0d\x0a is \r\n" -#--- -#- "\x0d\x0a is \r\n" -#--- -#{hex esc: "\x0d\x0a is \r\n"} -#--- -#["\x0d\x0a is \r\n"] -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t.docref(0)["unicode"].val(), csubstr(R"(☺✅𝄞)")); - EXPECT_EQ(t.docref(0)["control"].val(), csubstr("\b1998\t1999\t2000\n")); - //EXPECT_EQ(t.docref(0)["hex esc"].val(), csubstr("\r\n is \r\n")); TODO - //EXPECT_EQ(t.docref(1)[0].val(), csubstr("\r\n is \r\n")); - //EXPECT_EQ(t.docref(2)[0].val(), csubstr("\r\n is \r\n")); - //EXPECT_EQ(t.docref(3)[0].val(), csubstr("\r\n is \r\n")); - }); -} - -TEST(double_quoted, test_suite_KSS4) -{ - csubstr yaml = R"( ---- -"quoted -string" ---- "quoted -string" ---- -- "quoted - string" ---- -- "quoted -string" ---- -"quoted - string": "quoted - string" ---- -"quoted -string": "quoted -string" -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t.docref(0).val(), "quoted string"); - EXPECT_EQ(t.docref(1).val(), "quoted string"); - EXPECT_EQ(t.docref(2)[0].val(), "quoted string"); - EXPECT_EQ(t.docref(3)[0].val(), "quoted string"); - EXPECT_EQ(t.docref(4)["quoted string"].val(), "quoted string"); - EXPECT_EQ(t.docref(5)["quoted string"].val(), "quoted string"); - }); -} - -TEST(double_quoted, test_suite_NAT4) -{ - csubstr yaml = R"( -a: ' - ' -b: ' - ' -c: " - " -d: " - " -e: ' - - ' -f: " - - " -g: ' - - - ' -h: " - - - " -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t["a"].val(), csubstr(" ")); - EXPECT_EQ(t["b"].val(), csubstr(" ")); - EXPECT_EQ(t["c"].val(), csubstr(" ")); - EXPECT_EQ(t["d"].val(), csubstr(" ")); - EXPECT_EQ(t["e"].val(), csubstr("\n")); - EXPECT_EQ(t["f"].val(), csubstr("\n")); - EXPECT_EQ(t["g"].val(), csubstr("\n\n")); - EXPECT_EQ(t["h"].val(), csubstr("\n\n")); - }); -} - -TEST(double_quoted, test_suite_NP9H) -{ - csubstr yaml = R"( -"folded -to a space, - -to a line feed, or \ - \ non-content" -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_val()); - EXPECT_EQ(t.rootref().val(), csubstr("folded to a space,\nto a line feed, or \t \tnon-content")); - }); -} - -TEST(double_quoted, test_suite_Q8AD) -{ - csubstr yaml = R"( -"folded -to a space, - -to a line feed, or \ - \ non-content" -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_val()); - EXPECT_EQ(t.rootref().val(), csubstr("folded to a space,\nto a line feed, or \t \tnon-content")); - }); -} - -TEST(double_quoted, test_suite_R4YG) -{ - csubstr yaml = R"( -- " - -detected - -" - -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t[0].val(), csubstr("\t\ndetected\n")); - }); -} - - -//----------------------------------------------------------------------------- - -void verify_error_is_reported(csubstr case_name, csubstr yaml, Location loc={}) -{ - SCOPED_TRACE(case_name); - SCOPED_TRACE(yaml); - Tree tree; - ExpectError::do_check(&tree, [&](){ - parse_in_arena(yaml, &tree); - }, loc); -} - -TEST(double_quoted, error_on_unmatched_quotes) -{ - verify_error_is_reported("map block", R"(foo: "' -bar: "")"); - verify_error_is_reported("seq block", R"(- "' -- "")"); - verify_error_is_reported("map flow", R"({foo: "', bar: ""})"); - verify_error_is_reported("seq flow", R"(["', ""])"); -} - -TEST(double_quoted, error_on_unmatched_quotes_with_escapes) -{ - verify_error_is_reported("map block", R"(foo: "\"' -bar: "")"); - verify_error_is_reported("seq block", R"(- "\"' -- "")"); - verify_error_is_reported("map flow", R"({foo: "\"', bar: ""})"); - verify_error_is_reported("seq flow", R"(["\"', ""])"); -} - -TEST(double_quoted, error_on_unmatched_quotes_at_end) -{ - verify_error_is_reported("map block", R"(foo: "" -bar: "')"); - verify_error_is_reported("seq block", R"(- "" -- "')"); - verify_error_is_reported("map flow", R"({foo: "", bar: "'})"); - verify_error_is_reported("seq flow", R"(["", "'])"); -} - -TEST(double_quoted, error_on_unmatched_quotes_at_end_with_escapes) -{ - verify_error_is_reported("map block", R"(foo: "" -bar: "\"')"); - verify_error_is_reported("seq block", R"(- "" -- "\"')"); - verify_error_is_reported("map flow", R"({foo: "", bar: "\"'})"); - verify_error_is_reported("seq flow", R"(["", "\"'])"); -} - -TEST(double_quoted, error_on_unclosed_quotes) -{ - verify_error_is_reported("map block", R"(foo: ", -bar: what)"); - verify_error_is_reported("seq block", R"(- " -- what)"); - verify_error_is_reported("map flow", R"({foo: ", bar: what})"); - verify_error_is_reported("seq flow", R"([", what])"); -} - -TEST(double_quoted, error_on_unclosed_quotes_with_escapes) -{ - verify_error_is_reported("map block", R"(foo: "\", -bar: what)"); - verify_error_is_reported("seq block", R"(- "\" -- what)"); - verify_error_is_reported("map flow", R"({foo: "\", bar: what})"); - verify_error_is_reported("seq flow", R"(["\", what])"); -} - -TEST(double_quoted, error_on_unclosed_quotes_at_end) -{ - verify_error_is_reported("map block", R"(foo: what -bar: ")"); - verify_error_is_reported("seq block", R"(- what -- ")"); - verify_error_is_reported("map flow", R"({foo: what, bar: "})"); - verify_error_is_reported("seq flow", R"([what, "])"); -} - -TEST(double_quoted, error_on_unclosed_quotes_at_end_with_escapes) -{ - verify_error_is_reported("map block", R"(foo: what -bar: "\")"); - verify_error_is_reported("seq block", R"(- what -- "\")"); - verify_error_is_reported("map flow", R"({foo: what, bar: "\"})"); - verify_error_is_reported("seq flow", R"([what, "\"])"); -} - -TEST(double_quoted, error_on_bad_utf_codepoints) -{ - verify_error_is_reported("incomplete \\x 0", R"(foo: "\x")"); - verify_error_is_reported("incomplete \\x 1", R"(foo: "\x1")"); - verify_error_is_reported("bad value \\x" , R"(foo: "\xko")"); - verify_error_is_reported("incomplete \\u 0", R"(foo: "\u")"); - verify_error_is_reported("incomplete \\u 1", R"(foo: "\u1")"); - verify_error_is_reported("incomplete \\u 2", R"(foo: "\u12")"); - verify_error_is_reported("incomplete \\u 3", R"(foo: "\u123")"); - verify_error_is_reported("bad value \\u" , R"(foo: "\ukoko")"); - verify_error_is_reported("incomplete \\U 0", R"(foo: "\U")"); - verify_error_is_reported("incomplete \\U 1", R"(foo: "\U1")"); - verify_error_is_reported("incomplete \\U 2", R"(foo: "\U12")"); - verify_error_is_reported("incomplete \\U 3", R"(foo: "\U123")"); - verify_error_is_reported("incomplete \\U 4", R"(foo: "\U1234")"); - verify_error_is_reported("incomplete \\U 5", R"(foo: "\U12345")"); - verify_error_is_reported("incomplete \\U 6", R"(foo: "\U123456")"); - verify_error_is_reported("incomplete \\U 7", R"(foo: "\U1234567")"); - verify_error_is_reported("bad value \\U" , R"(foo: "\Ukokokoko")"); -} - -TEST(double_quoted, github253) -{ - { - Tree tree; - NodeRef root = tree.rootref(); - root |= MAP; - root["t"] = "t't\\nt"; - root["t"] |= _WIP_VAL_DQUO; - std::string s = emitrs_yaml(tree); - Tree tree2 = parse_in_arena(to_csubstr(s)); - EXPECT_EQ(tree2["t"].val(), tree["t"].val()); - } - { - Tree tree; - NodeRef root = tree.rootref(); - root |= MAP; - root["t"] = "t't\\nt"; - root["t"] |= _WIP_VAL_SQUO; - std::string s = emitrs_yaml(tree); - Tree tree2 = parse_in_arena(to_csubstr(s)); - EXPECT_EQ(tree2["t"].val(), tree["t"].val()); - } - { - Tree tree; - NodeRef root = tree.rootref(); - root |= MAP; - root["s"] = "t\rt"; - root["s"] |= _WIP_VAL_DQUO; - std::string s = emitrs_yaml(tree); - EXPECT_EQ(s, "s: \"t\\rt\"\n"); - Tree tree2 = parse_in_arena(to_csubstr(s)); - EXPECT_EQ(tree2["s"].val(), tree["s"].val()); - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -CASE_GROUP(DOUBLE_QUOTED) -{ - -ADD_CASE_TO_GROUP("dquoted, only text", -R"("Some text without any quotes." -)", - N(DOCVAL | VALQUO, "Some text without any quotes.") -); - -ADD_CASE_TO_GROUP("dquoted, with single quotes", -R"("Some text 'with single quotes'")", - N(DOCVAL|VALQUO, "Some text 'with single quotes'") -); - -ADD_CASE_TO_GROUP("dquoted, with double quotes", -R"("Some \"text\" \"with double quotes\"")", - N(DOCVAL|VALQUO, "Some \"text\" \"with double quotes\"") -); - -ADD_CASE_TO_GROUP("dquoted, with single and double quotes", -R"("Some text 'with single quotes' \"and double quotes\".")", - N(DOCVAL|VALQUO, "Some text 'with single quotes' \"and double quotes\".") -); - -ADD_CASE_TO_GROUP("dquoted, with escapes", -R"("Some text with escapes \\n \\r \\t")", - N(DOCVAL|VALQUO, "Some text with escapes \\n \\r \\t") -); - -ADD_CASE_TO_GROUP("dquoted, with newline", -R"("Some text with\nnewline")", - N(DOCVAL|VALQUO, "Some text with\nnewline") -); - -ADD_CASE_TO_GROUP("dquoted, with tabs", -R"("\tSome\ttext\twith\ttabs\t")", - N(DOCVAL|VALQUO, "\tSome\ttext\twith\ttabs\t") -); - -ADD_CASE_TO_GROUP("dquoted, with tabs 4ZYM", -R"(plain: text - lines -quoted: "text - lines" -block: | - text - lines -)", - L{N("plain", "text lines"), - N(KEYVAL|VALQUO, "quoted", "text lines"), - N(KEYVAL|VALQUO,"block", "text\n \tlines\n")} -); - -ADD_CASE_TO_GROUP("dquoted, with tabs 7A4E", -R"(" 1st non-empty - - 2nd non-empty - 3rd non-empty ")", - N(DOCVAL|VALQUO, " 1st non-empty\n2nd non-empty 3rd non-empty ") -); - -ADD_CASE_TO_GROUP("dquoted, with tabs TL85", -R"(" - foo - - bar - - baz -")", N(DOCVAL|VALQUO, " foo\nbar\nbaz ")); - -ADD_CASE_TO_GROUP("dquoted, all", -R"("Several lines of text, -containing 'single quotes' and \"double quotes\". \ -Escapes (like \\n) work.\nIn addition, -newlines can be esc\ -aped to prevent them from being converted to a space. - -Newlines can also be added by leaving a blank line. - Leading whitespace on lines is ignored." -)", - N(DOCVAL|VALQUO, "Several lines of text, containing 'single quotes' and \"double quotes\". Escapes (like \\n) work.\nIn addition, newlines can be escaped to prevent them from being converted to a space.\nNewlines can also be added by leaving a blank line. Leading whitespace on lines is ignored.") -); - -ADD_CASE_TO_GROUP("dquoted, empty", -R"("")", - N(DOCVAL|VALQUO, "") -); - -ADD_CASE_TO_GROUP("dquoted, blank", -R"( -- "" -- " " -- " " -- " " -- " " -)", - L{N(QV, ""), N(QV, " "), N(QV, " "), N(QV, " "), N(QV, " ")} -); - -ADD_CASE_TO_GROUP("dquoted, numbers", // these should not be quoted when emitting -R"( -- -1 -- -1.0 -- +1.0 -- 1e-2 -- 1e+2 -)", - L{N("-1"), N("-1.0"), N("+1.0"), N("1e-2"), N("1e+2")} -); - -ADD_CASE_TO_GROUP("dquoted, trailing space", -R"('a aaaa ')", - N(DOCVAL|VALQUO, "a aaaa ") -); - -ADD_CASE_TO_GROUP("dquoted, leading space", -R"(' a aaaa')", - N(DOCVAL|VALQUO, " a aaaa") -); - -ADD_CASE_TO_GROUP("dquoted, trailing and leading space", -R"(' 012345 ')", - N(DOCVAL|VALQUO, " 012345 ") -); - -ADD_CASE_TO_GROUP("dquoted, 1 dquote", -R"("\"")", - N(DOCVAL|VALQUO, "\"") -); - -ADD_CASE_TO_GROUP("dquoted, 2 dquotes", -R"("\"\"")", - N(DOCVAL|VALQUO, "\"\"") -); - -ADD_CASE_TO_GROUP("dquoted, 3 dquotes", -R"("\"\"\"")", - N(DOCVAL|VALQUO, "\"\"\"") -); - -ADD_CASE_TO_GROUP("dquoted, 4 dquotes", -R"("\"\"\"\"")", - N(DOCVAL|VALQUO, "\"\"\"\"") -); - -ADD_CASE_TO_GROUP("dquoted, 5 dquotes", -R"("\"\"\"\"\"")", - N(DOCVAL|VALQUO, "\"\"\"\"\"") -); - -ADD_CASE_TO_GROUP("dquoted, example 2", -R"("This is a key\nthat has multiple lines\n": and this is its value -)", - L{N(QK, "This is a key\nthat has multiple lines\n", "and this is its value")} -); - -ADD_CASE_TO_GROUP("dquoted, example 2.1", -R"("This is a key - -that has multiple lines - -": and this is its value -)", - L{N(QK, "This is a key\nthat has multiple lines\n", "and this is its value")} -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_emit.cpp b/thirdparty/ryml/test/test_emit.cpp deleted file mode 100644 index 3166dfaed..000000000 --- a/thirdparty/ryml/test/test_emit.cpp +++ /dev/null @@ -1,491 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/std/std.hpp" -#include "c4/yml/parse.hpp" -#include "c4/yml/emit.hpp" -#include -#include -#include -#endif -#include - -#include "./test_case.hpp" - -#include - -namespace c4 { -namespace yml { - -template -std::string emit2file(Emit &&fn) -{ - C4_SUPPRESS_WARNING_MSVC_WITH_PUSH(4996) // fopen unsafe - std::string filename = fs::tmpnam(); - FILE *f = fopen(filename.c_str(), "wb"); - C4_CHECK(f != nullptr); - fn(f); - fflush(f); - fclose(f); - std::string result = fs::file_get_contents(filename.c_str()); - fs::rmfile(filename.c_str()); - return result; - C4_SUPPRESS_WARNING_MSVC_POP -} - -template -std::string emit2stream(Emit &&fn) -{ - std::ostringstream ss; - fn(ss); - return ss.str(); -} - -template -std::string emit2buf(Emit &&fn) -{ - std::string buf; - buf.resize(2048); - substr out = fn(to_substr(buf)); - if(out.len > buf.size()) - { - buf.resize(out.len); - out = fn(to_substr(buf)); - } - buf.resize(out.len); - return buf; -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(as_json, basic) -{ - Tree et; - { - as_json j(et); - EXPECT_EQ(j.tree, &et); - } - Tree t = parse_in_arena("[foo, bar]"); - { - as_json j(t); - EXPECT_EQ(j.tree, &t); - EXPECT_EQ(j.node, t.root_id()); - } - { - as_json j(t, 2u); - EXPECT_EQ(j.tree, &t); - EXPECT_EQ(j.node, 2u); - } - { - as_json j(t[0]); - EXPECT_EQ(j.tree, &t); - EXPECT_EQ(j.node, 1u); - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_emits(Tree const& t, size_t id, std::string const& expected, std::string const& expected_json) -{ - EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, id, buf); }), expected); - EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, id, buf); }), expected_json); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, id, f); }), expected); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, id, f); }), expected_json); - EXPECT_EQ(emitrs_yaml(t, id), expected); - EXPECT_EQ(emitrs_json(t, id), expected_json); -} - -template -void test_emits(Tree const& t, std::string const& expected, std::string const& expected_json) -{ - EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, buf); }), expected); - EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, buf); }), expected_json); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, f); }), expected); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, f); }), expected_json); - EXPECT_EQ(emit2stream([&](std::ostream& s){ s << t; }), expected); - EXPECT_EQ(emit2stream([&](std::ostream& s){ s << as_json(t); }), expected_json); - EXPECT_EQ(emitrs_yaml(t), expected); - EXPECT_EQ(emitrs_json(t), expected_json); -} - -template -void test_emits(ConstNodeRef t, std::string const& expected, std::string const& expected_json) -{ - EXPECT_EQ(emit2buf([&](substr buf){ return emit_yaml(t, buf); }), expected); - EXPECT_EQ(emit2buf([&](substr buf){ return emit_json(t, buf); }), expected_json); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_yaml(t, f); }), expected); - EXPECT_EQ(emit2file([&](FILE *f){ return emit_json(t, f); }), expected_json); - EXPECT_EQ(emit2stream([&](std::ostream& s){ s << t; }), expected); - EXPECT_EQ(emit2stream([&](std::ostream& s){ s << as_json(t); }), expected_json); - EXPECT_EQ(emitrs_yaml(t), expected); - EXPECT_EQ(emitrs_json(t), expected_json); -} - - -TEST(emit, empty_tree) -{ - const Tree t; // must be const! - std::string expected = R"()"; - test_emits(t, expected, expected); -} - -TEST(emit, existing_tree) -{ - const Tree t = parse_in_arena("[foo, bar]"); - std::string expected = "- foo\n- bar\n"; - std::string expected_json = R"(["foo","bar"])"; - test_emits(t, expected, expected_json); -} - -TEST(emit, existing_seq_node) -{ - Tree nct = parse_in_arena("[foo, bar, [nested, seq], {nested: map}]"); - Tree const& t = nct; - { - std::string expected = "- foo\n- bar\n- - nested\n - seq\n- nested: map\n"; - std::string expected_json = R"(["foo","bar",["nested","seq"],{"nested": "map"}])"; - { - SCOPED_TRACE("rootref"); - test_emits(t.crootref(), expected, expected_json); - } - { - SCOPED_TRACE("t"); - test_emits(t, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, t.root_id(), expected, expected_json); - } - } - { - ConstNodeRef n = t[0]; - std::string expected = "foo\n"; - std::string expected_json = "\"foo\""; - { - SCOPED_TRACE("noderef"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - nct._add_flags(n.id(), _WIP_STYLE_FLOW_SL); - expected = "foo"; - { - SCOPED_TRACE("t, id"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - } - { - ConstNodeRef n = t[1]; - std::string expected = "bar\n"; - std::string expected_json = "\"bar\""; - { - SCOPED_TRACE("noderef"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - nct._add_flags(n.id(), _WIP_STYLE_FLOW_SL); - expected = "bar"; - { - SCOPED_TRACE("t, id"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - - } - { - ConstNodeRef n = t[2]; - std::string expected = "- nested\n- seq\n"; - std::string expected_json = "[\"nested\",\"seq\"]"; - { - SCOPED_TRACE("noderef"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - expected = "[nested,seq]"; - nct._add_flags(n.id(), _WIP_STYLE_FLOW_SL); - { - SCOPED_TRACE("t, id"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - } - { - ConstNodeRef n = t[3]; - std::string expected = "nested: map\n"; - std::string expected_json = "{\"nested\": \"map\"}"; - { - SCOPED_TRACE("noderef"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - expected = "{nested: map}"; - nct._add_flags(n.id(), _WIP_STYLE_FLOW_SL); - { - SCOPED_TRACE("t, id"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - } -} - -TEST(emit, existing_map_node) -{ - Tree nct = parse_in_arena("{0: foo, 1: bar, 2: [nested, seq], 3: {nested: map}}"); - Tree const& t = nct; - { - std::string expected = "0: foo\n1: bar\n2:\n - nested\n - seq\n3:\n nested: map\n"; - std::string expected_json = R"({"0": "foo","1": "bar","2": ["nested","seq"],"3": {"nested": "map"}})"; - { - SCOPED_TRACE("rootref"); - test_emits(t.rootref(), expected, expected_json); - } - { - SCOPED_TRACE("t"); - test_emits(t, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, t.root_id(), expected, expected_json); - } - } - { - ConstNodeRef n = t[0]; - std::string expected = "0: foo\n"; - std::string expected_json = "\"0\": \"foo\""; - { - SCOPED_TRACE("noderef"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - expected = "0: foo"; - nct._add_flags(n.id(), _WIP_STYLE_FLOW_SL); - { - SCOPED_TRACE("t, id"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - } - { - ConstNodeRef n = t[1]; - std::string expected = "1: bar\n"; - std::string expected_json = "\"1\": \"bar\""; - { - SCOPED_TRACE("noderef"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - expected = "1: bar"; - nct._add_flags(n.id(), _WIP_STYLE_FLOW_SL); - { - SCOPED_TRACE("t, id"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - } - { - ConstNodeRef n = t[2]; - std::string expected = "2:\n - nested\n - seq\n"; - std::string expected_json = "\"2\": [\"nested\",\"seq\"]"; - { - SCOPED_TRACE("noderef"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - expected = "2: [nested,seq]"; - nct._add_flags(n.id(), _WIP_STYLE_FLOW_SL); - { - SCOPED_TRACE("t, id"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - } - { - ConstNodeRef n = t[3]; - std::string expected = "3:\n nested: map\n"; - std::string expected_json = "\"3\": {\"nested\": \"map\"}"; - { - SCOPED_TRACE("noderef"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - expected = "3: {nested: map}"; - nct._add_flags(n.id(), _WIP_STYLE_FLOW_SL); - { - SCOPED_TRACE("t, id"); - test_emits(n, expected, expected_json); - } - { - SCOPED_TRACE("t, id"); - test_emits(t, n.id(), expected, expected_json); - } - } -} - -TEST(emit, percent_is_quoted) -{ - Tree ti = parse_in_arena("{}"); - ASSERT_TRUE(ti.rootref().is_map()); - ti["%ROOT"] = "%VAL"; - ti["%ROOT2"] |= SEQ; - ti["%ROOT2"][0] = "%VAL"; - ti["%ROOT2"][1] = "%VAL"; - std::string yaml = emitrs_yaml(ti); - test_check_emit_check(to_csubstr(yaml), [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_TRUE(t.rootref().has_child("%ROOT")); - ASSERT_TRUE(t.rootref().has_child("%ROOT2")); - ASSERT_EQ(t["%ROOT2"].num_children(), 2u); - EXPECT_TRUE(t["%ROOT"].is_key_quoted()); - EXPECT_TRUE(t["%ROOT"].is_val_quoted()); - EXPECT_TRUE(t["%ROOT2"].is_key_quoted()); - EXPECT_TRUE(t["%ROOT2"][0].is_val_quoted()); - EXPECT_TRUE(t["%ROOT2"][1].is_val_quoted()); - }); -} - -TEST(emit, at_is_quoted__issue_309) -{ - Tree ti = parse_in_arena("{at: [], backtick: []"); - ti["at"][0] << "@test"; - ti["at"][1] = "@test2"; - ti["at"][2] << "@"; - ti["at"][3] = "@"; - ti["backtick"][0] << "`test"; - ti["backtick"][1] = "`test2"; - ti["backtick"][2] << "`"; - ti["backtick"][3] = "`"; - std::string yaml = emitrs_yaml(ti); - test_check_emit_check(to_csubstr(yaml), [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_TRUE(t.rootref().has_child("at")); - ASSERT_TRUE(t.rootref().has_child("backtick")); - ASSERT_EQ(t["at"].num_children(), 4u); - ASSERT_EQ(t["backtick"].num_children(), 4u); - EXPECT_EQ(t["at"][0].val(), "@test"); - EXPECT_EQ(t["at"][1].val(), "@test2"); - EXPECT_EQ(t["at"][2].val(), "@"); - EXPECT_EQ(t["at"][3].val(), "@"); - EXPECT_TRUE(t["at"][0].is_val_quoted()); - EXPECT_TRUE(t["at"][1].is_val_quoted()); - EXPECT_TRUE(t["at"][2].is_val_quoted()); - EXPECT_TRUE(t["at"][3].is_val_quoted()); - EXPECT_EQ(t["backtick"][0].val(), "`test"); - EXPECT_EQ(t["backtick"][1].val(), "`test2"); - EXPECT_EQ(t["backtick"][2].val(), "`"); - EXPECT_EQ(t["backtick"][3].val(), "`"); - EXPECT_TRUE(t["backtick"][0].is_val_quoted()); - EXPECT_TRUE(t["backtick"][1].is_val_quoted()); - EXPECT_TRUE(t["backtick"][2].is_val_quoted()); - EXPECT_TRUE(t["backtick"][3].is_val_quoted()); - }); -} - -TEST(emit, at_is_quoted_only_in_the_beggining__issue_320) -{ - Tree ti = parse_in_arena("{at: [], backtick: []"); - ti["at"].append_child() << "@test"; - ti["at"].append_child() << "t@est"; - ti["at"].append_child() << "test@"; - ti["at"].append_child() = "@test2"; - ti["at"].append_child() = "t@est2"; - ti["at"].append_child() = "test2@"; - ti["backtick"].append_child() << "`test"; - ti["backtick"].append_child() << "t`est"; - ti["backtick"].append_child() << "test`"; - ti["backtick"].append_child() = "`test2"; - ti["backtick"].append_child() = "t`est2"; - ti["backtick"].append_child() = "test2`"; - std::string yaml = emitrs_yaml(ti); - test_check_emit_check(to_csubstr(yaml), [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_TRUE(t.rootref().has_child("at")); - ASSERT_TRUE(t.rootref().has_child("backtick")); - ASSERT_EQ(t["at"].num_children(), 6u); - ASSERT_EQ(t["backtick"].num_children(), 6u); - EXPECT_EQ(t["at"][0].val(), "@test"); - EXPECT_EQ(t["at"][1].val(), "t@est"); - EXPECT_EQ(t["at"][2].val(), "test@"); - EXPECT_EQ(t["at"][3].val(), "@test2"); - EXPECT_EQ(t["at"][4].val(), "t@est2"); - EXPECT_EQ(t["at"][5].val(), "test2@"); - EXPECT_TRUE( t["at"][0].is_val_quoted()); - EXPECT_TRUE( ! t["at"][1].is_val_quoted()); - EXPECT_TRUE( ! t["at"][2].is_val_quoted()); - EXPECT_TRUE( t["at"][3].is_val_quoted()); - EXPECT_TRUE( ! t["at"][4].is_val_quoted()); - EXPECT_TRUE( ! t["at"][5].is_val_quoted()); - EXPECT_EQ(t["backtick"][0].val(), "`test"); - EXPECT_EQ(t["backtick"][1].val(), "t`est"); - EXPECT_EQ(t["backtick"][2].val(), "test`"); - EXPECT_EQ(t["backtick"][3].val(), "`test2"); - EXPECT_EQ(t["backtick"][4].val(), "t`est2"); - EXPECT_EQ(t["backtick"][5].val(), "test2`"); - EXPECT_TRUE( t["backtick"][0].is_val_quoted()); - EXPECT_TRUE( ! t["backtick"][1].is_val_quoted()); - EXPECT_TRUE( ! t["backtick"][2].is_val_quoted()); - EXPECT_TRUE( t["backtick"][3].is_val_quoted()); - EXPECT_TRUE( ! t["backtick"][4].is_val_quoted()); - EXPECT_TRUE( ! t["backtick"][5].is_val_quoted()); - }); -} - - -//------------------------------------------- -// this is needed to use the test case library -Case const* get_case(csubstr /*name*/) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_empty_file.cpp b/thirdparty/ryml/test/test_empty_file.cpp deleted file mode 100644 index 38e66fbaf..000000000 --- a/thirdparty/ryml/test/test_empty_file.cpp +++ /dev/null @@ -1,79 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - - -CASE_GROUP(EMPTY_FILE) -{ - -ADD_CASE_TO_GROUP("empty0-nochars", -"", -NOTYPE); - - -ADD_CASE_TO_GROUP("empty0-multiline", R"( - - -)", NOTYPE); - - -ADD_CASE_TO_GROUP("empty0-multiline with spaces", R"( - - - - - - -)", NOTYPE); - - -ADD_CASE_TO_GROUP("empty0-multiline with spaces and tabs", R"( - - - - - - - -)", NOTYPE); - - -ADD_CASE_TO_GROUP("empty0-multiline-with-comments 0", R"( -# well hello sir, I see you are fine -# very fine thank you -# send my very best wishes -)", NOTYPE); - -ADD_CASE_TO_GROUP("empty0-multiline-with-comments 1", R"( - - - -# well hello sir, I see you are fine -# very fine thank you -# send my very best wishes - - - -)", NOTYPE); - -ADD_CASE_TO_GROUP("empty0-multiline-with-comments 2", R"( - - - - -# well hello sir, I see you are fine -# very fine thank you -# send my very best wishes - - - - - - -)", NOTYPE); - -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_empty_map.cpp b/thirdparty/ryml/test/test_empty_map.cpp deleted file mode 100644 index 44d818cc6..000000000 --- a/thirdparty/ryml/test/test_empty_map.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - - -CASE_GROUP(EMPTY_MAP) -{ - -ADD_CASE_TO_GROUP("empty map, explicit", -"{}", - MAP -); - - -ADD_CASE_TO_GROUP("empty map, explicit, whitespace", -" {}", - MAP -); - - -ADD_CASE_TO_GROUP("empty map, multiline", -R"({ - -} -)", - MAP -); - - -ADD_CASE_TO_GROUP("empty map, multilines", -R"({ -# ksjdfkjhsdfkjhsdfkjh - - -} -)", - MAP - ); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_empty_scalar.cpp b/thirdparty/ryml/test/test_empty_scalar.cpp deleted file mode 100644 index 52147569c..000000000 --- a/thirdparty/ryml/test/test_empty_scalar.cpp +++ /dev/null @@ -1,353 +0,0 @@ -#include "./test_group.hpp" -#include - -namespace c4 { -namespace yml { - -// See also: -// https://github.com/biojppm/rapidyaml/issues/263 -// https://github.com/biojppm/rapidyaml/pull/264 - -C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wuseless-cast") - -constexpr const NodeType_e DQV = (NodeType_e)(DOC | QV); - -TEST(empty_scalar, parse_zero_length_strings) -{ - char inp[] = R"( -seq: - - "" - - '' - - > - - | -map: - a: "" - b: '' - c: > - d: | -)"; - const Tree tr = parse_in_place(inp); - EXPECT_TRUE(tr["seq"].has_key()); - EXPECT_TRUE(tr["map"].has_key()); - EXPECT_TRUE(tr["seq"].is_seq()); - EXPECT_TRUE(tr["map"].is_map()); - for(const char *name : {"seq", "map"}) - { - ConstNodeRef node = tr[to_csubstr(name)]; - ASSERT_EQ(node.num_children(), 4); - for(const auto &child : node.children()) - { - EXPECT_TRUE(child.is_val_quoted()); - EXPECT_EQ(child.val().len, 0u); - EXPECT_NE(child.val().str, nullptr); - EXPECT_NE(child.val(), nullptr); - EXPECT_EQ(child.val(), ""); - EXPECT_FALSE(child.val_is_null()); - } - } -} - -TEST(empty_scalar, flow_seq) -{ - test_check_emit_check("['', '']", [&](Tree const &t){ - ASSERT_TRUE(t.rootref().has_children()); - for(ConstNodeRef ch : t.rootref().children()) - { - EXPECT_TRUE(ch.is_val_quoted()); - EXPECT_FALSE(ch.val_is_null()); - EXPECT_EQ(ch.val().len, 0); - EXPECT_NE(ch.val().str, nullptr); - EXPECT_NE(ch.val(), nullptr); - } - }); - test_check_emit_check("[ , ]", [&](Tree const &t){ - ASSERT_TRUE(t.rootref().has_children()); - for(ConstNodeRef ch : t.rootref().children()) - { - EXPECT_FALSE(ch.is_val_quoted()); - EXPECT_TRUE(ch.val_is_null()); - EXPECT_EQ(ch.val().len, 0); - EXPECT_EQ(ch.val().str, nullptr); - EXPECT_EQ(ch.val(), nullptr); - } - }); -} - -TEST(empty_scalar, parse_empty_strings) -{ - char inp[] = R"( -# use multiple empty entries to ensure the parser -# correctly deals with the several cases -seq: - - - - - - - - -map: - a: - b: - c: - d: -)"; - const Tree tr = parse_in_place(inp); - for(const char *name : {"seq", "map"}) - { - ConstNodeRef node = tr[to_csubstr(name)]; - ASSERT_EQ(node.num_children(), 4); - for(const auto &child : node.children()) - { - EXPECT_FALSE(child.type().is_val_quoted()); - EXPECT_EQ(child.val(), ""); - EXPECT_EQ(child.val(), nullptr); - EXPECT_EQ(child.val().str, nullptr); - EXPECT_EQ(child.val().len, 0u); - EXPECT_TRUE(child.val_is_null()); - } - } -} - -TEST(empty_scalar, std_string) -{ - std::string stdstr; - csubstr stdss = to_csubstr(stdstr); - csubstr nullss; - EXPECT_NE(stdss, nullptr); - EXPECT_NE(stdss.str, nullptr); - EXPECT_EQ(stdss.len, 0u); - EXPECT_EQ(nullss, nullptr); - EXPECT_EQ(nullss.str, nullptr); - EXPECT_EQ(nullss.len, 0u); - Tree tree = parse_in_arena("{ser: {}, eq: {}}"); - tree["ser"]["stdstr"] << stdss; - tree["ser"]["nullss"] << nullss; - tree["eq"]["stdstr"] = stdss; - tree["eq"]["nullss"] = nullss; - EXPECT_EQ(emitrs_yaml(tree), - "ser:\n" - " stdstr: ''\n" - " nullss: \n" - "eq:\n" - " stdstr: ''\n" - " nullss: \n" - ); -} - -TEST(empty_scalar, to_arena) -{ - Tree tr; - { - const char *val = ""; - size_t num = to_chars(substr{}, val); - ASSERT_EQ(num, 0u); - char buf_[10]; - csubstr serialized = to_chars_sub(buf_, val); - EXPECT_EQ(serialized.len, 0); - EXPECT_NE(serialized.str, nullptr); - EXPECT_NE(serialized, nullptr); - csubstr r = tr.to_arena(""); - EXPECT_EQ(r.len, 0u); - EXPECT_NE(r.str, nullptr); - EXPECT_NE(r, nullptr); - } - { - const char *val = nullptr; - size_t num = to_chars(substr{}, val); - ASSERT_EQ(num, 0u); - char buf_[10]; - csubstr serialized = to_chars_sub(buf_, val); - EXPECT_EQ(serialized.len, 0); - EXPECT_NE(serialized.str, nullptr); - EXPECT_NE(serialized, nullptr); - csubstr r = tr.to_arena(""); - EXPECT_EQ(r.len, 0u); - EXPECT_NE(r.str, nullptr); - EXPECT_NE(r, nullptr); - r = tr.to_arena(val); - EXPECT_EQ(r.len, 0u); - EXPECT_EQ(r.str, nullptr); - EXPECT_EQ(r, nullptr); - } - { - std::nullptr_t val = nullptr; - size_t num = to_chars(substr{}, val); - ASSERT_EQ(num, 0u); - csubstr r = tr.to_arena(val); - EXPECT_EQ(r.len, 0u); - EXPECT_EQ(r.str, nullptr); - EXPECT_EQ(r, nullptr); - } -} - -TEST(empty_scalar, gcc_error) -{ - Tree tr; - csubstr nullstr = {}; - ASSERT_EQ(nullstr.str, nullptr); - ASSERT_EQ(nullstr.len, 0); - std::cout << "\nserializing with empty arena...\n"; - csubstr result = tr.to_arena(nullstr); - EXPECT_EQ(result.str, nullptr); // fails! - EXPECT_EQ(result.len, 0); - std::cout << "\nserializing with nonempty arena...\n"; - result = tr.to_arena(nullstr); - EXPECT_EQ(result.str, nullptr); // fails! - EXPECT_EQ(result.len, 0); -} - -TEST(empty_scalar, build_zero_length_string) -{ - Tree tr; - NodeRef root = tr.rootref(); - root |= MAP; - auto addseq = [&root](csubstr name) { NodeRef n = root[name]; n |= SEQ; return n; }; - - // try both with nonnull-zero-length and null-zero-length - std::string stdstr; - csubstr stdss = to_csubstr(stdstr); - csubstr empty = csubstr("nonempty").first(0); - csubstr nullss = {}; - - // these are the conditions we wish to cover: - ASSERT_TRUE(stdss.str != nullptr); - ASSERT_TRUE(stdss.len == 0u); - ASSERT_TRUE(empty.str != nullptr); - ASSERT_TRUE(empty.len == 0u); - ASSERT_TRUE(nullss.str == nullptr); - ASSERT_TRUE(nullss.len == 0u); - - // = and << must have exactly the same behavior where nullity is - // regarded - - { - NodeRef quoted = addseq("quoted"); - {NodeRef r = quoted.append_child(); r = "" ; r.set_type(r.type() | VALQUO);} - {NodeRef r = quoted.append_child(); r << "" ; r.set_type(r.type() | VALQUO);} - {NodeRef r = quoted.append_child(); r = empty ; r.set_type(r.type() | VALQUO);} - {NodeRef r = quoted.append_child(); r << empty ; r.set_type(r.type() | VALQUO);} - {NodeRef r = quoted.append_child(); r = stdss ; r.set_type(r.type() | VALQUO);} - {NodeRef r = quoted.append_child(); r << stdss ; r.set_type(r.type() | VALQUO);} - } - { - NodeRef quoted_null = addseq("quoted_null"); - {NodeRef r = quoted_null.append_child(); r = nullss ; r.set_type(r.type() | VALQUO);} - {NodeRef r = quoted_null.append_child(); r << nullss ; r.set_type(r.type() | VALQUO);} - {NodeRef r = quoted_null.append_child(); r = nullptr ; r.set_type(r.type() | VALQUO);} - {NodeRef r = quoted_null.append_child(); r << nullptr; r.set_type(r.type() | VALQUO);} - } - { - NodeRef non_quoted = addseq("nonquoted"); - non_quoted.append_child() = ""; - non_quoted.append_child() << ""; - non_quoted.append_child() = empty; - non_quoted.append_child() << empty; - non_quoted.append_child() = stdss; - non_quoted.append_child() << stdss; - } - { - NodeRef non_quoted_null = addseq("nonquoted_null"); - non_quoted_null.append_child() = nullss; - non_quoted_null.append_child() << nullss; - non_quoted_null.append_child() = nullptr; - non_quoted_null.append_child() << nullptr; - } - - // quoted cases will never be null, regardless of the - // incoming scalar - auto test_quoted_empty = [](ConstNodeRef node){ - SCOPED_TRACE(node.key()); - ASSERT_TRUE(node.has_children()); - { - size_t pos = 0; - for(ConstNodeRef child : node.cchildren()) - { - EXPECT_TRUE(child.is_val_quoted()) << "pos=" << pos; - EXPECT_EQ(child.val().len, 0u) << "pos=" << pos; - EXPECT_NE(child.val().str, nullptr) << "pos=" << pos; - EXPECT_NE(child.val(), nullptr) << "pos=" << pos; - EXPECT_EQ(child.val(), "") << "pos=" << pos; - EXPECT_FALSE(child.val_is_null()) << "pos=" << pos; - pos++; - } - } - }; - auto test_quoted_null = [](ConstNodeRef node){ - SCOPED_TRACE(node.key()); - ASSERT_TRUE(node.has_children()); - size_t pos = 0; - for(ConstNodeRef child : node.cchildren()) - { - EXPECT_TRUE(child.is_val_quoted()) << "pos=" << pos; - EXPECT_FALSE(child.val_is_null()) << "pos=" << pos; // because it's quoted - EXPECT_EQ(child.val().len, 0u) << "pos=" << pos; - EXPECT_EQ(child.val().str, nullptr) << "pos=" << pos; - EXPECT_EQ(child.val(), nullptr) << "pos=" << pos; - EXPECT_EQ(child.val(), "") << "pos=" << pos; - pos++; - } - }; - // ... but according to the incoming scalar, non quoted cases may - // or may not be null - auto test_non_quoted_empty = [](ConstNodeRef node){ - SCOPED_TRACE(node.key()); - ASSERT_TRUE(node.has_children()); - size_t pos = 0; - for(ConstNodeRef child : node.cchildren()) - { - EXPECT_TRUE(child.is_val()) << "pos=" << pos; - EXPECT_FALSE(child.val_is_null()) << "pos=" << pos; // because it's quoted - EXPECT_EQ(child.val(), "") << "pos=" << pos; - EXPECT_NE(child.val(), nullptr) << "pos=" << pos; - EXPECT_EQ(child.val().len, 0u) << "pos=" << pos; - EXPECT_NE(child.val().str, nullptr) << "pos=" << pos; - ++pos; - } - }; - auto test_non_quoted_null = [](ConstNodeRef node){ - SCOPED_TRACE(node.key()); - ASSERT_TRUE(node.has_children()); - size_t pos = 0; - for(ConstNodeRef child : node.cchildren()) - { - EXPECT_TRUE(child.is_val()) << "pos=" << pos; - EXPECT_EQ(child.val(), "") << "pos=" << pos; - EXPECT_EQ(child.val(), nullptr) << "pos=" << pos; - EXPECT_EQ(child.val().len, 0u) << "pos=" << pos; - EXPECT_EQ(child.val().str, nullptr) << "pos=" << pos; - EXPECT_TRUE(child.val_is_null()) << "pos=" << pos; - ++pos; - } - }; - - { - SCOPED_TRACE("input tree"); - test_quoted_empty(tr["quoted"]); - // in the built tree, the values will be quoted and null - test_quoted_null(tr["quoted_null"]); - test_non_quoted_empty(tr["nonquoted"]); - test_non_quoted_null(tr["nonquoted_null"]); - } - - std::string yaml = emitrs_yaml(tr); - std::cout << yaml; - test_check_emit_check(to_csubstr(yaml), [&](Tree const &t){ - SCOPED_TRACE("output tree"); - test_quoted_empty(t["quoted"]); - // after a roundtrip, they will be nonnull, because the quotes win. - test_quoted_empty(t["quoted_null"]); - test_non_quoted_empty(t["nonquoted"]); - test_non_quoted_null(t["nonquoted_null"]); - }); -} - -CASE_GROUP(EMPTY_SCALAR) -{ -ADD_CASE_TO_GROUP("empty scalar, single quoted", - "''", - N(DQV, "") -); -} - -} // namespace yml -} // namespace c4 - -C4_SUPPRESS_WARNING_GCC_POP diff --git a/thirdparty/ryml/test/test_empty_seq.cpp b/thirdparty/ryml/test/test_empty_seq.cpp deleted file mode 100644 index 2b8bcab7b..000000000 --- a/thirdparty/ryml/test/test_empty_seq.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -CASE_GROUP(EMPTY_SEQ) -{ - -ADD_CASE_TO_GROUP("empty seq, explicit", -"[]", - SEQ -); - - -ADD_CASE_TO_GROUP("empty seq, explicit, whitespace", -" []", - SEQ -); - - -ADD_CASE_TO_GROUP("empty seq, multiline", -R"([ -] -)", - SEQ -); - -ADD_CASE_TO_GROUP("empty seq, multilines", -R"([ -# ksjdfkjhsdfkjhsdfkjh - - -] -)", - SEQ -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_explicit_key.cpp b/thirdparty/ryml/test/test_explicit_key.cpp deleted file mode 100644 index a9aefd59e..000000000 --- a/thirdparty/ryml/test/test_explicit_key.cpp +++ /dev/null @@ -1,419 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - - -TEST(explicit_key, test_suite_5WE3) -{ - csubstr yaml = R"( -? explicit key # Empty value -? | - block key -: - one # Explicit compact - - two # block value -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_NE(t.find_child(t.root_id(), "explicit key"), (size_t)NONE); - ASSERT_NE(t.find_child(t.root_id(), "block key\n"), (size_t)NONE); - EXPECT_EQ(t["explicit key"].val(), csubstr{}); - EXPECT_TRUE(t["block key\n"].is_seq()); - EXPECT_EQ(t["block key\n"][0], csubstr("one")); - EXPECT_EQ(t["block key\n"][1], csubstr("two")); - }); -} - - -TEST(explicit_key, test_suite_DFF7_v1) -{ - csubstr yaml = R"( -{ -? explicit: entry, -implicit: entry, -? -} -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_EQ(t.rootref().num_children(), 3u); - ASSERT_TRUE(t.rootref().has_child("explicit")); - EXPECT_EQ(t["explicit"].val(), csubstr("entry")); - ASSERT_TRUE(t.rootref().has_child("implicit")); - EXPECT_EQ(t["explicit"].val(), csubstr("entry")); - ASSERT_TRUE(t.rootref().has_child(csubstr{})); - EXPECT_EQ(t[csubstr{}].val(), csubstr{}); - }); -} - -TEST(explicit_key, test_suite_DFF7_v2) -{ - csubstr yaml = R"( -{ -? - key on next line -: - val on next line -, -? - # no key -: - val on next line -} -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_EQ(t.rootref().num_children(), 2u); - ASSERT_TRUE(t.rootref().has_child("key on next line")); - EXPECT_EQ(t[0].key(), "key on next line"); - EXPECT_EQ(t[0].val(), "val on next line"); - EXPECT_EQ(t[1].key(), csubstr{}); - EXPECT_EQ(t[1].val(), "val on next line"); - }); -} - - -TEST(explicit_key, test_suite_FRK4) -{ - csubstr yaml = R"( -{ - ? foo :, - : bar, -} -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_TRUE(t.rootref().has_child("foo")); - EXPECT_EQ(t["foo"].val(), csubstr{}); - ASSERT_TRUE(t.rootref().has_child(csubstr{})); - EXPECT_EQ(t[csubstr{}].val(), csubstr("bar")); - }); -} - - -TEST(explicit_key, test_suite_M2N8) -{ - csubstr yaml = R"( -- ? : x -- ? : -- ? : -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_seq()); - ASSERT_EQ(t.rootref().num_children(), 3u); - ASSERT_EQ(t[0].num_children(), 1u); - EXPECT_EQ(t[0][0].key(), csubstr{}); - EXPECT_EQ(t[0][0].val(), "x"); - ASSERT_EQ(t[1].num_children(), 1u); - EXPECT_EQ(t[1][0].key(), csubstr{}); - EXPECT_EQ(t[1][0].val(), csubstr{}); - ASSERT_EQ(t[2].num_children(), 1u); - EXPECT_EQ(t[2][0].key(), csubstr{}); - EXPECT_EQ(t[2][0].val(), csubstr{}); - }); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -CASE_GROUP(EXPLICIT_KEY) -{ -// -ADD_CASE_TO_GROUP("explicit key, last value missing", -R"( -? a -? b -? ---- !!set # test that we do not add any last item -? a -? b ---- !!set # test that we do add the last item -? a -? b -? -... -)", -N(STREAM, L{ - N(DOCMAP, L{ - N(KEYVAL, "a", {}), - N(KEYVAL, "b", {}), - N(KEYVAL, "", {}) - }), - N(DOCMAP, TL("!!set", L{ - N(KEYVAL, "a", {}), - N(KEYVAL, "b", {}), - })), - N(DOCMAP, TL("!!set", L{ - N(KEYVAL, "a", {}), - N(KEYVAL, "b", {}), - N(KEYVAL, "", {}) - })), - }) -); - -ADD_CASE_TO_GROUP("explicit key, ambiguity 2EBW", -R"( -a!"#$%&'()*+,-./09:;<=>?@AZ[\]^_`az{|}~: safe -?foo: safe question mark -:foo: safe colon --foo: safe dash -this is#not: a comment -)", -L{ - N("a!\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~", "safe"), - N("?foo", "safe question mark"), - N(":foo", "safe colon"), - N("-foo", "safe dash"), - N("this is#not", "a comment"), -}); - -ADD_CASE_TO_GROUP("explicit key, ambiguity 2EBW, expl", -R"({ - a!"#$%&'()*+-./09:;<=>?@AZ[\]^_`az{|~: safe, - ?foo: safe question mark, - :foo: safe colon, - -foo: safe dash, - this is#not: a comment, -})", -L{ - N("a!\"#$%&'()*+-./09:;<=>?@AZ[\\]^_`az{|~", "safe"), - N("?foo", "safe question mark"), - N(":foo", "safe colon"), - N("-foo", "safe dash"), - N("this is#not", "a comment"), -}); - -ADD_CASE_TO_GROUP("explicit key, ambiguity 2EBW, impl seq", -R"( -- a!"#$%&'()*+,-./09:;<=>?@AZ[\]^_`az{|}~ -- ?foo -- :foo -- -foo -- this is#not:a comment -)", -L{ - N("a!\"#$%&'()*+,-./09:;<=>?@AZ[\\]^_`az{|}~"), - N("?foo"), - N(":foo"), - N("-foo"), - N("this is#not:a comment"), -}); - -ADD_CASE_TO_GROUP("explicit key, ambiguity 2EBW, expl seq", -R"([ - a!"#$%&'()*+-./09:;<=>?@AZ[\^_`az{|}~, - ?foo, - :foo, - -foo, - this is#not:a comment, -])", -L{ - N("a!\"#$%&'()*+-./09:;<=>?@AZ[\\^_`az{|}~"), - N("?foo"), - N(":foo"), - N("-foo"), - N("this is#not:a comment"), -}); - -ADD_CASE_TO_GROUP("explicit key with line break in between", -R"( -? an explicit key -: its value -)", - L{N("an explicit key", "its value")} -); - -ADD_CASE_TO_GROUP("explicit key 2nd, inside explicit map", -R"( -{ - a simple key: a value, - ? an explicit key: another value, -} -)", - L{ - N("a simple key", "a value"), - N("an explicit key", "another value"), - } -); - -ADD_CASE_TO_GROUP("explicit key 1st, inside explicit map", -R"( -{ - ? an explicit key: another value, - a simple key: a value, -} -)", - L{ - N("an explicit key", "another value"), - N("a simple key", "a value"), - } -); - -ADD_CASE_TO_GROUP("explicit key 2nd", -R"( -a simple key: a value -? an explicit key: another value -)", - L{ - N("a simple key", "a value"), - N("an explicit key", "another value"), - } -); - -ADD_CASE_TO_GROUP("explicit key 1st", -R"( -? an explicit key: another value -a simple key: a value -)", - L{ - N("an explicit key", "another value"), - N("a simple key", "a value"), - } -); - -ADD_CASE_TO_GROUP("explicit key nested in a map, 1st", -R"( -map: - ? an explicit key: another value - a simple key: a value -? an explicit key deindented: its value -)", - L{ - N("map", L{ - N("an explicit key", "another value"), - N("a simple key", "a value"), - }), - N("an explicit key deindented", "its value") - } -); - -ADD_CASE_TO_GROUP("explicit key nested in a seq, 1st", -R"( -- ? an explicit key: another value - a simple key: a value -- ? another explicit key: its value -)", - L{ - N(L{ - N("an explicit key", "another value"), - N("a simple key", "a value"), - }), - N(L{N("another explicit key", "its value")}) - } -); - -ADD_CASE_TO_GROUP("explicit block key, literal, clip", -R"(? | - This is a key - that has multiple lines - -: and this is its value -)", - L{ - N(QK, "This is a key\nthat has multiple lines\n", "and this is its value") - } -); - -ADD_CASE_TO_GROUP("explicit block key, literal, keep", -R"(? |+ - This is a key - that has multiple lines - -: and this is its value -)", - L{ - N(QK, "This is a key\nthat has multiple lines\n\n", "and this is its value") - } -); - -ADD_CASE_TO_GROUP("explicit block key, literal, strip", -R"(? |- - This is a key - that has multiple lines - -: and this is its value -)", - L{ - N(QK, "This is a key\nthat has multiple lines", "and this is its value") - } -); - -ADD_CASE_TO_GROUP("explicit block key, folded, clip", -R"(? > - This is a key - that has multiple lines - -: and this is its value -)", - L{ - N(QK, "This is a key that has multiple lines\n", "and this is its value") - } -); - -ADD_CASE_TO_GROUP("explicit block key, folded, keep", -R"(? >+ - This is a key - that has multiple lines - -: and this is its value -)", - L{ - N(QK, "This is a key that has multiple lines\n\n", "and this is its value") - } -); - -ADD_CASE_TO_GROUP("explicit block key, folded, strip", -R"(? >- - This is a key - that has multiple lines - -: and this is its value -)", - L{ - N(QK, "This is a key that has multiple lines", "and this is its value") - } -); - -ADD_CASE_TO_GROUP("explicit key, missing val 7W2P", -R"( -? a -? b -c: -? d -e: -)", -N(MAP, L{ - N(KEYVAL, "a", {}), - N(KEYVAL, "b", {}), - N(KEYVAL, "c", {}), - N(KEYVAL, "d", {}), - N(KEYVAL, "e", {}), - }) -); - -ADD_CASE_TO_GROUP("explicit key, missing val ZWK4", -R"( -a: 1 -? b -&anchor c: 3 -? d -!!str e: 4 -? f -)", -N(MAP, L{ - N("a", "1"), - N(KEYVAL, "b", {}), - N("c", AR(KEYANCH, "anchor"), "3"), - N(KEYVAL, "d", {}), - N(TS("!!str", "e"), "4"), - N(KEYVAL, "f", {}), - }) -); - -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_generic_map.cpp b/thirdparty/ryml/test/test_generic_map.cpp deleted file mode 100644 index 273c05cb4..000000000 --- a/thirdparty/ryml/test/test_generic_map.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - - -CASE_GROUP(GENERIC_MAP) -{ - -ADD_CASE_TO_GROUP("generic map", -R"( -a simple key: a value # The KEY token is produced here. -? a complex key -: another value -a mapping: - key 1: value 1 - key 2: value 2 -a sequence: - - item 1 - - item 2 -)", - L{ - N("a simple key", "a value"), - N("a complex key", "another value"), - N("a mapping", L{N("key 1", "value 1"), N("key 2", "value 2")}), - N("a sequence", L{N("item 1"), N("item 2")}), - } -); - - -ADD_CASE_TO_GROUP("seq nested in map", -R"( -items: - - part_no: A4786 - descrip: Water Bucket (Filled) - price: 1.47 - quantity: 4 - - part_no: E1628 - descrip: High Heeled "Ruby" Slippers - size: 8 - price: 133.7 - quantity: 1 -)", -L{ - N{"items", L{ - N{L{N{"part_no", "A4786"}, - N{"descrip", "Water Bucket (Filled)"}, - N{"price", "1.47"}, - N{"quantity", "4"},}}, - N{L{N{"part_no", "E1628"}, - N{"descrip", "High Heeled \"Ruby\" Slippers"}, - N{"size", "8"}, - N{"price", "133.7"}, - N{"quantity", "1"},}}}}, - } -); - -ADD_CASE_TO_GROUP("seq nested in map, v2", -R"( -items: - - - part_no: A4786 - descrip: Water Bucket (Filled) - price: 1.47 - quantity: 4 - - - part_no: E1628 - descrip: High Heeled "Ruby" Slippers - size: 8 - price: 133.7 - quantity: 1 -)", -L{ - N{"items", L{ - N{L{N{"part_no", "A4786"}, - N{"descrip", "Water Bucket (Filled)"}, - N{"price", "1.47"}, - N{"quantity", "4"},}}, - N{L{N{"part_no", "E1628"}, - N{"descrip", "High Heeled \"Ruby\" Slippers"}, - N{"size", "8"}, - N{"price", "133.7"}, - N{"quantity", "1"},}}}}, - } -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_generic_seq.cpp b/thirdparty/ryml/test/test_generic_seq.cpp deleted file mode 100644 index 45f9c1d3e..000000000 --- a/thirdparty/ryml/test/test_generic_seq.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -CASE_GROUP(GENERIC_SEQ) -{ - -ADD_CASE_TO_GROUP("generic seq v0", -R"( -- item 1 -- item 2 -- - item 3.1 - - item 3.2 -- key 1: value 1 - key 2: value 2 -)", - L{ - N("item 1"), - N("item 2"), - N(L{N("item 3.1"), N("item 3.2")}), - N(L{N("key 1", "value 1"), N("key 2", "value 2")}) - } -); - -ADD_CASE_TO_GROUP("generic seq v1", -R"( -- item 1 -- item 2 -- - - item 3.1 - - item 3.2 -- - key 1: value 1 - key 2: value 2 -)", - L{ - N("item 1"), - N("item 2"), - N(L{N("item 3.1"), N("item 3.2")}), - N(L{N("key 1", "value 1"), N("key 2", "value 2")}) - } -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_github_issues.cpp b/thirdparty/ryml/test/test_github_issues.cpp deleted file mode 100644 index 3c12307fc..000000000 --- a/thirdparty/ryml/test/test_github_issues.cpp +++ /dev/null @@ -1,590 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -TEST(github, 268) -{ - Tree tree = parse_in_arena(R"( - list: - - &bar bar - map: - node: *bar - )"); - tree.resolve(); - auto root = tree.rootref(); - ASSERT_TRUE(root["map"].is_map()); - ASSERT_TRUE(root["map"].has_child("node")); - ASSERT_EQ(root["map"]["node"], "bar"); -} - -TEST(github, 277) -{ - Tree tree = parse_in_arena(R"( - A: &A - V: 3 - W: 4 - B: - <<: *A - V: 5 - X: 6 - )"); - const char *keys[] = {"V", "W", "X"}; - const char *vals[] = {"5", "4", "6"}; - tree.resolve(); - auto root = tree.rootref(); - ASSERT_TRUE(root["B"].is_map()); - size_t num_childs = root["B"].num_children(); - size_t child = 0; - ASSERT_EQ(num_childs, 3); - for (const auto node : root["B"].children()) - { - EXPECT_EQ(node.key(), csubstr(keys[child], 1)); - EXPECT_EQ(node.val(), csubstr(vals[child], 1)); - child++; - } - // test whether the tree is corrupted - test_invariants(tree); - child = num_childs; - for (size_t n = tree.last_child(root["B"].id()); n != NONE; n = tree.prev_sibling(n)) - { - ASSERT_NE(child, 0); - EXPECT_EQ(tree.key(n), csubstr(keys[child - 1], 1)); - child--; - } -} - - -TEST(github, 78) -{ - Tree t = parse_in_arena("{foo: 1, bar: [2, 3]}"); - EXPECT_EQ(t["foo"].val(), "1"); - EXPECT_EQ(t["bar"][0].val(), "2"); - EXPECT_EQ(t["bar"][1].val(), "3"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(github, 60) -{ - Tree tree = parse_in_arena(R"( - traits: - roleBonuses: - - bonus: 5 - bonusText: - de: Bonus auf die Virusstärke von Relikt- - und Datenanalysatoren - en: bonus to Relic and Data - Analyzer virus strength - fr: de bonus à la puissance du virus des analyseurs - de reliques et des analyseurs de données - ja: 遺物アナライザーデータアナライザーのウイルス強度が増加 - ru: повышается степень опасности вирусов, применяемых в комплексах - анализа данных и комплексах анализа - артефактов - zh: 遗迹分析仪数据分析仪病毒强度加成 - importance: 1 - unitID: 139 -)"); - auto root = tree.rootref(); - ASSERT_TRUE(root.is_map()); - ASSERT_TRUE(root.has_child("traits")); - auto rb = root["traits"]["roleBonuses"][0]; - ASSERT_TRUE(rb.valid()); - EXPECT_EQ(rb["bonus"].val(), "5"); - auto txt = rb["bonusText"]; - ASSERT_TRUE(txt.valid()); - ASSERT_TRUE(txt.is_map()); - EXPECT_TRUE(txt.has_child("de")); - EXPECT_TRUE(txt.has_child("en")); - EXPECT_TRUE(txt.has_child("fr")); - EXPECT_TRUE(txt.has_child("ja")); - EXPECT_TRUE(txt.has_child("ru")); - EXPECT_TRUE(txt.has_child("zh")); - EXPECT_EQ(txt["de"].val(), "Bonus auf die Virusstärke von Relikt- und Datenanalysatoren"); - EXPECT_EQ(txt["en"].val(), "bonus to Relic and Data Analyzer virus strength"); - EXPECT_EQ(txt["fr"].val(), "de bonus à la puissance du virus des analyseurs de reliques et des analyseurs de données"); - EXPECT_EQ(txt["ja"].val(), "遺物アナライザーデータアナライザーのウイルス強度が増加"); - EXPECT_EQ(txt["ru"].val(), "повышается степень опасности вирусов, применяемых в комплексах анализа данных и комплексах анализа артефактов"); - EXPECT_EQ(txt["zh"].val(), "遗迹分析仪数据分析仪病毒强度加成"); - - - tree = parse_in_arena(R"(208: - basePrice: 3000.0 - description: - de: Ursprünglich als Rakete für den Fangschuss entworfen, um einem beschädigten - Schiff den Todesstoß zu geben, hat die Inferno Heavy Missile seither eine - Reihe technischer Upgrades durchlaufen. Die neueste Version hat eine leichtere - Sprengladung als das Original, aber stark verbesserte Lenksysteme. - en: Originally designed as a 'finisher' - the killing blow to a crippled ship - - the Inferno heavy missile has since gone through various technological - upgrades. The latest version has a lighter payload than the original, - but much improved guidance systems. - fr: Conçu à l'origine pour donner le coup de grâce, le missile lourd Inferno - a depuis subi de nombreuses améliorations techniques. La dernière version - emporte une charge utile réduite par rapport à l'originale, mais est dotée - de systèmes de guidage améliorés. - ja: 元々「フィニッシャー」―大破した船にとどめを刺す兵器として設計されたインフェルノヘビーミサイルは、以来各種の技術改良を経てきた。現行型は初期型より軽い弾頭を採用しているが、それを補って余りある優れた誘導システムを持つ。 - ru: Тяжелая ракета Inferno изначально была спроектирована как «оружие последнего - удара» для уничтожения подбитых кораблей. С тех пор было выпущено несколько - ее модификаций. В последней модификации используется заряд меньшей мощности, - но более совершенная система наведения. - zh: 炼狱重型导弹历经多种技术改良,原本被设计为给予落魄敌舰最后一击的“终结者”角色。相比原型,最新版导弹载荷较轻,但装配了大幅改进的制导系统。 - graphicID: 20048 - groupID: 385 - iconID: 188 - marketGroupID: 924 - mass: 1000.0 - name: - de: Inferno Heavy Missile - en: Inferno Heavy Missile - fr: Missile lourd Inferno - ja: インフェルノヘビーミサイル - ru: Inferno Heavy Missile - zh: 炼狱重型导弹 - portionSize: 100 - published: true - radius: 300.0 - volume: 0.03 -)"); - root = tree.rootref()["208"]; - EXPECT_EQ(root["description"]["ja"].val(), "元々「フィニッシャー」―大破した船にとどめを刺す兵器として設計されたインフェルノヘビーミサイルは、以来各種の技術改良を経てきた。現行型は初期型より軽い弾頭を採用しているが、それを補って余りある優れた誘導システムを持つ。"); - EXPECT_EQ(root["name"]["ja"].val(), "インフェルノヘビーミサイル"); - EXPECT_EQ(root["name"]["zh"].val(), "炼狱重型导弹"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(github, 31) -{ - Tree tree; - NodeRef r = tree.rootref(); - r |= MAP; - - auto meas = r["meas"]; - meas |= MAP; - - auto plist = meas["createParameterList"]; - plist |= SEQ; - - { - auto lumi = plist.append_child(); - lumi << "Lumi"; - EXPECT_TRUE(lumi.is_val()); - } - - { - auto lumi = plist.append_child(); - lumi |= MAP; - lumi["value"] << 1; - lumi["relErr"] << 0.1; - EXPECT_TRUE(lumi.is_map()); - } - - { - ExpectError::check_assertion(&tree, [&](){ - auto lumi = plist.append_child(); - lumi << "Lumi"; - lumi |= MAP; - }); - } - - { - ExpectError::check_assertion(&tree, [&](){ - auto lumi = plist.append_child(); - lumi << "Lumi"; - lumi |= SEQ; - }); - } - - { - ExpectError::check_assertion(&tree, [&](){ - auto lumi = plist.append_child(); - lumi |= MAP; - lumi << "Lumi"; - }); - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -CASE_GROUP(GITHUB_ISSUES) -{ - -ADD_CASE_TO_GROUP("github3-problem1", -R"( -translation: [-2, -2, 5])", -L{N("translation", L{N("-2"), N("-2"), N("5")})} -); - -// these must work without quotes -ADD_CASE_TO_GROUP("github3-problem2-ex1", -R"( -audio resource: -)", -L{N(KEYVAL, "audio resource", /*"~"*/{})} -); -ADD_CASE_TO_GROUP("github3-problem2-ex2", -R"( -audio resource: -more: - example: y -)", -L{N(KEYVAL, "audio resource", /*"~"*/{}), N("more", L{N("example", "y")})} -); - -ADD_CASE_TO_GROUP("github3-problem3", -R"(component: - type: perspective camera component - some_data: {} # this was working - data: - {} # but this was not working -)", -L{N("component", L{ - N("type", "perspective camera component"), - N(KEYMAP, "some_data", L{}), - N(KEYMAP, "data", L{}) - } -)} -); - -/* THIS IS CAUSING VS TO CRASH OUT OF HEAP SPACE -ADD_CASE_TO_GROUP("github3", -R"( -universe: - objects: - object: - uuid: A7AB039C0EF3A74480A1B398247039A7 - components: - - component: - type: name component - data: - object name: Root Node - - component: - type: transform component - data: - translation: [-2, -2, 5] - rotation: [0, 0, 0, 1] - scaling: [1, 1, 1] - - component: - type: perspective camera component - data: - {} - - component: - type: mesh component - data: - mesh resource: TODO - - component: - type: lua script component - data: - {} - - component: - type: audio component - data: - audio resource: '' - type: 0 - current sample: 184102 - spatialized: true - children: - - object: - uuid: E1C364A925D649408E83C8EEF5179A87 - components: - - component: - type: name component - data: - object name: Prepend - children: - [] - - object: - uuid: 377DBA885AF4CD42B8A56BB3471F60E5 - components: - - component: - type: name component - data: - object name: pivot - children: - [] - - object: - uuid: 6DD1835797DADB4F95232CE7E9DE41BA - components: - - component: - type: name component - data: - object name: Append - children: - [] -)", - L{N("universe", L{ - N("objects", L{ - N("object", L{ - N("uuid", "A7AB039C0EF3A74480A1B398247039A7"), - N("components", L{ - N(L{N("component", L{N("type", "name component"), N("data", L{N("object name", "Root Node")}), }), }), - N(L{N("component", L{N("type", "transform component"), N("data", L{N("translation", L{N("-2"), N("-2"), N("5")}), N("rotation", L{N("0"), N("0"), N("0"), N("1")}), N("scaling", L{N("1"), N("1"), N("1")}),}), }), }), - N(L{N("component", L{N("type", "perspective camera component"), N(KEYMAP, "data", L{}), }), }), - N(L{N("component", L{N("type", "mesh component"), N("data", L{N("mesh resource", "TODO")}), }), }), - N(L{N("component", L{N("type", "lua script component"), N(KEYMAP, "data", L{}), }), }), - N(L{N("component", L{N("type", "audio component"), N("data", L{N("audio resource", ""), N("type", "0"), N("current sample", "184102"), N("spatialized", "true"), }), }), }), // component - }), // components - N("children", L{ - N(L{N("object", L{ - N("uuid", "E1C364A925D649408E83C8EEF5179A87"), - N("components", L{N(L{N("component", L{N("type", "name component"), N("data", L{N("object name", "Prepend")}), }), }), }), - N(KEYSEQ, "children", L{}), - }), }), // object - N(L{N("object", L{ - N("uuid", "377DBA885AF4CD42B8A56BB3471F60E5"), - N("components", L{N(L{N("component", L{N("type", "name component"), N("data", L{N("object name", "pivot")}), }), }), }), - N(KEYSEQ, "children", L{}), - }), }), // object - N(L{N("object", L{ - N("uuid", "6DD1835797DADB4F95232CE7E9DE41BA"), - N("components", L{N(L{N("component", L{N("type", "name component"), N("data", L{N("object name", "Append")}), }), }), }), - N(KEYSEQ, "children", L{}), - }), }), // object - }), // children - }), // object - }) // objects - }) // universe - } -); -*/ - -ADD_CASE_TO_GROUP("github6-problem1", -R"( -- UQxRibHKEDI: - - 0.mp4 - - 1.mp4 - - 2.mp4 - - 3.mp4 -- DcYsg8VFdC0: - - 0.mp4 - - 1.mp4 - - 2.mp4 - - 3.mp4 -- Yt3ymqZXzLY: - - 0.mp4 - - 1.mp4 - - 2.mp4 - - 3.mp4 -)", -L{ -N(L{N("UQxRibHKEDI", L{N("0.mp4"), N("1.mp4"), N("2.mp4"), N("3.mp4")})}), -N(L{N("DcYsg8VFdC0", L{N("0.mp4"), N("1.mp4"), N("2.mp4"), N("3.mp4")})}), -N(L{N("Yt3ymqZXzLY", L{N("0.mp4"), N("1.mp4"), N("2.mp4"), N("3.mp4")})}), -} -); - -ADD_CASE_TO_GROUP("github6", -R"(videos: -- UQxRibHKEDI: - - 0.mp4 - - 1.mp4 - - 2.mp4 - - 3.mp4 -- DcYsg8VFdC0: - - 0.mp4 - - 1.mp4 - - 2.mp4 - - 3.mp4 -- Yt3ymqZXzLY: - - 0.mp4 - - 1.mp4 - - 2.mp4 - - 3.mp4 -)", -L{N("videos", L{ -N(L{N("UQxRibHKEDI", L{N("0.mp4"), N("1.mp4"), N("2.mp4"), N("3.mp4")})}), -N(L{N("DcYsg8VFdC0", L{N("0.mp4"), N("1.mp4"), N("2.mp4"), N("3.mp4")})}), -N(L{N("Yt3ymqZXzLY", L{N("0.mp4"), N("1.mp4"), N("2.mp4"), N("3.mp4")})}), -})} -); - -ADD_CASE_TO_GROUP("github34/ex1", -R"( -# correct: -MessageID1: 'MapRegion_HyrulePrairie' -MessageID2: "MapRegion_HyrulePrairie" -MessageID3: 'MapRegion_HyrulePrairie' -MessageID4: "MapRegion_HyrulePrairie" -# incorrect: uninitialised memory? -MessageID5: 'MapRegion_HyrulePrairie' -MessageID6: "MapRegion_HyrulePrairie" -MessageID7: 'MapRegion_HyrulePrairie' -MessageID8: "MapRegion_HyrulePrairie" -MessageID9: 'MapRegion_HyrulePrairie' -MessageID0: "MapRegion_HyrulePrairie" -)", -L{ - N(QV, "MessageID1", "MapRegion_HyrulePrairie"), - N(QV, "MessageID2", "MapRegion_HyrulePrairie"), - N(QV, "MessageID3", "MapRegion_HyrulePrairie"), - N(QV, "MessageID4", "MapRegion_HyrulePrairie"), - N(QV, "MessageID5", "MapRegion_HyrulePrairie"), - N(QV, "MessageID6", "MapRegion_HyrulePrairie"), - N(QV, "MessageID7", "MapRegion_HyrulePrairie"), - N(QV, "MessageID8", "MapRegion_HyrulePrairie"), - N(QV, "MessageID9", "MapRegion_HyrulePrairie"), - N(QV, "MessageID0", "MapRegion_HyrulePrairie"), -} -); - -ADD_CASE_TO_GROUP("github34/ex2", -R"( -# correct: -- MessageID1: 'MapRegion_HyrulePrairie' -- MessageID2: "MapRegion_HyrulePrairie" -- MessageID3: 'MapRegion_HyrulePrairie' -- MessageID4: "MapRegion_HyrulePrairie" -# incorrect: uninitialised memory? -- MessageID5: 'MapRegion_HyrulePrairie' -- MessageID6: "MapRegion_HyrulePrairie" -- MessageID7: 'MapRegion_HyrulePrairie' -- MessageID8: "MapRegion_HyrulePrairie" -- MessageID9: 'MapRegion_HyrulePrairie' -- MessageID0: "MapRegion_HyrulePrairie" -)", -L{ - N(L{N(QV, "MessageID1", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID2", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID3", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID4", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID5", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID6", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID7", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID8", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID9", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID0", "MapRegion_HyrulePrairie")}), -} -); - -ADD_CASE_TO_GROUP("github34", -R"( -# incorrect: uninitialised memory? -- MessageID1: 'MapRegion_HyrulePrairie' -- MessageID2: "MapRegion_HyrulePrairie" - -# incorrect: uninitialised memory? -- MessageID3: 'MapRegion_HyrulePrairie ' -- MessageID4: "MapRegion_HyrulePrairie " - -# incorrect: for some reason the ' is included in the string -- MessageID5: 'MapRegion_HyrulePrairie ' -- MessageID6: 'MapRegion_HyrulePrairie ' -- MessageID7: "MapRegion_HyrulePrairie " -- MessageID8: "MapRegion_HyrulePrairie " - -# incorrect: same issue -- MessageID9: 'MapRegion_HyrulePrairie ' -- MessageID10: "MapRegion_HyrulePrairie " - -# incorrect: still has the trailing quote -- MessageID11: 'MapRegion_HyrulePrairie' -- MessageID12: "MapRegion_HyrulePrairie" - -# the string is parsed correctly in this case -- key1: true1 - MessageID1: 'MapRegion_HyrulePrairie1 ' -- key2: true2 - MessageID2: "MapRegion_HyrulePrairie2 " -)", -L{ - N(L{N(QV, "MessageID1", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID2", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID3", "MapRegion_HyrulePrairie ")}), - N(L{N(QV, "MessageID4", "MapRegion_HyrulePrairie ")}), - N(L{N(QV, "MessageID5", "MapRegion_HyrulePrairie ")}), - N(L{N(QV, "MessageID6", "MapRegion_HyrulePrairie ")}), - N(L{N(QV, "MessageID7", "MapRegion_HyrulePrairie ")}), - N(L{N(QV, "MessageID8", "MapRegion_HyrulePrairie ")}), - N(L{N(QV, "MessageID9", "MapRegion_HyrulePrairie ")}), - N(L{N(QV, "MessageID10", "MapRegion_HyrulePrairie ")}), - N(L{N(QV, "MessageID11", "MapRegion_HyrulePrairie")}), - N(L{N(QV, "MessageID12", "MapRegion_HyrulePrairie")}), - N(L{N("key1", "true1"), N(QV, "MessageID1", "MapRegion_HyrulePrairie1 ")}), - N(L{N("key2", "true2"), N(QV, "MessageID2", "MapRegion_HyrulePrairie2 ")}), -} -); - -ADD_CASE_TO_GROUP("github35/expected_error11", EXPECT_PARSE_ERROR, -R"( -# *segfault* // not anymore! -- key1: true1 - MessageID1: 'MapRegion_HyrulePrairie1 ' -)", - LineCol(4, 1) -); - -ADD_CASE_TO_GROUP("github35/expected_error12", EXPECT_PARSE_ERROR, -R"( -# *segfault* // not anymore! -- key2: true2 - MessageID2: "MapRegion_HyrulePrairie2 " -)", - LineCol(4, 1) -); - -ADD_CASE_TO_GROUP("github35/expected_error21", EXPECT_PARSE_ERROR, -R"( -# *segfault* // not anymore! -- key1: true1 - MessageID1: 'MapRegion_HyrulePrairie1 ' -)", - LineCol(4, 15) -); - -ADD_CASE_TO_GROUP("github35/expected_error22", EXPECT_PARSE_ERROR, -R"( -# *segfault* // not anymore! -- key2: true2 - MessageID2: "MapRegion_HyrulePrairie2 " -)", - LineCol(4, 15) -); - -ADD_CASE_TO_GROUP("github128/1", RESOLVE_REFS | EXPECT_PARSE_ERROR, "a: *invalid"); -ADD_CASE_TO_GROUP("github128/2", RESOLVE_REFS/* | HAS_PARSE_ERROR*/, "*", N(DOCVAL, "*")); - -ADD_CASE_TO_GROUP("github129", RESOLVE_REFS, R"( -ref: &ref ref_val -a: *ref # resolve the reference -b: '*ref' # don't resolve, it's just a string -c: "*ref" # don't resolve, it's just a string -d: > # don't resolve, it's just a string - *ref -e: >- # don't resolve, it's just a string - *ref -f: >+ # don't resolve, it's just a string - *ref -g: | # don't resolve, it's just a string - *ref -h: |- # don't resolve, it's just a string - *ref -i: |+ # don't resolve, it's just a string - *ref -)", L{ - N("ref", "ref_val"), - N("a", "ref_val"), // this should be resolved - N(QV, "b", "*ref"), // this should not be resolved (just a string) - N(QV, "c", "*ref"), // this should not be resolved (just a string) - N(QV, "d", "*ref\n"), // this should not be resolved (just a string) - N(QV, "e", "*ref"), // this should not be resolved (just a string) - N(QV, "f", "*ref\n"), // this should not be resolved (just a string) - N(QV, "g", "*ref\n"), // this should not be resolved (just a string) - N(QV, "h", "*ref"), // this should not be resolved (just a string) - N(QV, "i", "*ref\n"), // this should not be resolved (just a string) - } -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_group.cpp b/thirdparty/ryml/test/test_group.cpp deleted file mode 100644 index 77ef5fa23..000000000 --- a/thirdparty/ryml/test/test_group.cpp +++ /dev/null @@ -1,732 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/detail/print.hpp" -#endif -#include "test_group.hpp" -#include "test_case.hpp" -#include -#include -#include - -#define RYML_NFO (RYML_DBG || 0) - -//----------------------------------------------------------------------------- -namespace c4 { -namespace yml { - -void YmlTestCase::_test_parse_using_ryml(CaseDataLineEndings *cd) -{ - #ifdef RYML_NFO - std::cout << "---------------\n"; - std::cout << c->src; - std::cout << "---------------\n"; - #endif - - if(c->flags & EXPECT_PARSE_ERROR) - { - auto flags = c->flags; - ExpectError::do_check(&cd->parsed_tree, [this, cd, flags](){ - parse_in_place(c->fileline, cd->src, &cd->parsed_tree); - if(flags & RESOLVE_REFS) - cd->parsed_tree.resolve(); - #ifdef RYML_DBG - // if this point was reached, then it means that the expected - // error failed to occur. So print debugging info. - std::cout << "failed to catch expected error while parsing.\nPARSED TREE:\n"; - print_tree(cd->parsed_tree); - #endif - }, c->expected_location); - return; - } - - cd->parsed_tree.clear(); - parse_in_place(c->fileline, cd->src, &cd->parsed_tree); - - #ifdef RYML_NFO - std::cout << "REF TREE:\n"; - print_tree(c->root); - std::cout << "PARSED TREE:\n"; - print_tree(cd->parsed_tree); - #endif - - { - SCOPED_TRACE("checking tree invariants of unresolved parsed tree"); - test_invariants(cd->parsed_tree); - } - { - SCOPED_TRACE("checking node invariants of unresolved parsed tree"); - test_invariants(cd->parsed_tree.rootref()); - } - - if(c->flags & RESOLVE_REFS) - { - cd->parsed_tree.resolve(); - #ifdef RYML_NFO - std::cout << "resolved tree!!!\n"; - print_tree(cd->parsed_tree); - #endif - { - SCOPED_TRACE("checking tree invariants of resolved parsed tree"); - test_invariants(cd->parsed_tree); - } - { - SCOPED_TRACE("checking node invariants of resolved parsed tree"); - test_invariants(cd->parsed_tree.rootref()); - } - } - - { - SCOPED_TRACE("comparing parsed tree to ref tree"); - EXPECT_GE(cd->parsed_tree.capacity(), c->root.reccount()); - EXPECT_EQ(cd->parsed_tree.size(), c->root.reccount()); - c->root.compare(cd->parsed_tree.rootref()); - } - - if(c->flags & RESOLVE_REFS) - { - cd->parsed_tree.reorder(); - #ifdef RYML_NFO - std::cout << "reordered tree!!!\n"; - print_tree(cd->parsed_tree); - #endif - { - SCOPED_TRACE("checking tree invariants of reordered parsed tree after resolving"); - test_invariants(cd->parsed_tree); - } - { - SCOPED_TRACE("checking node invariants of reordered parsed tree after resolving"); - test_invariants(cd->parsed_tree.rootref()); - } - - { - SCOPED_TRACE("comparing parsed tree to ref tree"); - EXPECT_GE(cd->parsed_tree.capacity(), c->root.reccount()); - EXPECT_EQ(cd->parsed_tree.size(), c->root.reccount()); - c->root.compare(cd->parsed_tree.rootref()); - } - } -} - - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_yml_stdout(CaseDataLineEndings *cd) -{ - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit(cd); - cd->numbytes_stdout = emit_yaml(cd->parsed_tree); - EXPECT_EQ(cd->numbytes_stdout, cd->emitted_yml.size()); -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_json_stdout(CaseDataLineEndings *cd) -{ - if(!(c->flags & JSON_ALSO)) - return; - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit_json(cd); - cd->numbytes_stdout_json = emit_json(cd->parsed_tree); - EXPECT_EQ(cd->numbytes_stdout_json, cd->emitted_json.size()); -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_yml_cout(CaseDataLineEndings *cd) -{ - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit(cd); - std::cout << cd->parsed_tree; - std::cout << cd->parsed_tree.rootref(); -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_json_cout(CaseDataLineEndings *cd) -{ - if(!(c->flags & JSON_ALSO)) - return; - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit_json(cd); - std::cout << as_json(cd->parsed_tree); - std::cout << as_json(cd->parsed_tree.rootref()); -} - - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_yml_stringstream(CaseDataLineEndings *cd) -{ - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit(cd); - { - std::stringstream ss; - ss << cd->parsed_tree; - std::string actual = ss.str(); - EXPECT_EQ(actual, cd->emit_buf); - } - { - std::stringstream ss; - ss << cd->parsed_tree.rootref(); - std::string actual = ss.str(); - EXPECT_EQ(actual, cd->emit_buf); - } -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_json_stringstream(CaseDataLineEndings *cd) -{ - if(!(c->flags & JSON_ALSO)) - return; - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit_json(cd); - { - std::stringstream ss; - ss << as_json(cd->parsed_tree); - std::string actual = ss.str(); - EXPECT_EQ(actual, cd->emitjson_buf); - } - { - std::stringstream ss; - ss << as_json(cd->parsed_tree.rootref()); - std::string actual = ss.str(); - EXPECT_EQ(actual, cd->emitjson_buf); - } -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_yml_ofstream(CaseDataLineEndings *cd) -{ - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit(cd); - { - auto fn = fs::tmpnam(); - { - std::ofstream f(fn, std::ios::binary); - f << cd->parsed_tree; - } - auto actual = fs::file_get_contents(fn.c_str()); - EXPECT_EQ(actual, cd->emit_buf); - fs::rmfile(fn.c_str()); - } - { - auto fn = fs::tmpnam(); - { - std::ofstream f(fn, std::ios::binary); - f << cd->parsed_tree.rootref(); - } - auto actual = fs::file_get_contents(fn.c_str()); - EXPECT_EQ(actual, cd->emit_buf); - fs::rmfile(fn.c_str()); - } -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_json_ofstream(CaseDataLineEndings *cd) -{ - if(!(c->flags & JSON_ALSO)) - return; - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit_json(cd); - { - auto fn = fs::tmpnam(); - { - std::ofstream f(fn, std::ios::binary); - f << as_json(cd->parsed_tree); - } - auto actual = fs::file_get_contents(fn.c_str()); - EXPECT_EQ(actual, cd->emitjson_buf); - fs::rmfile(fn.c_str()); - } - { - auto fn = fs::tmpnam(); - { - std::ofstream f(fn, std::ios::binary); - f << as_json(cd->parsed_tree.rootref()); - } - auto actual = fs::file_get_contents(fn.c_str()); - EXPECT_EQ(actual, cd->emitjson_buf); - fs::rmfile(fn.c_str()); - } -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_yml_string(CaseDataLineEndings *cd) -{ - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit(cd); - auto em = emitrs_yaml(cd->parsed_tree, &cd->emit_buf); - EXPECT_EQ(em.len, cd->emit_buf.size()); - EXPECT_EQ(em.len, cd->numbytes_stdout); - #ifdef RYML_NFO - std::cout << em; - #endif -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emit_json_string(CaseDataLineEndings *cd) -{ - if(!(c->flags & JSON_ALSO)) - return; - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit_json(cd); - auto em = emitrs_json(cd->parsed_tree, &cd->emit_buf); - EXPECT_EQ(em.len, cd->emitjson_buf.size()); - EXPECT_EQ(em.len, cd->numbytes_stdout_json); - #ifdef RYML_NFO - std::cout << em; - #endif -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emitrs(CaseDataLineEndings *cd) -{ - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - using vtype = std::vector; - using stype = std::string; - vtype vv, v = emitrs_yaml(cd->parsed_tree); - stype ss, s = emitrs_yaml(cd->parsed_tree); - EXPECT_EQ(to_csubstr(v), to_csubstr(s)); - csubstr svv = emitrs_yaml(cd->parsed_tree, &vv); - csubstr sss = emitrs_yaml(cd->parsed_tree, &ss); - EXPECT_EQ(svv, sss); -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emitrs_json(CaseDataLineEndings *cd) -{ - if(!(c->flags & JSON_ALSO)) - return; - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - using vtype = std::vector; - using stype = std::string; - vtype vv, v = emitrs_json(cd->parsed_tree); - stype ss, s = emitrs_json(cd->parsed_tree); - EXPECT_EQ(to_csubstr(v), to_csubstr(s)); - csubstr svv = emitrs_json(cd->parsed_tree, &vv); - csubstr sss = emitrs_json(cd->parsed_tree, &ss); - EXPECT_EQ(svv, sss); -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emitrs_cfile(CaseDataLineEndings *cd) -{ - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - auto s = emitrs_yaml(cd->parsed_tree); - std::string r; - { - c4::fs::ScopedTmpFile f; - emit_yaml(cd->parsed_tree, f.m_file); - fflush(f.m_file); - r = f.contents(); - } - EXPECT_EQ(s, r); -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_emitrs_json_cfile(CaseDataLineEndings *cd) -{ - if(!(c->flags & JSON_ALSO)) - return; - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - auto s = emitrs_json(cd->parsed_tree); - std::string r; - { - c4::fs::ScopedTmpFile f; - emit_json(cd->parsed_tree, f.m_file); - fflush(f.m_file); - r = f.contents(); - } - EXPECT_EQ(s, r); -} - - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_complete_round_trip(CaseDataLineEndings *cd) -{ - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit(cd); - { - SCOPED_TRACE("parsing emitted yml"); - cd->parse_buf = cd->emit_buf; - cd->parsed_yml = to_substr(cd->parse_buf); - parse_in_place(c->fileline, cd->parsed_yml, &cd->emitted_tree); - } - #ifdef RYML_NFO - std::cout << "~~~~~~~~~~~~~~ src yml:\n"; - _c4presc(cd->src); - std::cout << "~~~~~~~~~~~~~~ parsed tree:\n"; - print_tree(cd->parsed_tree); - std::cout << "~~~~~~~~~~~~~~ emitted yml:\n"; - _c4presc(cd->emitted_yml); - std::cout << "~~~~~~~~~~~~~~ emitted tree:\n"; - print_tree(cd->emitted_tree); - std::cout << "~~~~~~~~~~~~~~\n"; - #endif - { - SCOPED_TRACE("checking node invariants of emitted tree"); - test_invariants(cd->parsed_tree.rootref()); - } - { - SCOPED_TRACE("checking node invariants of emitted tree"); - test_invariants(cd->emitted_tree.rootref()); - } - { - SCOPED_TRACE("comparing emitted and parsed tree"); - test_compare(cd->emitted_tree, cd->parsed_tree); - } - { - SCOPED_TRACE("checking tree invariants of emitted tree"); - test_invariants(cd->emitted_tree); - } - { - SCOPED_TRACE("comparing parsed tree to ref tree"); - EXPECT_GE(cd->parsed_tree.capacity(), c->root.reccount()); - EXPECT_EQ(cd->parsed_tree.size(), c->root.reccount()); - c->root.compare(cd->parsed_tree.rootref()); - } - { - SCOPED_TRACE("comparing emitted tree to ref tree"); - EXPECT_GE(cd->emitted_tree.capacity(), c->root.reccount()); - EXPECT_EQ(cd->emitted_tree.size(), c->root.reccount()); - // in this case, we can ignore whether scalars are quoted. - // Because it can happen that a scalar was quoted in the - // original file, but the re-emitted data does not quote the - // scalars. - c->root.compare(cd->emitted_tree.rootref(), true); - } -} - - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_complete_round_trip_json(CaseDataLineEndings *cd) -{ - if(!(c->flags & JSON_ALSO)) - return; - if(c->flags & EXPECT_PARSE_ERROR) - return; - _ensure_parse(cd); - _ensure_emit_json(cd); - { - SCOPED_TRACE("parsing emitted json"); - cd->parse_buf_json = cd->emitjson_buf; - cd->parsed_json = to_substr(cd->parse_buf_json); - parse_in_place(c->fileline, cd->parsed_json, &cd->emitted_tree_json); - } - #ifdef RYML_NFO - std::cout << "~~~~~~~~~~~~~~ src yml:\n"; - _c4presc(cd->src); - std::cout << "~~~~~~~~~~~~~~ parsed tree:\n"; - print_tree(cd->parsed_tree); - std::cout << "~~~~~~~~~~~~~~ emitted json:\n"; - _c4presc(cd->emitted_json); - std::cout << "~~~~~~~~~~~~~~ emitted json tree:\n"; - print_tree(cd->emitted_tree_json); - std::cout << "~~~~~~~~~~~~~~\n"; - #endif - { - SCOPED_TRACE("checking node invariants of emitted tree"); - test_invariants(cd->parsed_tree.rootref()); - } - { - SCOPED_TRACE("checking node invariants of emitted json tree"); - test_invariants(cd->emitted_tree_json.rootref()); - } - { - SCOPED_TRACE("comparing emitted json and parsed tree"); - test_compare(cd->emitted_tree_json, cd->parsed_tree); - } - { - SCOPED_TRACE("checking tree invariants of emitted json tree"); - test_invariants(cd->emitted_tree_json); - } - { - SCOPED_TRACE("comparing parsed tree to ref tree"); - EXPECT_GE(cd->parsed_tree.capacity(), c->root.reccount()); - EXPECT_EQ(cd->parsed_tree.size(), c->root.reccount()); - c->root.compare(cd->parsed_tree.rootref()); - } - { - SCOPED_TRACE("comparing emitted tree to ref tree"); - EXPECT_GE(cd->emitted_tree_json.capacity(), c->root.reccount()); - EXPECT_EQ(cd->emitted_tree_json.size(), c->root.reccount()); - // in this case, we can ignore whether scalars are quoted. - // Because it can happen that a scalar was quoted in the - // original file, but the re-emitted data does not quote the - // scalars. - c->root.compare(cd->emitted_tree_json.rootref(), true); - } -} - -//----------------------------------------------------------------------------- -void YmlTestCase::_test_recreate_from_ref(CaseDataLineEndings *cd) -{ - if(c->flags & EXPECT_PARSE_ERROR) - return; - if(cd->parsed_tree.empty()) - parse_in_place(c->fileline, cd->src, &cd->parsed_tree); - if(cd->emit_buf.empty()) - cd->emitted_yml = emitrs_yaml(cd->parsed_tree, &cd->emit_buf); - { - SCOPED_TRACE("recreating a new tree from the ref tree"); - cd->recreated.reserve(cd->parsed_tree.size()); - NodeRef r = cd->recreated.rootref(); - c->root.recreate(&r); - } - #ifdef RYML_NFO - std::cout << "REF TREE:\n"; - print_tree(c->root); - std::cout << "RECREATED TREE:\n"; - print_tree(cd->recreated); - #endif - { - SCOPED_TRACE("checking node invariants of recreated tree"); - test_invariants(cd->recreated.rootref()); - } - { - SCOPED_TRACE("checking tree invariants of recreated tree"); - test_invariants(cd->recreated); - } - { - SCOPED_TRACE("comparing recreated tree to ref tree"); - c->root.compare(cd->recreated.rootref()); - } -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, parse_unix) -{ - SCOPED_TRACE("unix style"); - _test_parse_using_ryml(&d->unix_style); -} - -TEST_P(YmlTestCase, parse_windows) -{ - SCOPED_TRACE("windows style"); - _test_parse_using_ryml(&d->windows_style); -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, emit_yml_unix_stdout) -{ - SCOPED_TRACE("unix style"); - _test_emit_yml_stdout(&d->unix_style); -} -TEST_P(YmlTestCase, emit_json_unix_stdout) -{ - SCOPED_TRACE("unix style json"); - _test_emit_json_stdout(&d->unix_style_json); -} - -TEST_P(YmlTestCase, emit_yml_windows_stdout) -{ - SCOPED_TRACE("windows style"); - _test_emit_yml_stdout(&d->windows_style); -} -TEST_P(YmlTestCase, emit_json_windows_stdout) -{ - SCOPED_TRACE("windows style json"); - _test_emit_json_stdout(&d->windows_style_json); -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, emit_yml_unix_cout) -{ - SCOPED_TRACE("unix style"); - _test_emit_yml_cout(&d->unix_style); -} -TEST_P(YmlTestCase, emit_json_unix_cout) -{ - SCOPED_TRACE("unix style json"); - _test_emit_json_cout(&d->unix_style_json); -} - -TEST_P(YmlTestCase, emit_yml_windows_cout) -{ - SCOPED_TRACE("windows style"); - _test_emit_yml_cout(&d->windows_style); -} -TEST_P(YmlTestCase, emit_json_windows_cout) -{ - SCOPED_TRACE("windows style json"); - _test_emit_json_cout(&d->windows_style_json); -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, emit_yml_unix_stringstream) -{ - SCOPED_TRACE("unix style"); - _test_emit_yml_stringstream(&d->unix_style); -} -TEST_P(YmlTestCase, emit_json_unix_stringstream) -{ - SCOPED_TRACE("unix style json"); - _test_emit_json_stringstream(&d->unix_style_json); -} - -TEST_P(YmlTestCase, emit_yml_windows_stringstream) -{ - SCOPED_TRACE("windows style"); - _test_emit_yml_stringstream(&d->windows_style); -} -TEST_P(YmlTestCase, emit_json_windows_stringstream) -{ - SCOPED_TRACE("windows style json"); - _test_emit_json_stringstream(&d->windows_style_json); -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, emit_yml_unix_ofstream) -{ - SCOPED_TRACE("unix style"); - _test_emit_yml_ofstream(&d->unix_style); -} -TEST_P(YmlTestCase, emit_json_unix_ofstream) -{ - SCOPED_TRACE("unix style json"); - _test_emit_json_ofstream(&d->unix_style_json); -} - -TEST_P(YmlTestCase, emit_yml_windows_ofstream) -{ - SCOPED_TRACE("windows style"); - _test_emit_yml_ofstream(&d->windows_style); -} -TEST_P(YmlTestCase, emit_json_windows_ofstream) -{ - SCOPED_TRACE("windows style json"); - _test_emit_json_ofstream(&d->windows_style_json); -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, emit_yml_unix_string) -{ - SCOPED_TRACE("unix style"); - _test_emit_yml_string(&d->unix_style); -} -TEST_P(YmlTestCase, emit_json_unix_string) -{ - SCOPED_TRACE("unix style json"); - _test_emit_json_string(&d->unix_style_json); -} - -TEST_P(YmlTestCase, emit_yml_windows_string) -{ - SCOPED_TRACE("windows style"); - _test_emit_yml_string(&d->windows_style); -} -TEST_P(YmlTestCase, emit_json_windows_string) -{ - SCOPED_TRACE("windows style json"); - _test_emit_json_string(&d->windows_style_json); -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, unix_emitrs) -{ - SCOPED_TRACE("unix style"); - _test_emitrs(&d->unix_style); -} -TEST_P(YmlTestCase, unix_emitrs_json) -{ - SCOPED_TRACE("unix style json"); - _test_emitrs_json(&d->unix_style_json); -} - -TEST_P(YmlTestCase, windows_emitrs) -{ - SCOPED_TRACE("windows style"); - _test_emitrs(&d->windows_style); -} -TEST_P(YmlTestCase, windows_emitrs_json) -{ - SCOPED_TRACE("windows style json"); - _test_emitrs_json(&d->windows_style_json); -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, unix_emitrs_cfile) -{ - SCOPED_TRACE("unix style"); - _test_emitrs_cfile(&d->unix_style); -} -TEST_P(YmlTestCase, unix_emitrs_json_cfile) -{ - SCOPED_TRACE("unix style json"); - _test_emitrs_json_cfile(&d->unix_style_json); -} - -TEST_P(YmlTestCase, windows_emitrs_cfile) -{ - SCOPED_TRACE("windows style"); - _test_emitrs_cfile(&d->windows_style); -} -TEST_P(YmlTestCase, windows_emitrs_json_cfile) -{ - SCOPED_TRACE("windows style json"); - _test_emitrs_json_cfile(&d->windows_style_json); -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, complete_unix_round_trip) -{ - SCOPED_TRACE("unix style"); - _test_complete_round_trip(&d->unix_style); -} -TEST_P(YmlTestCase, complete_unix_round_trip_json) -{ - SCOPED_TRACE("unix style json"); - _test_complete_round_trip_json(&d->unix_style_json); -} - -TEST_P(YmlTestCase, complete_windows_round_trip) -{ - SCOPED_TRACE("windows style"); - _test_complete_round_trip(&d->windows_style); -} -TEST_P(YmlTestCase, complete_windows_round_trip_json) -{ - SCOPED_TRACE("windows style json"); - _test_complete_round_trip_json(&d->windows_style_json); -} - -//----------------------------------------------------------------------------- -TEST_P(YmlTestCase, unix_recreate_from_ref) -{ - SCOPED_TRACE("unix style"); - _test_recreate_from_ref(&d->unix_style); -} - -TEST_P(YmlTestCase, windows_recreate_from_ref) -{ - SCOPED_TRACE("windows style"); - _test_recreate_from_ref(&d->windows_style); -} - - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_group.hpp b/thirdparty/ryml/test/test_group.hpp deleted file mode 100644 index f661ec9b5..000000000 --- a/thirdparty/ryml/test/test_group.hpp +++ /dev/null @@ -1,210 +0,0 @@ -#pragma once -#ifndef C4_RYML_TEST_GROUP_HPP_ -#define C4_RYML_TEST_GROUP_HPP_ - -#include "./test_case.hpp" -#include "c4/span.hpp" -#include - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4068/*unknown pragma*/) -#elif defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunknown-pragmas" -//# pragma GCC diagnostic ignored "-Wpragma-system-header-outside-header" -#endif - - -namespace c4 { -namespace yml { - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// a fixture for running the tests -struct YmlTestCase : public ::testing::TestWithParam -{ - csubstr const name; - Case const* c; - CaseData * d; - - YmlTestCase() : name(to_csubstr(GetParam())) - { - c = get_case(name); - d = get_data(name); - } - - void SetUp() override - { - // Code here will be called immediately after the constructor (right - // before each test). - std::cout << "-------------------------------------------\n"; - std::cout << "running test case '" << name << "'\n"; - std::cout << "-------------------------------------------\n"; - } - - void _test_parse_using_ryml(CaseDataLineEndings *cd); - - void _test_emit_yml_stdout(CaseDataLineEndings *cd); - void _test_emit_json_stdout(CaseDataLineEndings *cd); - - void _test_emit_yml_cout(CaseDataLineEndings *cd); - void _test_emit_json_cout(CaseDataLineEndings *cd); - - void _test_emit_yml_stringstream(CaseDataLineEndings *cd); - void _test_emit_json_stringstream(CaseDataLineEndings *cd); - - void _test_emit_yml_ofstream(CaseDataLineEndings *cd); - void _test_emit_json_ofstream(CaseDataLineEndings *cd); - - void _test_emit_yml_string(CaseDataLineEndings *cd); - void _test_emit_json_string(CaseDataLineEndings *cd); - - void _test_emitrs(CaseDataLineEndings *cd); - void _test_emitrs_json(CaseDataLineEndings *cd); - - void _test_emitrs_cfile(CaseDataLineEndings *cd); - void _test_emitrs_json_cfile(CaseDataLineEndings *cd); - - void _test_complete_round_trip(CaseDataLineEndings *cd); - void _test_complete_round_trip_json(CaseDataLineEndings *cd); - - void _test_recreate_from_ref(CaseDataLineEndings *cd); - - void _ensure_parse(CaseDataLineEndings *cd) - { - if(cd->parsed_tree.empty()) - parse_in_place(c->fileline, cd->src, &cd->parsed_tree); - } - void _ensure_emit(CaseDataLineEndings *cd) - { - _ensure_parse(cd); - if(cd->emit_buf.empty()) - { - cd->emitted_yml = emitrs_yaml(cd->parsed_tree, &cd->emit_buf); - ASSERT_EQ(cd->emitted_yml.size(), cd->emit_buf.size()); - if(cd->emitted_yml.size()) - { - ASSERT_EQ(cd->emitted_yml.data(), cd->emit_buf.data()); - } - } - } - void _ensure_emit_json(CaseDataLineEndings *cd) - { - _ensure_parse(cd); - if(cd->emitjson_buf.empty()) - { - cd->emitted_json = emitrs_json(cd->parsed_tree, &cd->emitjson_buf); - ASSERT_EQ(cd->emitted_json.size(), cd->emitjson_buf.size()); - if(cd->emitted_json.size()) - { - ASSERT_EQ(cd->emitted_json.data(), cd->emitjson_buf.data()); - } - } - } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -// facilities for declaring test data - -using N = CaseNode; -using L = CaseNode::iseqmap; -using TS = TaggedScalar; -using TL = CaseNode::TaggedList; -using AR = AnchorRef; - -constexpr const NodeType_e QK = (NodeType_e)(VAL | KEYQUO); -constexpr const NodeType_e QV = (NodeType_e)(VAL | VALQUO); -constexpr const NodeType_e QKV = (NodeType_e)(VAL | KEYQUO | VALQUO); - -#ifdef __GNUC__ -#if __GNUC__ == 4 && __GNUC_MINOR__ >= 8 -struct CaseAdder { - std::vector *group_cases; - const csubstr file; - const int line; - - template - void operator ()(Args... parameters) const { - group_cases->emplace_back(csubstr(file), line, parameters...); - } -}; - -/* all arguments are to the constructor of Case */ -#define ADD_CASE_TO_GROUP CaseAdder{group_cases__, csubstr(__FILE__), __LINE__+1} -#endif -#endif - -#ifndef ADD_CASE_TO_GROUP -#define ADD_CASE_TO_GROUP(...) \ - group_cases__->emplace_back(csubstr(__FILE__), __LINE__+1, __VA_ARGS__) -#endif - -#define CASE_GROUP(group_name) \ - \ -/* fwd declaration to fill the container with cases */ \ -void add_cases_##group_name(std::vector *group_cases); \ - \ -/* container with the cases */ \ -std::vector const& get_cases_##group_name() \ -{ \ - static std::vector cases_##group_name; \ - if(cases_##group_name.empty()) \ - add_cases_##group_name(&cases_##group_name); \ - return cases_##group_name; \ -} \ - \ -/* container with the case names */ \ -std::vector const& get_case_names_##group_name() \ -{ \ - static std::vector case_names_##group_name; \ - if(case_names_##group_name.empty()) \ - { \ - for(auto const& c : get_cases_##group_name()) \ - case_names_##group_name.emplace_back(c.name); \ - /* check repetitions */ \ - std::vector cp = case_names_##group_name; \ - std::sort(cp.begin(), cp.end()); \ - for(size_t i = 0; i+1 < cp.size(); ++i) \ - if(cp[i] == cp[i+1]) \ - C4_ERROR("duplicate case name: '%.*s'", _c4prsp(cp[i])); \ - } \ - return case_names_##group_name; \ -} \ - \ -INSTANTIATE_TEST_SUITE_P(group_name, YmlTestCase, ::testing::ValuesIn(get_case_names_##group_name())); \ -GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(YmlTestCase); \ - \ -/* used by the fixture to obtain a case by name */ \ -Case const* get_case(csubstr name) \ -{ \ - for(Case const& c : get_cases_##group_name()) \ - if(c.name == name) \ - return &c; \ - C4_ERROR("case not found: '%.*s'", _c4prsp(name)); \ - return nullptr; \ -} \ - \ -/* finally, define the cases by calling ADD_CASE_TO_GROUP() */ \ -void add_cases_##group_name(std::vector *group_cases__) - - -} // namespace yml -} // namespace c4 - -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif - -#endif // C4_RYML_TEST_GROUP_HPP_ diff --git a/thirdparty/ryml/test/test_indentation.cpp b/thirdparty/ryml/test/test_indentation.cpp deleted file mode 100644 index 46b6f9bff..000000000 --- a/thirdparty/ryml/test/test_indentation.cpp +++ /dev/null @@ -1,340 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -CASE_GROUP(INDENTATION) -{ - -ADD_CASE_TO_GROUP("indented doc", R"( - # this is an indented doc - --- - - foo - - bar - - baz -)", -N(STREAM, L{N(DOCSEQ, L{N("foo"), N("bar"), N("baz")})}) -); - -ADD_CASE_TO_GROUP("4 chars", -R"( -key: - value -another_key: - sub_key0: - - val0 - - val1 - sub_key1: - - val2 - - val3 - sub_key2: - - val4 - - val5 -)", -L{ - N("key", "value"), - N("another_key", L{ - N("sub_key0", L{N("val0"), N("val1")}), - N("sub_key1", L{N("val2"), N("val3")}), - N("sub_key2", L{N("val4"), N("val5")}), - }) -}); - -ADD_CASE_TO_GROUP("2 chars + 4 chars, ex0", -R"( -key: - value -another_key: - sub_key0: - - val0 - - val1 - sub_key1: - - val2 - - val3 - sub_key2: - - val4 - - val5 -)", -L{ - N("key", "value"), - N("another_key", L{ - N("sub_key0", L{N("val0"), N("val1")}), - N("sub_key1", L{N("val2"), N("val3")}), - N("sub_key2", L{N("val4"), N("val5")}), - }) -}); - -ADD_CASE_TO_GROUP("2 chars + 4 chars, ex1", -R"( -key: - value -another_key: - sub_key0: - - val0 - - val1 - sub_key1: - - val2 - - val3 - sub_key2: - - val4 - - val5 -)", -L{ - N("key", "value"), - N("another_key", L{ - N("sub_key0", L{N("val0"), N("val1")}), - N("sub_key1", L{N("val2"), N("val3")}), - N("sub_key2", L{N("val4"), N("val5")}), - }) -}); - -ADD_CASE_TO_GROUP("2 chars + 4 chars, ex2", -R"( -key: - value -another_key: - sub_key0: - - val0 - - val1 - sub_key1: - - val2 - - val3 - sub_key2: - - val4 - - val5 -)", -L{ - N("key", "value"), - N("another_key", L{ - N("sub_key0", L{N("val0"), N("val1")}), - N("sub_key1", L{N("val2"), N("val3")}), - N("sub_key2", L{N("val4"), N("val5")}), - }) -}); - -ADD_CASE_TO_GROUP("non-indented blank lines", -R"( -matrix: - - include: # next line is blank - - - env01 - - env02 - - env03 - - env04 # next line has one space - - - env11 - - env12 - - env13 - - env14 # next line has two spaces - - - env21 - - env22 - - env23 - - env24 # next line has three spaces - - - env31 - - env32 - - env33 - - env34 # next line has four spaces - - - env41 - - env42 - - env43 - - env44 # next line has five spaces - - - env51 - - env52 - - env53 - - env54 # next line has six spaces - - - env61 - - env62 - - env63 - - env64 # next line has five spaces -)", -L{N("matrix", L{ - N("include", L{ - N("env01"), N("env02"), N("env03"), N("env04"), - N("env11"), N("env12"), N("env13"), N("env14"), - N("env21"), N("env22"), N("env23"), N("env24"), - N("env31"), N("env32"), N("env33"), N("env34"), - N("env41"), N("env42"), N("env43"), N("env44"), - N("env51"), N("env52"), N("env53"), N("env54"), - N("env61"), N("env62"), N("env63"), N("env64"), - } - )}) -}); - -ADD_CASE_TO_GROUP("unnecessary indentation", -R"( -skip_commits: - files: - - a - - b - - c - - d - - e - - f - more_files: - - a - - b - even_more_files: - - a - - b -more_skip: - files: - - a - - b - - c - - d - - e - - f - more_files: - - a - - b -)", -L{ - N("skip_commits", L{ - N("files", L{N("a"), N("b"), N("c"), N("d"), N("e"), N("f"),}), - N("more_files", L{N("a"), N("b"),}), - N("even_more_files", L{N("a"), N("b"),}), - }), - N("more_skip", L{ - N("files", L{N("a"), N("b"), N("c"), N("d"), N("e"), N("f"),}), - N("more_files", L{N("a"), N("b"),}), - }) -}); - - -ADD_CASE_TO_GROUP("blank lines indented, 1 - at same scope", -R"( -skip_commits: - files: - - a # next line has 22 spaces (aligns with -) - - - b # next line has 23 spaces (aligns with #) - - - c # next line has 3 spaces - - - d -)", -L{ - N("skip_commits", L{ - N("files", L{N("a"), N("b"), N("c"), N("d"),}), - }), -}); - -ADD_CASE_TO_GROUP("indentation at start", -R"( - foo: - - a - - b - bar: - - c - - d -)", -L{ - N("foo", L{N("a"), N("b"),}), - N("bar", L{N("c"), N("d"),}), -}); - -ADD_CASE_TO_GROUP("unaligned comments", -R"( - stand2sit: - map: mirror - dat: - - a - - b -# - - b1 - # - - b2 - # - # - # - - b3 - # - # - # - - b4 - # - # - c - #- d - - b5 - #- d2 - #- d3 - #- d4 - - b6 - #- d41 - # - - b61 - # - # - - b62 - # - # - # - - b63 - # - - b64 - # - - b65 - # - # - # - - b66 - # - # - # - # - #- d41 - #- d5 - #- d6 - #- d7 - - b7 - #- d8 - # - # - # - - b8 - # - # - # - - b9 - # - # - - b10 - # -# - - e - - f - - g -)", -L{ - N("stand2sit", L{ - N("map", "mirror"), - N("dat", L{N("a"), N("b"), N("b1"), N("b2"), N("b3"), N("b4"), N("b5"), N("b6"), N("b61"), N("b62"), N("b63"), N("b64"), N("b65"), N("b66"), N("b7"), N("b8"), N("b9"), N("b10"), N("e"), N("f"), N("g")}), - }), -}); - -ADD_CASE_TO_GROUP("issue83", -R"( -e: - - f -g: h -a: - - b - -c: d -)", -L{ -N("e", L{N("f")}), -N("g", "h"), -N("a", L{N("b")}), -N("c", "d"), -}); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_json.cpp b/thirdparty/ryml/test/test_json.cpp deleted file mode 100644 index 8be31b794..000000000 --- a/thirdparty/ryml/test/test_json.cpp +++ /dev/null @@ -1,516 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/std/std.hpp" -#include "c4/yml/parse.hpp" -#include "c4/yml/emit.hpp" -#include -#include -#include -#endif - -#include "./test_case.hpp" - -#include - -namespace foo { - -template -struct vec2 -{ - T x, y; -}; -template -struct vec3 -{ - T x, y, z; -}; -template -struct vec4 -{ - T x, y, z, w; -}; - -template size_t to_chars(c4::substr buf, vec2 v) { return c4::format(buf, "({},{})", v.x, v.y); } -template size_t to_chars(c4::substr buf, vec3 v) { return c4::format(buf, "({},{},{})", v.x, v.y, v.z); } -template size_t to_chars(c4::substr buf, vec4 v) { return c4::format(buf, "({},{},{},{})", v.x, v.y, v.z, v.w); } - -template bool from_chars(c4::csubstr buf, vec2 *v) { size_t ret = c4::unformat(buf, "({},{})", v->x, v->y); return ret != c4::yml::npos; } -template bool from_chars(c4::csubstr buf, vec3 *v) { size_t ret = c4::unformat(buf, "({},{},{})", v->x, v->y, v->z); return ret != c4::yml::npos; } -template bool from_chars(c4::csubstr buf, vec4 *v) { size_t ret = c4::unformat(buf, "({},{},{},{})", v->x, v->y, v->z, v->w); return ret != c4::yml::npos; } - -TEST(serialize, type_as_str) -{ - c4::yml::Tree t; - - auto r = t.rootref(); - r |= c4::yml::MAP; - - vec2 v2in{10, 11}; - vec2 v2out; - r["v2"] << v2in; - r["v2"] >> v2out; - EXPECT_EQ(v2in.x, v2out.x); - EXPECT_EQ(v2in.y, v2out.y); - - vec3 v3in{100, 101, 102}; - vec3 v3out; - r["v3"] << v3in; - r["v3"] >> v3out; - EXPECT_EQ(v3in.x, v3out.x); - EXPECT_EQ(v3in.y, v3out.y); - EXPECT_EQ(v3in.z, v3out.z); - - vec4 v4in{1000, 1001, 1002, 1003}; - vec4 v4out; - r["v4"] << v4in; - r["v4"] >> v4out; - EXPECT_EQ(v4in.x, v4out.x); - EXPECT_EQ(v4in.y, v4out.y); - EXPECT_EQ(v4in.z, v4out.z); - EXPECT_EQ(v4in.w, v4out.w); - - char buf[256]; - c4::csubstr interm = c4::yml::emit_json(t, buf); - EXPECT_EQ(interm, R"_({"v2": "(10,11)","v3": "(100,101,102)","v4": "(1000,1001,1002,1003)"})_"); -} -} // namespace foo - -namespace c4 { -namespace yml { - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(general, emitting) -{ - std::string cmpbuf; - std::string cmpbuf2; - - Tree tree; - auto r = tree.rootref(); - - r |= MAP; // this is needed to make the root a map - - r["foo"] = "1"; // ryml works only with strings. - // Note that the tree will be __pointing__ at the - // strings "foo" and "1" used here. You need - // to make sure they have at least the same - // lifetime as the tree. - - auto s = r["seq"]; // does not change the tree until s is written to. - s |= SEQ; - r["seq"].append_child() = "bar0"; // value of this child is now __pointing__ at "bar0" - r["seq"].append_child() = "bar1"; - r["seq"].append_child() = "bar2"; - - //print_tree(tree); - - // emit to stdout (can also emit to FILE* or ryml::span) - emitrs_json(tree, &cmpbuf); - EXPECT_EQ(cmpbuf, R"({"foo": 1,"seq": ["bar0","bar1","bar2"]})"); - - // serializing: using operator<< instead of operator= - // will make the tree serialize the value into a char - // arena inside the tree. This arena can be reserved at will. - int ch3 = 33, ch4 = 44; - s.append_child() << ch3; - s.append_child() << ch4; - - { - std::string tmp = "child5"; - s.append_child() << tmp; - // now tmp can go safely out of scope, as it was - // serialized to the tree's internal string arena - } - - emitrs_json(tree, &cmpbuf); - EXPECT_EQ(cmpbuf, R"({"foo": 1,"seq": ["bar0","bar1","bar2",33,44,"child5"]})"); - - // to serialize keys: - int k = 66; - r.append_child() << key(k) << 7; - emitrs_json(tree, &cmpbuf); - EXPECT_EQ(cmpbuf, R"({"foo": 1,"seq": ["bar0","bar1","bar2",33,44,"child5"],"66": 7})"); -} - -TEST(general, map_to_root) -{ - std::string cmpbuf; const char *exp; - std::map m({{"bar", 2}, {"foo", 1}}); - Tree t; - t.rootref() << m; - - emitrs_json(t, &cmpbuf); - exp = "{\"bar\": 2,\"foo\": 1}"; - EXPECT_EQ(cmpbuf, exp); - - t["foo"] << 10; - t["bar"] << 20; - - m.clear(); - t.rootref() >> m; - - EXPECT_EQ(m["foo"], 10); - EXPECT_EQ(m["bar"], 20); -} - -TEST(general, json_stream_operator) -{ - std::map out, m({{"bar", 2}, {"foo", 1}, {"foobar_barfoo:barfoo_foobar", 1001}, {"asdfjkl;", 42}, {"00000000000000000000000000000000000000000000000000000000000000", 1}}); - Tree t; - t.rootref() << m; - std::string str; - { - std::stringstream ss; - ss << as_json(t); - str = ss.str(); - } - Tree res = c4::yml::parse_in_place(to_substr(str)); - EXPECT_EQ(res["foo"].val(), "1"); - EXPECT_EQ(res["bar"].val(), "2"); - EXPECT_EQ(res["foobar_barfoo:barfoo_foobar"].val(), "1001"); - EXPECT_EQ(res["asdfjkl;"].val(), "42"); - EXPECT_EQ(res["00000000000000000000000000000000000000000000000000000000000000"].val(), "1"); - res.rootref() >> out; - EXPECT_EQ(out["foo"], 1); - EXPECT_EQ(out["bar"], 2); - EXPECT_EQ(out["foobar_barfoo:barfoo_foobar"], 1001); - EXPECT_EQ(out["asdfjkl;"], 42); - EXPECT_EQ(out["00000000000000000000000000000000000000000000000000000000000000"], 1); -} - -TEST(emit_json, issue72) -{ - Tree t; - NodeRef r = t.rootref(); - - r |= MAP; - r["1"] = "null"; - r["2"] = "true"; - r["3"] = "false"; - r["null"] = "1"; - r["true"] = "2"; - r["false"] = "3"; - - std::string out; - emitrs_json(t, &out); - - EXPECT_EQ(out, R"({"1": null,"2": true,"3": false,"null": 1,"true": 2,"false": 3})"); -} - - -TEST(emit_json, issue121) -{ - Tree t = parse_in_arena(R"( -string_value: "string" -number_value: "9001" -broken_value: "0.30.2" -)"); - EXPECT_TRUE(t["string_value"].get()->m_type.type & VALQUO); - EXPECT_TRUE(t["number_value"].get()->m_type.type & VALQUO); - EXPECT_TRUE(t["broken_value"].get()->m_type.type & VALQUO); - std::string out; - emitrs_json(t, &out); - EXPECT_EQ(out, R"({"string_value": "string","number_value": "9001","broken_value": "0.30.2"})"); - out.clear(); - emitrs_yaml(t, &out); - EXPECT_EQ(out, R"(string_value: 'string' -number_value: '9001' -broken_value: '0.30.2' -)"); -} - -TEST(emit_json, issue291) -{ - Tree t = parse_in_arena("{}"); - t["james"] = "045"; - auto s = emitrs_json(t); - EXPECT_EQ(s, "{\"james\": \"045\"}"); -} - -TEST(emit_json, issue292) -{ - EXPECT_FALSE(csubstr("0.0.0").is_number()); - EXPECT_FALSE(csubstr("0.0.0").is_integer()); - EXPECT_FALSE(csubstr("0.0.0").is_real()); - EXPECT_FALSE(csubstr("0.1.0").is_number()); - EXPECT_FALSE(csubstr("0.1.0").is_integer()); - EXPECT_FALSE(csubstr("0.1.0").is_real()); - EXPECT_FALSE(csubstr("0.6.1").is_number()); - EXPECT_FALSE(csubstr("0.6.1").is_integer()); - EXPECT_FALSE(csubstr("0.6.1").is_real()); - EXPECT_FALSE(csubstr("1.1.9").is_number()); - EXPECT_FALSE(csubstr("1.1.9").is_integer()); - EXPECT_FALSE(csubstr("1.1.9").is_real()); - EXPECT_FALSE(csubstr("1.2.3").is_number()); - EXPECT_FALSE(csubstr("1.2.3").is_integer()); - EXPECT_FALSE(csubstr("1.2.3").is_real()); - Tree t = parse_in_arena("{}"); - t["james"] = "0.0.0"; - EXPECT_EQ(emitrs_json(t), "{\"james\": \"0.0.0\"}"); - t["james"] = "0.1.0"; - EXPECT_EQ(emitrs_json(t), "{\"james\": \"0.1.0\"}"); - t["james"] = "0.6.1"; - EXPECT_EQ(emitrs_json(t), "{\"james\": \"0.6.1\"}"); - t["james"] = "1.1.9"; - EXPECT_EQ(emitrs_json(t), "{\"james\": \"1.1.9\"}"); - t["james"] = "1.2.3"; - EXPECT_EQ(emitrs_json(t), "{\"james\": \"1.2.3\"}"); -} - -TEST(emit_json, issue297) -{ - char yml_buf[] = R"( -comment: | - abc - def -)"; - Tree t = parse_in_place(yml_buf); - auto s = emitrs_json(t); - EXPECT_EQ(s, "{\"comment\": \"abc\\ndef\\n\"}"); -} - -TEST(emit_json, issue297_escaped_chars) -{ - Tree t = parse_in_arena("{}"); - t["quote"] = "abc\"def"; - t["newline"] = "abc\ndef"; - t["tab"] = "abc\tdef"; - t["carriage"] = "abc\rdef"; - t["backslash"] = "abc\\def"; - t["backspace"] = "abc\bdef"; - t["formfeed"] = "abc\fdef"; - std::string expected = R"({"quote": "abc\"def","newline": "abc\ndef","tab": "abc\tdef","carriage": "abc\rdef","backslash": "abc\\def","backspace": "abc\bdef","formfeed": "abc\fdef"})"; - auto actual = emitrs_json(t); - EXPECT_EQ(actual, expected); -} - -namespace { -std::string k(ConstNodeRef node) { return std::string(node.key().str, node.key().len); } -std::string v(ConstNodeRef node) { return std::string(node.val().str, node.val().len); } -} -TEST(emit_json, issue313_quoted_numbers__1) -{ - EXPECT_TRUE(csubstr("0.99356698989868164").is_number()); // [WEIRD0][0] - EXPECT_TRUE(csubstr("0.99356698989868164").is_real()); // [WEIRD0][0] - EXPECT_FALSE(csubstr("0.99356698989868164").is_integer()); // [WEIRD0][0] - EXPECT_TRUE(csubstr("0.0064908224157989025").is_number()); // [WEIRD2][0] - EXPECT_TRUE(csubstr("0.0064908224157989025").is_real()); // [WEIRD2][0] - EXPECT_FALSE(csubstr("0.0064908224157989025").is_integer()); // [WEIRD2][0] - EXPECT_TRUE(csubstr("0.0064917667768895626").is_number()); // [WEIRD2][1] - EXPECT_TRUE(csubstr("0.0064917667768895626").is_real()); // [WEIRD2][1] - EXPECT_FALSE(csubstr("0.0064917667768895626").is_integer()); // [WEIRD2][1] - EXPECT_TRUE(csubstr("0.0064947893843054771").is_number()); // [WEIRD2][2] - EXPECT_TRUE(csubstr("0.0064947893843054771").is_real()); // [WEIRD2][2] - EXPECT_FALSE(csubstr("0.0064947893843054771").is_integer()); // [WEIRD2][2] - EXPECT_TRUE(csubstr("0.91054189205169678").is_number()); // [WEIRD4][0] - EXPECT_TRUE(csubstr("0.91054189205169678").is_real()); // [WEIRD4][0] - EXPECT_FALSE(csubstr("0.91054189205169678").is_integer()); // [WEIRD4][0] - EXPECT_TRUE(csubstr("0.13215841352939606").is_number()); // [REALLY_WEIRD5][9][0] - EXPECT_TRUE(csubstr("0.13215841352939606").is_real()); // [REALLY_WEIRD5][9][0] - EXPECT_FALSE(csubstr("0.13215841352939606").is_integer()); // [REALLY_WEIRD5][9][0] - Tree t0 = parse_in_arena(R"([ - 0.99356698989868164, - 0.0064908224157989025, - 0.0064917667768895626, - 0.0064947893843054771, - 0.91054189205169678, - 0.13215841352939606, -])"); - std::string yaml = emitrs_json(t0); - test_check_emit_check(to_csubstr(yaml), [&](Tree const &t){ - for(ConstNodeRef number : t.rootref().children()) - { - ASSERT_TRUE(number.is_val()); - EXPECT_FALSE(number.is_val_quoted()) << "tree[" << t.rootref().child_pos(number) << "]=" << v(number); - } - }); -} - - -TEST(emit_json, issue313_quoted_numbers__2) -{ - Tree ti = parse_in_arena(R"({ -WEIRD0: [0.99356698989868164, 1.0605627298355103], -OK1: [0, 0, 0], -WEIRD2: [0.0064908224157989025, 0.0064917667768895626, 0.0064947893843054771], -OK3: [6.6227097511291504, 6.8674740791320801, 7.0403199195861816, 7.5792555809020996, 7.9916787147521973, 8.136042594909668, 8.5505847930908203, 8.701807975769043, 8.926518440246582, 8.9484291076660156, 9.0740194320678711, 9.3788108825683594, 9.406926155090332], -WEIRD4: [0.91054189205169678, 0.98725020885467529, 1.070807933807373], -REALLY_WEIRD5: [ - [1.5158847570419312, 1.6361792087554932], # 0 - [1.0741721391677856, 1.1791903972625732], # 1 - [1.4423576593399048, 1.7063977718353271], # 2 - [1.1791903972625732], # 3 - [1.1493504047393799, 1.1791903972625732], # 4 - [1.1791903972625732, 1.3334760665893555], # 5 - [1.0655292272567749, 1.4933452606201172], # 6 - [1.0712906122207642, 1.1791903972625732], # 7 - [1.1791903972625732, 1.830910325050354], # 8 - [0.13215841352939606, 1.4161584377288818], # 9 - [1.1791903972625732, 1.5179581642150879], # 10 - [1.1791903972625732, 1.2864601612091064], # 11 - [1.1791903972625732, 1.6865267753601074], # 12 - [1.1791903972625732, 1.2192368507385254], # 13 - [1.1130030155181885, 1.5196701288223267], # 14 - [1.0621790885925293, 1.1791903972625732] # 15 -]})"); - std::string yaml = emitrs_json(ti); - test_check_emit_check(to_csubstr(yaml), [](Tree const &t){ - for(ConstNodeRef node : t.rootref().children()) - { - ASSERT_TRUE(node.is_seq()); - ASSERT_TRUE(node.has_key()); - EXPECT_TRUE(node.is_key_quoted()) << "tree[" << k(node) << "]"; - if(node.key() != "REALLY_WEIRD5") - { - for(ConstNodeRef number : node.children()) - { - ASSERT_TRUE(number.is_val()); - EXPECT_FALSE(number.is_val_quoted()) << "tree[" << k(node) << "][" << node.child_pos(number) << "]=" << v(number); - } - } - else - { - for(ConstNodeRef seq : node.children()) - { - ASSERT_TRUE(seq.is_seq()); - for(ConstNodeRef number : seq.children()) - { - ASSERT_TRUE(number.is_val()); - EXPECT_FALSE(number.is_val_quoted()) << "tree[" << k(node) << "][" << node.child_pos(seq) << "][" << seq.child_pos(number) << "]=" << v(number); - } - } - } - } - }); -} - - -#define _test(actual_src, expected_src) \ - { \ - SCOPED_TRACE(__LINE__); \ - csubstr file = __FILE__ ":" C4_XQUOTE(__LINE__); \ - Tree actual = parse_in_arena(file, actual_src); \ - Tree expected = parse_in_arena(file, expected_src); \ - test_compare(actual, expected); \ - } - - -TEST(json, basic) -{ - _test("", ""); - _test("{}", "{}"); - _test(R"("a":"b")", - R"("a": "b")"); - _test(R"('a':'b')", - R"('a': 'b')"); - _test(R"({'a':'b'})", - R"({'a': 'b'})"); - _test(R"({"a":"b"})", - R"({"a": "b"})"); - - _test(R"({"a":{"a":"b"}})", - R"({"a": {"a": "b"}})"); - _test(R"({'a':{'a':'b'}})", - R"({'a': {'a': 'b'}})"); -} - -TEST(json, github142) -{ - _test(R"({"A":"B}"})", - R"({"A": "B}"})"); - _test(R"({"A":"{B"})", - R"({"A": "{B"})"); - _test(R"({"A":"{B}"})", - R"({"A": "{B}"})"); - _test(R"({ "A":"B}" })", - R"({ "A": "B}" })"); - _test(R"({"A":["B]","[C","[D]"]})", - R"({"A": ["B]","[C","[D]"]})"); - //_test(R"({"A":["B\"]","[\"C","\"[D]\""]})", // VS2019 chokes on this. - // R"({"A": ["B\"]","[\"C","\"[D]\""]})"); - - _test(R"({'A':'B}'})", - R"({'A': 'B}'})"); - _test(R"({'A':'{B'})", - R"({'A': '{B'})"); - _test(R"({'A':'{B}'})", - R"({'A': '{B}'})"); - _test(R"({ 'A':'B}' })", - R"({ 'A': 'B}' })"); - _test(R"({'A':['B]','[C','[D]']})", - R"({'A': ['B]','[C','[D]']})"); - _test(R"({'A':['B'']','[''C','''[D]''']})", - R"({'A': ['B'']','[''C','''[D]''']})"); -} - -TEST(json, github52) -{ - _test(R"({"a": "b","c": 42,"d": "e"})", - R"({"a": "b","c": 42,"d": "e"})"); - _test(R"({"aaaa": "bbbb","cccc": 424242,"dddddd": "eeeeeee"})", - R"({"aaaa": "bbbb","cccc": 424242,"dddddd": "eeeeeee"})"); - - _test(R"({"a":"b","c":42,"d":"e"})", - R"({"a": "b","c": 42,"d": "e"})"); - _test(R"({"aaaaa":"bbbbb","ccccc":424242,"ddddd":"eeeee"})", - R"({"aaaaa": "bbbbb","ccccc": 424242,"ddddd": "eeeee"})"); - _test(R"({"a":"b","c":{},"d":"e"})", - R"({"a": "b","c": {},"d": "e"})"); - _test(R"({"aaaaa":"bbbbb","ccccc":{ },"ddddd":"eeeee"})", - R"({"aaaaa": "bbbbb","ccccc": { },"ddddd": "eeeee"})"); - _test(R"({"a":"b","c":true,"d":"e"})", - R"({"a": "b","c": true,"d": "e"})"); - _test(R"({"a":"b","c":false,"d":"e"})", - R"({"a": "b","c": false,"d": "e"})"); - _test(R"({"a":"b","c":true,"d":"e"})", - R"({"a": "b","c": true,"d": "e"})"); - _test(R"({"a":"b","c":null,"d":"e"})", - R"({"a": "b","c": null,"d": "e"})"); - _test(R"({"aaaaa":"bbbbb","ccccc":false,"ddddd":"eeeee"})", - R"({"aaaaa": "bbbbb","ccccc": false,"ddddd": "eeeee"})"); - _test(R"({"a":"b","c":false,"d":"e"})", - R"({"a": "b","c": false,"d": "e"})"); - _test(R"({"aaaaa":"bbbbb","ccccc":true,"ddddd":"eeeee"})", - R"({"aaaaa": "bbbbb","ccccc": true,"ddddd": "eeeee"})"); -} - -TEST(json, nested) -{ - _test(R"({"a":"b","c":{"a":"b","c":{},"d":"e"},"d":"e"})", - R"({"a": "b","c": {"a": "b","c": {},"d": "e"},"d": "e"})"); - _test(R"({"a":"b","c":{"a":"b","c":{"a":"b","c":{},"d":"e"},"d":"e"},"d":"e"})", - R"({"a": "b","c": {"a": "b","c": {"a": "b","c": {},"d": "e"},"d": "e"},"d": "e"})"); - _test(R"({"a":"b","c":{"a":"b","c":{"a":"b","c":{"a":"b","c":{},"d":"e"},"d":"e"},"d":"e"},"d":"e"})", - R"({"a": "b","c": {"a": "b","c": {"a": "b","c": {"a": "b","c": {},"d": "e"},"d": "e"},"d": "e"},"d": "e"})"); - _test(R"({"a":"b","c":{"a":"b","c":{"a":"b","c":{"a":"b","c":{"a":"b","c":{},"d":"e"},"d":"e"},"d":"e"},"d":"e"},"d":"e"})", - R"({"a": "b","c": {"a": "b","c": {"a": "b","c": {"a": "b","c": {"a": "b","c": {},"d": "e"},"d": "e"},"d": "e"},"d": "e"},"d": "e"})"); - - _test(R"({"a":"b","c":["a","c","d","e"],"d":"e"})", - R"({"a": "b","c": ["a","c","d","e"],"d": "e"})"); -} - -TEST(json, nested_end) -{ - _test(R"({"a":"b","d":"e","c":{"a":"b","d":"e","c":{}}})", - R"({"a": "b","d": "e","c": {"a": "b","d": "e","c": {}}})"); - _test(R"({"a":"b","d":"e","c":{"a":"b","d":"e","c":{"a":"b","d":"e","c":{}}}})", - R"({"a": "b","d": "e","c": {"a": "b","d": "e","c": {"a": "b","d": "e","c": {}}}})"); - _test(R"({"a":"b","d":"e","c":{"a":"b","d":"e","c":{"a":"b","d":"e","c":{"a":"b","d":"e","c":{}}}}})", - R"({"a": "b","d": "e","c": {"a": "b","d": "e","c": {"a": "b","d": "e","c": {"a": "b","d": "e","c": {}}}}})"); - _test(R"({"a":"b","d":"e","c":{"a":"b","d":"e","c":{"a":"b","d":"e","c":{"a":"b","d":"e","c":{"a":"b","d":"e","c":{}}}}}})", - R"({"a": "b","d": "e","c": {"a": "b","d": "e","c": {"a": "b","d": "e","c": {"a": "b","d": "e","c": {"a": "b","d": "e","c": {}}}}}})"); -} - -#undef _test - - -//------------------------------------------- -// this is needed to use the test case library -Case const* get_case(csubstr /*name*/) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_location.cpp b/thirdparty/ryml/test/test_location.cpp deleted file mode 100644 index e05407eeb..000000000 --- a/thirdparty/ryml/test/test_location.cpp +++ /dev/null @@ -1,720 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include -#include -#endif -#include "./test_case.hpp" -#include - - -namespace c4 { -namespace yml { - -TEST(locations, default_is_no_location) -{ - { - ParserOptions opts; - EXPECT_EQ(opts.locations(), false); - } - { - Parser parser; - EXPECT_EQ(parser.options().locations(), false); - } - { - Parser parser(ParserOptions{}); - EXPECT_EQ(parser.options().locations(), false); - } - { - Parser parser(ParserOptions().locations(false)); - EXPECT_EQ(parser.options().locations(), false); - } - { - Parser parser(ParserOptions().locations(true)); - EXPECT_EQ(parser.options().locations(), true); - } -} - - -TEST(locations, error_is_triggered_querying_with_locations_disabled) -{ - bool parsed_ok = false; - ExpectError::do_check([&]{ - Parser parser(ParserOptions().locations(false)); - Tree t = parser.parse_in_arena("test", "foo: bar"); - parsed_ok = true; - (void)parser.location(t["foo"]); - }); - EXPECT_TRUE(parsed_ok); -} - - - -#define _checkloc(node, line_, col_, str) \ - { \ - const Location loc = parser.location(node); \ - EXPECT_EQ(loc.name, "myfile.yml"); \ - EXPECT_EQ(loc.line, line_); \ - EXPECT_EQ(loc.col, col_); \ - EXPECT_EQ(t.arena().sub(loc.offset, csubstr(str).len), csubstr(str)); \ - } - -TEST(locations, no_error_is_triggered_querying_with_locations) -{ - Parser parser(ParserOptions().locations(true)); - EXPECT_EQ(parser.options().locations(), true); - Tree t = parser.parse_in_arena("myfile.yml", "foo: bar"); - _checkloc(t["foo"], 0, 0, "foo"); -} - - -TEST(locations, docval) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - Tree t = parser.parse_in_arena("myfile.yml", "docval"); - _checkloc(t.rootref(), 0u, 0u, "docval"); - t = parser.parse_in_arena("myfile.yml", "\n docval"); - _checkloc(t.rootref(), 1u, 1u, "docval"); - t = parser.parse_in_arena("myfile.yml", "\n\n docval"); - _checkloc(t.rootref(), 2u, 1u, "docval"); -} - -TEST(locations, docval_null) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - Tree t = parser.parse_in_arena("myfile.yml", "~"); - _checkloc(t.rootref(), 0u, 0u, "~"); - t = parser.parse_in_arena("myfile.yml", ""); - _checkloc(t.rootref(), 0u, 0u, ""); - t = parser.parse_in_arena("myfile.yml", R"(# -# -# -# -# -)"); - _checkloc(t.rootref(), 0u, 0u, ""); -} - -TEST(locations, seq_block) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - csubstr yaml = R"( -- this -- is -- a -- seq -- - and - - this - - - as - - well - - - # this one works as well - # even with a comment in between - the scalar value is here - - and here's another value - - - - another val - - yet another val -)"; - Tree t = parser.parse_in_arena("myfile.yml", yaml); - ConstNodeRef seq = t.rootref(); - ASSERT_TRUE(seq.is_seq()); - _checkloc(seq , 1u, 0u, "- "); - _checkloc(seq[0] , 1u, 2u, "this"); - _checkloc(seq[1] , 2u, 2u, "is"); - _checkloc(seq[2] , 3u, 2u, "a"); - _checkloc(seq[3] , 4u, 2u, "seq"); - _checkloc(seq[4] , 5u, 2u, "- "); - _checkloc(seq[4][0] , 5u, 4u, "and"); - _checkloc(seq[4][1] , 6u, 4u, "this"); - _checkloc(seq[4][2] , 7u, 4u, "- "); - _checkloc(seq[4][2][0], 7u, 6u, "as"); - _checkloc(seq[4][2][1], 8u, 6u, "well"); - _checkloc(seq[4][3] , 9u, 4u, "- # this one works as well"); - _checkloc(seq[4][3][0], 11u, 6u, "the scalar value is here"); - _checkloc(seq[4][3][1], 12u, 8u, "and here's another value"); - _checkloc(seq[4][3][2], 14u, 6u, "- "); - _checkloc(seq[4][3][2][0], 14u, 8u, "another val"); - _checkloc(seq[4][3][2][1], 15u, 8u, "yet another val"); -} - -TEST(locations, map_block) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - csubstr yaml = R"( -this: ~ -is: ~ -a: ~ -map: ~ -and: - this: - as: ~ - well: ~ - aswell: # this one works as well - # even with a comment in between - val: here - hah: here -)"; - Tree t = parser.parse_in_arena("myfile.yml", yaml); - ConstNodeRef map = t.rootref(); - ASSERT_TRUE(map.is_map()); - _checkloc(map , 1u, 0u, "this:"); - _checkloc(map["this"] , 1u, 0u, "this:"); - _checkloc(map["is"] , 2u, 0u, "is:"); - _checkloc(map["a"] , 3u, 0u, "a:"); - _checkloc(map["map"] , 4u, 0u, "map:"); - _checkloc(map["and"] , 5u, 0u, "and:"); - _checkloc(map["and"]["this"] , 6u, 2u, "this:"); - _checkloc(map["and"]["this"]["as"] , 7u, 4u, "as:"); - _checkloc(map["and"]["this"]["well"] , 8u, 4u, "well:"); - _checkloc(map["and"]["aswell"] , 9u, 2u, "aswell:"); - _checkloc(map["and"]["aswell"]["val"] , 11u, 4u, "val:"); - _checkloc(map["and"]["aswell"]["hah"] , 12u, 4u, "hah:"); -} - -TEST(locations, seq_block_null) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - const Tree t = parser.parse_in_arena("myfile.yml", R"(--- -- ~ -- ~ -- notnull -- ~ -- ~ ---- -- ~ -- - ~ -- - - ~ -- - - - ~ -- - - - - ~ -- - - - ~ -- - - ~ -- - ~ -- ~ ---- -- -- -- -- - - - - - - - - - - - - - - - - - - - - - - - - -- -)"); - _checkloc(t.rootref() , 0u, 0u, "---"); - _checkloc(t.docref(0) , 1u, 0u, "- "); - _checkloc(t.docref(0)[0], 1u, 2u, "~"); - _checkloc(t.docref(0)[1], 2u, 2u, "~"); - _checkloc(t.docref(0)[2], 3u, 2u, "notnull"); - _checkloc(t.docref(0)[3], 4u, 2u, "~"); - _checkloc(t.docref(0)[4], 5u, 2u, "~"); - _checkloc(t.docref(1) , 7u, 0u, "- "); - _checkloc(t.docref(1)[0], 7u, 2u, "~"); - _checkloc(t.docref(1)[1], 8u, 2u, "- "); - _checkloc(t.docref(1)[1][0], 8u, 4u, "~"); - _checkloc(t.docref(1)[2], 9u, 2u, "- "); - _checkloc(t.docref(1)[2][0], 9u, 4u, "- "); - _checkloc(t.docref(1)[2][0][0], 9u, 6u, "~"); - _checkloc(t.docref(1)[3], 10u, 2u, "- "); - _checkloc(t.docref(1)[3][0], 10u, 4u, "- "); - _checkloc(t.docref(1)[3][0][0], 10u, 6u, "- "); - _checkloc(t.docref(1)[3][0][0][0], 10u, 8u, "~"); - _checkloc(t.docref(1)[4], 11u, 2u, "- "); - _checkloc(t.docref(1)[4][0], 11u, 4u, "- "); - _checkloc(t.docref(1)[4][0][0], 11u, 6u, "- "); - _checkloc(t.docref(1)[4][0][0][0], 11u, 8u, "- "); - _checkloc(t.docref(1)[4][0][0][0][0], 11u, 10u, "~"); - _checkloc(t.docref(1)[5], 12u, 2u, "- "); - _checkloc(t.docref(1)[5][0], 12u, 4u, "- "); - _checkloc(t.docref(1)[5][0][0], 12u, 6u, "- "); - _checkloc(t.docref(1)[5][0][0][0], 12u, 8u, "~"); - _checkloc(t.docref(1)[6], 13u, 2u, "- "); - _checkloc(t.docref(1)[6][0], 13u, 4u, "- "); - _checkloc(t.docref(1)[6][0][0], 13u, 6u, "~"); - _checkloc(t.docref(1)[7], 14u, 2u, "- "); - _checkloc(t.docref(1)[7][0], 14u, 4u, "~"); - _checkloc(t.docref(1)[8], 15u, 2u, "~"); - _checkloc(t.docref(2) , 17u, 0u, "-"); - _checkloc(t.docref(2)[0], 17u, 0u, "-"); - _checkloc(t.docref(2)[1], 17u, 0u, "-"); - _checkloc(t.docref(2)[2], 21u, 2u, "-"); - _checkloc(t.docref(2)[3], 21u, 2u, "-"); - _checkloc(t.docref(2)[3][0], 21u, 2u, "-"); - _checkloc(t.docref(2)[3][1], 24u, 4u, "-"); - _checkloc(t.docref(2)[3][2], 24u, 4u, "-"); - _checkloc(t.docref(2)[3][2][0], 25u, 6u, "-"); - _checkloc(t.docref(2)[3][2][0][0], 26u, 8u, "-"); - _checkloc(t.docref(2)[3][2][0][0][0], 26u, 8u, "-"); - _checkloc(t.docref(2)[3][2][0][0][1], 26u, 8u, "-"); - _checkloc(t.docref(2)[3][2][1], 25u, 6u, "-"); - _checkloc(t.docref(2)[3][3], 24u, 4u, "-"); // fix: this should be after the previous child - _checkloc(t.docref(2)[3][4], 21u, 2u, "-"); // fix: this should be after the previous child - _checkloc(t.docref(2)[3][5], 21u, 2u, "-"); // fix: this should be after the previous child -} - -TEST(locations, map_block_null) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - Tree t = parser.parse_in_arena("myfile.yml", R"(--- -~: v ---- -null: v ---- - : v -)"); - _checkloc(t.rootref() , 0u, 0u, "---"); - _checkloc(t.docref(0) , 1u, 0u, ""); - _checkloc(t.docref(0)[0], 1u, 0u, "~"); - _checkloc(t.docref(1) , 3u, 0u, "null"); - _checkloc(t.docref(1)[0], 3u, 0u, "null"); - _checkloc(t.docref(2) , 5u, 1u, ""); - _checkloc(t.docref(2)[0], 5u, 3u, ""); -} - -TEST(locations, empty_seq) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - Tree t = parser.parse_in_arena("myfile.yml", R"(--- -- [] -- [] -- notnull -- [] -- [] ---- -- [] ---- -[] ---- -key0: [] -key1: [] -key2: notnull -key3: [] -key4: [] ---- -key: [] -)"); - _checkloc(t.rootref() , 0u, 0u, "---"); - _checkloc(t.docref(0) , 1u, 0u, "- "); - _checkloc(t.docref(0)[0], 1u, 2u, "[]"); - _checkloc(t.docref(0)[1], 2u, 2u, "[]"); - _checkloc(t.docref(0)[2], 3u, 2u, "notnull"); - _checkloc(t.docref(0)[3], 4u, 2u, "[]"); - _checkloc(t.docref(0)[4], 5u, 2u, "[]"); - _checkloc(t.docref(1) , 7u, 0u, "- "); - _checkloc(t.docref(1)[0], 7u, 2u, "[]"); - _checkloc(t.docref(2) , 9u, 0u, "[]"); - _checkloc(t.docref(3) , 11u, 0u, "key0"); // WTF - _checkloc(t.docref(3)["key0"], 11u, 0u, "key0"); - _checkloc(t.docref(3)["key1"], 12u, 0u, "key1"); - _checkloc(t.docref(3)["key2"], 13u, 0u, "key2"); - _checkloc(t.docref(3)["key3"], 14u, 0u, "key3"); - _checkloc(t.docref(3)["key4"], 15u, 0u, "key4"); -} - -TEST(locations, empty_map) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - Tree t = parser.parse_in_arena("myfile.yml", R"(--- -- {} -- {} -- notnull -- {} -- {} ---- -- {} ---- -{} ---- -key0: {} -key1: {} -key2: notnull -key3: {} -key4: {} ---- -key: {} -)"); - _checkloc(t.rootref() , 0u, 0u, "---"); - _checkloc(t.docref(0) , 1u, 0u, "- "); - _checkloc(t.docref(0)[0], 1u, 2u, "{}"); - _checkloc(t.docref(0)[1], 2u, 2u, "{}"); - _checkloc(t.docref(0)[2], 3u, 2u, "notnull"); - _checkloc(t.docref(0)[3], 4u, 2u, "{}"); - _checkloc(t.docref(0)[4], 5u, 2u, "{}"); - _checkloc(t.docref(1) , 7u, 0u, "- "); - _checkloc(t.docref(1)[0], 7u, 2u, "{}"); - _checkloc(t.docref(2) , 9u, 0u, "{}"); - _checkloc(t.docref(3) , 11u, 0u, "key0"); // WTF - _checkloc(t.docref(3)["key0"], 11u, 0u, "key0"); - _checkloc(t.docref(3)["key1"], 12u, 0u, "key1"); - _checkloc(t.docref(3)["key2"], 13u, 0u, "key2"); - _checkloc(t.docref(3)["key3"], 14u, 0u, "key3"); - _checkloc(t.docref(3)["key4"], 15u, 0u, "key4"); -} - - -TEST(locations, seq_flow) -{ - Tree t; - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - csubstr yaml = R"([one,two,three,four,items])"; - parser.parse_in_arena("myfile.yml", yaml, &t); - ConstNodeRef seq = t.rootref(); - ASSERT_TRUE(seq.is_seq()); - _checkloc(seq , 0u, 0u, "["); - _checkloc(seq[0], 0u, 1u, "one"); - _checkloc(seq[1], 0u, 5u, "two"); - _checkloc(seq[2], 0u, 9u, "three"); - _checkloc(seq[3], 0u, 15u, "four"); - _checkloc(seq[4], 0u, 20u, "items"); -} - -TEST(locations, map_flow) -{ - Tree t; - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - csubstr yaml = R"({one: item,two: items,three: items,four: items})"; - parser.parse_in_arena("myfile.yml", yaml, &t); - ConstNodeRef map = t.rootref(); - ASSERT_TRUE(map.is_map()); - _checkloc(map , 0u, 0u, "{"); - _checkloc(map[0], 0u, 1u, "one:"); - _checkloc(map[1], 0u, 11u, "two:"); - _checkloc(map[2], 0u, 22u, "three:"); - _checkloc(map[3], 0u, 35u, "four:"); -} - -TEST(locations, seq_flow_nested) -{ - Tree t; - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - csubstr yaml = R"([ - one, - two, - [woops, there, [ goes, master]], - five, - {wait: is, this: { really: a map? }, nope: [ a seq!, { had: you there, eehh: no. } ]}, - yep, - it, - was -])"; - parser.parse_in_arena("myfile.yml", yaml, &t); - ConstNodeRef seq = t.rootref(); - ASSERT_TRUE(seq.is_seq()); - _checkloc(seq , 0u, 0u, "["); - _checkloc(seq[0] , 1u, 2u, "one"); - _checkloc(seq[1] , 2u, 2u, "two"); - _checkloc(seq[2] , 3u, 2u, "["); - _checkloc(seq[2][0] , 3u, 3u, "woops"); - _checkloc(seq[2][1] , 3u, 14u, "there"); - _checkloc(seq[2][2] , 3u, 24u, "["); - _checkloc(seq[2][2][0] , 3u, 27u, "goes"); - _checkloc(seq[2][2][1] , 3u, 37u, "master"); - _checkloc(seq[3] , 4u, 2u, "five"); - _checkloc(seq[4] , 5u, 2u, "{"); - _checkloc(seq[4]["wait"] , 5u, 3u, "wait"); - _checkloc(seq[4]["this"] , 5u, 15u, "this"); - _checkloc(seq[4]["this"]["really"] , 5u, 27u, "really"); - _checkloc(seq[4]["nope"] , 5u, 47u, "nope"); - _checkloc(seq[4]["nope"][0] , 5u, 56u, "a seq!"); - _checkloc(seq[4]["nope"][1] , 5u, 64u, "{"); - _checkloc(seq[4]["nope"][1]["had"] , 5u, 66u, "had"); - _checkloc(seq[4]["nope"][1]["eehh"], 5u, 83u, "eehh"); - _checkloc(seq[5] , 6u, 2u, "yep"); - _checkloc(seq[6] , 7u, 2u, "it"); - _checkloc(seq[7] , 8u, 2u, "was"); -} - -TEST(locations, grow_array) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - Tree t = parser.parse_in_arena("myfile.yml", "docval"); - _checkloc(t.rootref(), 0u, 0u, "docval"); - t = parser.parse_in_arena("myfile.yml", "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\ndocval"); - _checkloc(t.rootref(), 47u, 0u, "docval"); -} - -// do a test with a buffer size up to 30 lines to ensure hitting -// the binary search path -TEST(locations, small_array) -{ - Tree t; - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - csubstr yaml = R"(--- -foo: yes -bar: - - 1 - - 2 -baz: - - 1_ - - 2_ - - 3_ -)"; - parser.parse_in_arena("myfile.yml", yaml, &t); - ConstNodeRef stream = t.rootref(); - ConstNodeRef map = t.docref(0); - ASSERT_TRUE(map.is_map()); - ASSERT_TRUE(map.is_doc()); - _checkloc(stream , 0u, 0u, "---"); - _checkloc(map , 1u, 0u, "foo"); - _checkloc(map["foo"] , 1u, 0u, "foo"); - _checkloc(map["bar"] , 2u, 0u, "bar"); - _checkloc(map["bar"][0], 3u, 4u, "1"); - _checkloc(map["bar"][1], 4u, 4u, "2"); - _checkloc(map["baz"] , 5u, 0u, "baz"); - _checkloc(map["baz"][0], 6u, 6u, "1_"); - _checkloc(map["baz"][1], 7u, 6u, "2_"); - _checkloc(map["baz"][2], 8u, 10u, "3_"); -} - -// do a test with a buffer of at least 30 lines to ensure hitting -// the binary search path -TEST(locations, large_array) -{ - Tree t; - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - csubstr yaml = R"(--- -foo1: definitely # 1 -bar1: - - 1 - - 2 -baz1: - - 1_ - - 2_ - - 3_ - - - -foo2: definitely # 12 -bar2: - - 1 - - 2 -baz2: - - 1_ - - 2_ - - 3_ - - - -foo3: definitely # 23 -bar3: - - 1 - - 2 -baz3: - - 1_ - - 2_ - - 3_ - - - -foo4: definitely # 34 -bar4: - - 1 - - 2 -baz4: - - 1_ - - 2_ - - 3_ - - - -foo5: definitely # 45 -bar5: - - 1 - - 2 -baz5: - - 1_ - - 2_ - - 3_ - - - -foo6: definitely # 56 -bar6: - - 1 - - 2 -baz6: - - 1_ - - 2_ - - 3_ -)"; - parser.parse_in_arena("myfile.yml", yaml, &t); - ConstNodeRef map = t.docref(0); - ASSERT_TRUE(map.is_map()); - ASSERT_TRUE(map.is_doc()); - _checkloc(t.rootref() , 0u, 0u, "---"); - _checkloc(map , 1u, 0u, "foo1"); - _checkloc(map["foo1"] , 0u+1u, 0u, "foo1"); - _checkloc(map["bar1"] , 0u+2u, 0u, "bar1"); - _checkloc(map["bar1"][0], 0u+3u, 4u, "1"); - _checkloc(map["bar1"][1], 0u+4u, 4u, "2"); - _checkloc(map["baz1"] , 0u+5u, 0u, "baz"); - _checkloc(map["baz1"][0], 0u+6u, 6u, "1_"); - _checkloc(map["baz1"][1], 0u+7u, 6u, "2_"); - _checkloc(map["baz1"][2], 0u+8u, 10u, "3_"); - // - _checkloc(map["foo2"] , 11u+1u, 0u, "foo2"); - _checkloc(map["bar2"] , 11u+2u, 0u, "bar2"); - _checkloc(map["bar2"][0], 11u+3u, 4u, "1"); - _checkloc(map["bar2"][1], 11u+4u, 4u, "2"); - _checkloc(map["baz2"] , 11u+5u, 0u, "baz2"); - _checkloc(map["baz2"][0], 11u+6u, 6u, "1_"); - _checkloc(map["baz2"][1], 11u+7u, 6u, "2_"); - _checkloc(map["baz2"][2], 11u+8u, 10u, "3_"); - // - _checkloc(map["foo3"] , 22u+1u, 0u, "foo3"); - _checkloc(map["bar3"] , 22u+2u, 0u, "bar3"); - _checkloc(map["bar3"][0], 22u+3u, 4u, "1"); - _checkloc(map["bar3"][1], 22u+4u, 4u, "2"); - _checkloc(map["baz3"] , 22u+5u, 0u, "baz3"); - _checkloc(map["baz3"][0], 22u+6u, 6u, "1_"); - _checkloc(map["baz3"][1], 22u+7u, 6u, "2_"); - _checkloc(map["baz3"][2], 22u+8u, 10u, "3_"); - // - _checkloc(map["foo4"] , 33u+1u, 0u, "foo4"); - _checkloc(map["bar4"] , 33u+2u, 0u, "bar4"); - _checkloc(map["bar4"][0], 33u+3u, 4u, "1"); - _checkloc(map["bar4"][1], 33u+4u, 4u, "2"); - _checkloc(map["baz4"] , 33u+5u, 0u, "baz4"); - _checkloc(map["baz4"][0], 33u+6u, 6u, "1_"); - _checkloc(map["baz4"][1], 33u+7u, 6u, "2_"); - _checkloc(map["baz4"][2], 33u+8u, 10u, "3_"); - // - _checkloc(map["foo5"] , 44u+1u, 0u, "foo5"); - _checkloc(map["bar5"] , 44u+2u, 0u, "bar5"); - _checkloc(map["bar5"][0], 44u+3u, 4u, "1"); - _checkloc(map["bar5"][1], 44u+4u, 4u, "2"); - _checkloc(map["baz5"] , 44u+5u, 0u, "baz5"); - _checkloc(map["baz5"][0], 44u+6u, 6u, "1_"); - _checkloc(map["baz5"][1], 44u+7u, 6u, "2_"); - _checkloc(map["baz5"][2], 44u+8u, 10u, "3_"); - // - _checkloc(map["foo6"] , 55u+1u, 0u, "foo6"); - _checkloc(map["bar6"] , 55u+2u, 0u, "bar6"); - _checkloc(map["bar6"][0], 55u+3u, 4u, "1"); - _checkloc(map["bar6"][1], 55u+4u, 4u, "2"); - _checkloc(map["baz6"] , 55u+5u, 0u, "baz6"); - _checkloc(map["baz6"][0], 55u+6u, 6u, "1_"); - _checkloc(map["baz6"][1], 55u+7u, 6u, "2_"); - _checkloc(map["baz6"][2], 55u+8u, 10u, "3_"); -} - - -TEST(locations, issue260_0) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - Tree tree = parser.parse_in_arena("source.yml", R"(Body: - - Id: 1 - Name: Apple - Script: | - Line One - Line Two - - Id: 2 - Name: Cat - Script: | - Line One - Line Two - - Id: 3 - Name: Dog - Script: | - Line One - Line Two)"); - EXPECT_EQ(tree["Body"][2]["Name"].val(), "Dog"); - EXPECT_EQ(parser.location(tree["Body"][2]["Name"]).line, 12u); -} - -TEST(locations, issue260_1) -{ - ParserOptions opts = ParserOptions().locations(true); - Parser parser(opts); - Tree tree = parser.parse_in_arena("source.yml", R"(Body: # 0 - - Id: 1 # line 1 - Name: Apple - - Id: 2 # line 3 - Name: Cat - Script: | - Line One - Line Two - - Id: 3 # line 8 - Name: Cat2 - Script: > - Line One - Line Two - - Id: 4 # line 13 - Name: Cat3 - Script: " - Line One - Line Two" - - Id: 5 # line 18 - Name: Cat4 - Script: ' - Line One - Line Two' - - Id: 5 # line 23 - Name: Cat5 - Script: - Line One - Line Two - - Id: 6 # line 28 - Name: Dog - Script: | - Line One - Line Two)"); - EXPECT_EQ(parser.location(tree["Body"][0]).line, 1u); - EXPECT_EQ(parser.location(tree["Body"][1]).line, 3u); - EXPECT_EQ(parser.location(tree["Body"][2]).line, 8u); - EXPECT_EQ(parser.location(tree["Body"][3]).line, 13u); - EXPECT_EQ(parser.location(tree["Body"][4]).line, 18u); - EXPECT_EQ(parser.location(tree["Body"][5]).line, 23u); - EXPECT_EQ(parser.location(tree["Body"][6]).line, 28u); - EXPECT_EQ(parser.location(tree["Body"][0]["Id"]).line, 1u); - EXPECT_EQ(parser.location(tree["Body"][1]["Id"]).line, 3u); - EXPECT_EQ(parser.location(tree["Body"][2]["Id"]).line, 8u); - EXPECT_EQ(parser.location(tree["Body"][3]["Id"]).line, 13u); - EXPECT_EQ(parser.location(tree["Body"][4]["Id"]).line, 18u); - EXPECT_EQ(parser.location(tree["Body"][5]["Id"]).line, 23u); - EXPECT_EQ(parser.location(tree["Body"][6]["Id"]).line, 28u); - EXPECT_EQ(parser.location(tree["Body"][0]["Name"]).line, 1u+1u); - EXPECT_EQ(parser.location(tree["Body"][1]["Name"]).line, 3u+1u); - EXPECT_EQ(parser.location(tree["Body"][2]["Name"]).line, 8u+1u); - EXPECT_EQ(parser.location(tree["Body"][3]["Name"]).line, 13u+1u); - EXPECT_EQ(parser.location(tree["Body"][4]["Name"]).line, 18u+1u); - EXPECT_EQ(parser.location(tree["Body"][5]["Name"]).line, 23u+1u); - EXPECT_EQ(parser.location(tree["Body"][6]["Name"]).line, 28u+1u); -} - - - -// The other test executables are written to contain the declarative-style -// YmlTestCases. This executable does not have any but the build setup -// assumes it does, and links with the test lib, which requires an existing -// get_case() function. So this is here to act as placeholder until (if?) -// proper test cases are added here. This was detected in #47 (thanks -// @cburgard). -Case const* get_case(csubstr) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_map_of_seq.cpp b/thirdparty/ryml/test/test_map_of_seq.cpp deleted file mode 100644 index fc577223e..000000000 --- a/thirdparty/ryml/test/test_map_of_seq.cpp +++ /dev/null @@ -1,201 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - - -CASE_GROUP(MAP_OF_SEQ) -{ - -ADD_CASE_TO_GROUP("map of empty seqs", -R"({foo: [], bar: [], baz: []})", - L{ - N(KEYSEQ, "foo", L()), - N(KEYSEQ, "bar", L()), - N(KEYSEQ, "baz", L()), - } -); - -ADD_CASE_TO_GROUP("map of seqs, one line", -R"({men: [John Smith, Bill Jones], women: [Mary Smith, Susan Williams]})", - L{ - N("men", L{N{"John Smith"}, N{"Bill Jones"}}), - N("women", L{N{"Mary Smith"}, N{"Susan Williams"}}) - } -); - -ADD_CASE_TO_GROUP("map of seqs", -R"( -men: - - John Smith - - Bill Jones -women: - - Mary Smith - - Susan Williams -)", - L{ - N("men", L{N{"John Smith"}, N{"Bill Jones"}}), - N("women", L{N{"Mary Smith"}, N{"Susan Williams"}}) - } -); - -ADD_CASE_TO_GROUP("map of seqs, not indented", -R"( -men: -- John Smith -- Bill Jones -women: -- Mary Smith -- Susan Williams -)", - L{ - N("men", L{N{"John Smith"}, N{"Bill Jones"}}), - N("women", L{N{"Mary Smith"}, N{"Susan Williams"}}) - } -); - -ADD_CASE_TO_GROUP("map of seqs, not indented, more", -R"( -product: -- sku: BL4438H - quantity: 1 - description: Super Hoop - price: 2392.00 # jumping one level here would be wrong. -tax: 1234.5 # we must jump two levels -product2: - subproduct1: - - sku: BL4438H - quantity: 1 - description: Super Hoop - price: 2392.00 # jumping one level here would be wrong. - subproduct2: - - sku: BL4438H - quantity: 1 - description: Super Hoop - price: 2392.00 # jumping one level here would be wrong. - tax2: 789.10 # we must jump two levels -tax3: 1234.5 -product3: - subproduct1: - - sku: BL4438H - quantity: 1 - description: Super Hoop - price: 2392.00 # jumping one level here would be wrong. - subproduct2: - - sku: BL4438H - quantity: 1 - description: Super Hoop - price: 2392.00 # jumping one level here would be wrong. - # a comment here, will it ruin parsing? - tax2: 789.10 # we must jump two levels -tax4: 1234.5 -product4: - subproduct1: - - sku: BL4438H - quantity: 1 - description: Super Hoop - price: 2392.00 # jumping one level here would be wrong. - subproduct2: - - sku: BL4438H - quantity: 1 - description: Super Hoop - price: 2392.00 # jumping one level here would be wrong. - # what about here? - tax2: 789.10 # we must jump two levels -tax5: 1234.5 -)", -L{ - N("product", L{ - N(L{N("sku", "BL4438H"), N("quantity", "1"), N("description", "Super Hoop"), N("price", "2392.00")}), - }), - N("tax", "1234.5"), - N("product2", L{ - N("subproduct1", L{ - N(L{N("sku", "BL4438H"), N("quantity", "1"), N("description", "Super Hoop"), N("price", "2392.00")}), - }), - N("subproduct2", L{ - N(L{N("sku", "BL4438H"), N("quantity", "1"), N("description", "Super Hoop"), N("price", "2392.00")}), - }), - N("tax2", "789.10"), - }), - N("tax3", "1234.5"), - N("product3", L{ - N("subproduct1", L{ - N(L{N("sku", "BL4438H"), N("quantity", "1"), N("description", "Super Hoop"), N("price", "2392.00")}), - }), - N("subproduct2", L{ - N(L{N("sku", "BL4438H"), N("quantity", "1"), N("description", "Super Hoop"), N("price", "2392.00")}), - }), - N("tax2", "789.10"), - }), - N("tax4", "1234.5"), - N("product4", L{ - N("subproduct1", L{ - N(L{N("sku", "BL4438H"), N("quantity", "1"), N("description", "Super Hoop"), N("price", "2392.00")}), - }), - N("subproduct2", L{ - N(L{N("sku", "BL4438H"), N("quantity", "1"), N("description", "Super Hoop"), N("price", "2392.00")}), - }), - N("tax2", "789.10"), - }), - N("tax5", "1234.5"), -}); - -ADD_CASE_TO_GROUP("map of seqs, next line", -R"( -men: - - - John Smith - - - Bill Jones -women: - - - Mary Smith - - - Susan Williams -)", - L{ - N("men", L{N{"John Smith"}, N{"Bill Jones"}}), - N("women", L{N{"Mary Smith"}, N{"Susan Williams"}}) - } -); - -ADD_CASE_TO_GROUP("map of seqs, next line without space", -R"( -men: - - - John Smith - - - Bill Jones -women: - - - Mary Smith - - - Susan Williams -)", - L{ - N("men", L{N{"John Smith"}, N{"Bill Jones"}}), - N("women", L{N{"Mary Smith"}, N{"Susan Williams"}}) - } -); - -ADD_CASE_TO_GROUP("map of seqs, deal with unk", -R"( -skip_commits: - files: - - a - - b - - c - - d - - e -)", -L{ - N("skip_commits", L{N("files", - L{N("a"), N("b"), N("c"), N("d"), N("e")} - )}), -} -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_merge.cpp b/thirdparty/ryml/test/test_merge.cpp deleted file mode 100644 index 873412115..000000000 --- a/thirdparty/ryml/test/test_merge.cpp +++ /dev/null @@ -1,225 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include -#include -#endif -#include - -#include "./test_case.hpp" - -namespace c4 { -namespace yml { - -// The other test executables are written to contain the declarative-style -// YmlTestCases. This executable does not have any but the build setup -// assumes it does, and links with the test lib, which requires an existing -// get_case() function. So this is here to act as placeholder until (if?) -// proper test cases are added here. This was detected in #47 (thanks -// @cburgard). -Case const* get_case(csubstr) -{ - return nullptr; -} - - -void test_merge(std::initializer_list li, csubstr expected) -{ - Tree loaded, merged, ref; - - parse_in_arena(expected, &ref); - - // make sure the arena in the loaded tree is never resized - size_t arena_dim = 2; - for(csubstr src : li) - { - arena_dim += src.len; - } - loaded.reserve_arena(arena_dim); - - for(csubstr src : li) - { - loaded.clear(); // do not clear the arena of the loaded tree - parse_in_arena(src, &loaded); - merged.merge_with(&loaded); - } - - auto buf_result = emitrs_yaml(merged); - auto buf_expected = emitrs_yaml(ref); - - EXPECT_EQ(buf_result, buf_expected); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(merge, basic) -{ - test_merge( - { - "{a: 0, b: 1}", - "{a: 1, c: 20}" - }, - "{a: 1, b: 1, c: 20}" - ); -} - -TEST(merge, val_to_seq) -{ - test_merge( - { - "{a: 0, b: 1}", - "{a: [1, 2]}" - }, - "{a: [1, 2], b: 1}" - ); -} - -TEST(merge, seq_to_val) -{ - test_merge( - { - "{a: [1, 2]}", - "{a: 0, b: 1}", - }, - "{a: 0, b: 1}" - ); -} - -TEST(merge, val_to_map) -{ - test_merge( - { - "{a: 0, b: 1}", - "{a: {c: 10, d: 20}}" - }, - "{a: {c: 10, d: 20}, b: 1}" - ); -} - -TEST(merge, map_to_val) -{ - test_merge( - { - "{a: {c: 10, d: 20}}", - "{a: 0, b: 1}", - }, - "{a: 0, b: 1}" - ); -} - -TEST(merge, seq_no_overlap_explicit) -{ - test_merge( - {"[0, 1, 2]", "[3, 4, 5]", "[6, 7, 8]"}, - "[0, 1, 2, 3, 4, 5, 6, 7, 8]" - ); -} - - -TEST(merge, seq_no_overlap_implicit) -{ - test_merge( - {"0, 1, 2", "3, 4, 5", "6, 7, 8"}, - "0, 1, 2, 3, 4, 5, 6, 7, 8" - ); -} - - -TEST(merge, seq_overlap_explicit) -{ - test_merge( - {"[0, 1, 2]", "[1, 2, 3]", "[2, 3, 4]"}, - "[0, 1, 2, 1, 2, 3, 2, 3, 4]" - // or this? "[0, 1, 2, 3, 4]" - ); -} - - -TEST(merge, seq_overlap_implicit) -{ - // now a bit more difficult - test_merge( - {"0, 1, 2", "1, 2, 3", "2, 3, 4"}, - "0, 1, 2, 1, 2, 3, 2, 3, 4" - // or this? "0, 1, 2, 3, 4" - ); -} - - -TEST(merge, map_orthogonal) -{ - test_merge( - {"a: 0", "b: 1", "c: 2"}, - "{a: 0, b: 1, c: 2}" - ); -} - - -TEST(merge, map_overriding) -{ - test_merge( - { - "a: 0", - "{a: 1, b: 1}", - "c: 2" - }, - "{a: 1, b: 1, c: 2}" - ); -} - -TEST(merge, map_overriding_multiple) -{ - test_merge( - { - "a: 0", - "{a: 1, b: 1}", - "c: 2", - "a: 2", - "a: 3", - "c: 4", - "c: 5", - "a: 4", - }, - "{a: 4, b: 1, c: 5}" - ); -} - - -TEST(merge, seq_nested_in_map) -{ - test_merge( - { - "{a: 0, seq: [a, b, c], d: 2}", - "{a: 1, seq: [d, e, f], d: 3, c: 3}" - }, - "{a: 1, seq: [a, b, c, d, e, f], d: 3, c: 3}" - ); -} - - -TEST(merge, seq_nested_in_map_override_with_map) -{ - test_merge( - { - "{a: 0, ovr: [a, b, c], d: 2}", - "{a: 1, ovr: {d: 0, b: 1, c: 2}, d: 3, c: 3}" - }, - "{a: 1, ovr: {d: 0, b: 1, c: 2}, d: 3, c: 3}" - ); -} - - -TEST(merge, seq_nested_in_map_override_with_keyval) -{ - test_merge( - { - "{a: 0, ovr: [a, b, c], d: 2}", - "{a: 1, ovr: foo, d: 3, c: 3}" - }, - "{a: 1, ovr: foo, d: 3, c: 3}" - ); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_nested_mapx2.cpp b/thirdparty/ryml/test/test_nested_mapx2.cpp deleted file mode 100644 index b1856fa08..000000000 --- a/thirdparty/ryml/test/test_nested_mapx2.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -CASE_GROUP(NESTED_MAPX2) -{ - -ADD_CASE_TO_GROUP("nested map x2, explicit, same line", -R"({foo: {foo0: 00, bar0: 01, baz0: 02}, bar: {foo1: 10, bar1: 11, baz1: 12}, baz: {foo2: 20, bar2: 21, baz2: 22}})", - L{ - N{"foo", L{N{"foo0", "00"}, N{"bar0", "01"}, N{"baz0", "02"}}}, - N{"bar", L{N{"foo1", "10"}, N{"bar1", "11"}, N{"baz1", "12"}}}, - N{"baz", L{N{"foo2", "20"}, N{"bar2", "21"}, N{"baz2", "22"}}}, - } -); - -ADD_CASE_TO_GROUP("nested map x2, explicit", -R"({ -foo: {foo0: 00, bar0: 01, baz0: 02}, -bar: {foo1: 10, bar1: 11, baz1: 12}, -baz: {foo2: 20, bar2: 21, baz2: 22} -})", - L{ - N{"foo", L{N{"foo0", "00"}, N{"bar0", "01"}, N{"baz0", "02"}}}, - N{"bar", L{N{"foo1", "10"}, N{"bar1", "11"}, N{"baz1", "12"}}}, - N{"baz", L{N{"foo2", "20"}, N{"bar2", "21"}, N{"baz2", "22"}}}, - } -); - -ADD_CASE_TO_GROUP("nested map x2", -R"( -foo: - foo0: 00 - bar0: 01 - baz0: 02 -bar: - foo1: 10 - bar1: 11 - baz1: 12 -baz: - foo2: 20 - bar2: 21 - baz2: 22 -)", - L{ - N{"foo", L{N{"foo0", "00"}, N{"bar0", "01"}, N{"baz0", "02"}}}, - N{"bar", L{N{"foo1", "10"}, N{"bar1", "11"}, N{"baz1", "12"}}}, - N{"baz", L{N{"foo2", "20"}, N{"bar2", "21"}, N{"baz2", "22"}}}, - } -); - - -ADD_CASE_TO_GROUP("nested map x2, commented", - R"( -send_to: - #host: 192.168.1.100 - #port: 7000 - host: 192.168.1.101 - port: 7001 - #host: 192.168.1.102 - #port: 7002 -)", - L{ - N("send_to", L{ - N("host", "192.168.1.101"), - N("port", "7001") }) - } -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_nested_mapx3.cpp b/thirdparty/ryml/test/test_nested_mapx3.cpp deleted file mode 100644 index 2ae163475..000000000 --- a/thirdparty/ryml/test/test_nested_mapx3.cpp +++ /dev/null @@ -1,103 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -CASE_GROUP(NESTED_MAPX3) -{ - -ADD_CASE_TO_GROUP("nested map x3, explicit", -R"({ - foo0: { - foo1: {foo2: 000, bar2: 001, baz2: 002}, - bar1: {foo2: 010, bar2: 011, baz2: 012}, - baz1: {foo2: 020, bar2: 021, baz2: 022} - }, - bar0: { - foo1: {foo2: 100, bar2: 101, baz2: 102}, - bar1: {foo2: 110, bar2: 111, baz2: 112}, - baz1: {foo2: 120, bar2: 121, baz2: 122} - }, - baz0: { - foo1: {foo2: 200, bar2: 201, baz2: 202}, - bar1: {foo2: 210, bar2: 211, baz2: 212}, - baz1: {foo2: 220, bar2: 221, baz2: 222} - } -})", - L{ - N{"foo0", L{ - N{"foo1", L{N{"foo2", "000"}, N{"bar2", "001"}, N{"baz2", "002"}}}, - N{"bar1", L{N{"foo2", "010"}, N{"bar2", "011"}, N{"baz2", "012"}}}, - N{"baz1", L{N{"foo2", "020"}, N{"bar2", "021"}, N{"baz2", "022"}}} }}, - N{"bar0", L{ - N{"foo1", L{N{"foo2", "100"}, N{"bar2", "101"}, N{"baz2", "102"}}}, - N{"bar1", L{N{"foo2", "110"}, N{"bar2", "111"}, N{"baz2", "112"}}}, - N{"baz1", L{N{"foo2", "120"}, N{"bar2", "121"}, N{"baz2", "122"}}} }}, - N{"baz0", L{ - N{"foo1", L{N{"foo2", "200"}, N{"bar2", "201"}, N{"baz2", "202"}}}, - N{"bar1", L{N{"foo2", "210"}, N{"bar2", "211"}, N{"baz2", "212"}}}, - N{"baz1", L{N{"foo2", "220"}, N{"bar2", "221"}, N{"baz2", "222"}}} }}, - } -); - -ADD_CASE_TO_GROUP("nested map x3", -R"( -foo0: - foo1: - foo2: 000 - bar2: 001 - baz2: 002 - bar1: - foo2: 010 - bar2: 011 - baz2: 012 - baz1: - foo2: 020 - bar2: 021 - baz2: 022 -bar0: - foo1: - foo2: 100 - bar2: 101 - baz2: 102 - bar1: - foo2: 110 - bar2: 111 - baz2: 112 - baz1: - foo2: 120 - bar2: 121 - baz2: 122 -baz0: - foo1: - foo2: 200 - bar2: 201 - baz2: 202 - bar1: - foo2: 210 - bar2: 211 - baz2: 212 - baz1: - foo2: 220 - bar2: 221 - baz2: 222 -)", - L{ - N{"foo0", L{ - N{"foo1", L{N{"foo2", "000"}, N{"bar2", "001"}, N{"baz2", "002"}}}, - N{"bar1", L{N{"foo2", "010"}, N{"bar2", "011"}, N{"baz2", "012"}}}, - N{"baz1", L{N{"foo2", "020"}, N{"bar2", "021"}, N{"baz2", "022"}}} }}, - N{"bar0", L{ - N{"foo1", L{N{"foo2", "100"}, N{"bar2", "101"}, N{"baz2", "102"}}}, - N{"bar1", L{N{"foo2", "110"}, N{"bar2", "111"}, N{"baz2", "112"}}}, - N{"baz1", L{N{"foo2", "120"}, N{"bar2", "121"}, N{"baz2", "122"}}} }}, - N{"baz0", L{ - N{"foo1", L{N{"foo2", "200"}, N{"bar2", "201"}, N{"baz2", "202"}}}, - N{"bar1", L{N{"foo2", "210"}, N{"bar2", "211"}, N{"baz2", "212"}}}, - N{"baz1", L{N{"foo2", "220"}, N{"bar2", "221"}, N{"baz2", "222"}}} }}, - } -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_nested_mapx4.cpp b/thirdparty/ryml/test/test_nested_mapx4.cpp deleted file mode 100644 index 1d40f2cb1..000000000 --- a/thirdparty/ryml/test/test_nested_mapx4.cpp +++ /dev/null @@ -1,190 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -CASE_GROUP(NESTED_MAPX4) -{ - -ADD_CASE_TO_GROUP("nested map x4, explicit", -R"({ - foo0: { - foo1: { foo2: {foo3: 0000, bar3: 0001, baz3: 0002}, bar2: {foo3: 0010, bar3: 0011, baz3: 0012}, baz2: {foo3: 0020, bar3: 0021, baz3: 0022} }, - bar1: { foo2: {foo3: 0100, bar3: 0101, baz3: 0102}, bar2: {foo3: 0110, bar3: 0111, baz3: 0112}, baz2: {foo3: 0120, bar3: 0121, baz3: 0122} }, - baz1: { foo2: {foo3: 0200, bar3: 0201, baz3: 0202}, bar2: {foo3: 0210, bar3: 0211, baz3: 0212}, baz2: {foo3: 0220, bar3: 0221, baz3: 0222} }, - }, - bar0: { - foo1: { foo2: {foo3: 1000, bar3: 1001, baz3: 1002}, bar2: {foo3: 1010, bar3: 1011, baz3: 1012}, baz2: {foo3: 1020, bar3: 1021, baz3: 1022} }, - bar1: { foo2: {foo3: 1100, bar3: 1101, baz3: 1102}, bar2: {foo3: 1110, bar3: 1111, baz3: 1112}, baz2: {foo3: 1120, bar3: 1121, baz3: 1122} }, - baz1: { foo2: {foo3: 1200, bar3: 1201, baz3: 1202}, bar2: {foo3: 1210, bar3: 1211, baz3: 1212}, baz2: {foo3: 1220, bar3: 1221, baz3: 1222} }, - }, - baz0: { - foo1: { foo2: {foo3: 2000, bar3: 2001, baz3: 2002}, bar2: {foo3: 2010, bar3: 2011, baz3: 2012}, baz2: {foo3: 2020, bar3: 2021, baz3: 2022} }, - bar1: { foo2: {foo3: 2100, bar3: 2101, baz3: 2102}, bar2: {foo3: 2110, bar3: 2111, baz3: 2112}, baz2: {foo3: 2120, bar3: 2121, baz3: 2122} }, - baz1: { foo2: {foo3: 2200, bar3: 2201, baz3: 2202}, bar2: {foo3: 2210, bar3: 2211, baz3: 2212}, baz2: {foo3: 2220, bar3: 2221, baz3: 2222} }, - }, -})", - L{ - N("foo0", L{ - N("foo1", L{N("foo2", L{N("foo3", "0000"), N("bar3", "0001"), N("baz3", "0002")}), N("bar2", L{N("foo3", "0010"), N("bar3", "0011"), N("baz3", "0012")}), N("baz2", L{N("foo3", "0020"), N("bar3", "0021"), N("baz3", "0022")})}), - N("bar1", L{N("foo2", L{N("foo3", "0100"), N("bar3", "0101"), N("baz3", "0102")}), N("bar2", L{N("foo3", "0110"), N("bar3", "0111"), N("baz3", "0112")}), N("baz2", L{N("foo3", "0120"), N("bar3", "0121"), N("baz3", "0122")})}), - N("baz1", L{N("foo2", L{N("foo3", "0200"), N("bar3", "0201"), N("baz3", "0202")}), N("bar2", L{N("foo3", "0210"), N("bar3", "0211"), N("baz3", "0212")}), N("baz2", L{N("foo3", "0220"), N("bar3", "0221"), N("baz3", "0222")})}), - }), - N("bar0", L{ - N("foo1", L{N("foo2", L{N("foo3", "1000"), N("bar3", "1001"), N("baz3", "1002")}), N("bar2", L{N("foo3", "1010"), N("bar3", "1011"), N("baz3", "1012")}), N("baz2", L{N("foo3", "1020"), N("bar3", "1021"), N("baz3", "1022")})}), - N("bar1", L{N("foo2", L{N("foo3", "1100"), N("bar3", "1101"), N("baz3", "1102")}), N("bar2", L{N("foo3", "1110"), N("bar3", "1111"), N("baz3", "1112")}), N("baz2", L{N("foo3", "1120"), N("bar3", "1121"), N("baz3", "1122")})}), - N("baz1", L{N("foo2", L{N("foo3", "1200"), N("bar3", "1201"), N("baz3", "1202")}), N("bar2", L{N("foo3", "1210"), N("bar3", "1211"), N("baz3", "1212")}), N("baz2", L{N("foo3", "1220"), N("bar3", "1221"), N("baz3", "1222")})}), - }), - N("baz0", L{ - N("foo1", L{N("foo2", L{N("foo3", "2000"), N("bar3", "2001"), N("baz3", "2002")}), N("bar2", L{N("foo3", "2010"), N("bar3", "2011"), N("baz3", "2012")}), N("baz2", L{N("foo3", "2020"), N("bar3", "2021"), N("baz3", "2022")})}), - N("bar1", L{N("foo2", L{N("foo3", "2100"), N("bar3", "2101"), N("baz3", "2102")}), N("bar2", L{N("foo3", "2110"), N("bar3", "2111"), N("baz3", "2112")}), N("baz2", L{N("foo3", "2120"), N("bar3", "2121"), N("baz3", "2122")})}), - N("baz1", L{N("foo2", L{N("foo3", "2200"), N("bar3", "2201"), N("baz3", "2202")}), N("bar2", L{N("foo3", "2210"), N("bar3", "2211"), N("baz3", "2212")}), N("baz2", L{N("foo3", "2220"), N("bar3", "2221"), N("baz3", "2222")})}), - }) - } -); - -ADD_CASE_TO_GROUP("nested map x4", -R"( -foo0: - foo1: - foo2: - foo3: 0000 - bar3: 0001 - baz3: 0002 - bar2: - foo3: 0010 - bar3: 0011 - baz3: 0012 - baz2: - foo3: 0020 - bar3: 0021 - baz3: 0022 - bar1: - foo2: - foo3: 0100 - bar3: 0101 - baz3: 0102 - bar2: - foo3: 0110 - bar3: 0111 - baz3: 0112 - baz2: - foo3: 0120 - bar3: 0121 - baz3: 0122 - baz1: - foo2: - foo3: 0200 - bar3: 0201 - baz3: 0202 - bar2: - foo3: 0210 - bar3: 0211 - baz3: 0212 - baz2: - foo3: 0220 - bar3: 0221 - baz3: 0222 -bar0: - foo1: - foo2: - foo3: 1000 - bar3: 1001 - baz3: 1002 - bar2: - foo3: 1010 - bar3: 1011 - baz3: 1012 - baz2: - foo3: 1020 - bar3: 1021 - baz3: 1022 - bar1: - foo2: - foo3: 1100 - bar3: 1101 - baz3: 1102 - bar2: - foo3: 1110 - bar3: 1111 - baz3: 1112 - baz2: - foo3: 1120 - bar3: 1121 - baz3: 1122 - baz1: - foo2: - foo3: 1200 - bar3: 1201 - baz3: 1202 - bar2: - foo3: 1210 - bar3: 1211 - baz3: 1212 - baz2: - foo3: 1220 - bar3: 1221 - baz3: 1222 -baz0: - foo1: - foo2: - foo3: 2000 - bar3: 2001 - baz3: 2002 - bar2: - foo3: 2010 - bar3: 2011 - baz3: 2012 - baz2: - foo3: 2020 - bar3: 2021 - baz3: 2022 - bar1: - foo2: - foo3: 2100 - bar3: 2101 - baz3: 2102 - bar2: - foo3: 2110 - bar3: 2111 - baz3: 2112 - baz2: - foo3: 2120 - bar3: 2121 - baz3: 2122 - baz1: - foo2: - foo3: 2200 - bar3: 2201 - baz3: 2202 - bar2: - foo3: 2210 - bar3: 2211 - baz3: 2212 - baz2: - foo3: 2220 - bar3: 2221 - baz3: 2222 -)", - L{ - N("foo0", L{ - N("foo1", L{N("foo2", L{N("foo3", "0000"), N("bar3", "0001"), N("baz3", "0002")}), N("bar2", L{N("foo3", "0010"), N("bar3", "0011"), N("baz3", "0012")}), N("baz2", L{N("foo3", "0020"), N("bar3", "0021"), N("baz3", "0022")})}), - N("bar1", L{N("foo2", L{N("foo3", "0100"), N("bar3", "0101"), N("baz3", "0102")}), N("bar2", L{N("foo3", "0110"), N("bar3", "0111"), N("baz3", "0112")}), N("baz2", L{N("foo3", "0120"), N("bar3", "0121"), N("baz3", "0122")})}), - N("baz1", L{N("foo2", L{N("foo3", "0200"), N("bar3", "0201"), N("baz3", "0202")}), N("bar2", L{N("foo3", "0210"), N("bar3", "0211"), N("baz3", "0212")}), N("baz2", L{N("foo3", "0220"), N("bar3", "0221"), N("baz3", "0222")})}), - }), - N("bar0", L{ - N("foo1", L{N("foo2", L{N("foo3", "1000"), N("bar3", "1001"), N("baz3", "1002")}), N("bar2", L{N("foo3", "1010"), N("bar3", "1011"), N("baz3", "1012")}), N("baz2", L{N("foo3", "1020"), N("bar3", "1021"), N("baz3", "1022")})}), - N("bar1", L{N("foo2", L{N("foo3", "1100"), N("bar3", "1101"), N("baz3", "1102")}), N("bar2", L{N("foo3", "1110"), N("bar3", "1111"), N("baz3", "1112")}), N("baz2", L{N("foo3", "1120"), N("bar3", "1121"), N("baz3", "1122")})}), - N("baz1", L{N("foo2", L{N("foo3", "1200"), N("bar3", "1201"), N("baz3", "1202")}), N("bar2", L{N("foo3", "1210"), N("bar3", "1211"), N("baz3", "1212")}), N("baz2", L{N("foo3", "1220"), N("bar3", "1221"), N("baz3", "1222")})}), - }), - N("baz0", L{ - N("foo1", L{N("foo2", L{N("foo3", "2000"), N("bar3", "2001"), N("baz3", "2002")}), N("bar2", L{N("foo3", "2010"), N("bar3", "2011"), N("baz3", "2012")}), N("baz2", L{N("foo3", "2020"), N("bar3", "2021"), N("baz3", "2022")})}), - N("bar1", L{N("foo2", L{N("foo3", "2100"), N("bar3", "2101"), N("baz3", "2102")}), N("bar2", L{N("foo3", "2110"), N("bar3", "2111"), N("baz3", "2112")}), N("baz2", L{N("foo3", "2120"), N("bar3", "2121"), N("baz3", "2122")})}), - N("baz1", L{N("foo2", L{N("foo3", "2200"), N("bar3", "2201"), N("baz3", "2202")}), N("bar2", L{N("foo3", "2210"), N("bar3", "2211"), N("baz3", "2212")}), N("baz2", L{N("foo3", "2220"), N("bar3", "2221"), N("baz3", "2222")})}), - }) - } -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_nested_seqx2.cpp b/thirdparty/ryml/test/test_nested_seqx2.cpp deleted file mode 100644 index 1361ae0ab..000000000 --- a/thirdparty/ryml/test/test_nested_seqx2.cpp +++ /dev/null @@ -1,133 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -CASE_GROUP(NESTED_SEQX2) -{ - -ADD_CASE_TO_GROUP("nested seq x2, empty, oneline", -R"([[], [], []])", - L{SEQ, SEQ, SEQ} -); - -ADD_CASE_TO_GROUP("nested seq x2, explicit, same line", -R"([[00, 01, 02], [10, 11, 12], [20, 21, 22]])", - L{ - N{L{N{"00"}, N{"01"}, N{"02"}}}, - N{L{N{"10"}, N{"11"}, N{"12"}}}, - N{L{N{"20"}, N{"21"}, N{"22"}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x2, explicit first+last level, same line, no spaces", -R"([[00,01,02],[10,11,12],[20,21,22]])", - L{ - N{L{N{"00"}, N{"01"}, N{"02"}}}, - N{L{N{"10"}, N{"11"}, N{"12"}}}, - N{L{N{"20"}, N{"21"}, N{"22"}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x2, explicit", -R"([ -[00, 01, 02], -[10, 11, 12], -[20, 21, 22], -])", - L{ - N{L{N{"00"}, N{"01"}, N{"02"}}}, - N{L{N{"10"}, N{"11"}, N{"12"}}}, - N{L{N{"20"}, N{"21"}, N{"22"}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x2", -R"( -- - 00 - - 01 - - 02 -- - 10 - - 11 - - 12 -- - 20 - - 21 - - 22 -)", - L{ - N{L{N{"00"}, N{"01"}, N{"02"}}}, - N{L{N{"10"}, N{"11"}, N{"12"}}}, - N{L{N{"20"}, N{"21"}, N{"22"}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x2, next line", -R"( -- - - 00 - - 01 - - 02 -- - - 10 - - 11 - - 12 -- - - 20 - - 21 - - 22 -)", - L{ - N{L{N{"00"}, N{"01"}, N{"02"}}}, - N{L{N{"10"}, N{"11"}, N{"12"}}}, - N{L{N{"20"}, N{"21"}, N{"22"}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x2, all next line", -R"( -- - - - 00 - - - 01 - - - 02 -- - - - 10 - - - 11 - - - 12 -- - - - 20 - - - 21 - - - 22 -)", - L{ - N{L{N{"00"}, N{"01"}, N{"02"}}}, - N{L{N{"10"}, N{"11"}, N{"12"}}}, - N{L{N{"20"}, N{"21"}, N{"22"}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x2, implicit first, explicit last level", -R"( -- [00, 01, 02] -- [10, 11, 12] -- [20, 21, 22] -)", - L{ - N{L{N{"00"}, N{"01"}, N{"02"}}}, - N{L{N{"10"}, N{"11"}, N{"12"}}}, - N{L{N{"20"}, N{"21"}, N{"22"}}}, - } -); -} - - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_nested_seqx3.cpp b/thirdparty/ryml/test/test_nested_seqx3.cpp deleted file mode 100644 index d1cc0beec..000000000 --- a/thirdparty/ryml/test/test_nested_seqx3.cpp +++ /dev/null @@ -1,187 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - - -CASE_GROUP(NESTED_SEQX3) -{ - -ADD_CASE_TO_GROUP("nested seq x3, explicit", -R"([ -[[000, 001, 002], [010, 011, 012], [020, 021, 022]], -[[100, 101, 102], [110, 111, 112], [120, 121, 122]], -[[200, 201, 202], [210, 211, 212], [220, 221, 222]], -])", - L{ - N{L{N{L{N{"000"}, N{"001"}, N{"002"}}}, N{L{N{"010"}, N{"011"}, N{"012"}}}, N{L{N{"020"}, N{"021"}, N{"022"}}}}}, - N{L{N{L{N{"100"}, N{"101"}, N{"102"}}}, N{L{N{"110"}, N{"111"}, N{"112"}}}, N{L{N{"120"}, N{"121"}, N{"122"}}}}}, - N{L{N{L{N{"200"}, N{"201"}, N{"202"}}}, N{L{N{"210"}, N{"211"}, N{"212"}}}, N{L{N{"220"}, N{"221"}, N{"222"}}}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x3", -R"( -- - - 000 - - 001 - - 002 - - - 010 - - 011 - - 012 - - - 020 - - 021 - - 022 -- - - 100 - - 101 - - 102 - - - 110 - - 111 - - 112 - - - 120 - - 121 - - 122 -- - - 200 - - 201 - - 202 - - - 210 - - 211 - - 212 - - - 220 - - 221 - - 222 -)", - L{ - N{L{N{L{N{"000"}, N{"001"}, N{"002"}}}, N{L{N{"010"}, N{"011"}, N{"012"}}}, N{L{N{"020"}, N{"021"}, N{"022"}}}}}, - N{L{N{L{N{"100"}, N{"101"}, N{"102"}}}, N{L{N{"110"}, N{"111"}, N{"112"}}}, N{L{N{"120"}, N{"121"}, N{"122"}}}}}, - N{L{N{L{N{"200"}, N{"201"}, N{"202"}}}, N{L{N{"210"}, N{"211"}, N{"212"}}}, N{L{N{"220"}, N{"221"}, N{"222"}}}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x3, continued on next line", -R"( -- - - - - 000 - - 001 - - 002 - - - - 010 - - 011 - - 012 - - - - 020 - - 021 - - 022 -- - - - - 100 - - 101 - - 102 - - - - 110 - - 111 - - 112 - - - - 120 - - 121 - - 122 -- - - - - 200 - - 201 - - 202 - - - - 210 - - 211 - - 212 - - - - 220 - - 221 - - 222 -)", - L{ - N{L{N{L{N{"000"}, N{"001"}, N{"002"}}}, N{L{N{"010"}, N{"011"}, N{"012"}}}, N{L{N{"020"}, N{"021"}, N{"022"}}}}}, - N{L{N{L{N{"100"}, N{"101"}, N{"102"}}}, N{L{N{"110"}, N{"111"}, N{"112"}}}, N{L{N{"120"}, N{"121"}, N{"122"}}}}}, - N{L{N{L{N{"200"}, N{"201"}, N{"202"}}}, N{L{N{"210"}, N{"211"}, N{"212"}}}, N{L{N{"220"}, N{"221"}, N{"222"}}}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x3, all continued on next line", -R"( -- - - - - - 000 - - - 001 - - - 002 - - - - - 010 - - - 011 - - - 012 - - - - - 020 - - - 021 - - - 022 -- - - - - - 100 - - - 101 - - - 102 - - - - - 110 - - - 111 - - - 112 - - - - - 120 - - - 121 - - - 122 -- - - - - - 200 - - - 201 - - - 202 - - - - - 210 - - - 211 - - - 212 - - - - - 220 - - - 221 - - - 222 -)", - L{ - N{L{N{L{N{"000"}, N{"001"}, N{"002"}}}, N{L{N{"010"}, N{"011"}, N{"012"}}}, N{L{N{"020"}, N{"021"}, N{"022"}}}}}, - N{L{N{L{N{"100"}, N{"101"}, N{"102"}}}, N{L{N{"110"}, N{"111"}, N{"112"}}}, N{L{N{"120"}, N{"121"}, N{"122"}}}}}, - N{L{N{L{N{"200"}, N{"201"}, N{"202"}}}, N{L{N{"210"}, N{"211"}, N{"212"}}}, N{L{N{"220"}, N{"221"}, N{"222"}}}}}, - } -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_nested_seqx4.cpp b/thirdparty/ryml/test/test_nested_seqx4.cpp deleted file mode 100644 index b63c4bac8..000000000 --- a/thirdparty/ryml/test/test_nested_seqx4.cpp +++ /dev/null @@ -1,124 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - - -CASE_GROUP(NESTED_SEQX4) -{ - -ADD_CASE_TO_GROUP("nested seq x4, explicit", -R"([ -[[[0000, 0001, 0002], [0010, 0011, 0012], [0020, 0021, 0022]], - [[0100, 0101, 0102], [0110, 0111, 0112], [0120, 0121, 0122]], - [[0200, 0201, 0202], [0210, 0211, 0212], [0220, 0221, 0222]]], - -[[[1000, 1001, 1002], [1010, 1011, 1012], [1020, 1021, 1022]], - [[1100, 1101, 1102], [1110, 1111, 1112], [1120, 1121, 1122]], - [[1200, 1201, 1202], [1210, 1211, 1212], [1220, 1221, 1222]]], - -[[[2000, 2001, 2002], [2010, 2011, 2012], [2020, 2021, 2022]], - [[2100, 2101, 2102], [2110, 2111, 2112], [2120, 2121, 2122]], - [[2200, 2201, 2202], [2210, 2211, 2212], [2220, 2221, 2222]]], -])", - L{ - N{L{N{L{N{L{N{"0000"}, N{"0001"}, N{"0002"}}}, N{L{N{"0010"}, N{"0011"}, N{"0012"}}}, N{L{N{"0020"}, N{"0021"}, N{"0022"}}}}}, N{L{N{L{N{"0100"}, N{"0101"}, N{"0102"}}}, N{L{N{"0110"}, N{"0111"}, N{"0112"}}}, N{L{N{"0120"}, N{"0121"}, N{"0122"}}}}}, N{L{N{L{N{"0200"}, N{"0201"}, N{"0202"}}}, N{L{N{"0210"}, N{"0211"}, N{"0212"}}}, N{L{N{"0220"}, N{"0221"}, N{"0222"}}}}}}}, - N{L{N{L{N{L{N{"1000"}, N{"1001"}, N{"1002"}}}, N{L{N{"1010"}, N{"1011"}, N{"1012"}}}, N{L{N{"1020"}, N{"1021"}, N{"1022"}}}}}, N{L{N{L{N{"1100"}, N{"1101"}, N{"1102"}}}, N{L{N{"1110"}, N{"1111"}, N{"1112"}}}, N{L{N{"1120"}, N{"1121"}, N{"1122"}}}}}, N{L{N{L{N{"1200"}, N{"1201"}, N{"1202"}}}, N{L{N{"1210"}, N{"1211"}, N{"1212"}}}, N{L{N{"1220"}, N{"1221"}, N{"1222"}}}}}}}, - N{L{N{L{N{L{N{"2000"}, N{"2001"}, N{"2002"}}}, N{L{N{"2010"}, N{"2011"}, N{"2012"}}}, N{L{N{"2020"}, N{"2021"}, N{"2022"}}}}}, N{L{N{L{N{"2100"}, N{"2101"}, N{"2102"}}}, N{L{N{"2110"}, N{"2111"}, N{"2112"}}}, N{L{N{"2120"}, N{"2121"}, N{"2122"}}}}}, N{L{N{L{N{"2200"}, N{"2201"}, N{"2202"}}}, N{L{N{"2210"}, N{"2211"}, N{"2212"}}}, N{L{N{"2220"}, N{"2221"}, N{"2222"}}}}}}}, - } -); - -ADD_CASE_TO_GROUP("nested seq x4", -R"( -- - - - 0000 - - 0001 - - 0002 - - - 0010 - - 0011 - - 0012 - - - 0020 - - 0021 - - 0022 - - - - 0100 - - 0101 - - 0102 - - - 0110 - - 0111 - - 0112 - - - 0120 - - 0121 - - 0122 - - - - 0200 - - 0201 - - 0202 - - - 0210 - - 0211 - - 0212 - - - 0220 - - 0221 - - 0222 -- - - - 1000 - - 1001 - - 1002 - - - 1010 - - 1011 - - 1012 - - - 1020 - - 1021 - - 1022 - - - - 1100 - - 1101 - - 1102 - - - 1110 - - 1111 - - 1112 - - - 1120 - - 1121 - - 1122 - - - - 1200 - - 1201 - - 1202 - - - 1210 - - 1211 - - 1212 - - - 1220 - - 1221 - - 1222 -- - - - 2000 - - 2001 - - 2002 - - - 2010 - - 2011 - - 2012 - - - 2020 - - 2021 - - 2022 - - - - 2100 - - 2101 - - 2102 - - - 2110 - - 2111 - - 2112 - - - 2120 - - 2121 - - 2122 - - - - 2200 - - 2201 - - 2202 - - - 2210 - - 2211 - - 2212 - - - 2220 - - 2221 - - 2222 -)", - L{ - N{L{N{L{N{L{N{"0000"}, N{"0001"}, N{"0002"}}}, N{L{N{"0010"}, N{"0011"}, N{"0012"}}}, N{L{N{"0020"}, N{"0021"}, N{"0022"}}}}}, N{L{N{L{N{"0100"}, N{"0101"}, N{"0102"}}}, N{L{N{"0110"}, N{"0111"}, N{"0112"}}}, N{L{N{"0120"}, N{"0121"}, N{"0122"}}}}}, N{L{N{L{N{"0200"}, N{"0201"}, N{"0202"}}}, N{L{N{"0210"}, N{"0211"}, N{"0212"}}}, N{L{N{"0220"}, N{"0221"}, N{"0222"}}}}}}}, - N{L{N{L{N{L{N{"1000"}, N{"1001"}, N{"1002"}}}, N{L{N{"1010"}, N{"1011"}, N{"1012"}}}, N{L{N{"1020"}, N{"1021"}, N{"1022"}}}}}, N{L{N{L{N{"1100"}, N{"1101"}, N{"1102"}}}, N{L{N{"1110"}, N{"1111"}, N{"1112"}}}, N{L{N{"1120"}, N{"1121"}, N{"1122"}}}}}, N{L{N{L{N{"1200"}, N{"1201"}, N{"1202"}}}, N{L{N{"1210"}, N{"1211"}, N{"1212"}}}, N{L{N{"1220"}, N{"1221"}, N{"1222"}}}}}}}, - N{L{N{L{N{L{N{"2000"}, N{"2001"}, N{"2002"}}}, N{L{N{"2010"}, N{"2011"}, N{"2012"}}}, N{L{N{"2020"}, N{"2021"}, N{"2022"}}}}}, N{L{N{L{N{"2100"}, N{"2101"}, N{"2102"}}}, N{L{N{"2110"}, N{"2111"}, N{"2112"}}}, N{L{N{"2120"}, N{"2121"}, N{"2122"}}}}}, N{L{N{L{N{"2200"}, N{"2201"}, N{"2202"}}}, N{L{N{"2210"}, N{"2211"}, N{"2212"}}}, N{L{N{"2220"}, N{"2221"}, N{"2222"}}}}}}}, - } -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_noderef.cpp b/thirdparty/ryml/test/test_noderef.cpp deleted file mode 100644 index cfd3363c3..000000000 --- a/thirdparty/ryml/test/test_noderef.cpp +++ /dev/null @@ -1,813 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/std/std.hpp" -#include "c4/yml/parse.hpp" -#include "c4/yml/emit.hpp" -#include -#include -#include -#endif -#include "./test_case.hpp" -#include "./callbacks_tester.hpp" - -#include - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4389) // signed/unsigned mismatch -#elif defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wdollar-in-identifier-extension" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -#endif - -namespace c4 { -namespace yml { - -TEST(NodeRef, general) -{ - Tree t; - - NodeRef root(&t); - - //using S = csubstr; - //using V = NodeScalar; - using N = NodeInit; - - root = N{MAP}; - root.append_child({"a", "0"}); - root.append_child({MAP, "b"}); - root["b"].append_child({SEQ, "seq"}); - root["b"]["seq"].append_child({"0"}); - root["b"]["seq"].append_child({"1"}); - root["b"]["seq"].append_child({"2"}); - root["b"]["seq"].append_child({NodeScalar{"!!str", "3"}}); - auto ch4 = root["b"]["seq"][3].append_sibling({"4"}); - EXPECT_EQ(ch4.id(), root["b"]["seq"][4].id()); - EXPECT_EQ(ch4.get(), root["b"]["seq"][4].get()); - EXPECT_EQ((type_bits)root["b"]["seq"][4].type(), (type_bits)VAL); - EXPECT_EQ(root["b"]["seq"][4].val(), "4"); - root["b"]["seq"].append_sibling({NodeScalar{"!!str", "aaa"}, NodeScalar{"!!int", "0"}}); - EXPECT_EQ((type_bits)root["b"]["seq"][4].type(), (type_bits)VAL); - EXPECT_EQ(root["b"]["seq"][4].val(), "4"); - - root["b"]["key"] = "val"; - auto seq = root["b"]["seq"]; - auto seq2 = root["b"]["seq2"]; - EXPECT_TRUE(seq2.is_seed()); - root["b"]["seq2"] = N(SEQ); - seq2 = root["b"]["seq2"]; - EXPECT_FALSE(seq2.is_seed()); - EXPECT_TRUE(seq2.is_seq()); - EXPECT_EQ(seq2.num_children(), 0); - EXPECT_EQ(root["b"]["seq2"].get(), seq2.get()); - auto seq20 = seq2[0]; - EXPECT_TRUE(seq20.is_seed()); - EXPECT_TRUE(seq2[0].is_seed()); - EXPECT_EQ(seq2.num_children(), 0); - EXPECT_TRUE(seq2[0].is_seed()); - EXPECT_TRUE(seq20.is_seed()); - EXPECT_NE(seq.get(), seq2.get()); - seq20 = root["b"]["seq2"][0]; - EXPECT_TRUE(seq20.is_seed()); - root["b"]["seq2"][0] = "00"; - seq20 = root["b"]["seq2"][0]; - EXPECT_FALSE(seq20.is_seed()); - NodeRef before = root["b"]["key"]; - EXPECT_EQ(before.key(), "key"); - EXPECT_EQ(before.val(), "val"); - root["b"]["seq2"][1] = "01"; - NodeRef after = root["b"]["key"]; - EXPECT_EQ(before.key(), "key"); - EXPECT_EQ(before.val(), "val"); - EXPECT_EQ(after.key(), "key"); - EXPECT_EQ(after.val(), "val"); - root["b"]["seq2"][2] = "02"; - root["b"]["seq2"][3] = "03"; - int iv = 0; - root["b"]["seq2"][4] << 55; root["b"]["seq2"][4] >> iv; - EXPECT_EQ(iv, 55); - size_t zv = 0; - root["b"]["seq2"][5] << size_t(55); root["b"]["seq2"][5] >> zv; - EXPECT_EQ(zv, size_t(55)); - float fv = 0; - root["b"]["seq2"][6] << 2.0f; root["b"]["seq2"][6] >> fv; - EXPECT_EQ(fv, 2.f); - float dv = 0; - root["b"]["seq2"][7] << 2.0; root["b"]["seq2"][7] >> dv; - EXPECT_EQ(dv, 2.0); - - EXPECT_EQ(root["b"]["key"].key(), "key"); - EXPECT_EQ(root["b"]["key"].val(), "val"); - - - emit_yaml(t); - - EXPECT_TRUE(root.type().is_map()); - EXPECT_TRUE(root["a"].type().is_keyval()); - EXPECT_EQ(root["a"].key(), "a"); - EXPECT_EQ(root["a"].val(), "0"); - - EXPECT_TRUE(root["b"].type().has_key()); - EXPECT_TRUE(root["b"].type().is_map()); - - EXPECT_TRUE(root["b"]["seq"].type().has_key()); - EXPECT_TRUE(root["b"]["seq"].type().is_seq()); - EXPECT_EQ (root["b"]["seq"].key(), "seq"); - EXPECT_TRUE(root["b"]["seq"][0].type().is_val()); - EXPECT_EQ( root["b"]["seq"][0].val(), "0"); - EXPECT_TRUE(root["b"]["seq"][1].type().is_val()); - EXPECT_EQ( root["b"]["seq"][1].val(), "1"); - EXPECT_TRUE(root["b"]["seq"][2].type().is_val()); - EXPECT_EQ( root["b"]["seq"][2].val(), "2"); - EXPECT_TRUE(root["b"]["seq"][3].type().is_val()); - EXPECT_EQ( root["b"]["seq"][3].val(), "3"); - EXPECT_EQ( root["b"]["seq"][3].val_tag(), "!!str"); - EXPECT_TRUE(root["b"]["seq"][4].type().is_val()); - EXPECT_EQ( root["b"]["seq"][4].val(), "4"); - - int tv; - EXPECT_EQ(root["b"]["key"].key(), "key"); - EXPECT_EQ(root["b"]["key"].val(), "val"); - EXPECT_EQ(root["b"]["seq2"][0].val(), "00"); root["b"]["seq2"][0] >> tv; EXPECT_EQ(tv, 0); - EXPECT_EQ(root["b"]["seq2"][1].val(), "01"); root["b"]["seq2"][1] >> tv; EXPECT_EQ(tv, 1); - EXPECT_EQ(root["b"]["seq2"][2].val(), "02"); root["b"]["seq2"][2] >> tv; EXPECT_EQ(tv, 2); - EXPECT_EQ(root["b"]["seq2"][3].val(), "03"); root["b"]["seq2"][3] >> tv; EXPECT_EQ(tv, 3); - EXPECT_EQ(root["b"]["seq2"][4].val(), "55"); EXPECT_EQ(iv, 55); - EXPECT_EQ(root["b"]["seq2"][5].val(), "55"); EXPECT_EQ(zv, size_t(55)); - EXPECT_EQ(root["b"]["seq2"][6].val(), "2"); EXPECT_EQ(fv, 2.f); - EXPECT_EQ(root["b"]["seq2"][6].val(), "2"); EXPECT_EQ(dv, 2.); - - root["b"]["seq"][2].set_val_serialized(22); - - emit_yaml(t); - - EXPECT_TRUE(root["b"]["aaa"].type().is_keyval()); - EXPECT_EQ(root["b"]["aaa"].key_tag(), "!!str"); - EXPECT_EQ(root["b"]["aaa"].key(), "aaa"); - EXPECT_EQ(root["b"]["aaa"].val_tag(), "!!int"); - EXPECT_EQ(root["b"]["aaa"].val(), "0"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void noderef_check_tree(ConstNodeRef const& root) -{ - test_invariants(*root.tree()); - - EXPECT_EQ(root.tree()->size(), 7u); - EXPECT_EQ(root.num_children(), 6u); - EXPECT_EQ(root.is_container(), true); - EXPECT_EQ(root.is_seq(), true); - - EXPECT_TRUE(root[0].type().is_val()); - EXPECT_EQ( root[0].val(), "0"); - EXPECT_TRUE(root[1].type().is_val()); - EXPECT_EQ( root[1].val(), "1"); - EXPECT_TRUE(root[2].type().is_val()); - EXPECT_EQ( root[2].val(), "2"); - EXPECT_TRUE(root[3].type().is_val()); - EXPECT_EQ( root[3].val(), "3"); - EXPECT_TRUE(root[4].type().is_val()); - EXPECT_EQ( root[4].val(), "4"); - EXPECT_TRUE(root[5].type().is_val()); - EXPECT_EQ( root[5].val(), "5"); -} - -TEST(NodeRef, append_child) -{ - Tree t; - - NodeRef root(&t); - - root |= SEQ; - root.append_child({"0"}); - root.append_child({"1"}); - root.append_child({"2"}); - root.append_child({"3"}); - root.append_child({"4"}); - root.append_child({"5"}); - - noderef_check_tree(root); -} - -TEST(NodeRef, prepend_child) -{ - Tree t; - - NodeRef root(&t); - - root |= SEQ; - root.prepend_child({"5"}); - root.prepend_child({"4"}); - root.prepend_child({"3"}); - root.prepend_child({"2"}); - root.prepend_child({"1"}); - root.prepend_child({"0"}); - - noderef_check_tree(root); -} - -TEST(NodeRef, insert_child) -{ - Tree t; - - NodeRef root(&t); - NodeRef none(&t, NONE); - - root |= SEQ; - root.insert_child({"3"}, none); - root.insert_child({"4"}, root[0]); - root.insert_child({"0"}, none); - root.insert_child({"5"}, root[2]); - root.insert_child({"1"}, root[0]); - root.insert_child({"2"}, root[1]); - - noderef_check_tree(root); -} - -TEST(NodeRef, remove_child) -{ - Tree t; - - NodeRef root(&t); - NodeRef none(&t, NONE); - - root |= SEQ; - root.insert_child({"3"}, none); - root.insert_child({"4"}, root[0]); - root.insert_child({"0"}, none); - root.insert_child({"5"}, root[2]); - root.insert_child({"1"}, root[0]); - root.insert_child({"2"}, root[1]); - - std::vector vec({10, 20, 30, 40, 50, 60, 70, 80, 90}); - root.insert_child(root[0]) << vec; // 1 - root.insert_child(root[2]) << vec; // 3 - root.insert_child(root[4]) << vec; // 5 - root.insert_child(root[6]) << vec; // 7 - root.insert_child(root[8]) << vec; // 9 - root.append_child() << vec; // 10 - - root.remove_child(11); - root.remove_child(9); - root.remove_child(7); - root.remove_child(5); - root.remove_child(3); - root.remove_child(1); - - noderef_check_tree(root); - - std::vector> vec2({{100, 200}, {300, 400}, {500, 600}, {700, 800, 900}}); - root.prepend_child() << vec2; // 0 - root.insert_child(root[1]) << vec2; // 2 - root.insert_child(root[3]) << vec2; // 4 - root.insert_child(root[5]) << vec2; // 6 - root.insert_child(root[7]) << vec2; // 8 - root.insert_child(root[9]) << vec2; // 10 - root.append_child() << vec2; // 12 - - root.remove_child(12); - root.remove_child(10); - root.remove_child(8); - root.remove_child(6); - root.remove_child(4); - root.remove_child(2); - root.remove_child(0); - - noderef_check_tree(root); -} - -TEST(NodeRef, move_in_same_parent) -{ - Tree t; - NodeRef r = t; - - std::vector> vec2({{100, 200}, {300, 400}, {500, 600}, {700, 800, 900}}); - std::map map2({{"foo", 100}, {"bar", 200}, {"baz", 300}}); - - r |= SEQ; - r.append_child() << vec2; - r.append_child() << map2; - r.append_child() << "elm2"; - r.append_child() << "elm3"; - - auto s = r[0]; - auto m = r[1]; - EXPECT_TRUE(s.is_seq()); - EXPECT_TRUE(m.is_map()); - EXPECT_EQ(s.num_children(), vec2.size()); - EXPECT_EQ(m.num_children(), map2.size()); - //printf("fonix"); print_tree(t); emit_yaml(r); - r[0].move(r[1]); - //printf("fonix"); print_tree(t); emit_yaml(r); - EXPECT_EQ(r[0].get(), m.get()); - EXPECT_EQ(r[0].num_children(), map2.size()); - EXPECT_EQ(r[1].get(), s.get()); - EXPECT_EQ(r[1].num_children(), vec2.size()); - test_invariants(t); -} - -TEST(NodeRef, move_in_same_parent_to_first_position) -{ - Tree t = parse_in_arena("[1, 2, 3, 0, 4]"); - NodeRef r = t; - - EXPECT_TRUE(r[0].val() == "1"); - EXPECT_TRUE(r[1].val() == "2"); - EXPECT_TRUE(r[2].val() == "3"); - EXPECT_TRUE(r[3].val() == "0"); - EXPECT_TRUE(r[4].val() == "4"); - r[3].move({}); - EXPECT_TRUE(r[0].val() == "0"); - EXPECT_TRUE(r[1].val() == "1"); - EXPECT_TRUE(r[2].val() == "2"); - EXPECT_TRUE(r[3].val() == "3"); - EXPECT_TRUE(r[4].val() == "4"); - test_invariants(t); - r[0].move({}); // should have no effect - EXPECT_TRUE(r[0].val() == "0"); - EXPECT_TRUE(r[1].val() == "1"); - EXPECT_TRUE(r[2].val() == "2"); - EXPECT_TRUE(r[3].val() == "3"); - EXPECT_TRUE(r[4].val() == "4"); - test_invariants(t); - r[4].move({}); - EXPECT_TRUE(r[0].val() == "4"); - EXPECT_TRUE(r[1].val() == "0"); - EXPECT_TRUE(r[2].val() == "1"); - EXPECT_TRUE(r[3].val() == "2"); - EXPECT_TRUE(r[4].val() == "3"); - test_invariants(t); -} - -TEST(NodeRef, move_to_other_parent) -{ - Tree t; - NodeRef r = t; - - std::vector> vec2({{100, 200}, {300, 400}, {500, 600}, {700, 800, 900}}); - std::map map2({{"foo", 100}, {"bar", 200}, {"baz", 300}}); - - r |= SEQ; - r.append_child() << vec2; - r.append_child() << map2; - r.append_child() << "elm2"; - r.append_child() << "elm3"; - - NodeData *elm2 = r[2].get(); - EXPECT_EQ(r[2].val(), "elm2"); - //printf("fonix"); print_tree(t); emit_yaml(r); - r[2].move(r[0], r[0][0]); - EXPECT_EQ(r[0][1].get(), elm2); - EXPECT_EQ(r[0][1].val(), "elm2"); - //printf("fonix"); print_tree(t); emit_yaml(r); - test_invariants(t); -} - -TEST(NodeRef, move_to_other_parent_to_first_position) -{ - Tree t = parse_in_arena("[[0, 1, 2, 3, 4], [00, 10, 20, 30, 40]]"); - NodeRef r = t; - - EXPECT_TRUE(r[0][0].val() == "0"); - EXPECT_TRUE(r[0][1].val() == "1"); - EXPECT_TRUE(r[0][2].val() == "2"); - EXPECT_TRUE(r[0][3].val() == "3"); - EXPECT_TRUE(r[0][4].val() == "4"); - EXPECT_TRUE(r[1][0].val() == "00"); - EXPECT_TRUE(r[1][1].val() == "10"); - EXPECT_TRUE(r[1][2].val() == "20"); - EXPECT_TRUE(r[1][3].val() == "30"); - EXPECT_TRUE(r[1][4].val() == "40"); - test_invariants(t); - r[0][0].move(r[1], {}); - EXPECT_TRUE(r[0][0].val() == "1"); - EXPECT_TRUE(r[0][1].val() == "2"); - EXPECT_TRUE(r[0][2].val() == "3"); - EXPECT_TRUE(r[0][3].val() == "4"); - EXPECT_TRUE(r[1][0].val() == "0"); - EXPECT_TRUE(r[1][1].val() == "00"); - EXPECT_TRUE(r[1][2].val() == "10"); - EXPECT_TRUE(r[1][3].val() == "20"); - EXPECT_TRUE(r[1][4].val() == "30"); - EXPECT_TRUE(r[1][5].val() == "40"); - test_invariants(t); - r[1][0].move(r[0], {}); - EXPECT_TRUE(r[0][0].val() == "0"); - EXPECT_TRUE(r[0][1].val() == "1"); - EXPECT_TRUE(r[0][2].val() == "2"); - EXPECT_TRUE(r[0][3].val() == "3"); - EXPECT_TRUE(r[0][4].val() == "4"); - EXPECT_TRUE(r[1][0].val() == "00"); - EXPECT_TRUE(r[1][1].val() == "10"); - EXPECT_TRUE(r[1][2].val() == "20"); - EXPECT_TRUE(r[1][3].val() == "30"); - EXPECT_TRUE(r[1][4].val() == "40"); - test_invariants(t); -} - -TEST(NodeRef, move_to_other_tree) -{ - Tree t0 = parse_in_arena("[0, 1, 2, 3, 4]"); - Tree t1 = parse_in_arena("[00, 10, 20, 30, 40]"); - NodeRef r0 = t0; - NodeRef r1 = t1; - - EXPECT_TRUE(r0[0].val() == "0"); - EXPECT_TRUE(r0[1].val() == "1"); - EXPECT_TRUE(r0[2].val() == "2"); - EXPECT_TRUE(r0[3].val() == "3"); - EXPECT_TRUE(r0[4].val() == "4"); - EXPECT_TRUE(r1[0].val() == "00"); - EXPECT_TRUE(r1[1].val() == "10"); - EXPECT_TRUE(r1[2].val() == "20"); - EXPECT_TRUE(r1[3].val() == "30"); - EXPECT_TRUE(r1[4].val() == "40"); - r0[0].move(r1, r1[0]); - test_invariants(t0); - test_invariants(t1); - EXPECT_TRUE(r0[0].val() == "1"); - EXPECT_TRUE(r0[1].val() == "2"); - EXPECT_TRUE(r0[2].val() == "3"); - EXPECT_TRUE(r0[3].val() == "4"); - EXPECT_TRUE(r1[0].val() == "00"); - EXPECT_TRUE(r1[1].val() == "0"); - EXPECT_TRUE(r1[2].val() == "10"); - EXPECT_TRUE(r1[3].val() == "20"); - EXPECT_TRUE(r1[4].val() == "30"); - EXPECT_TRUE(r1[5].val() == "40"); - test_invariants(t0); - test_invariants(t1); - r1[1].move(r0, r0[0]); - EXPECT_TRUE(r0[0].val() == "1"); - EXPECT_TRUE(r0[1].val() == "0"); - EXPECT_TRUE(r0[2].val() == "2"); - EXPECT_TRUE(r0[3].val() == "3"); - EXPECT_TRUE(r0[4].val() == "4"); - EXPECT_TRUE(r1[0].val() == "00"); - EXPECT_TRUE(r1[1].val() == "10"); - EXPECT_TRUE(r1[2].val() == "20"); - EXPECT_TRUE(r1[3].val() == "30"); - EXPECT_TRUE(r1[4].val() == "40"); - test_invariants(t0); - test_invariants(t1); -} - -TEST(NodeRef, move_to_other_tree_to_first_position) -{ - Tree t0 = parse_in_arena("[0, 1, 2, 3, 4]"); - Tree t1 = parse_in_arena("[00, 10, 20, 30, 40]"); - NodeRef r0 = t0; - NodeRef r1 = t1; - - EXPECT_TRUE(r0[0].val() == "0"); - EXPECT_TRUE(r0[1].val() == "1"); - EXPECT_TRUE(r0[2].val() == "2"); - EXPECT_TRUE(r0[3].val() == "3"); - EXPECT_TRUE(r0[4].val() == "4"); - EXPECT_TRUE(r1[0].val() == "00"); - EXPECT_TRUE(r1[1].val() == "10"); - EXPECT_TRUE(r1[2].val() == "20"); - EXPECT_TRUE(r1[3].val() == "30"); - EXPECT_TRUE(r1[4].val() == "40"); - test_invariants(t0); - test_invariants(t1); - r0[0].move(r1, {}); - EXPECT_TRUE(r0[0].val() == "1"); - EXPECT_TRUE(r0[1].val() == "2"); - EXPECT_TRUE(r0[2].val() == "3"); - EXPECT_TRUE(r0[3].val() == "4"); - EXPECT_TRUE(r1[0].val() == "0"); - EXPECT_TRUE(r1[1].val() == "00"); - EXPECT_TRUE(r1[2].val() == "10"); - EXPECT_TRUE(r1[3].val() == "20"); - EXPECT_TRUE(r1[4].val() == "30"); - EXPECT_TRUE(r1[5].val() == "40"); - test_invariants(t0); - test_invariants(t1); - r1[0].move(r0, {}); - EXPECT_TRUE(r0[0].val() == "0"); - EXPECT_TRUE(r0[1].val() == "1"); - EXPECT_TRUE(r0[2].val() == "2"); - EXPECT_TRUE(r0[3].val() == "3"); - EXPECT_TRUE(r0[4].val() == "4"); - EXPECT_TRUE(r1[0].val() == "00"); - EXPECT_TRUE(r1[1].val() == "10"); - EXPECT_TRUE(r1[2].val() == "20"); - EXPECT_TRUE(r1[3].val() == "30"); - EXPECT_TRUE(r1[4].val() == "40"); - test_invariants(t0); - test_invariants(t1); -} - -TEST(NodeRef, duplicate_to_same_tree) -{ - Tree t = parse_in_arena("[{a0: [b0, c0], a1: [b1, c1], a2: [b2, c2], a3: [b3, c3]}]"); - auto checkseq = [](ConstNodeRef const& s){ - ASSERT_EQ(s.num_children(), 4u); - ASSERT_EQ(s[0].num_children(), 2u); - ASSERT_EQ(s[1].num_children(), 2u); - ASSERT_EQ(s[2].num_children(), 2u); - ASSERT_EQ(s[3].num_children(), 2u); - EXPECT_EQ(s[0].key(), "a0"); - EXPECT_EQ(s[0][0].val(), "b0"); - EXPECT_EQ(s[0][1].val(), "c0"); - EXPECT_EQ(s[1].key(), "a1"); - EXPECT_EQ(s[1][0].val(), "b1"); - EXPECT_EQ(s[1][1].val(), "c1"); - EXPECT_EQ(s[2].key(), "a2"); - EXPECT_EQ(s[2][0].val(), "b2"); - EXPECT_EQ(s[2][1].val(), "c2"); - EXPECT_EQ(s[3].key(), "a3"); - EXPECT_EQ(s[3][0].val(), "b3"); - EXPECT_EQ(s[3][1].val(), "c3"); - }; - { - SCOPED_TRACE("at the beginning"); - t[0].duplicate({}); - test_check_emit_check(t, [&checkseq](ConstNodeRef r){ - checkseq(r[0]); - checkseq(r[1]); - }); - } - { - SCOPED_TRACE("at the end"); - t[0].duplicate(t.rootref().last_child()); - test_check_emit_check(t, [&checkseq](ConstNodeRef r){ - checkseq(r[0]); - checkseq(r[1]); - checkseq(r[2]); - }); - } - { - SCOPED_TRACE("in the middle"); - t[0].duplicate(t.rootref().first_child()); - test_check_emit_check(t, [&checkseq](ConstNodeRef r){ - checkseq(r[0]); - checkseq(r[1]); - checkseq(r[2]); - }); - } -} - -TEST(NodeRef, duplicate_to_different_tree) -{ - Tree t = parse_in_arena("[{a0: [b0, c0], a1: [b1, c1], a2: [b2, c2], a3: [b3, c3]}]"); - auto checkseq = [](ConstNodeRef const& s){ - ASSERT_EQ(s.num_children(), 4u); - ASSERT_EQ(s[0].num_children(), 2u); - ASSERT_EQ(s[1].num_children(), 2u); - ASSERT_EQ(s[2].num_children(), 2u); - ASSERT_EQ(s[3].num_children(), 2u); - EXPECT_EQ(s[0].key(), "a0"); - EXPECT_EQ(s[0][0].val(), "b0"); - EXPECT_EQ(s[0][1].val(), "c0"); - EXPECT_EQ(s[1].key(), "a1"); - EXPECT_EQ(s[1][0].val(), "b1"); - EXPECT_EQ(s[1][1].val(), "c1"); - EXPECT_EQ(s[2].key(), "a2"); - EXPECT_EQ(s[2][0].val(), "b2"); - EXPECT_EQ(s[2][1].val(), "c2"); - EXPECT_EQ(s[3].key(), "a3"); - EXPECT_EQ(s[3][0].val(), "b3"); - EXPECT_EQ(s[3][1].val(), "c3"); - }; - auto check_orig = [&checkseq](ConstNodeRef const& r){ - ASSERT_TRUE(r.is_seq()); - ASSERT_GE(r.num_children(), 1u); - checkseq(r[0]); - }; - Tree d = parse_in_arena("[]"); - { - SCOPED_TRACE("at the beginning"); - t[0].duplicate(d, {}); - test_check_emit_check(t, check_orig); - test_check_emit_check(d, check_orig); - } - { - SCOPED_TRACE("at the end"); - t[0].duplicate(d, d.rootref().last_child()); - test_check_emit_check(t, check_orig); - test_check_emit_check(d, check_orig); - test_check_emit_check(d, [&checkseq](ConstNodeRef r){ - checkseq(r[1]); - }); - } - { - SCOPED_TRACE("in the middle"); - t[0].duplicate(d, d.rootref().first_child()); - test_check_emit_check(t, check_orig); - test_check_emit_check(d, check_orig); - test_check_emit_check(d, [&checkseq](ConstNodeRef r){ - checkseq(r[1]); - checkseq(r[2]); - }); - } -} - -TEST(NodeRef, intseq) -{ - Tree t = parse_in_arena("iseq: [8, 10]"); - NodeRef n = t["iseq"]; - int a, b; - n[0] >> a; - n[1] >> b; - EXPECT_EQ(a, 8); - EXPECT_EQ(b, 10); - test_invariants(t); -} - -TEST(NodeRef, vsConstNodeRef) -{ - Tree t = parse_in_arena("iseq: [8, 10]"); - Tree const& ct = t; - NodeRef mseq = t["iseq"]; - ConstNodeRef seq = t["iseq"]; - EXPECT_EQ(mseq.tree(), seq.tree()); - EXPECT_EQ(mseq.id(), seq.id()); - EXPECT_TRUE(mseq == seq); - EXPECT_FALSE(mseq != seq); - EXPECT_TRUE(seq == mseq); - EXPECT_FALSE(seq != mseq); - // mseq = ct["iseq"]; // deliberate compile error - seq = ct["iseq"]; // ok - // mseq = seq; // deliberate compilation error - seq = mseq; // ok - { - NodeData *nd = mseq.get(); - // nd = seq.get(); // deliberate compile error - C4_UNUSED(nd); - } - { - NodeData const* nd = seq.get(); - nd = seq.get(); // ok - C4_UNUSED(nd); - } - test_invariants(t); -} - - -// see https://github.com/biojppm/rapidyaml/issues/294 -TEST(NodeRef, overload_sets) -{ - // doc() - { - Tree t = parse_in_arena("a\n---\nb"); - NodeRef n = t; - NodeRef const nc = t; - ConstNodeRef const cn = t; - EXPECT_EQ(n.doc(0), nc.doc(0)); - EXPECT_EQ(n.doc(0), cn.doc(0)); - } - Tree t = parse_in_arena("{iseq: [8, 10], imap: {a: b, c: d}}"); - NodeRef n = t; - NodeRef const nc = t; - ConstNodeRef const cn = t; - // get() - { - EXPECT_EQ(n["iseq"].get(), nc["iseq"].get()); - EXPECT_EQ(n["iseq"].get(), cn["iseq"].get()); - } - // parent() - { - EXPECT_EQ(n["iseq"].parent(), nc["iseq"].parent()); - EXPECT_EQ(n["iseq"].parent(), cn["iseq"].parent()); - } - // child_pos() - { - EXPECT_EQ(n["iseq"].child_pos(n["iseq"][0]), nc["iseq"].child_pos(n["iseq"][0])); - EXPECT_EQ(n["iseq"].child_pos(n["iseq"][0]), cn["iseq"].child_pos(n["iseq"][0])); - } - // num_children() - { - EXPECT_EQ(n["iseq"].num_children(), nc["iseq"].num_children()); - EXPECT_EQ(n["iseq"].num_children(), cn["iseq"].num_children()); - } - // first_child() - { - EXPECT_EQ(n["iseq"].first_child(), nc["iseq"].first_child()); - EXPECT_EQ(n["iseq"].first_child(), cn["iseq"].first_child()); - } - // last_child() - { - EXPECT_EQ(n["iseq"].last_child(), nc["iseq"].last_child()); - EXPECT_EQ(n["iseq"].last_child(), cn["iseq"].last_child()); - } - // child() - { - EXPECT_EQ(n["iseq"].child(0), nc["iseq"].child(0)); - EXPECT_EQ(n["iseq"].child(0), cn["iseq"].child(0)); - } - // find_child() - { - EXPECT_EQ(n.find_child("iseq"), nc.find_child("iseq")); - EXPECT_EQ(n.find_child("iseq"), cn.find_child("iseq")); - } - // prev_sibling() - { - EXPECT_EQ(n["iseq"][1].prev_sibling(), nc["iseq"][1].prev_sibling()); - EXPECT_EQ(n["iseq"][1].prev_sibling(), cn["iseq"][1].prev_sibling()); - } - // next_sibling() - { - EXPECT_EQ(n["iseq"][0].next_sibling(), nc["iseq"][0].next_sibling()); - EXPECT_EQ(n["iseq"][0].next_sibling(), cn["iseq"][0].next_sibling()); - } - // first_sibling() - { - EXPECT_EQ(n["iseq"][1].first_sibling(), nc["iseq"][1].first_sibling()); - EXPECT_EQ(n["iseq"][1].first_sibling(), cn["iseq"][1].first_sibling()); - } - // last_sibling() - { - EXPECT_EQ(n["iseq"][0].last_sibling(), nc["iseq"][0].last_sibling()); - EXPECT_EQ(n["iseq"][0].last_sibling(), cn["iseq"][0].last_sibling()); - } - // sibling() - { - EXPECT_EQ(n["iseq"][1].sibling(0), nc["iseq"][1].sibling(0)); - EXPECT_EQ(n["iseq"][1].sibling(0), cn["iseq"][1].sibling(0)); - } - // find_sibling() - { - EXPECT_EQ(n["iseq"].find_sibling("imap"), nc["iseq"].find_sibling("imap")); - EXPECT_EQ(n["iseq"].find_sibling("imap"), cn["iseq"].find_sibling("imap")); - } - // operator[](csubstr) - { - EXPECT_EQ(n["iseq"].id(), nc["iseq"].id()); - EXPECT_EQ(n["iseq"].id(), cn["iseq"].id()); - } - // operator[](size_t) - { - EXPECT_EQ(n["iseq"][0].id(), nc["iseq"][0].id()); - EXPECT_EQ(n["iseq"][0].id(), cn["iseq"][0].id()); - } - // begin() - { - EXPECT_EQ(n["iseq"].begin().m_child_id, nc["iseq"].begin().m_child_id); - EXPECT_EQ(n["iseq"].begin().m_child_id, cn["iseq"].begin().m_child_id); - } - // end() - { - EXPECT_EQ(n["iseq"].end().m_child_id, nc["iseq"].end().m_child_id); - EXPECT_EQ(n["iseq"].end().m_child_id, cn["iseq"].end().m_child_id); - } - // cbegin() - { - EXPECT_EQ(n["iseq"].cbegin().m_child_id, nc["iseq"].cbegin().m_child_id); - EXPECT_EQ(n["iseq"].cbegin().m_child_id, cn["iseq"].cbegin().m_child_id); - } - // cend() - { - EXPECT_EQ(n["iseq"].cend().m_child_id, nc["iseq"].cend().m_child_id); - EXPECT_EQ(n["iseq"].cend().m_child_id, cn["iseq"].cend().m_child_id); - } - // children() - { - EXPECT_EQ(n["iseq"].children().b.m_child_id, nc["iseq"].children().b.m_child_id); - EXPECT_EQ(n["iseq"].children().b.m_child_id, cn["iseq"].children().b.m_child_id); - } - // cchildren() - { - EXPECT_EQ(n["iseq"].cchildren().b.m_child_id, nc["iseq"].cchildren().b.m_child_id); - EXPECT_EQ(n["iseq"].cchildren().b.m_child_id, cn["iseq"].cchildren().b.m_child_id); - } - // siblings() - { - EXPECT_EQ(n["iseq"][0].siblings().b.m_child_id, nc["iseq"][0].siblings().b.m_child_id); - EXPECT_EQ(n["iseq"][0].siblings().b.m_child_id, cn["iseq"][0].siblings().b.m_child_id); - EXPECT_EQ(n.siblings().b.m_child_id, nc.siblings().b.m_child_id); - EXPECT_EQ(n.siblings().b.m_child_id, cn.siblings().b.m_child_id); - } - // csiblings() - { - EXPECT_EQ(n["iseq"][0].csiblings().b.m_child_id, nc["iseq"][0].csiblings().b.m_child_id); - EXPECT_EQ(n["iseq"][0].csiblings().b.m_child_id, cn["iseq"][0].csiblings().b.m_child_id); - EXPECT_EQ(n.csiblings().b.m_child_id, nc.csiblings().b.m_child_id); - EXPECT_EQ(n.csiblings().b.m_child_id, cn.csiblings().b.m_child_id); - } -} - - -//------------------------------------------- -// this is needed to use the test case library -Case const* get_case(csubstr /*name*/) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 - -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/test/test_null_val.cpp b/thirdparty/ryml/test/test_null_val.cpp deleted file mode 100644 index 4bdd6d419..000000000 --- a/thirdparty/ryml/test/test_null_val.cpp +++ /dev/null @@ -1,519 +0,0 @@ -#include "./test_group.hpp" -#include "c4/error.hpp" - -namespace c4 { -namespace yml { - -C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wuseless-cast") - -csubstr getafter(csubstr yaml, csubstr pattern) -{ - size_t pos = yaml.find(pattern); - RYML_ASSERT(pos != npos); - RYML_ASSERT(yaml.sub(pos).begins_with(pattern)); - return yaml.sub(pos + pattern.len); -} - -#define _check_null_pointing_at(expr_, which, pattern, arena) \ - do \ - { \ - csubstr expr = expr_.which(); \ - if(expr.empty()) \ - { \ - EXPECT_EQ(expr, nullptr); \ - EXPECT_EQ(expr.len, 0u); \ - EXPECT_EQ(expr.str, nullptr); \ - } \ - EXPECT_TRUE(expr_.which##_is_null()); \ - } while(0) - - -TEST(null_val, simple) -{ - Tree tree = parse_in_arena("{foo: , bar: '', baz: [,,,], bat: [ , , , ], two: [,,], one: [,], empty: []}"); - _check_null_pointing_at(tree["foo"], val, " ,", tree.arena()); - ASSERT_EQ(tree["baz"].num_children(), 3u); - _check_null_pointing_at(tree["baz"][0], val, "[,,,]", tree.arena()); - _check_null_pointing_at(tree["baz"][1], val, ",,,]", tree.arena()); - _check_null_pointing_at(tree["baz"][2], val, ",,]", tree.arena()); - ASSERT_EQ(tree["bat"].num_children(), 3u); - _check_null_pointing_at(tree["bat"][0], val, " , , , ]", tree.arena()); - _check_null_pointing_at(tree["bat"][1], val, " , , ]", tree.arena()); - _check_null_pointing_at(tree["bat"][2], val, " , ]", tree.arena()); - ASSERT_EQ(tree["two"].num_children(), 2u); - _check_null_pointing_at(tree["two"][0], val, "[,,]", tree.arena()); - _check_null_pointing_at(tree["two"][1], val, ",,]", tree.arena()); - ASSERT_EQ(tree["one"].num_children(), 1u); - _check_null_pointing_at(tree["one"][0], val, "[,]", tree.arena()); - ASSERT_EQ(tree["empty"].num_children(), 0u); -} - -TEST(null_val, block_seq) -{ - csubstr yaml = R"( -# nospace -- -- -- -# onespace -- -- -- -# null -- null -- null -- null -- ~ -)"; - ASSERT_EQ(yaml.count('\r'), 0u); - Tree tree = parse_in_arena(yaml); - ASSERT_EQ(tree.rootref().num_children(), 10u); - // FIXME: empty vals in block seqs are pointing at the next item! - _check_null_pointing_at(tree[0], val, after("nospace\n-\n"), tree.arena()); - _check_null_pointing_at(tree[1], val, after("nospace\n-\n-\n"), tree.arena()); - _check_null_pointing_at(tree[2], val, after("nospace\n-\n-\n-\n# onespace\n"), tree.arena()); - _check_null_pointing_at(tree[3], val, after("onespace\n- \n"), tree.arena()); - _check_null_pointing_at(tree[4], val, after("onespace\n- \n- \n"), tree.arena()); - _check_null_pointing_at(tree[5], val, after("onespace\n- \n- \n- \n# null\n"), tree.arena()); - // but explicitly null vals are ok: - _check_null_pointing_at(tree[6], val, "null\n- null\n- null\n- ~\n", tree.arena()); - _check_null_pointing_at(tree[7], val, "null\n- null\n- ~", tree.arena()); - _check_null_pointing_at(tree[8], val, "null\n- ~\n", tree.arena()); - _check_null_pointing_at(tree[9], val, "~\n", tree.arena()); -} - -TEST(null_val, block_map) -{ - csubstr yaml = R"( -# nospace -val0: -val1: -val2: -# onespace -val3: -val4: -val5: -# null -val6: null -val7: null -val8: null -val9: ~ -)"; - ASSERT_EQ(yaml.count('\r'), 0u); - Tree tree = parse_in_arena(yaml); - ASSERT_EQ(tree.rootref().num_children(), 10u); - // FIXME: empty vals in block seqs are pointing at the next item! - _check_null_pointing_at(tree["val0"], val, after("val0:"), tree.arena()); - _check_null_pointing_at(tree["val1"], val, after("val1:"), tree.arena()); - _check_null_pointing_at(tree["val2"], val, after("val2:\n# onespace"), tree.arena()); - _check_null_pointing_at(tree["val3"], val, after("val3: "), tree.arena()); - _check_null_pointing_at(tree["val4"], val, after("val4: "), tree.arena()); - _check_null_pointing_at(tree["val5"], val, after("val5: \n# null"), tree.arena()); - // but explicitly null vals are ok: - _check_null_pointing_at(tree["val6"], val, "null\nval7:", tree.arena()); - _check_null_pointing_at(tree["val7"], val, "null\nval8:", tree.arena()); - _check_null_pointing_at(tree["val8"], val, "null\nval9:", tree.arena()); - _check_null_pointing_at(tree["val9"], val, "~\n", tree.arena()); -} - - -TEST(null_val, issue103) -{ - C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wuseless-cast") - Tree tree; - - tree = parse_in_arena(R"({null: null})"); - ASSERT_EQ(tree.size(), 2u); - EXPECT_EQ(tree.root_id(), 0u); - EXPECT_EQ(tree.first_child(0), 1u); - EXPECT_TRUE(tree.type(1).is_keyval()); - EXPECT_EQ(tree.key(1), "null"); - EXPECT_EQ(tree.val(1), "null"); - EXPECT_TRUE(tree.key_is_null(1)); - EXPECT_TRUE(tree.val_is_null(1)); - EXPECT_TRUE(tree[0].key_is_null()); - EXPECT_TRUE(tree[0].val_is_null()); - _check_null_pointing_at(tree[0], key, "null:", tree.arena()); - _check_null_pointing_at(tree[0], val, "null}", tree.arena()); - - tree = parse_in_arena(R"({Null: Null})"); - ASSERT_EQ(tree.size(), 2u); - EXPECT_EQ(tree.root_id(), 0u); - EXPECT_EQ(tree.first_child(0), 1u); - EXPECT_TRUE(tree.type(1).is_keyval()); - EXPECT_EQ(tree.key(1), "Null"); - EXPECT_EQ(tree.val(1), "Null"); - EXPECT_TRUE(tree.key_is_null(1)); - EXPECT_TRUE(tree.val_is_null(1)); - EXPECT_TRUE(tree[0].key_is_null()); - EXPECT_TRUE(tree[0].val_is_null()); - _check_null_pointing_at(tree[0], key, "Null:", tree.arena()); - _check_null_pointing_at(tree[0], val, "Null}", tree.arena()); - - tree = parse_in_arena(R"({NULL: NULL})"); - ASSERT_EQ(tree.size(), 2u); - EXPECT_EQ(tree.root_id(), 0u); - EXPECT_EQ(tree.first_child(0), 1u); - EXPECT_TRUE(tree.type(1).is_keyval()); - EXPECT_EQ(tree.key(1), "NULL"); - EXPECT_EQ(tree.val(1), "NULL"); - EXPECT_TRUE(tree.key_is_null(1)); - EXPECT_TRUE(tree.val_is_null(1)); - EXPECT_TRUE(tree[0].key_is_null()); - EXPECT_TRUE(tree[0].val_is_null()); - _check_null_pointing_at(tree[0], key, "NULL:", tree.arena()); - _check_null_pointing_at(tree[0], val, "NULL}", tree.arena()); - - tree = parse_in_arena(R"({ : })"); - ASSERT_EQ(tree.size(), 2u); - EXPECT_EQ(tree.root_id(), 0u); - EXPECT_EQ(tree.first_child(0), 1u); - EXPECT_TRUE(tree.type(1).is_keyval()); - EXPECT_EQ(tree.key(1), nullptr); - EXPECT_EQ(tree.val(1), nullptr); - EXPECT_TRUE(tree.key_is_null(1)); - EXPECT_TRUE(tree.val_is_null(1)); - EXPECT_TRUE(tree[0].key_is_null()); - EXPECT_TRUE(tree[0].val_is_null()); - _check_null_pointing_at(tree[0], key, ": }", tree.arena()); - _check_null_pointing_at(tree[0], val, " }", tree.arena()); - - tree = parse_in_arena(R"({~: ~})"); - ASSERT_EQ(tree.size(), 2u); - EXPECT_EQ(tree.root_id(), 0u); - EXPECT_EQ(tree.first_child(0), 1u); - EXPECT_TRUE(tree.type(1).is_keyval()); - EXPECT_EQ(tree.key(1), "~"); - EXPECT_EQ(tree.val(1), "~"); - EXPECT_TRUE(tree.key_is_null(1)); - EXPECT_TRUE(tree.val_is_null(1)); - EXPECT_TRUE(tree[0].key_is_null()); - EXPECT_TRUE(tree[0].val_is_null()); - _check_null_pointing_at(tree[0], key, "~:", tree.arena()); - _check_null_pointing_at(tree[0], val, "~}", tree.arena()); - - tree = parse_in_arena(R"({"~": "~"})"); - ASSERT_EQ(tree.size(), 2u); - EXPECT_EQ(tree.root_id(), 0u); - EXPECT_EQ(tree.first_child(0), 1u); - EXPECT_TRUE(tree.type(1).is_keyval()); - EXPECT_EQ(tree.key(1), "~"); - EXPECT_EQ(tree.val(1), "~"); - EXPECT_NE(tree.key(1), nullptr); - EXPECT_NE(tree.val(1), nullptr); - EXPECT_FALSE(tree.key_is_null(1)); - EXPECT_FALSE(tree.val_is_null(1)); - EXPECT_FALSE(tree[0].key_is_null()); - EXPECT_FALSE(tree[0].val_is_null()); - - tree = parse_in_arena(R"({"null": "null"})"); - ASSERT_EQ(tree.size(), 2u); - EXPECT_EQ(tree.root_id(), 0u); - EXPECT_EQ(tree.first_child(0), 1u); - EXPECT_TRUE(tree.type(1).is_keyval()); - EXPECT_EQ(tree.key(1), "null"); - EXPECT_EQ(tree.val(1), "null"); - EXPECT_NE(tree.key(1), nullptr); - EXPECT_NE(tree.val(1), nullptr); - EXPECT_FALSE(tree[0].key_is_null()); - EXPECT_FALSE(tree[0].val_is_null()); - - tree = parse_in_arena(R"({"Null": "Null"})"); - ASSERT_EQ(tree.size(), 2u); - EXPECT_EQ(tree.root_id(), 0u); - EXPECT_EQ(tree.first_child(0), 1u); - EXPECT_TRUE(tree.type(1).is_keyval()); - EXPECT_EQ(tree.key(1), "Null"); - EXPECT_EQ(tree.val(1), "Null"); - EXPECT_NE(tree.key(1), nullptr); - EXPECT_NE(tree.val(1), nullptr); - EXPECT_FALSE(tree.key_is_null(1)); - EXPECT_FALSE(tree.val_is_null(1)); - EXPECT_FALSE(tree[0].key_is_null()); - EXPECT_FALSE(tree[0].val_is_null()); - - tree = parse_in_arena(R"({"NULL": "NULL"})"); - ASSERT_EQ(tree.size(), 2u); - EXPECT_EQ(tree.root_id(), 0u); - EXPECT_EQ(tree.first_child(0), 1u); - EXPECT_TRUE(tree.type(1).is_keyval()); - EXPECT_EQ(tree.key(1), "NULL"); - EXPECT_EQ(tree.val(1), "NULL"); - EXPECT_NE(tree.key(1), nullptr); - EXPECT_NE(tree.val(1), nullptr); - EXPECT_FALSE(tree.key_is_null(1)); - EXPECT_FALSE(tree.val_is_null(1)); - EXPECT_FALSE(tree[0].key_is_null()); - EXPECT_FALSE(tree[0].val_is_null()); - - C4_SUPPRESS_WARNING_GCC_POP -} - - -TEST(null_val, null_key) -{ - auto tree = parse_in_arena(R"({null: null})"); - - ASSERT_EQ(tree.size(), 2u); - _check_null_pointing_at(tree[0], key, "null: ", tree.arena()); - _check_null_pointing_at(tree[0], val, "null}", tree.arena()); -} - - -TEST(null_val, readme_example) -{ - csubstr yaml = R"( -seq: - - ~ - - null - - - - - # a comment - - -map: - val0: ~ - val1: null - val2: - val3: - # a comment - val4: -)"; - Parser p; - Tree t = p.parse_in_arena("file.yml", yaml); - // as expected: (len is null, str is pointing at the value where the node starts) - EXPECT_EQ(t["seq"][0].val(), "~"); - EXPECT_EQ(t["seq"][1].val(), "null"); - EXPECT_EQ(t["seq"][2].val(), nullptr); - EXPECT_EQ(t["seq"][3].val(), nullptr); - EXPECT_EQ(t["seq"][4].val(), nullptr); - EXPECT_EQ(t["map"][0].val(), "~"); - EXPECT_EQ(t["map"][1].val(), "null"); - EXPECT_EQ(t["map"][2].val(), nullptr); - EXPECT_EQ(t["map"][3].val(), nullptr); - EXPECT_EQ(t["map"][4].val(), nullptr); - // standard null values point at the expected location: - EXPECT_EQ(csubstr(t["seq"][0].val().str, 1), csubstr("~")); - EXPECT_EQ(csubstr(t["seq"][1].val().str, 4), csubstr("null")); - EXPECT_EQ(csubstr(t["map"]["val0"].val().str, 1), csubstr("~")); - EXPECT_EQ(csubstr(t["map"]["val1"].val().str, 4), csubstr("null")); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -CASE_GROUP(NULL_VAL) -{ - -ADD_CASE_TO_GROUP("all null", -R"( -- -- # with space -- null -- Null -- NULL -- ~ -- null: null -- Null: Null -- NULL: NULL -- ~: ~ -- ~: null -- null: ~ -)", -L{ -N(VAL, nullptr), -N(VAL, nullptr), -N(VAL, "null"), -N(VAL, "Null"), -N(VAL, "NULL"), -N(VAL, "~"), -N(MAP, L{N(KEYVAL, "null", "null")}), -N(MAP, L{N(KEYVAL, "Null", "Null")}), -N(MAP, L{N(KEYVAL, "NULL", "NULL")}), -N(MAP, L{N(KEYVAL, "~", "~")}), -N(MAP, L{N(KEYVAL, "~", "null")}), -N(MAP, L{N(KEYVAL, "null", "~")}), -}); - -ADD_CASE_TO_GROUP("null map vals, expl", -R"({foo: , bar: , baz: } -)", -L{N(KEYVAL, "foo", nullptr), N(KEYVAL, "bar", nullptr), N(KEYVAL, "baz", nullptr)} -); - -ADD_CASE_TO_GROUP("null map vals, impl", -R"( -foo: -bar: -baz: -)", -L{N(KEYVAL, "foo", nullptr), N(KEYVAL, "bar", nullptr), N(KEYVAL, "baz", nullptr)} -); - -ADD_CASE_TO_GROUP("null seq vals, impl", -R"(- -- -- -)", -L{N(VAL, nullptr), N(VAL, nullptr), N(VAL, nullptr)} -); - -ADD_CASE_TO_GROUP("null seq vals in map, impl, mixed 1", -R"( -foo: - - - - - - -bar: -baz: -)", -L{N("foo", L{N(VAL, nullptr), N(VAL, nullptr), N(VAL, nullptr)}), N(KEYVAL, "bar", nullptr), N(KEYVAL, "baz", nullptr)} -); - -ADD_CASE_TO_GROUP("null seq vals in map, impl, mixed 2", -R"( -foo: -bar: - - - - - - -baz: -)", -L{N(KEYVAL, "foo", nullptr), N("bar", L{N(VAL, nullptr), N(VAL, nullptr), N(VAL, nullptr)}), N(KEYVAL, "baz", nullptr)} -); - -ADD_CASE_TO_GROUP("null seq vals in map, impl, mixed 3", -R"( -foo: -bar: -baz: - - - - - - -)", -L{N(KEYVAL, "foo", nullptr), N(KEYVAL, "bar", nullptr), N("baz", L{N(VAL, nullptr), N(VAL, nullptr), N(VAL, nullptr)})} -); - -ADD_CASE_TO_GROUP("null map vals in seq, impl, mixed 1", -R"( -- foo: - bar: - baz: -- -- -)", -L{N(L{N(KEYVAL, "foo", nullptr), N(KEYVAL, "bar", nullptr), N(KEYVAL, "baz", nullptr)}), N(VAL, nullptr), N(VAL, nullptr)} -); - -ADD_CASE_TO_GROUP("null map vals in seq, impl, mixed 2", -R"( -- -- foo: - bar: - baz: -- -)", -L{N(VAL, nullptr), N(L{N(KEYVAL, "foo", nullptr), N(KEYVAL, "bar", nullptr), N(KEYVAL, "baz", nullptr)}), N(VAL, nullptr)} -); - -ADD_CASE_TO_GROUP("null map vals in seq, impl, mixed 3", -R"( -- -- -- foo: - bar: - baz: -)", -L{N(VAL, nullptr), N(VAL, nullptr), N(L{N(KEYVAL, "foo", nullptr), N(KEYVAL, "bar", nullptr), N(KEYVAL, "baz", nullptr)})} -); - -ADD_CASE_TO_GROUP("issue84.1", -R"( -fixed case: - foo: a - bar: -your case: - foo: a - bar: '' -whatever: baz -)", -L{ -N("fixed case", L{N("foo", "a"), N(KEYVAL, "bar", nullptr)}), -N("your case", L{N("foo", "a"), N(QV, "bar", "")}), -N("whatever", "baz"), -}); - -ADD_CASE_TO_GROUP("issue84.2", -R"( -version: 0 -type: xml -param_root: - objects: - System: {SameGroupActorName: '', IsGetItemSelf: false} - General: - Speed: 1.0 - Life: 100 - IsLifeInfinite: false - ElectricalDischarge: 1.0 - IsBurnOutBorn: false - BurnOutBornName: - IsBurnOutBornIdent: false - ChangeDropTableName: '' -)", -L{ -N("version", "0"), -N("type", "xml"), -N("param_root", L{ - N("objects", L{ - N("System", L{ - N(QV, "SameGroupActorName", ""), - N("IsGetItemSelf", "false") - }), - N("General", L{ - N("Speed", "1.0"), - N("Life", "100"), - N("IsLifeInfinite", "false"), - N("ElectricalDischarge", "1.0"), - N("IsBurnOutBorn", "false"), - N(KEYVAL, "BurnOutBornName", nullptr), - N("IsBurnOutBornIdent", "false"), - N(QV, "ChangeDropTableName", ""), - }), - }) -}), -}); - -ADD_CASE_TO_GROUP("issue84.3", -R"( -version: 10 -type: test -param_root: - objects: - TestContent: - Str64_empty: '' - Str64_empty2: - Str64_empty3: '' - lists: {} -)", -L{ -N("version", "10"), -N("type", "test"), -N("param_root", L{ - N("objects", L{ - N("TestContent", L{ - N(QV, "Str64_empty", ""), - N(KEYVAL, "Str64_empty2", nullptr), - N(QV, "Str64_empty3", ""), - }), - }), - N(KEYMAP, "lists", L{}) -}), -}); - -} - -} // namespace yml -} // namespace c4 - -C4_SUPPRESS_WARNING_GCC_POP diff --git a/thirdparty/ryml/test/test_number.cpp b/thirdparty/ryml/test/test_number.cpp deleted file mode 100644 index 41c388ae6..000000000 --- a/thirdparty/ryml/test/test_number.cpp +++ /dev/null @@ -1,217 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -template -auto mkvals() -> typename std::enable_if::value, std::vector>::type -{ - return std::vector({std::numeric_limits::min(), -10, -5, -1, 0, 1, 5, 10, std::numeric_limits::max(),}); -} -template -auto mkvals() -> typename std::enable_if::value, std::vector>::type -{ - return std::vector({0, 1, 5, 10, std::numeric_limits::max(),}); -} -template -void test_ints() -{ - C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wuseless-cast") - std::vector values = mkvals(); - Tree t = parse_in_arena("{dec: [], hex: [], bin: [], oct: [], versions: ['0.1', 0.1.2, 0.1.2.3, 0.1.2.3.4]}"); - NodeRef r = t.rootref(); - for(I val : values) - { - I out; - r["dec"].append_child() << val; - r["hex"].append_child() << fmt::hex(val); - r["bin"].append_child() << fmt::bin(val); - r["oct"].append_child() << fmt::oct(val); - out = (I)(val + I(1)); - r["dec"].last_child() >> out; - EXPECT_EQ(out, val); - out = (I)(val + I(1)); - r["hex"].last_child() >> out; - EXPECT_EQ(out, val); - out = (I)(val + I(1)); - r["bin"].last_child() >> out; - EXPECT_EQ(out, val); - out = (I)(val + I(1)); - r["oct"].last_child() >> out; - EXPECT_EQ(out, val); - } - { - std::string emitted = emitrs_yaml(t); - Tree parsed = parse_in_place(to_substr(emitted)); - ASSERT_EQ(parsed["dec"].num_children(), values.size()); - ASSERT_EQ(parsed["hex"].num_children(), values.size()); - ASSERT_EQ(parsed["bin"].num_children(), values.size()); - ASSERT_EQ(parsed["oct"].num_children(), values.size()); - ASSERT_EQ(parsed["versions"].num_children(), 4u); - size_t pos = 0; - for(I val : values) - { - I out = (I)(val + I(1)); - parsed["dec"][pos] >> out; - EXPECT_EQ(out, val); - out = (I)(val + I(1)); - parsed["hex"][pos] >> out; - EXPECT_EQ(out, val); - out = (I)(val + I(1)); - parsed["bin"][pos]>> out; - EXPECT_EQ(out, val); - out = (I)(val + I(1)); - parsed["oct"][pos] >> out; - EXPECT_EQ(out, val); - ++pos; - } - EXPECT_EQ(parsed["versions"][0], "0.1"); - EXPECT_EQ(parsed["versions"][1], "0.1.2"); - EXPECT_EQ(parsed["versions"][2], "0.1.2.3"); - EXPECT_EQ(parsed["versions"][3], "0.1.2.3.4"); - } - { - std::string emitted = emitrs_json(t); - Tree parsed = parse_in_place(to_substr(emitted)); - ASSERT_EQ(parsed["dec"].num_children(), values.size()); - ASSERT_EQ(parsed["hex"].num_children(), values.size()); - ASSERT_EQ(parsed["bin"].num_children(), values.size()); - ASSERT_EQ(parsed["oct"].num_children(), values.size()); - ASSERT_EQ(parsed["versions"].num_children(), 4u); - size_t pos = 0; - for(I val : values) - { - I out = (I)(val + I(1)); - parsed["dec"][pos] >> out; - EXPECT_EQ(out, val); - out = (I)(val + I(1)); - parsed["hex"][pos] >> out; - EXPECT_EQ(out, val); - out = (I)(val + I(1)); - parsed["bin"][pos]>> out; - EXPECT_EQ(out, val); - out = (I)(val + I(1)); - parsed["oct"][pos] >> out; - EXPECT_EQ(out, val); - ++pos; - } - EXPECT_EQ(parsed["versions"][0], "0.1"); - EXPECT_EQ(parsed["versions"][1], "0.1.2"); - EXPECT_EQ(parsed["versions"][2], "0.1.2.3"); - EXPECT_EQ(parsed["versions"][3], "0.1.2.3.4"); - } - C4_SUPPRESS_WARNING_GCC_POP -} - -TEST(number, idec) -{ - test_ints(); - test_ints(); - test_ints(); - test_ints(); - test_ints(); - test_ints(); - test_ints(); - test_ints(); -} - - - -CASE_GROUP(NUMBER) -{ - -ADD_CASE_TO_GROUP("integer numbers, flow", JSON_ALSO, -R"(translation: [-2, -2, 5, 0xa, -0xb, 0XA, -0XA, 0b10, -0b10, 0B10, -0B10, 0o17, -0o17, 0O17, -0O17])", -L{N("translation", L{ - N("-2"), N("-2"), N("5"), - N("0xa"), N("-0xb"), - N("0XA"), N("-0XA"), - N("0b10"), N("-0b10"), - N("0B10"), N("-0B10"), - N("0o17"), N("-0o17"), - N("0O17"), N("-0O17"), -})}); - -ADD_CASE_TO_GROUP("integer numbers, block", JSON_ALSO, -R"(translation: - - -2 - - -2 - - -5 -)", -L{N("translation", L{N("-2"), N("-2"), N("-5")})} -); - -ADD_CASE_TO_GROUP("floating point numbers, flow", JSON_ALSO, -R"([-2.0, -2.1, 0.1, .1, -.2, -2.e+6, -3e-6, 1.12345e+011])", -L{N("-2.0"), N("-2.1"), N("0.1"), N(".1"), N("-.2"), N("-2.e+6"), N("-3e-6"), N("1.12345e+011")} -); - -ADD_CASE_TO_GROUP("floating point numbers, block", JSON_ALSO, -R"( -- -2.0 -- -2.1 -- 0.1 -- .1 -- -.2 -- -2.e+6 -- -3e-6 -- 1.12345e+011 -)", -L{N("-2.0"), N("-2.1"), N("0.1"), N(".1"), N("-.2"), N("-2.e+6"), N("-3e-6"), N("1.12345e+011")} -); - -ADD_CASE_TO_GROUP("hex floating point numbers, block", JSON_ALSO, -R"( -- -2.0 -- -2.1 -- 0.1 -- .1 -- -.2 -- -2.e+6 -- -3e-6 -- 1.12345e+011 -)", -L{N("-2.0"), N("-2.1"), N("0.1"), N(".1"), N("-.2"), N("-2.e+6"), N("-3e-6"), N("1.12345e+011")} -); - -ADD_CASE_TO_GROUP("version numbers", JSON_ALSO, -R"( -- 1.2.3 -- 1.2.3.4 -- [1.2.3, 4.5.6] -- [1.2.3.4, 4.5.6.7] -- - 1.2.3 - - 4.5.6 -- - 1.2.3.4 - - 4.5.6.7 -- a: 1.2.3 -- a: 1.2.3.4 -- {a: 1.2.3} -- {a: 1.2.3.4} -- a: 1.2.3 - b: 4.5.6 -- a: 1.2.3.4 - b: 4.5.6.7 -- {a: 1.2.3, b: 4.5.6} -- {a: 1.2.3.4, b: 4.5.6.7} -)", -L{ - N("1.2.3"), - N("1.2.3.4"), - N(L{N("1.2.3"), N("4.5.6")}), - N(L{N("1.2.3.4"), N("4.5.6.7")}), - N(L{N("1.2.3"), N("4.5.6")}), - N(L{N("1.2.3.4"), N("4.5.6.7")}), - N(L{N("a", "1.2.3")}), - N(L{N("a", "1.2.3.4")}), - N(L{N("a", "1.2.3")}), - N(L{N("a", "1.2.3.4")}), - N(L{N("a", "1.2.3"), N("b", "4.5.6")}), - N(L{N("a", "1.2.3.4"), N("b", "4.5.6.7")}), - N(L{N("a", "1.2.3"), N("b", "4.5.6")}), - N(L{N("a", "1.2.3.4"), N("b", "4.5.6.7")}), -}); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_parser.cpp b/thirdparty/ryml/test/test_parser.cpp deleted file mode 100644 index 3ef9ee145..000000000 --- a/thirdparty/ryml/test/test_parser.cpp +++ /dev/null @@ -1,566 +0,0 @@ -#ifdef RYML_SINGLE_HEADER -#include "ryml_all.hpp" -#else -#include "c4/yml/parse.hpp" -#endif -#include -#include "./callbacks_tester.hpp" - - -namespace c4 { -namespace yml { - -// TODO: add this as a method to csubstr -bool is_same(csubstr lhs, csubstr rhs) -{ - return lhs.str == rhs.str && lhs.len == rhs.len; -} - -void mklarge(Parser *p, Callbacks const& cb) -{ - p->~Parser(); - new ((void*)p) Parser(cb); - p->reserve_stack(20); // cause an allocation - p->reserve_locations(128); // cause an allocation - p->reserve_filter_arena(128); // cause an allocation -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(Parser, empty_ctor) -{ - Parser parser; - EXPECT_EQ(parser.callbacks(), get_callbacks()); -} - -TEST(Parser, callbacks_ctor) -{ - CallbacksTester cbt; - { - Parser parser(cbt.callbacks()); - EXPECT_EQ(parser.callbacks(), cbt.callbacks()); - } - EXPECT_EQ(cbt.num_allocs, 0u); - EXPECT_EQ(cbt.num_deallocs, 0u); -} - -TEST(Parser, reserve_capacity) -{ - CallbacksTester cbt("test", 20000/*Bytes*/); - { - Parser parser(cbt.callbacks()); - EXPECT_EQ(cbt.num_allocs, 0u); - EXPECT_EQ(cbt.num_deallocs, 0u); - parser.reserve_stack(18); - EXPECT_EQ(cbt.num_allocs, 1u); - EXPECT_EQ(cbt.num_deallocs, 0u); - parser.reserve_stack(24); - EXPECT_EQ(cbt.num_allocs, 2u); - EXPECT_EQ(cbt.num_deallocs, 1u); - parser.reserve_stack(28); - EXPECT_EQ(cbt.num_allocs, 3u); - EXPECT_EQ(cbt.num_deallocs, 2u); - } - EXPECT_EQ(cbt.num_allocs, 3u); - EXPECT_EQ(cbt.num_deallocs, 3u); - cbt.check(); -} - -TEST(Parser, reserve_locations) -{ - CallbacksTester ts; - { - Parser parser(ts.callbacks()); - EXPECT_EQ(parser.callbacks(), ts.callbacks()); - EXPECT_EQ(ts.num_allocs, 0u); - EXPECT_EQ(ts.num_deallocs, 0u); - parser.reserve_locations(128); - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(ts.alloc_size, 128u * sizeof(size_t)); - EXPECT_EQ(ts.dealloc_size, 0u); - } - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 1u); - EXPECT_EQ(ts.alloc_size, 128u * sizeof(size_t)); - EXPECT_EQ(ts.dealloc_size, 128u * sizeof(size_t)); -} - -TEST(Parser, reserve_filter_arena) -{ - size_t cap = 256u; - CallbacksTester ts; - { - Parser parser(ts.callbacks()); - EXPECT_EQ(parser.filter_arena_capacity(), 0u); - EXPECT_EQ(parser.callbacks(), ts.callbacks()); - EXPECT_EQ(ts.num_allocs, 0u); - EXPECT_EQ(ts.num_deallocs, 0u); - parser.reserve_filter_arena(cap); - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(ts.alloc_size, cap); - EXPECT_EQ(ts.dealloc_size, 0u); - } - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 1u); - EXPECT_EQ(ts.alloc_size, cap); - EXPECT_EQ(ts.dealloc_size, cap); -} - -TEST(Parser, copy_ctor) -{ - { - Parser src; - mklarge(&src, get_callbacks()); - EXPECT_EQ(src.callbacks(), get_callbacks()); - Parser dst(src); - EXPECT_EQ(src.callbacks(), get_callbacks()); - EXPECT_EQ(dst.callbacks(), get_callbacks()); - } - { - CallbacksTester ts; - { - Parser src; - mklarge(&src, ts.callbacks()); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - size_t nbefore = ts.num_allocs; - EXPECT_GT(ts.num_allocs, 0u); - Parser dst(src); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), ts.callbacks()); - EXPECT_GT(ts.num_allocs, nbefore); - } - EXPECT_EQ(ts.num_allocs, ts.num_deallocs); - EXPECT_EQ(ts.alloc_size, ts.dealloc_size); - } -} - -TEST(Parser, move_ctor) -{ - { - Parser src; - mklarge(&src, get_callbacks()); - EXPECT_EQ(src.callbacks(), get_callbacks()); - Parser dst(std::move(src)); - EXPECT_EQ(src.callbacks(), get_callbacks()); - EXPECT_EQ(dst.callbacks(), get_callbacks()); - } - { - CallbacksTester ts; - { - Parser src; - mklarge(&src, ts.callbacks()); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - size_t nbefore = ts.num_allocs; - EXPECT_GT(ts.num_allocs, 0u); - Parser dst(std::move(src)); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), ts.callbacks()); - EXPECT_EQ(ts.num_allocs, nbefore); - } - EXPECT_EQ(ts.num_allocs, ts.num_deallocs); - EXPECT_EQ(ts.alloc_size, ts.dealloc_size); - } -} - -TEST(Parser, copy_assign_same_callbacks) -{ - CallbacksTester ts; - { - Parser src; - Parser dst; - mklarge(&src, ts.callbacks()); - mklarge(&dst, ts.callbacks()); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), ts.callbacks()); - size_t nbefore = ts.num_allocs; - EXPECT_GT(ts.num_allocs, 0u); - EXPECT_GT(ts.num_allocs, 0u); - dst = src; - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), ts.callbacks()); - EXPECT_GT(ts.num_allocs, nbefore); - } - EXPECT_EQ(ts.num_allocs, ts.num_deallocs); - EXPECT_EQ(ts.alloc_size, ts.dealloc_size); -} - -TEST(Parser, copy_assign_diff_callbacks) -{ - CallbacksTester ts("src"); - CallbacksTester td("dst"); - { - Parser src; - Parser dst; - mklarge(&src, ts.callbacks()); - mklarge(&dst, td.callbacks()); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), td.callbacks()); - size_t nbefore = ts.num_allocs; - EXPECT_GT(ts.num_allocs, 0u); - EXPECT_GT(td.num_allocs, 0u); - dst = src; - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), ts.callbacks()); - EXPECT_GT(ts.num_allocs, nbefore); - EXPECT_EQ(td.num_allocs, nbefore); - } - EXPECT_EQ(ts.num_allocs, ts.num_deallocs); - EXPECT_EQ(ts.alloc_size, ts.dealloc_size); - EXPECT_EQ(td.num_allocs, td.num_deallocs); - EXPECT_EQ(td.alloc_size, td.dealloc_size); -} - -TEST(Parser, move_assign_same_callbacks) -{ - CallbacksTester ts; - { - Parser src; - Parser dst; - mklarge(&src, ts.callbacks()); - mklarge(&dst, ts.callbacks()); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), ts.callbacks()); - size_t nbefore = ts.num_allocs; - EXPECT_GT(ts.num_allocs, 0u); - EXPECT_GT(ts.num_allocs, 0u); - dst = std::move(src); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), ts.callbacks()); - EXPECT_EQ(ts.num_allocs, nbefore); - } - EXPECT_EQ(ts.num_allocs, ts.num_deallocs); - EXPECT_EQ(ts.alloc_size, ts.dealloc_size); -} - -TEST(Parser, move_assign_diff_callbacks) -{ - CallbacksTester ts("src"); - CallbacksTester td("dst"); - { - Parser src; - Parser dst; - mklarge(&src, ts.callbacks()); - mklarge(&dst, td.callbacks()); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), td.callbacks()); - size_t nbefore = ts.num_allocs; - EXPECT_GT(ts.num_allocs, 0u); - EXPECT_GT(td.num_allocs, 0u); - dst = std::move(src); - ASSERT_EQ(src.callbacks(), ts.callbacks()); - ASSERT_EQ(dst.callbacks(), ts.callbacks()); - EXPECT_EQ(td.num_allocs, nbefore); // dst frees with td - EXPECT_EQ(ts.num_allocs, nbefore); // dst moves from ts - } - EXPECT_EQ(ts.num_allocs, ts.num_deallocs); - EXPECT_EQ(ts.alloc_size, ts.dealloc_size); - EXPECT_EQ(td.num_allocs, td.num_deallocs); - EXPECT_EQ(td.alloc_size, td.dealloc_size); -} - -TEST(Parser, new_tree_receives_callbacks) -{ - char src_[] = "{a: b}"; - substr src = src_; - csubstr csrc = src_; - { - { - Parser parser; - EXPECT_EQ(parser.callbacks(), get_callbacks()); - Tree t = parser.parse_in_arena("file0", csrc); - EXPECT_EQ(t.callbacks(), get_callbacks()); - } - CallbacksTester cbt("test", 20000/*Bytes*/); - { - Parser parser(cbt.callbacks()); - EXPECT_EQ(parser.callbacks(), cbt.callbacks()); - Tree t = parser.parse_in_arena("file1", csrc); - EXPECT_EQ(t.callbacks(), cbt.callbacks()); - } - cbt.check(); - } - { - { - Parser parser; - EXPECT_EQ(parser.callbacks(), get_callbacks()); - Tree t = parser.parse_in_place("file", src); - EXPECT_EQ(t.callbacks(), get_callbacks()); - } - CallbacksTester cbt("test", 20000/*Bytes*/); - { - Parser parser(cbt.callbacks()); - EXPECT_EQ(parser.callbacks(), cbt.callbacks()); - Tree t = parser.parse_in_place("file", src); - EXPECT_EQ(t.callbacks(), cbt.callbacks()); - } - cbt.check(); - } -} - -TEST(Parser, existing_tree_overwrites_parser_callbacks) -{ - char src_[] = "{a: b}"; - substr src = src_; - csubstr csrc = src_; - { - CallbacksTester cbp("parser"); - CallbacksTester cbt("tree"); - { - Tree tree(cbt.callbacks()); - Parser parser(cbp.callbacks()); - EXPECT_EQ(tree.callbacks(), cbt.callbacks()); - EXPECT_EQ(parser.callbacks(), cbp.callbacks()); - parser.parse_in_arena("file", csrc, &tree); - EXPECT_EQ(tree.callbacks(), cbt.callbacks()); - EXPECT_EQ(parser.callbacks(), cbp.callbacks()); - } - cbp.check(); - cbt.check(); - } - { - CallbacksTester cbp("parser"); - CallbacksTester cbt("tree"); - { - Tree tree(cbt.callbacks()); - Parser parser(cbp.callbacks()); - EXPECT_EQ(tree.callbacks(), cbt.callbacks()); - EXPECT_EQ(parser.callbacks(), cbp.callbacks()); - parser.parse_in_place("file", src, &tree); - EXPECT_EQ(tree.callbacks(), cbt.callbacks()); - EXPECT_EQ(parser.callbacks(), cbp.callbacks()); - } - cbp.check(); - cbt.check(); - } -} - -TEST(Parser, filename_and_buffer_are_stored) -{ - char src_[] = "{a: b}"; - substr src = src_; - csubstr csrc = src_; - Parser parser; - EXPECT_EQ(parser.filename(), csubstr{}); - { - Tree tree = parser.parse_in_place("file0", src); - EXPECT_EQ(parser.filename(), "file0"); - EXPECT_TRUE(is_same(parser.source(), src)); - } - { - Tree tree = parser.parse_in_arena("file1", csrc); - EXPECT_EQ(parser.filename(), "file1"); - EXPECT_TRUE(!is_same(parser.source(), src)); - } - { - Tree tree = parser.parse_in_place("file2", src); - EXPECT_EQ(parser.filename(), "file2"); - EXPECT_TRUE(is_same(parser.source(), src)); - } - { - Tree tree = parser.parse_in_arena({}, csrc); - EXPECT_EQ(parser.filename(), csubstr{}); - EXPECT_TRUE(!is_same(parser.source(), src)); - } -} - -TEST(parse_in_place, overloads) -{ - char src1_[] = "{a: b}"; - char src2_[] = "{c: d, e: {}}"; - { - Tree tree = parse_in_place(src1_); - EXPECT_EQ(tree["a"].val(), "b"); - } - { - Tree tree = parse_in_place("src1", src1_); - EXPECT_EQ(tree["a"].val(), "b"); - } - { - Tree tree; - parse_in_place(src1_, &tree); - EXPECT_EQ(tree["a"].val(), "b"); - } - { - Tree tree; - parse_in_place("src1", src1_, &tree); - EXPECT_EQ(tree["a"].val(), "b"); - } - { - Tree tree = parse_in_place(src2_); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].is_map(), true); - EXPECT_EQ(tree["e"].has_children(), false); - size_t e = tree.find_child(tree.root_id(), "e"); - ASSERT_NE(e, (size_t)NONE); - parse_in_place(src1_, &tree, e); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].has_children(), true); - EXPECT_EQ(tree["e"]["a"].val(), "b"); - } - { - Tree tree = parse_in_place("src2", src2_); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].is_map(), true); - EXPECT_EQ(tree["e"].has_children(), false); - size_t e = tree.find_child(tree.root_id(), "e"); - ASSERT_NE(e, (size_t)NONE); - parse_in_place("src1", src1_, &tree, e); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].has_children(), true); - EXPECT_EQ(tree["e"]["a"].val(), "b"); - } - { - Tree tree = parse_in_place(src2_); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].is_map(), true); - EXPECT_EQ(tree["e"].has_children(), false); - parse_in_place(src1_, tree["e"]); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].has_children(), true); - EXPECT_EQ(tree["e"]["a"].val(), "b"); - } - { - Tree tree = parse_in_place("src2", src2_); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].is_map(), true); - EXPECT_EQ(tree["e"].has_children(), false); - parse_in_place("src1", src1_, tree["e"]); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].has_children(), true); - EXPECT_EQ(tree["e"]["a"].val(), "b"); - } -} - -TEST(parse_in_arena, overloads) -{ - csubstr src1 = "{a: b}"; - csubstr src2 = "{c: d, e: {}}"; - { - Tree tree = parse_in_arena(src1); - EXPECT_EQ(tree["a"].val(), "b"); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src1), (size_t)npos); - } - { - Tree tree = parse_in_arena("src1", src1); - EXPECT_EQ(tree["a"].val(), "b"); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src1), (size_t)npos); - } - { - Tree tree; - parse_in_arena(src1, &tree); - EXPECT_EQ(tree["a"].val(), "b"); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src1), (size_t)npos); - } - { - Tree tree; - parse_in_arena("src1", src1, &tree); - EXPECT_EQ(tree["a"].val(), "b"); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src1), (size_t)npos); - } - { - Tree tree = parse_in_arena(src2); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].is_map(), true); - EXPECT_EQ(tree["e"].has_children(), false); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src2), (size_t)npos); - size_t e = tree.find_child(tree.root_id(), "e"); - ASSERT_NE(e, (size_t)NONE); - parse_in_arena(src1, &tree, e); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].has_children(), true); - EXPECT_EQ(tree["e"]["a"].val(), "b"); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src1), (size_t)npos); - EXPECT_NE(tree.arena().find(src2), (size_t)npos); - } - { - Tree tree = parse_in_arena("src2", src2); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].is_map(), true); - EXPECT_EQ(tree["e"].has_children(), false); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src2), (size_t)npos); - size_t e = tree.find_child(tree.root_id(), "e"); - ASSERT_NE(e, (size_t)NONE); - parse_in_arena("src1", src1, &tree, e); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].has_children(), true); - EXPECT_EQ(tree["e"]["a"].val(), "b"); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src1), (size_t)npos); - EXPECT_NE(tree.arena().find(src2), (size_t)npos); - } - { - Tree tree = parse_in_arena(src2); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].is_map(), true); - EXPECT_EQ(tree["e"].has_children(), false); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src2), (size_t)npos); - parse_in_arena(src1, tree["e"]); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].has_children(), true); - EXPECT_EQ(tree["e"]["a"].val(), "b"); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src1), (size_t)npos); - EXPECT_NE(tree.arena().find(src2), (size_t)npos); - } - { - Tree tree = parse_in_arena("src2", src2); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].is_map(), true); - EXPECT_EQ(tree["e"].has_children(), false); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src2), (size_t)npos); - parse_in_arena("src1", src1, tree["e"]); - EXPECT_EQ(tree["c"].val(), "d"); - EXPECT_EQ(tree["e"].has_children(), true); - EXPECT_EQ(tree["e"]["a"].val(), "b"); - EXPECT_FALSE(tree.arena().empty()); - EXPECT_NE(tree.arena().find(src1), (size_t)npos); - EXPECT_NE(tree.arena().find(src2), (size_t)npos); - } -} - -TEST(parse_in_place, version_numbers) -{ - char src1_[] = "{a: 1.2.3}"; - { - Tree tree = parse_in_place(src1_); - EXPECT_EQ(tree["a"].val(), "1.2.3"); - } -} - -} // namespace yml -} // namespace c4 - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// this is needed to use the test case library - -#ifndef RYML_SINGLE_HEADER -#include "c4/substr.hpp" -#endif - -namespace c4 { -namespace yml { -struct Case; -Case const* get_case(csubstr /*name*/) -{ - return nullptr; -} -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_plain_scalar.cpp b/thirdparty/ryml/test/test_plain_scalar.cpp deleted file mode 100644 index ec147c5d8..000000000 --- a/thirdparty/ryml/test/test_plain_scalar.cpp +++ /dev/null @@ -1,800 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -TEST(plain_scalar, issue153_seq) -{ - Tree t = parse_in_arena("- A\n \n"); - EXPECT_EQ(t[0].val(), "A"); -} - -TEST(plain_scalar, issue153_map) -{ - Tree t = parse_in_arena("foo: A\n \n"); - EXPECT_EQ(t["foo"].val(), "A"); -} - - -TEST(plain_scalar, test_suite_7TMG) -{ - csubstr yaml = R"(--- -word1 -# comment ---- -# first value is NOT a multiline plain scalar -[ word1 -# comment -, word2] -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_TRUE(t.rootref().is_stream()); - ConstNodeRef doc = t.rootref().first_child(); - ASSERT_TRUE(doc.is_doc()); - ASSERT_TRUE(doc.is_val()); - EXPECT_EQ(doc.val(), "word1"); - doc = t.rootref().child(1); - ASSERT_TRUE(doc.is_doc()); - ASSERT_TRUE(doc.is_seq()); - EXPECT_EQ(doc[0].val(), "word1"); - EXPECT_EQ(doc[1].val(), "word2"); - }); -} - - -TEST(plain_scalar, test_suite_82AN) -{ - csubstr yaml = R"( ----word1 -word2 -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_doc()); - ASSERT_TRUE(t.rootref().is_val()); - EXPECT_EQ(t.rootref().val(), csubstr("---word1 word2")); - }); -} - -TEST(plain_scalar, test_suite_EXG3) -{ - csubstr yaml = R"( ---- ----word1 -word2 -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_stream()); - ASSERT_TRUE(t.rootref().first_child().is_doc()); - ASSERT_TRUE(t.rootref().first_child().is_val()); - EXPECT_EQ(t.rootref().first_child().val(), csubstr("---word1 word2")); - }); -} - - -TEST(plain_scalar, test_suite_9YRD) -{ - csubstr yaml = R"( -a -b - c -d - -e -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_doc()); - ASSERT_TRUE(t.rootref().is_val()); - EXPECT_EQ(t.rootref().val(), csubstr("a b c d\ne")); - }); -} - -TEST(plain_scalar, test_suite_EX5H) -{ - csubstr yaml = R"( ---- -a -b - c -d - -e -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_stream()); - ASSERT_TRUE(t.rootref().child(0).is_doc()); - ASSERT_TRUE(t.rootref().child(0).is_val()); - EXPECT_EQ(t.rootref().child(0).val(), csubstr("a b c d\ne")); - }); -} - - -TEST(plain_scalar, test_suite_M7A3) -{ - csubstr yaml = R"( -Bare -document -... -# No document -... -| -%!PS-Adobe-2.0 # Not the first line -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_stream()); - ASSERT_EQ(t.rootref().num_children(), 2u); - EXPECT_EQ(t.rootref().child(0).val(), csubstr("Bare document")); - EXPECT_EQ(t.rootref().child(1).val(), csubstr("%!PS-Adobe-2.0 # Not the first line\n")); - }); -} - - -TEST(plain_scalar, test_suite_HS5T) -{ - csubstr yaml = R"( -1st non-empty - - 2nd non-empty - 3rd non-empty -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_doc()); - ASSERT_TRUE(t.rootref().is_val()); - EXPECT_EQ(t.rootref().val(), csubstr("1st non-empty\n2nd non-empty 3rd non-empty")); - }); -} - -TEST(plain_scalar, test_suite_NB6Z) -{ - csubstr yaml = R"( -key: - value - with - - tabs - tabs - - foo - - bar - baz - -key1: - value - with - - tabs - tabs - - foo - - bar - baz - -key2: something - else -key3: something - else -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_TRUE(t.rootref().has_child("key")); - ASSERT_TRUE(t.rootref().has_child("key1")); - ASSERT_TRUE(t.rootref().has_child("key2")); - ASSERT_TRUE(t.rootref().has_child("key3")); - EXPECT_EQ(t["key"].val(), csubstr("value with\ntabs tabs\nfoo\nbar baz")); - EXPECT_EQ(t["key1"].val(), csubstr("value with\ntabs tabs\nfoo\nbar baz")); - EXPECT_EQ(t["key2"].val(), csubstr("something else")); - EXPECT_EQ(t["key3"].val(), csubstr("something else")); - }); -} - -TEST(plain_scalar, test_suite_NB6Z_seq) -{ - csubstr yaml = R"( -- value - with - - tabs - tabs - - foo - - bar - baz - -- value - with - - tabs - tabs - - foo - - bar - baz - -- more - value -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_seq()); - ASSERT_EQ(t.rootref().num_children(), 3u); - EXPECT_EQ(t[0].val(), csubstr("value with\ntabs tabs\nfoo\nbar baz")); - EXPECT_EQ(t[1].val(), csubstr("value with\ntabs tabs\nfoo\nbar baz")); - EXPECT_EQ(t[2].val(), csubstr("more value")); - }); -} - -TEST(plain_scalar, test_suite_NB6Z_docval) -{ - csubstr yaml = R"( -value -with - -tabs -tabs - - foo - - bar - baz - -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_doc()); - ASSERT_TRUE(t.rootref().is_val()); - EXPECT_EQ(t.rootref().val(), csubstr("value with\ntabs tabs\nfoo\nbar baz")); - }); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -CASE_GROUP(PLAIN_SCALAR) -{ -// -ADD_CASE_TO_GROUP("plain scalar, 1 word only", -R"(a_single_word_scalar_to_test)", - N(DOCVAL, "a_single_word_scalar_to_test") -); - -ADD_CASE_TO_GROUP("plain scalar, 1 line with spaces", -R"(a scalar with spaces in it all in one line)", - N(DOCVAL, "a scalar with spaces in it all in one line") -); - -ADD_CASE_TO_GROUP("plain scalar, multiline", -R"( -a scalar with several lines in it - of course also with spaces but for now there are no quotes - and also no blank lines to speak of)", - N(DOCVAL, "a scalar with several lines in it of course also with spaces but for now there are no quotes and also no blank lines to speak of") -); - -ADD_CASE_TO_GROUP("plain scalar, multiline, unindented", -R"( -a scalar with several lines in it - of course also with spaces but for now there are no quotes - and also no blank lines to speak of)", - N(DOCVAL, "a scalar with several lines in it of course also with spaces but for now there are no quotes and also no blank lines to speak of") -); - -ADD_CASE_TO_GROUP("plain scalar, multiline, quotes, escapes", -R"( -a scalar with several lines in it and also 'single quotes' - and "double quotes" and assorted escapes such as \r or \n)", - N(DOCVAL, "a scalar with several lines in it and also 'single quotes' and \"double quotes\" and assorted escapes such as \\r or \\n") -); - -ADD_CASE_TO_GROUP("plain scalar, multiline, quotes, escapes, blank lines middle", -R"( -A scalar with several lines in it and also 'single quotes'. - A blank line follows after this one. - - And "double quotes" and assorted escapes such as \r or \n)", - N(DOCVAL, "A scalar with several lines in it and also 'single quotes'. A blank line follows after this one.\nAnd \"double quotes\" and assorted escapes such as \\r or \\n") -); - -ADD_CASE_TO_GROUP("plain scalar, multiline, quotes, escapes, blank lines first", -R"( -A scalar with several lines in it and also 'single quotes'. - - A blank line precedes this one. - And "double quotes" and assorted escapes such as \r or \n)", - N(DOCVAL, "A scalar with several lines in it and also 'single quotes'.\nA blank line precedes this one. And \"double quotes\" and assorted escapes such as \\r or \\n") -); - -ADD_CASE_TO_GROUP("plain scalar, multiline, quotes, escapes, blank lines last", -R"( -A scalar with several lines in it and also 'single quotes'. - And "double quotes" and assorted escapes such as \r or \n. - A blank line follows after this one. - - )", - N(DOCVAL, "A scalar with several lines in it and also 'single quotes'. And \"double quotes\" and assorted escapes such as \\r or \\n. A blank line follows after this one.") -); - -ADD_CASE_TO_GROUP("plain scalar, example", -R"( -Several lines of text - with some "quotes" of various 'types'. - Escapes (like \n) don't do anything. - - Newlines can be added by leaving a blank line. - Additional leading whitespace is ignored.)", - N(DOCVAL, "Several lines of text with some \"quotes\" of various 'types'. Escapes (like \\n) don't do anything.\nNewlines can be added by leaving a blank line. Additional leading whitespace is ignored.") -); - -ADD_CASE_TO_GROUP("plain scalar, map example 1", -R"( -example: Several lines of text, - with some "quotes" of various 'types'. - Escapes (like \n) don't do anything. - - Newlines can be added by leaving a blank line. - Additional leading whitespace is ignored. - -another example: Several lines of text, - - but the second line is empty, and _indented_. - There are more lines that follow. - -yet another example: Several lines of text, - - but the second line is empty, and _unindented_. - There are more lines that follow. -final example: Several lines of text, - - - but the second line is empty, and _unindented_. - There are more lines that follow. And the last line - terminates at the end of the file.)", - L{ - N("example", "Several lines of text, with some \"quotes\" of various 'types'. " - "Escapes (like \\n) don't do anything.\n" - "Newlines can be added by leaving a blank line. " - "Additional leading whitespace is ignored."), - N("another example", "Several lines of text,\n" - "but the second line is empty, and _indented_. " - "There are more lines that follow."), - N("yet another example", "Several lines of text,\n" - "but the second line is empty, and _unindented_. " - "There are more lines that follow."), - N("final example", "Several lines of text,\n\n" - "but the second line is empty, and _unindented_. " - "There are more lines that follow. " - "And the last line terminates at the end of the file."), - } -); - -/* -ADD_CASE_TO_GROUP("plain scalar, map example 2", IGNORE_LIBYAML_PARSE_FAIL|IGNORE_YAMLCPP_PARSE_FAIL, -R"( -example: - Several lines of text, - with some "quotes" of various 'types'. - Escapes (like \n) don't do anything. - - Newlines can be added by leaving a blank line. - Additional leading whitespace is ignored. -)", - L{N("example", "Several lines of text, with some \"quotes\" of various 'types'. Escapes (like \\n) don't do anything.\nNewlines can be added by leaving a blank line. Additional leading whitespace is ignored.")} -); -*/ - -ADD_CASE_TO_GROUP("plain scalar, seq example 1", -R"( -- Several lines of text, - with some "quotes" of various 'types'. - Escapes (like \n) don't do anything. - - Newlines can be added by leaving a blank line. - Additional leading whitespace is ignored.)", - L{N("Several lines of text, with some \"quotes\" of various 'types'. " - "Escapes (like \\n) don't do anything.\n" - "Newlines can be added by leaving a blank line. " - "Additional leading whitespace is ignored.")} -); - -/* -ADD_CASE_TO_GROUP("plain scalar, seq example 2", IGNORE_LIBYAML_PARSE_FAIL|IGNORE_YAMLCPP_PARSE_FAIL, -R"( -- - Several lines of text, - with some "quotes" of various 'types'. - Escapes (like \n) don't do anything. - - Newlines can be added by leaving a blank line. - Additional leading whitespace is ignored. -)", - L{N("Several lines of text, with some \"quotes\" of various 'types'. Escapes (like \\n) don't do anything.\nNewlines can be added by leaving a blank line. Additional leading whitespace is ignored.")} -); -*/ - -ADD_CASE_TO_GROUP("plain scalar, special characters 1", -R"( -- Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. - How about empty lines? - - Can we also have [] or {} inside? - Guess we can. - And how about at the beginning? - { - for example } - [ - for example ] - - - for example - ::- for example - - and now two empty lines - - - - and now three empty lines - - - - - and an empty line, unindented - - - followed by more text - and another four at the end - - - - - -)", - L{N("Several lines of text, with special:characters, like:this-or-this - - and some \"quotes\" of various 'types'. " - "How about empty lines?\n" - "Can we also have [] or {} inside? Guess we can. " - "And how about at the beginning? { - for example } [ - for example ] - - for example ::- for example\n" - "and now two empty lines -\n\n" - "and now three empty lines -\n\n\n" - "and an empty line, unindented -\n" - "followed by more text " - "and another four at the end -" - )} -); - -ADD_CASE_TO_GROUP("plain scalar, special characters 3MYT", -R"(--- # ZWK4 -a: 1 -? b -&anchor c: 3 # the anchor is for the scalar 'c' -? d -!!str e: 4 -? f ---- -k:#foo &a !t s ---- -"k:#foo &a !t s" ---- -'k:#foo &a !t s' - ---- # 3MYT -k:#foo - &a !t s ---- -k:#foo - &a !t s ---- -k:#foo - &a !t s ---- -k:#foo - &a !t s - ---- # 3MYT -k:#foo - !t s ---- -k:#foo - !t s ---- -k:#foo - !t s ---- -k:#foo - !t s -)", - N(STREAM, L{ - N(DOCMAP, L{ - N("a", "1"), - N(KEYVAL, "b", {}), - N("c", AR(KEYANCH, "anchor"), "3"), - N(KEYVAL, "d", {}), - N(TS("!!str", "e"), "4"), - N(KEYVAL, "f", {}), - }), - - N(DOCVAL, "k:#foo &a !t s"), - N(DOCVAL|VALQUO, "k:#foo &a !t s"), - N(DOCVAL|VALQUO, "k:#foo &a !t s"), - - N(DOCVAL, "k:#foo &a !t s"), - N(DOCVAL, "k:#foo &a !t s"), - N(DOCVAL, "k:#foo &a !t s"), - N(DOCVAL, "k:#foo &a !t s"), - - N(DOCVAL, "k:#foo !t s"), - N(DOCVAL, "k:#foo !t s"), - N(DOCVAL, "k:#foo !t s"), - N(DOCVAL, "k:#foo !t s"), - }) - ); - -// make sure there is no ambiguity with this case -ADD_CASE_TO_GROUP("plain scalar, sequence ambiguity", -R"( -- - some text - - and this is a sequence -- some text - - and this is /not/ a sequence -- - some text - - and this is a sequence -- some text - - and this is /not/ a sequence -)", - L{ - N(L{N("some text"), N("and this is a sequence")}), - N("some text - and this is /not/ a sequence"), - N(L{N("some text"), N("and this is a sequence")}), - N("some text - and this is /not/ a sequence"), - } -); - -ADD_CASE_TO_GROUP("plain scalar, empty lines at the beginning", -R"( -- - - - Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. -- - - Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. -- - Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. -)", - L{ - N("Several lines of text, with special:characters, like:this-or-this - - and some \"quotes\" of various 'types'."), - N("Several lines of text, with special:characters, like:this-or-this - - and some \"quotes\" of various 'types'."), - N("Several lines of text, with special:characters, like:this-or-this - - and some \"quotes\" of various 'types'."), - } -); - -ADD_CASE_TO_GROUP("plain scalar, empty continuation lines", -R"( -- the next lines have 2cols, 0cols, 2cols, - - - - and this line has some text in it. -> 0 - - now 0, 0, 2, 2, 0, 1, 1, 0, 4, 4, 0, 0 - - - - - - - - - - - - - and finally some more text -)", - L{ - N("the next lines have 2cols, 0cols, 2cols," - "\n\n\n" - "and this line has some text in it. -> 0" - "\n" - "now 0, 0, 2, 2, 0, 1, 1, 0, 4, 4, 0, 0" - "\n\n\n\n\n\n\n\n\n\n\n\n" - "and finally some more text"), - } -); - - -ADD_CASE_TO_GROUP("plain scalar, indented first line", -R"( -- Several lines of text, - - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. -- - - Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. -- - Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. -)", - L{ - N("Several lines of text,\nwith special:characters, like:this-or-this - - and some \"quotes\" of various 'types'."), - N("Several lines of text, with special:characters, like:this-or-this - - and some \"quotes\" of various 'types'."), - N("Several lines of text, with special:characters, like:this-or-this - - and some \"quotes\" of various 'types'."), - } -); - -ADD_CASE_TO_GROUP("plain scalar, do not accept ': ' mid line", EXPECT_PARSE_ERROR, -R"(- Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. - But this: must cause a parse error. -)", - LineCol(4, 11) -); - -ADD_CASE_TO_GROUP("plain scalar, do not accept ': ' start line", EXPECT_PARSE_ERROR, -R"( -- Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. - But this must cause a parse error - - : foo bar -)", - LineCol(6, 3) -); - -ADD_CASE_TO_GROUP("plain scalar, do not accept ': ' at line end", EXPECT_PARSE_ERROR, -R"(- Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. - But this must cause a parse error: -)", - LineCol(4, 36) -); - -ADD_CASE_TO_GROUP("plain scalar, do not accept ':' at line end", EXPECT_PARSE_ERROR, -R"(- Several lines of text, - with special:characters, like:this-or-this - - - and some "quotes" of various 'types'. - But this must cause a parse error: - - well, did it? -)", - LineCol(4, 36) -); - -ADD_CASE_TO_GROUP("plain scalar, accept ' #' at line start", -R"(- Several lines of text, - and this is valid - - #with special:characters, like:this-or-this - -)", - L{N("Several lines of text, and this is valid -"),} -); - -ADD_CASE_TO_GROUP("plain scalar, accept ' #' on first line", -R"(- Several lines of text, and this is valid - - #with special:characters, like:this-or-this - -)", - L{N("Several lines of text, and this is valid -")} -); - -ADD_CASE_TO_GROUP("plain scalar, accept ' #' at line end", -R"(- Several lines of text, - with special:characters, #comment at the end -)", - L{N("Several lines of text, with special:characters,")} -); - -ADD_CASE_TO_GROUP("plain scalar, accept '#'", -R"( -- Several lines of text, # with a comment -- Several lines of text, - with special#characters, like#this_#_-or-#-:this - - - and some "quotes" of various 'types'. -)", - L{ - N("Several lines of text,"), - N("Several lines of text, " - "with special#characters, like#this_#_-or-#-:this - " - "- and some \"quotes\" of various 'types'."), - } -); - -ADD_CASE_TO_GROUP("plain scalar, explicit", -R"( -[ - a plain scalar - with several lines - - and blank lines - - as well - , - and another plain scalar - , - and yet another one - - - -with many lines - -and yet more, deindented -] -)", - L{ - N("a plain scalar with several lines\nand blank lines\nas well"), - N("and another plain scalar"), - N("and yet another one\n\n\nwith many lines\nand yet more"), - N("deindented"), - } -); - -ADD_CASE_TO_GROUP("plain scalar, explicit, early end, seq", EXPECT_PARSE_ERROR, -R"([ - a plain scalar - with several lines -)", - LineCol(4, 1) -); - -ADD_CASE_TO_GROUP("plain scalar, explicit, early end, map", EXPECT_PARSE_ERROR, -R"({foo: - a plain scalar - with several lines -)", - LineCol(4, 1) -); - -ADD_CASE_TO_GROUP("plain scalar, multiple docs", -R"(--- -- a plain scalar - with several lines ---- -- a second plain scalar - with several lines -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a plain scalar with several lines")}), - N(DOCSEQ, L{N("a second plain scalar with several lines")}), - }) -); - -ADD_CASE_TO_GROUP("plain scalar, multiple docs, termination", -R"(--- -- a plain scalar - with several lines -... ---- -- a second plain scalar - with several lines -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a plain scalar with several lines")}), - N(DOCSEQ, L{N("a second plain scalar with several lines")}), - }) -); - -ADD_CASE_TO_GROUP("plain scalar, trailing whitespace", - R"(--- -foo ---- -foo - ---- -foo - - - -)", - N(STREAM, L{ - N(DOCVAL, "foo"), - N(DOCVAL, "foo"), - N(DOCVAL, "foo"), - }) - ); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_preprocess.cpp b/thirdparty/ryml/test/test_preprocess.cpp deleted file mode 100644 index 7f6719e5f..000000000 --- a/thirdparty/ryml/test/test_preprocess.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include -#include -#endif -#include "./test_case.hpp" -#include - -namespace c4 { -namespace yml { - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(preprocess, rxmap_basic) -{ - #define _test(val, expected) \ - EXPECT_EQ(preprocess_rxmap(val), expected) - - _test("{}", "{}"); - _test("a", "{a: 1}"); - _test("{a}", "{a: 1}"); - _test("a, b, c", "{a: 1, b: 1, c: 1}"); - _test("a,b,c", "{a,b,c: 1}"); - _test("a a a a, b, c", "{a a a a: 1, b: 1, c: 1}"); - _test(",", "{,}"); - - _test("a: [b, c, d]", "{a: [b, c, d]}"); - _test("a:b: [b, c, d]", "{a:b: [b, c, d]}"); - _test("a,b: [b, c, d]", "{a,b: [b, c, d]}"); - _test("a: {b, c, d}", "{a: {b, c, d}}"); - _test("a: {b: {f, g}, c: {h, i}, d: {j, k}}", - "{a: {b: {f, g}, c: {h, i}, d: {j, k}}}"); - _test("a: {b: {f g}, c: {f g}, d: {j, k}}", - "{a: {b: {f g}, c: {f g}, d: {j, k}}}"); - - #undef _test -} - - - -// The other test executables are written to contain the declarative-style -// YmlTestCases. This executable does not have any but the build setup -// assumes it does, and links with the test lib, which requires an existing -// get_case() function. So this is here to act as placeholder until (if?) -// proper test cases are added here. -Case const* get_case(csubstr) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_scalar_names.cpp b/thirdparty/ryml/test/test_scalar_names.cpp deleted file mode 100644 index b0d420349..000000000 --- a/thirdparty/ryml/test/test_scalar_names.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#include "./test_group.hpp" - -#if defined(_MSC_VER) -# pragma warning(push) -//# pragma warning(disable: 4127/*conditional expression is constant*/) -//# pragma warning(disable: 4389/*'==': signed/unsigned mismatch*/) -#elif defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wdollar-in-identifier-extension" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -#endif - -namespace c4 { -namespace yml { - -#define _(name) N(#name) // makes it simpler -#define __(name) N(#name, #name) // makes it simpler - -CASE_GROUP(SCALAR_NAMES) -{ - -ADD_CASE_TO_GROUP("funny names, seq", -R"( -- a -- b:b -- c{c -- cc{ -- c}c -- cc} -- c!c -- cc! -- .foo -- . -- -a -- +b -- /b -- :c -- $g -- "*" -- '*' -- >- - * -- "*a" -- '*a' -- >- - *a -)", -L{_(a), _(b:b), _(c{c), _(cc{), _(c}c), _(cc}), _(c!c), _(cc!), _(.foo), _(.), _(-a), _(+b), _(/b), _(:c), _($g), - N(QV, "*"), N(QV, "*"), N(QV, "*"), N(QV, "*a"), N(QV, "*a"), N(QV, "*a")} -); - -ADD_CASE_TO_GROUP("funny names, seq expl", -R"([a, b, c, .foo, ., -a, +b, /b, :c, $g])", -L{_(a), _(b), _(c), _(.foo), _(.), _(-a), _(+b), _(/b), _(:c), _($g)} -); - -ADD_CASE_TO_GROUP("funny names, map", -R"( -a: a -b: b -c: c -.foo: .foo -.: . --a: -a -+b: +b -/b: /b -:c: :c -$g: $g -'*': '*' -'*a': '*a' -)", -L{__(a), __(b), __(c), __(.foo), __(.), __(-a), __(+b), __(/b), __(:c), __($g), - N(QKV, "*", "*"), N(QKV, "*a", "*a")} -); - -ADD_CASE_TO_GROUP("funny names, map expl", -R"({a: a, b: b, c: c, .foo: .foo, .: ., -a: -a, +b: +b, /b: /b, :c: :c, $g: $g, - '*': '*', '*a':'*a'})", -L{__(a), __(b), __(c), __(.foo), __(.), __(-a), __(+b), __(/b), __(:c), __($g), - N(QKV, "*", "*"), N(QKV, "*a", "*a")} -); -} - -} // namespace yml -} // namespace c4 - -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/test/test_seq_of_map.cpp b/thirdparty/ryml/test/test_seq_of_map.cpp deleted file mode 100644 index 90bbcbd79..000000000 --- a/thirdparty/ryml/test/test_seq_of_map.cpp +++ /dev/null @@ -1,348 +0,0 @@ -#include "./test_group.hpp" -#include "test_case.hpp" - -namespace c4 { -namespace yml { - -TEST(seq_of_map, with_anchors) -{ - { - // this case is vanilla: - csubstr yaml = R"(- a0: v0 - &a1 a1: v1 - &a2 a2: v2 - &a3 a3: v3 -- a0: w0 - *a1: w1 - *a2: w2 - *a3: w3 -- &seq - a4: v4 -)"; - Tree t = parse_in_arena(yaml); - EXPECT_EQ(emitrs_yaml(t), yaml); - ASSERT_EQ(t.rootref().num_children(), 3u); - ASSERT_EQ(t[2].has_val_anchor(), true); - ASSERT_EQ(t[2].val_anchor(), "seq"); - } - { - // but this case may fail because the indentation - // may be set from the scalar instead of the tag: - csubstr yaml = R"(- &a1 a1: v1 - &a2 a2: v2 - &a3 a3: v3 -- *a1: w1 - *a2: w2 - *a3: w3 -)"; - Tree t = parse_in_arena(yaml); - EXPECT_EQ(emitrs_yaml(t), yaml); - } -} - -TEST(seq_of_map, with_tags) -{ - { - // this case is vanilla: - csubstr yaml = R"(- a0: v0 - !!str a1: v1 - !!str a2: v2 - !!str a3: v3 -- a0: w0 - !!int a1: !!str w1 - !!int a2: !!str w2 - !!int a3: !!str w3 -- a0: v1 - !foo a1: v1 - !foo a2: v2 - !foo a3: v3 -)"; - Tree t = parse_in_arena(yaml); - EXPECT_EQ(emitrs_yaml(t), yaml); - } - { - // but this case may fail because the indentation - // may be set from the scalar instead of the tag: - csubstr yaml = R"(- !!str a1: v1 - !!str a2: v2 - !!str a3: v3 -- !!int a1: !!str w1 - !!int a2: !!str w2 - !!int a3: !!str w3 -- !foo a1: v1 - !foo a2: v2 - !foo a3: v3 -)"; - Tree t = parse_in_arena(yaml); - EXPECT_EQ(emitrs_yaml(t), yaml); - } -} - -TEST(seq_of_map, missing_scalars_v1) -{ - Tree t = parse_in_arena(R"(a: - - ~: ~ - - ~: ~ -)"); - #ifdef RYML_DBG - print_tree(t); - #endif - ASSERT_EQ(t["a"].num_children(), 2u); - ASSERT_EQ(t["a"][0].num_children(), 1u); - EXPECT_EQ(t["a"][0].first_child().key(), "~"); - EXPECT_EQ(t["a"][0].first_child().val(), "~"); - ASSERT_EQ(t["a"][1].num_children(), 1u); - EXPECT_EQ(t["a"][1].first_child().key(), "~"); - EXPECT_EQ(t["a"][1].first_child().val(), "~"); -} - -TEST(seq_of_map, missing_scalars_v2) -{ - Tree t = parse_in_arena(R"(a: - - : - - : -)"); - #ifdef RYML_DBG - print_tree(t); - #endif - ASSERT_EQ(t["a"].num_children(), 2u); - ASSERT_EQ(t["a"][0].num_children(), 1u); - EXPECT_EQ(t["a"][0].first_child().key(), nullptr); - EXPECT_EQ(t["a"][0].first_child().val(), nullptr); - ASSERT_EQ(t["a"][1].num_children(), 1u); - EXPECT_EQ(t["a"][1].first_child().key(), nullptr); - EXPECT_EQ(t["a"][1].first_child().val(), nullptr); -} - -TEST(seq_of_map, missing_scalars_v3) -{ - Tree t = parse_in_arena(R"(a: - - : - - : -)"); - #ifdef RYML_DBG - print_tree(t); - #endif - ASSERT_EQ(t["a"].num_children(), 2u); - ASSERT_EQ(t["a"][0].num_children(), 1u); - EXPECT_EQ(t["a"][0].first_child().key(), nullptr); - EXPECT_EQ(t["a"][0].first_child().val(), nullptr); - ASSERT_EQ(t["a"][1].num_children(), 1u); - EXPECT_EQ(t["a"][1].first_child().key(), nullptr); - EXPECT_EQ(t["a"][1].first_child().val(), nullptr); -} - -#ifdef RYML_WITH_TAB_TOKENS -TEST(seq_of_map, test_suite_6BCT) -{ - Tree t = parse_in_arena(R"( -- foo0: bar0 -- foo1 : bar1 -- foo2 : bar2 -)"); - #ifdef RYML_DBG - print_tree(t); - #endif - ASSERT_TRUE(t[0].is_map()); - ASSERT_TRUE(t[1].is_map()); - ASSERT_TRUE(t[2].is_map()); - EXPECT_EQ(t[0]["foo0"].val(), csubstr("bar0")); - EXPECT_EQ(t[1]["foo1"].val(), csubstr("bar1")); - EXPECT_EQ(t[2]["foo2"].val(), csubstr("bar2")); -} -#endif - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -CASE_GROUP(SEQ_OF_MAP) -{ - -ADD_CASE_TO_GROUP("seq of empty maps, one line", -R"([{}, {}, {}])", - L{MAP, MAP, MAP} -); - -ADD_CASE_TO_GROUP("seq of maps, one line", -R"([{name: John Smith, age: 33}, {name: Mary Smith, age: 27}])", - L{ - N{L{N("name", "John Smith"), N("age", "33")}}, - N{L{N("name", "Mary Smith"), N("age", "27")}} - } -); - -ADD_CASE_TO_GROUP("seq of maps, implicit seq, explicit maps", -R"( -- {name: John Smith, age: 33} -- {name: Mary Smith, age: 27} -)", - L{ - N{L{N("name", "John Smith"), N("age", "33")}}, - N{L{N("name", "Mary Smith"), N("age", "27")}} - } -); - -ADD_CASE_TO_GROUP("seq of maps", -R"( -- name: John Smith - age: 33 -- name: Mary Smith - age: 27 -)", - L{ - N{L{N("name", "John Smith"), N("age", "33")}}, - N{L{N("name", "Mary Smith"), N("age", "27")}} - } -); - -ADD_CASE_TO_GROUP("seq of maps, next line", -R"( -- - name: - John Smith - age: - 33 -- - name: - Mary Smith - age: - 27 -)", - L{ - N{L{N("name", "John Smith"), N("age", "33")}}, - N{L{N("name", "Mary Smith"), N("age", "27")}} - } -); - -ADD_CASE_TO_GROUP("seq of maps, bug #32 ex1", -R"( -- 'a': 1 - b: 2 -)", - L{ - N{L{N(QK, "a", "1"), N("b", "2")}} - } -); - -ADD_CASE_TO_GROUP("seq of maps, bug #32 ex2", -R"( -- a: 1 - b: 2 -- b: 2 - 'a': 1 -- b: 2 - 'a': 1 - c: 3 -- {'a': 1, b: 2} -)", - L{ - N{L{N("a", "1"), N("b", "2")}}, - N{L{N("b", "2"), N(QK, "a", "1")}}, - N{L{N("b", "2"), N(QK, "a", "1"), N("c", "3")}}, - N{L{N(QK, "a", "1"), N("b", "2")}}, - } -); - -ADD_CASE_TO_GROUP("seq of maps, bug #32 ex3", -R"( -'a': 1 -b: 2 -b: 2 -'a': 1 -)", -L{ - N(QK, "a", "1"), N("b", "2"), N("b", "2"), N(QK, "a", "1"), -}); - - -ADD_CASE_TO_GROUP("seq of maps, implicit map in seq", -R"('implicit block key' : [ - 'implicit flow key 1' : value1, - 'implicit flow key 2' : value2, - 'implicit flow key 3' : value3, - 'implicit flow key m' : {key1: val1, key2: val2}, - 'implicit flow key s' : [val1, val2], -])", -L{N(KEYSEQ|KEYQUO, "implicit block key", L{ - N(L{N(KEYVAL|KEYQUO, "implicit flow key 1", "value1")}), - N(L{N(KEYVAL|KEYQUO, "implicit flow key 2", "value2")}), - N(L{N(KEYVAL|KEYQUO, "implicit flow key 3", "value3")}), - N(L{N(KEYMAP|KEYQUO, "implicit flow key m", L{N("key1", "val1"), N("key2", "val2")})}), - N(L{N(KEYSEQ|KEYQUO, "implicit flow key s", L{N("val1"), N("val2")})}), -})}); - - -ADD_CASE_TO_GROUP("seq of maps, implicit map in seq, missing scalar", -R"({a : [ - : foo -], -b : [ - : -foo -], -c : [ - : -, - : -]})", -L{ - N("a", L{N(MAP, L{N("", "foo")}),}), - N("b", L{N(MAP, L{N("", "foo")}),}), - N("c", L{N(MAP, L{N(KEYVAL, "", {})}), N(MAP, L{N(KEYVAL, "", {})}),}), -}); - - -ADD_CASE_TO_GROUP("seq of maps, implicit with anchors, unresolved", -R"( -- &a1 a1: v1 - &a2 a2: v2 - &a3 a3: v3 -- *a1: w1 - *a2: w2 - *a3: w3 -)", -L{ - N(L{N( "a1", AR(KEYANCH, "a1"), "v1"), N( "a2", AR(KEYANCH, "a2"), "v2"), N( "a3", AR(KEYANCH, "a3"), "v3")}), - N(L{N("*a1", AR(KEYREF, "*a1"), "w1"), N("*a2", AR(KEYREF, "*a2"), "w2"), N("*a3", AR(KEYREF, "*a3"), "w3")}), -}); - - -ADD_CASE_TO_GROUP("seq of maps, implicit with anchors, resolved", RESOLVE_REFS, -R"( -- &a1 a1: v1 - &a2 a2: v2 - &a3 a3: v3 -- *a1: w1 - *a2: w2 - *a3: w3 -)", -L{ - N(L{N("a1", "v1"), N("a2", "v2"), N("a3", "v3")}), - N(L{N("a1", "w1"), N("a2", "w2"), N("a3", "w3")}), -}); - - -ADD_CASE_TO_GROUP("seq of maps, implicit with tags", -R"( -- !!str a1: v1 - !!str a2: v2 - !!str a3: v3 -- a1: !!str w1 - a2: !!str w2 - a3: !!str w3 -- !foo a1: v1 - !foo a2: v2 - !foo a3: v3 -)", -L{ - N(L{N(TS("!!str", "a1"), "v1"), N(TS("!!str", "a2"), "v2"), N(TS("!!str", "a3"), "v3")}), - N(L{N("a1", TS("!!str", "w1")), N("a2", TS("!!str", "w2")), N("a3", TS("!!str", "w3"))}), - N(L{N(TS("!foo", "a1"), "v1"), N(TS("!foo", "a2"), "v2"), N(TS("!foo", "a3"), "v3")}), -}); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_serialize.cpp b/thirdparty/ryml/test/test_serialize.cpp deleted file mode 100644 index 979de7c79..000000000 --- a/thirdparty/ryml/test/test_serialize.cpp +++ /dev/null @@ -1,499 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/std/std.hpp" -#include "c4/yml/parse.hpp" -#include "c4/yml/emit.hpp" -#include -#include -#include -#endif - -#include "./test_case.hpp" - -#include - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4389) // signed/unsigned mismatch -#elif defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wdollar-in-identifier-extension" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -#endif - -namespace foo { - -template -struct vec2 -{ - T x, y; -}; -template -struct vec3 -{ - T x, y, z; -}; -template -struct vec4 -{ - T x, y, z, w; -}; - -template size_t to_chars(c4::substr buf, vec2 v) { return c4::format(buf, "({},{})", v.x, v.y); } -template size_t to_chars(c4::substr buf, vec3 v) { return c4::format(buf, "({},{},{})", v.x, v.y, v.z); } -template size_t to_chars(c4::substr buf, vec4 v) { return c4::format(buf, "({},{},{},{})", v.x, v.y, v.z, v.w); } - -template bool from_chars(c4::csubstr buf, vec2 *v) { size_t ret = c4::unformat(buf, "({},{})", v->x, v->y); return ret != c4::yml::npos; } -template bool from_chars(c4::csubstr buf, vec3 *v) { size_t ret = c4::unformat(buf, "({},{},{})", v->x, v->y, v->z); return ret != c4::yml::npos; } -template bool from_chars(c4::csubstr buf, vec4 *v) { size_t ret = c4::unformat(buf, "({},{},{},{})", v->x, v->y, v->z, v->w); return ret != c4::yml::npos; } - -TEST(serialize, type_as_str) -{ - c4::yml::Tree t; - - auto r = t.rootref(); - r |= c4::yml::MAP; - - vec2 v2in{10, 11}; - vec2 v2out{1, 2}; - r["v2"] << v2in; - r["v2"] >> v2out; - EXPECT_EQ(v2in.x, v2out.x); - EXPECT_EQ(v2in.y, v2out.y); - - vec3 v3in{100, 101, 102}; - vec3 v3out{1, 2, 3}; - r["v3"] << v3in; - r["v3"] >> v3out; - EXPECT_EQ(v3in.x, v3out.x); - EXPECT_EQ(v3in.y, v3out.y); - EXPECT_EQ(v3in.z, v3out.z); - - vec4 v4in{1000, 1001, 1002, 1003}; - vec4 v4out{1, 2, 3, 4}; - r["v4"] << v4in; - r["v4"] >> v4out; - EXPECT_EQ(v4in.x, v4out.x); - EXPECT_EQ(v4in.y, v4out.y); - EXPECT_EQ(v4in.z, v4out.z); - EXPECT_EQ(v4in.w, v4out.w); - - char buf[256]; - c4::csubstr ret = c4::yml::emit_yaml(t, buf); - EXPECT_EQ(ret, R"(v2: '(10,11)' -v3: '(100,101,102)' -v4: '(1000,1001,1002,1003)' -)"); -} -} // namespace foo - - -namespace c4 { -namespace yml { - -//------------------------------------------- -template -void do_test_serialize(Args&& ...args) -{ - using namespace c4::yml; - Container s(std::forward(args)...); - Container out; - - Tree t; - NodeRef n(&t); - - n << s; - //print_tree(t); - emit_yaml(t); - c4::yml::check_invariants(t); - n >> out; - EXPECT_EQ(s, out); -} - - -TEST(serialize, std_vector_int) -{ - using T = int; - using L = std::initializer_list; - do_test_serialize>(L{1, 2, 3, 4, 5}); -} -TEST(serialize, std_vector_bool) -{ - using T = bool; - using L = std::initializer_list; - do_test_serialize>(L{true, false, true, false, true, true}); -} -TEST(serialize, std_vector_string) -{ - using T = std::string; - using L = std::initializer_list; - do_test_serialize>(L{"0asdadk0", "1sdfkjdfgu1", "2fdfdjkhdfgkjhdfi2", "3e987dfgnfdg83", "4'd0fgºçdfg«4"}); -} -TEST(serialize, std_vector_std_vector_int) -{ - using T = std::vector; - using L = std::initializer_list; - do_test_serialize>(L{{1, 2, 3, 4, 5}, {6, 7, 8, 9, 0}}); -} - - -TEST(serialize, std_map__int_int) -{ - using M = std::map; - using L = std::initializer_list; - do_test_serialize(L{{10, 0}, {11, 1}, {22, 2}, {10001, 1000}, {20002, 2000}, {30003, 3000}}); -} -TEST(serialize, std_map__std_string_int) -{ - using M = std::map; - using L = std::initializer_list; - do_test_serialize(L{{"asdsdf", 0}, {"dfgdfgdfg", 1}, {"dfgjdfgkjh", 2}}); -} -TEST(serialize, std_map__string_vectori) -{ - using M = std::map>; - using L = std::initializer_list; - do_test_serialize(L{{"asdsdf", {0, 1, 2, 3}}, {"dfgdfgdfg", {4, 5, 6, 7}}, {"dfgjdfgkjh", {8, 9, 10, 11}}}); -} -TEST(serialize, std_vector__map_string_int) -{ - using V = std::vector< std::map>; - using M = typename V::value_type; - using L = std::initializer_list; - do_test_serialize(L{ - M{{"asdasf", 0}, {"dfgkjhdfg", 1}, {"fghffg", 2}, {"r5656kjnh9b'dfgwg+*", 3}}, - M{{"asdasf", 10}, {"dfgkjhdfg", 11}, {"fghffg", 12}, {"r5656kjnh9b'dfgwg+*", 13}}, - M{{"asdasf", 20}, {"dfgkjhdfg", 21}, {"fghffg", 22}, {"r5656kjnh9b'dfgwg+*", 23}}, - M{{"asdasf", 30}, {"dfgkjhdfg", 31}, {"fghffg", 32}, {"r5656kjnh9b'dfgwg+*", 33}}, - }); -} - - -TEST(serialize, bool) -{ - Tree t = parse_in_arena("{a: 0, b: false, c: 1, d: true}"); - bool v, w; - t["a"] >> v; - EXPECT_EQ(v, false); - t["b"] >> v; - EXPECT_EQ(v, false); - t["c"] >> v; - EXPECT_EQ(v, true); - t["d"] >> v; - EXPECT_EQ(v, true); - - t["e"] << true; - EXPECT_EQ(t["e"].val(), "1"); - t["e"] >> w; - EXPECT_EQ(w, true); - - t["e"] << false; - EXPECT_EQ(t["e"].val(), "0"); - t["e"] >> w; - EXPECT_EQ(w, false); - - t["e"] << fmt::boolalpha(true); - EXPECT_EQ(t["e"].val(), "true"); - t["e"] >> w; - EXPECT_EQ(w, true); - - t["e"] << fmt::boolalpha(false); - EXPECT_EQ(t["e"].val(), "false"); - t["e"] >> w; - EXPECT_EQ(w, false); -} - -TEST(serialize, nan) -{ - Tree t = parse_in_arena(R"( -good: - - .nan - - .nan - - .NaN - - .NAN - - nan - - - .nan -set: - - nothing - - nothing -})"); - t["set"][0] << std::numeric_limits::quiet_NaN(); - t["set"][1] << std::numeric_limits::quiet_NaN(); - EXPECT_EQ(t["set"][0].val(), ".nan"); - EXPECT_EQ(t["set"][1].val(), ".nan"); - EXPECT_EQ(t["good"][0].val(), ".nan"); - EXPECT_EQ(t["good"][1].val(), ".nan"); - EXPECT_EQ(t["good"][2].val(), ".NaN"); - EXPECT_EQ(t["good"][3].val(), ".NAN"); - EXPECT_EQ(t["good"][4].val(), "nan"); - EXPECT_EQ(t["good"][5].val(), ".nan"); - float f; - double d; - f = 0.f; - d = 0.; - t["good"][0] >> f; - t["good"][0] >> d; - EXPECT_TRUE(std::isnan(f)); - EXPECT_TRUE(std::isnan(d)); - f = 0.f; - d = 0.; - t["good"][1] >> f; - t["good"][1] >> d; - EXPECT_TRUE(std::isnan(f)); - EXPECT_TRUE(std::isnan(d)); - f = 0.f; - d = 0.; - t["good"][2] >> f; - t["good"][2] >> d; - EXPECT_TRUE(std::isnan(f)); - EXPECT_TRUE(std::isnan(d)); - f = 0.f; - d = 0.; - t["good"][3] >> f; - t["good"][3] >> d; - EXPECT_TRUE(std::isnan(f)); - EXPECT_TRUE(std::isnan(d)); - f = 0.f; - d = 0.; - t["good"][4] >> f; - t["good"][4] >> d; - EXPECT_TRUE(std::isnan(f)); - EXPECT_TRUE(std::isnan(d)); - f = 0.f; - d = 0.; - t["good"][5] >> f; - t["good"][5] >> d; - EXPECT_TRUE(std::isnan(f)); - EXPECT_TRUE(std::isnan(d)); -} - -TEST(serialize, inf) -{ - C4_SUPPRESS_WARNING_GCC_CLANG_WITH_PUSH("-Wfloat-equal"); - Tree t = parse_in_arena(R"( -good: - - .inf - - .inf - - .Inf - - .INF - - inf - - infinity - - - .inf -set: - - nothing - - nothing -})"); - float finf = std::numeric_limits::infinity(); - double dinf = std::numeric_limits::infinity(); - t["set"][0] << finf; - t["set"][1] << dinf; - EXPECT_EQ(t["set"][0].val(), ".inf"); - EXPECT_EQ(t["set"][1].val(), ".inf"); - EXPECT_EQ(t["good"][0].val(), ".inf"); - EXPECT_EQ(t["good"][1].val(), ".inf"); - EXPECT_EQ(t["good"][2].val(), ".Inf"); - EXPECT_EQ(t["good"][3].val(), ".INF"); - EXPECT_EQ(t["good"][4].val(), "inf"); - EXPECT_EQ(t["good"][5].val(), "infinity"); - EXPECT_EQ(t["good"][6].val(), ".inf"); - float f; - double d; - f = 0.f; - d = 0.; - t["good"][0] >> f; - t["good"][0] >> d; - EXPECT_TRUE(f == finf); - EXPECT_TRUE(d == dinf); - f = 0.f; - d = 0.; - t["good"][1] >> f; - t["good"][1] >> d; - EXPECT_TRUE(f == finf); - EXPECT_TRUE(d == dinf); - f = 0.f; - d = 0.; - t["good"][2] >> f; - t["good"][2] >> d; - EXPECT_TRUE(f == finf); - EXPECT_TRUE(d == dinf); - f = 0.f; - d = 0.; - t["good"][3] >> f; - t["good"][3] >> d; - EXPECT_TRUE(f == finf); - EXPECT_TRUE(d == dinf); - f = 0.f; - d = 0.; - t["good"][4] >> f; - t["good"][4] >> d; - EXPECT_TRUE(f == finf); - EXPECT_TRUE(d == dinf); - f = 0.f; - d = 0.; - t["good"][5] >> f; - t["good"][5] >> d; - EXPECT_TRUE(f == finf); - EXPECT_TRUE(d == dinf); - f = 0.f; - d = 0.; - t["good"][6] >> f; - t["good"][6] >> d; - EXPECT_TRUE(f == finf); - EXPECT_TRUE(d == dinf); - - t = parse_in_arena(R"( -good: - - -.inf - - -.inf - - -.Inf - - -.INF - - -inf - - -infinity - - - -.inf -set: - - nothing - - nothing -})"); - t["set"][0] << -finf; - t["set"][1] << -dinf; - EXPECT_EQ(t["set"][0].val(), "-.inf"); - EXPECT_EQ(t["set"][1].val(), "-.inf"); - EXPECT_EQ(t["good"][0].val(), "-.inf"); - EXPECT_EQ(t["good"][1].val(), "-.inf"); - EXPECT_EQ(t["good"][2].val(), "-.Inf"); - EXPECT_EQ(t["good"][3].val(), "-.INF"); - EXPECT_EQ(t["good"][4].val(), "-inf"); - EXPECT_EQ(t["good"][5].val(), "-infinity"); - EXPECT_EQ(t["good"][6].val(), "-.inf"); - f = 0.f; - d = 0.; - t["good"][0] >> f; - t["good"][0] >> d; - EXPECT_TRUE(f == -finf); - EXPECT_TRUE(d == -dinf); - f = 0.f; - d = 0.; - t["good"][1] >> f; - t["good"][1] >> d; - EXPECT_TRUE(f == -finf); - EXPECT_TRUE(d == -dinf); - f = 0.f; - d = 0.; - t["good"][2] >> f; - t["good"][2] >> d; - EXPECT_TRUE(f == -finf); - EXPECT_TRUE(d == -dinf); - f = 0.f; - d = 0.; - t["good"][3] >> f; - t["good"][3] >> d; - EXPECT_TRUE(f == -finf); - EXPECT_TRUE(d == -dinf); - f = 0.f; - d = 0.; - t["good"][4] >> f; - t["good"][4] >> d; - EXPECT_TRUE(f == -finf); - EXPECT_TRUE(d == -dinf); - f = 0.f; - d = 0.; - t["good"][5] >> f; - t["good"][5] >> d; - EXPECT_TRUE(f == -finf); - EXPECT_TRUE(d == -dinf); - f = 0.f; - d = 0.; - t["good"][6] >> f; - t["good"][6] >> d; - EXPECT_TRUE(f == -finf); - EXPECT_TRUE(d == -dinf); - C4_SUPPRESS_WARNING_GCC_CLANG_POP -} - -TEST(serialize, std_string) -{ - auto t = parse_in_arena("{foo: bar}"); - std::string s; - EXPECT_NE(s, "bar"); - t["foo"] >> s; - EXPECT_EQ(s, "bar"); -} - -TEST(serialize, anchor_and_ref_round_trip) -{ - const char yaml[] = R"(anchor_objects: - - &id001 - name: id001 - - &id002 - name: id002 - - name: id003 - - &id004 - name: id004 -references: - reference_key: *id001 - reference_list: - - *id002 - - *id004 -)"; - - Tree t = parse_in_arena(yaml); - std::string cmpbuf; - emitrs_yaml(t, &cmpbuf); - EXPECT_EQ(cmpbuf, yaml); -} - -TEST(serialize, create_anchor_ref_trip) -{ - const char expected_yaml[] = R"(anchor_objects: - - &id001 - name: a_name -reference_list: - - *id001 -)"; - - Tree tree; - auto root_id = tree.root_id(); - tree.to_map(root_id); - - auto anchor_list_id = tree.append_child(root_id); - tree.to_seq(anchor_list_id, "anchor_objects"); - - auto anchor_map0 = tree.append_child(anchor_list_id); - tree.to_map(anchor_map0); - tree.set_val_anchor(anchor_map0, "id001"); - - auto anchor_elem0 = tree.append_child(anchor_map0); - tree.to_keyval(anchor_elem0, "name", "a_name"); - - auto ref_list_id = tree.append_child(root_id); - tree.to_seq(ref_list_id, "reference_list"); - - auto elem0_id = tree.append_child(ref_list_id); - tree.set_val_ref(elem0_id, "id001"); - - std::string cmpbuf; - emitrs_yaml(tree, &cmpbuf); - EXPECT_EQ(cmpbuf, expected_yaml); -} - - -//------------------------------------------- -// this is needed to use the test case library -Case const* get_case(csubstr /*name*/) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 - -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/test/test_simple_anchor.cpp b/thirdparty/ryml/test/test_simple_anchor.cpp deleted file mode 100644 index 0ee4a629f..000000000 --- a/thirdparty/ryml/test/test_simple_anchor.cpp +++ /dev/null @@ -1,1405 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -TEST(anchors, circular) -{ - Tree t = parse_in_arena(R"(&x -- *x -)"); - ASSERT_TRUE(t.rootref().is_val_anchor()); - ASSERT_TRUE(t[0].is_val_ref()); - EXPECT_EQ(t.rootref().val_anchor(), "x"); - EXPECT_EQ(t[0].val_ref(), "x"); -} - -TEST(anchors, node_scalar_set_ref_when_empty) -{ - { - NodeScalar ns; - ns.set_ref_maybe_replacing_scalar("foo", /*has_scalar*/false); - EXPECT_EQ(ns.scalar, "foo"); - EXPECT_EQ(ns.anchor, "foo"); - } - { - NodeScalar ns; - ns.set_ref_maybe_replacing_scalar("*foo", /*has_scalar*/false); - EXPECT_EQ(ns.scalar, "*foo"); - EXPECT_EQ(ns.anchor, "foo"); - } -} - -TEST(anchors, node_scalar_set_ref_when_non_empty) -{ - { - NodeScalar ns; - ns.scalar = "whatever"; - ns.set_ref_maybe_replacing_scalar("foo", /*has_scalar*/true); - EXPECT_EQ(ns.scalar, "foo"); - EXPECT_EQ(ns.anchor, "foo"); - } - { - NodeScalar ns; - ns.scalar = "whatever"; - ns.set_ref_maybe_replacing_scalar("*foo", /*has_scalar*/true); - EXPECT_EQ(ns.scalar, "*foo"); - EXPECT_EQ(ns.anchor, "foo"); - ns.set_ref_maybe_replacing_scalar("foo", /*has_scalar*/true); - EXPECT_EQ(ns.scalar, "*foo"); // keep the previous as it is well formed - EXPECT_EQ(ns.anchor, "foo"); - ns.set_ref_maybe_replacing_scalar("bar", /*has_scalar*/true); - EXPECT_EQ(ns.scalar, "bar"); // replace previous as it is not well formed - EXPECT_EQ(ns.anchor, "bar"); - } -} - -TEST(anchors, no_ambiguity_when_key_scalars_begin_with_star) -{ - Tree t = parse_in_arena("{foo: &foo 1, *foo: 2, '*foo': 3}"); - - EXPECT_TRUE(t[1].is_key_ref()); - EXPECT_FALSE(t[2].is_key_ref()); - - EXPECT_FALSE(t[1].is_key_quoted()); - EXPECT_TRUE(t[2].is_key_quoted()); - - EXPECT_EQ(t[1].key(), "*foo"); - EXPECT_EQ(t[1].key_ref(), "foo"); - EXPECT_EQ(t[2].key(), "*foo"); - - EXPECT_EQ(emitrs_yaml(t), R"(foo: &foo 1 -*foo: 2 -'*foo': 3 -)"); - - t.resolve(); - - EXPECT_EQ(emitrs_yaml(t), R"(foo: 1 -1: 2 -'*foo': 3 -)"); -} - -TEST(anchors, no_ambiguity_when_val_scalars_begin_with_star) -{ - Tree t = parse_in_arena("{foo: &foo 1, ref: *foo, quo: '*foo'}"); - - EXPECT_TRUE(t["ref"].is_val_ref()); - EXPECT_FALSE(t["quo"].is_val_ref()); - - EXPECT_FALSE(t["ref"].is_val_quoted()); - EXPECT_TRUE(t["quo"].is_val_quoted()); - - EXPECT_EQ(t["ref"].val_ref(), "foo"); - EXPECT_EQ(t["ref"].val(), "*foo"); - EXPECT_EQ(t["quo"].val(), "*foo"); - - EXPECT_EQ(emitrs_yaml(t), R"(foo: &foo 1 -ref: *foo -quo: '*foo' -)"); - - t.resolve(); - - EXPECT_EQ(emitrs_yaml(t), R"(foo: 1 -ref: 1 -quo: '*foo' -)"); -} - -TEST(anchors, no_ambiguity_with_inheritance) -{ - Tree t = parse_in_arena("{foo: &foo {a: 1, b: 2}, bar: {<<: *foo}, sq: {'<<': haha}, dq: {\"<<\": hehe}}"); - - EXPECT_TRUE(t["bar"].has_child("<<")); - EXPECT_TRUE(t["bar"]["<<"].is_key_ref()); - EXPECT_TRUE(t["bar"]["<<"].is_val_ref()); - EXPECT_TRUE(t["sq"]["<<"].is_key_quoted()); - EXPECT_TRUE(t["dq"]["<<"].is_key_quoted()); - EXPECT_FALSE(t["sq"]["<<"].is_key_ref()); - EXPECT_FALSE(t["dq"]["<<"].is_key_ref()); - EXPECT_EQ(t["sq"]["<<"].key(), "<<"); - EXPECT_EQ(t["dq"]["<<"].key(), "<<"); - EXPECT_EQ(t["bar"]["<<"].key(), "<<"); - EXPECT_EQ(t["bar"]["<<"].val(), "*foo"); - EXPECT_EQ(t["bar"]["<<"].key_ref(), "<<"); - EXPECT_EQ(t["bar"]["<<"].val_ref(), "foo"); - - EXPECT_EQ(emitrs_yaml(t), R"(foo: &foo - a: 1 - b: 2 -bar: - <<: *foo -sq: - '<<': haha -dq: - '<<': hehe -)"); - t.resolve(); - EXPECT_EQ(emitrs_yaml(t), R"(foo: - a: 1 - b: 2 -bar: - a: 1 - b: 2 -sq: - '<<': haha -dq: - '<<': hehe -)"); -} - -TEST(anchors, programatic_key_ref) -{ - Tree t = parse_in_arena("{}"); - NodeRef r = t.rootref(); - r["kanchor"] = "2"; - r["kanchor"].set_key_anchor("kanchor"); - r["vanchor"] = "3"; - r["vanchor"].set_val_anchor("vanchor"); - r["*kanchor"] = "4"; - r["*vanchor"] = "5"; - NodeRef ch = r.append_child(); - ch.set_key_ref("kanchor"); - ch.set_val("6"); - ch = r.append_child(); - ch.set_key_ref("vanchor"); - ch.set_val("7"); - EXPECT_EQ(emitrs_yaml(t), R"(&kanchor kanchor: 2 -vanchor: &vanchor 3 -'*kanchor': 4 -'*vanchor': 5 -*kanchor: 6 -*vanchor: 7 -)"); - t.resolve(); - EXPECT_EQ(emitrs_yaml(t), R"(kanchor: 2 -vanchor: 3 -'*kanchor': 4 -'*vanchor': 5 -kanchor: 6 -3: 7 -)"); -} - -TEST(anchors, programatic_val_ref) -{ - Tree t = parse_in_arena("{}"); - t["kanchor"] = "2"; - t["kanchor"].set_key_anchor("kanchor"); - t["vanchor"] = "3"; - t["vanchor"].set_val_anchor("vanchor"); - - t["kref"].create(); - t["vref"].create(); - t["kref"].set_val_ref("kanchor"); - t["vref"].set_val_ref("vanchor"); - - EXPECT_EQ(emitrs_yaml(t), R"(&kanchor kanchor: 2 -vanchor: &vanchor 3 -kref: *kanchor -vref: *vanchor -)"); - t.resolve(); - EXPECT_EQ(emitrs_yaml(t), R"(kanchor: 2 -vanchor: 3 -kref: kanchor -vref: 3 -)"); -} - -TEST(anchors, programatic_inheritance) -{ - Tree t = parse_in_arena("{orig: &orig {foo: bar, baz: bat}, copy: {}, notcopy: {}, notref: {}}"); - - t["copy"]["<<"] = "*orig"; - t["copy"]["<<"].set_key_ref("<<"); - t["copy"]["<<"].set_val_ref("orig"); - - t["notcopy"]["test"] = "*orig"; - t["notcopy"]["test"].set_val_ref("orig"); - t["notcopy"]["<<"] = "*orig"; - t["notcopy"]["<<"].set_val_ref("orig"); - - t["notref"]["<<"] = "*orig"; - - EXPECT_EQ(emitrs_yaml(t), R"(orig: &orig - foo: bar - baz: bat -copy: - <<: *orig -notcopy: - test: *orig - '<<': *orig -notref: - '<<': '*orig' -)"); - t.resolve(); - EXPECT_EQ(emitrs_yaml(t), R"(orig: - foo: bar - baz: bat -copy: - foo: bar - baz: bat -notcopy: - test: - foo: bar - baz: bat - '<<': - foo: bar - baz: bat -notref: - '<<': '*orig' -)"); -} - -TEST(anchors, programatic_multiple_inheritance) -{ - Tree t = parse_in_arena("{orig1: &orig1 {foo: bar}, orig2: &orig2 {baz: bat}, orig3: &orig3 {and: more}, copy: {}}"); - - t["copy"]["<<"] |= SEQ; - t["copy"]["<<"].set_key_ref("<<"); - NodeRef ref1 = t["copy"]["<<"].append_child(); - NodeRef ref2 = t["copy"]["<<"].append_child(); - NodeRef ref3 = t["copy"]["<<"].append_child(); - ref1 = "*orig1"; - ref2 = "*orig2"; - ref3 = "*orig3"; - ref1.set_val_ref("orig1"); - ref2.set_val_ref("orig2"); - ref3.set_val_ref("orig3"); - - EXPECT_EQ(emitrs_yaml(t), R"(orig1: &orig1 - foo: bar -orig2: &orig2 - baz: bat -orig3: &orig3 - and: more -copy: - <<: - - *orig1 - - *orig2 - - *orig3 -)"); - t.resolve(); - EXPECT_EQ(emitrs_yaml(t), R"(orig1: - foo: bar -orig2: - baz: bat -orig3: - and: more -copy: - foo: bar - baz: bat - and: more -)"); -} - -TEST(anchors, set_anchor_leading_ampersand_is_optional) -{ - Tree t = parse_in_arena("{without: 0, with: 1}"); - - t["without"].set_key_anchor("without"); - t["with"].set_key_anchor("&with"); - EXPECT_EQ(t["without"].key_anchor(), "without"); - EXPECT_EQ(t["with"].key_anchor(), "with"); - EXPECT_EQ(emitrs_yaml(t), R"(&without without: 0 -&with with: 1 -)"); - - t["without"].set_val_anchor("without"); - t["with"].set_val_anchor("&with"); - EXPECT_EQ(t["without"].val_anchor(), "without"); - EXPECT_EQ(t["with"].val_anchor(), "with"); - EXPECT_EQ(emitrs_yaml(t), R"(&without without: &without 0 -&with with: &with 1 -)"); -} - -TEST(anchors, set_ref_leading_star_is_optional) -{ - Tree t = parse_in_arena("{}"); - - t["*without"] = "0"; - t["*with"] = "1"; - EXPECT_EQ(emitrs_yaml(t), R"('*without': 0 -'*with': 1 -)"); - - t["*without"].set_key_ref("without"); - t["*with"].set_key_ref("*with"); - EXPECT_EQ(t["*without"].key_ref(), "without"); - EXPECT_EQ(t["*with"].key_ref(), "with"); - EXPECT_EQ(emitrs_yaml(t), R"(*without: 0 -*with: 1 -)"); - - t["*without"].set_val_ref("without"); - t["*with"].set_val_ref("*with"); - EXPECT_EQ(t["*without"].val_ref(), "without"); - EXPECT_EQ(t["*with"].val_ref(), "with"); - EXPECT_EQ(emitrs_yaml(t), R"(*without: *without -*with: *with -)"); -} - -TEST(anchors, set_key_ref_also_sets_the_key_when_none_exists) -{ - Tree t = parse_in_arena("{}"); - NodeRef root = t.rootref(); - NodeRef without = root.append_child(); - NodeRef with = root.append_child(); - ASSERT_FALSE(without.has_key()); - ASSERT_FALSE(with.has_key()); - without.set_key_ref("without"); - with.set_key_ref("*with"); - without.set_val("0"); - with.set_val("1"); - ASSERT_TRUE(without.has_key()); - ASSERT_TRUE(with.has_key()); - EXPECT_EQ(without.key(), "without"); - EXPECT_EQ(with.key(), "*with"); - EXPECT_EQ(without.key_ref(), "without"); - EXPECT_EQ(with.key_ref(), "with"); - EXPECT_EQ(emitrs_yaml(t), R"(*without: 0 -*with: 1 -)"); -} - -TEST(anchors, set_val_ref_also_sets_the_val_when_none_exists) -{ - Tree t = parse_in_arena("{}"); - NodeRef root = t.rootref(); - NodeRef without = root.append_child(); - NodeRef with = root.append_child(); - without.set_key("without"); - with.set_key("with"); - ASSERT_FALSE(without.has_val()); - ASSERT_FALSE(with.has_val()); - without.set_val_ref("without"); - with.set_val_ref("*with"); - ASSERT_TRUE(without.has_val()); - ASSERT_TRUE(with.has_val()); - EXPECT_EQ(without.val(), "without"); - EXPECT_EQ(with.val(), "*with"); - EXPECT_EQ(without.val_ref(), "without"); - EXPECT_EQ(with.val_ref(), "with"); - EXPECT_EQ(emitrs_yaml(t), R"(without: *without -with: *with -)"); -} - -TEST(anchors, set_key_ref_replaces_existing_key) -{ - Tree t = parse_in_arena("{*foo: bar}"); - NodeRef root = t.rootref(); - EXPECT_TRUE(root.has_child("*foo")); - root["*foo"].set_key_ref("notfoo"); - EXPECT_FALSE(root.has_child("*foo")); - EXPECT_FALSE(root.has_child("*notfoo")); - EXPECT_TRUE(root.has_child("notfoo")); - EXPECT_EQ(emitrs_yaml(t), "*notfoo: bar\n"); -} - -TEST(anchors, set_val_ref_replaces_existing_key) -{ - Tree t = parse_in_arena("{foo: *bar}"); - NodeRef root = t.rootref(); - root["foo"].set_val_ref("notbar"); - EXPECT_EQ(root["foo"].val(), "notbar"); - root["foo"].set_val_ref("*notfoo"); - EXPECT_EQ(root["foo"].val(), "*notfoo"); - EXPECT_EQ(emitrs_yaml(t), "foo: *notfoo\n"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(weird_anchor_cases_from_suite, 2SXE) -{ - Tree t = parse_in_arena(R"(&a: key: &a value -foo: - *a: -)"); - t.resolve(); - #ifdef THIS_IS_A_KNOWN_LIMITATION // since we do not allow colon in anchors, this would fail: - EXPECT_EQ(emitrs(t), R"(key: value -foo: key -)"); - #endif - // so we get this instead: - EXPECT_EQ(emitrs_yaml(t), R"(key: value -foo: - value: -)"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// SIMPLE_ANCHOR/YmlTestCase.parse_using_ryml/0 - -C4_SUPPRESS_WARNING_GCC_WITH_PUSH("-Wuseless-cast") - -/** verify that the reference class is working correctly (yay, testing the tests) */ -TEST(CaseNode, anchors) -{ - const NodeType mask = KEYREF|KEYANCH|VALREF|VALANCH; - - { - auto n = N("*vref", AR(KEYREF, "vref"), "c"); - EXPECT_EQ(n.key, "*vref"); - EXPECT_EQ(n.val, "c"); - EXPECT_EQ((type_bits)(n.type & mask), (type_bits)KEYREF); - EXPECT_EQ((type_bits)n.key_anchor.type, (type_bits)KEYREF); - EXPECT_EQ((type_bits)n.val_anchor.type, (type_bits)NOTYPE); - EXPECT_EQ(n.key_anchor.str, "vref"); - EXPECT_EQ(n.val_anchor.str, ""); - } - - { - CaseNode n("<<", "*base", AR(VALANCH, "base")); - EXPECT_EQ(n.key, "<<"); - EXPECT_EQ(n.val, "*base"); - EXPECT_EQ((type_bits)(n.type & mask), (type_bits)VALANCH); - EXPECT_EQ((type_bits)n.key_anchor.type, (type_bits)NOTYPE); - EXPECT_EQ((type_bits)n.val_anchor.type, (type_bits)VALANCH); - EXPECT_EQ(n.key_anchor.str, ""); - EXPECT_EQ(n.val_anchor.str, "base"); - } - - { - CaseNode n("base", L{N("name", "Everyone has same name")}, AR(VALANCH, "base")); - EXPECT_EQ(n.key, "base"); - EXPECT_EQ(n.val, ""); - EXPECT_NE(n.type.is_seq(), true); - EXPECT_EQ((type_bits)(n.type & mask), (type_bits)VALANCH); - EXPECT_EQ((type_bits)n.key_anchor.type, (type_bits)NOTYPE); - EXPECT_EQ((type_bits)n.val_anchor.type, (type_bits)VALANCH); - EXPECT_EQ(n.key_anchor.str, ""); - EXPECT_EQ(n.val_anchor.str, "base"); - } - - { - L l{N("<<", "*base", AR(VALREF, "base"))}; - CaseNode const& base = *l.begin(); - EXPECT_EQ(base.key, "<<"); - EXPECT_EQ(base.val, "*base"); - EXPECT_EQ(base.type.is_keyval(), true); - EXPECT_EQ((type_bits)(base.type & mask), (type_bits)VALREF); - EXPECT_EQ((type_bits)base.key_anchor.type, (type_bits)NOTYPE); - EXPECT_EQ((type_bits)base.val_anchor.type, (type_bits)VALREF); - EXPECT_EQ(base.key_anchor.str, ""); - EXPECT_EQ(base.val_anchor.str, "base"); - } - - { - L l{N("<<", "*base", AR(VALREF, "base")), N("age", "10")}; - CaseNode const& base = *l.begin(); - CaseNode const& age = *(&base + 1); - EXPECT_EQ(base.key, "<<"); - EXPECT_EQ(base.val, "*base"); - EXPECT_EQ(base.type.is_keyval(), true); - EXPECT_EQ((type_bits)(base.type & mask), (type_bits)VALREF); - EXPECT_EQ((type_bits)base.key_anchor.type, (type_bits)NOTYPE); - EXPECT_EQ((type_bits)base.val_anchor.type, (type_bits)VALREF); - EXPECT_EQ(base.key_anchor.str, ""); - EXPECT_EQ(base.val_anchor.str, "base"); - - EXPECT_EQ(age.key, "age"); - EXPECT_EQ(age.val, "10"); - EXPECT_EQ(age.type.is_keyval(), true); - EXPECT_EQ((type_bits)(age.type & mask), (type_bits)0); - EXPECT_EQ((type_bits)age.key_anchor.type, (type_bits)NOTYPE); - EXPECT_EQ((type_bits)age.val_anchor.type, (type_bits)NOTYPE); - EXPECT_EQ(age.key_anchor.str, ""); - EXPECT_EQ(age.val_anchor.str, ""); - } - - { - CaseNode n("foo", L{N("<<", "*base", AR(VALREF, "base")), N("age", "10")}, AR(VALANCH, "foo")); - EXPECT_EQ(n.key, "foo"); - EXPECT_EQ(n.val, ""); - EXPECT_EQ(n.type.has_key(), true); - EXPECT_EQ(n.type.is_map(), true); - EXPECT_EQ((type_bits)(n.type & mask), (type_bits)VALANCH); - EXPECT_EQ((type_bits)n.key_anchor.type, (type_bits)NOTYPE); - EXPECT_EQ((type_bits)n.val_anchor.type, (type_bits)VALANCH); - EXPECT_EQ(n.key_anchor.str, ""); - EXPECT_EQ(n.val_anchor.str, "foo"); - - CaseNode const& base = n.children[0]; - EXPECT_EQ(base.key, "<<"); - EXPECT_EQ(base.val, "*base"); - EXPECT_EQ(base.type.has_key() && base.type.has_val(), true); - EXPECT_EQ((type_bits)(base.type & mask), (type_bits)VALREF); - EXPECT_EQ((type_bits)base.key_anchor.type, (type_bits)NOTYPE); - EXPECT_EQ((type_bits)base.val_anchor.type, (type_bits)VALREF); - EXPECT_EQ(base.key_anchor.str, ""); - EXPECT_EQ(base.val_anchor.str, "base"); - } - -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(simple_anchor, resolve_works_on_an_empty_tree) -{ - Tree t; - t.resolve(); - EXPECT_TRUE(t.empty()); -} - -TEST(simple_anchor, resolve_works_on_a_tree_without_refs) -{ - Tree t = parse_in_arena("[a, b, c, d, e, f]"); - size_t size_before = t.size(); - t.resolve(); - EXPECT_EQ(t.size(), size_before); -} - -TEST(simple_anchor, resolve_works_on_keyrefvalref) -{ - Tree t = parse_in_arena("{&a a: &b b, *b: *a}"); - EXPECT_EQ(t["a"].has_key_anchor(), true); - EXPECT_EQ(t["a"].has_val_anchor(), true); - EXPECT_EQ(t["a"].key_anchor(), "a"); - EXPECT_EQ(t["a"].val_anchor(), "b"); - EXPECT_EQ(t["*b"].is_key_ref(), true); - EXPECT_EQ(t["*b"].is_val_ref(), true); - EXPECT_EQ(t["*b"].key_ref(), "b"); - EXPECT_EQ(t["*b"].val_ref(), "a"); - EXPECT_EQ(emitrs_yaml(t), R"(&a a: &b b -*b: *a -)"); - t.resolve(); - EXPECT_EQ(t["a"].val(), "b"); - EXPECT_EQ(t["b"].val(), "a"); - EXPECT_EQ(emitrs_yaml(t), R"(a: b -b: a -)"); -} - -TEST(simple_anchor, anchors_of_first_child_key_implicit) -{ - csubstr yaml = R"(&anchor0 -&anchor4 top4: - key4: scalar4 -top5: &anchor5 - key5: scalar5 -top6: - &anchor6 key6: scalar6 -top61: - &anchor61 key61: - scalar61 -top62: - &anchor62 - key62: - scalar62 -)"; - Tree t = parse_in_arena(yaml); - EXPECT_EQ(t.rootref().has_val_anchor(), true); - EXPECT_EQ(t.rootref().val_anchor(), "anchor0"); - EXPECT_EQ(t["top4"].has_key_anchor(), true); - EXPECT_EQ(t["top4"].has_val_anchor(), false); - EXPECT_EQ(t["top4"].key_anchor(), "anchor4"); - EXPECT_EQ(t["top4"]["key4"].val(), "scalar4"); - EXPECT_EQ(t["top4"]["key4"].has_key_anchor(), false); - EXPECT_EQ(t["top5"].has_key_anchor(), false); - EXPECT_EQ(t["top5"].has_val_anchor(), true); - EXPECT_EQ(t["top5"].val_anchor(), "anchor5"); - EXPECT_EQ(t["top5"]["key5"].val(), "scalar5"); - EXPECT_EQ(t["top5"]["key5"].has_key_anchor(), false); - EXPECT_EQ(t["top6"].has_key_anchor(), false); - EXPECT_EQ(t["top6"].has_val_anchor(), false); - EXPECT_EQ(t["top6"]["key6"].val(), "scalar6"); - ASSERT_EQ(t["top6"]["key6"].has_key_anchor(), true); - EXPECT_EQ(t["top6"]["key6"].key_anchor(), "anchor6"); - EXPECT_EQ(t["top61"].has_key_anchor(), false); - EXPECT_EQ(t["top61"].has_val_anchor(), false); - EXPECT_EQ(t["top61"]["key61"].val(), "scalar61"); - ASSERT_EQ(t["top61"]["key61"].has_key_anchor(), true); - EXPECT_EQ(t["top61"]["key61"].key_anchor(), "anchor61"); - EXPECT_EQ(t["top62"].has_key_anchor(), false); - EXPECT_EQ(t["top62"].has_val_anchor(), true); - EXPECT_EQ(t["top62"].val_anchor(), "anchor62"); - EXPECT_EQ(t["top62"]["key62"].val(), "scalar62"); - ASSERT_EQ(t["top62"]["key62"].has_key_anchor(), false); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -CASE_GROUP(SIMPLE_ANCHOR) -{ - -ADD_CASE_TO_GROUP("merge example, unresolved", -R"(# https://yaml.org/type/merge.html -- &CENTER { x: 1, y: 2 } -- &LEFT { x: 0, y: 2 } -- &BIG { r: 10 } -- &SMALL { r: 1 } - -# All the following maps are equal: - -- # Explicit keys - x: 1 - y: 2 - r: 10 - label: center/big - -- # Merge one map - << : *CENTER - r: 10 - label: center/big - -- # Merge multiple maps - << : [ *CENTER, *BIG ] - label: center/big - -- # Override - << : [ *BIG, *LEFT, *SMALL ] - x: 1 - label: center/big -)", -L{ - N(L{N("x", "1" ), N("y", "2")}, AR(VALANCH, "CENTER")), - N(L{N("x", "0" ), N("y", "2")}, AR(VALANCH, "LEFT" )), - N(L{N("r", "10") }, AR(VALANCH, "BIG" )), - N(L{N("r", "1" ) }, AR(VALANCH, "SMALL" )), - N(L{N("x", "1" ), N("y", "2"), N("r", "10"), N("label", "center/big")}), - N(L{N("<<", AR(KEYREF, "<<"), "*CENTER", AR(VALREF, "*CENTER")), N("r", "10"), N("label", "center/big")}), - N(L{N("<<", AR(KEYREF, "<<"), L{N("*CENTER", AR(VALREF, "*CENTER")), N("*BIG", AR(VALREF, "*BIG"))}), N("label", "center/big")}), - N(L{N("<<", AR(KEYREF, "<<"), L{N("*BIG", AR(VALREF, "*BIG")), N("*LEFT", AR(VALREF, "*LEFT")), N("*SMALL", AR(VALREF, "*SMALL"))}), N("x", "1"), N("label", "center/big")}), -}); - -ADD_CASE_TO_GROUP("merge example, resolved", RESOLVE_REFS, -R"(# https://yaml.org/type/merge.html -- &CENTER { x: 1, y: 2 } -- &LEFT { x: 0, y: 2 } -- &BIG { r: 10 } -- &SMALL { r: 1 } - -# All the following maps are equal: - -- # Explicit keys - x: 1 - y: 2 - r: 10 - label: center/big - -- # Merge one map - << : *CENTER - r: 10 - label: center/big - -- # Merge multiple maps - << : [ *CENTER, *BIG ] - label: center/big - -- # Override - << : [ *SMALL, *LEFT, *BIG ] - x: 1 - label: center/big -)", -L{ - N(L{N("x", "1" ), N("y", "2")}), - N(L{N("x", "0" ), N("y", "2")}), - N(L{N("r", "10") }), - N(L{N("r", "1" ) }), - N(L{N("x", "1" ), N("y", "2"), N("r", "10"), N("label", "center/big")}), - N(L{N("x", "1" ), N("y", "2"), N("r", "10"), N("label", "center/big")}), - N(L{N("x", "1" ), N("y", "2"), N("r", "10"), N("label", "center/big")}), - N(L{N("x", "1" ), N("y", "2"), N("r", "10"), N("label", "center/big")}), -}); - -ADD_CASE_TO_GROUP("simple anchor 1, implicit, unresolved", -R"( -anchored_content: &anchor_name This string will appear as the value of two keys. -other_anchor: *anchor_name -anchors_in_seqs: - - &anchor_in_seq this value appears in both elements of the sequence - - *anchor_in_seq -base: &base - name: Everyone has same name -foo: &foo - <<: *base - age: 10 -bar: &bar - <<: *base - age: 20 -)", - L{ - N("anchored_content", "This string will appear as the value of two keys.", AR(VALANCH, "anchor_name")), - N("other_anchor", "*anchor_name", AR(VALREF, "anchor_name")), - N("anchors_in_seqs", L{ - N("this value appears in both elements of the sequence", AR(VALANCH, "anchor_in_seq")), - N("*anchor_in_seq", AR(VALREF, "anchor_in_seq")), - }), - N("base", L{N("name", "Everyone has same name")}, AR(VALANCH, "base")), - N("foo", L{N("<<", AR(KEYREF, "<<"), "*base", AR(VALREF, "base")), N("age", "10")}, AR(VALANCH, "foo")), - N("bar", L{N("<<", AR(KEYREF, "<<"), "*base", AR(VALREF, "base")), N("age", "20")}, AR(VALANCH, "bar")), - } -); - -ADD_CASE_TO_GROUP("simple anchor 1, explicit, unresolved", -R"({ -anchored_content: &anchor_name This string will appear as the value of two keys., -other_anchor: *anchor_name, -anchors_in_seqs: [ - &anchor_in_seq this value appears in both elements of the sequence, - *anchor_in_seq - ], -base: &base { - name: Everyone has same name - }, -foo: &foo { - <<: *base, - age: 10 - }, -bar: &bar { - <<: *base, - age: 20 - } -})", - L{ - N("anchored_content", "This string will appear as the value of two keys.", AR(VALANCH, "anchor_name")), - N("other_anchor", "*anchor_name", AR(VALREF, "anchor_name")), - N("anchors_in_seqs", L{ - N("this value appears in both elements of the sequence", AR(VALANCH, "anchor_in_seq")), - N("*anchor_in_seq", AR(VALREF, "anchor_in_seq")), - }), - N("base", L{N("name", "Everyone has same name")}, AR(VALANCH, "base")), - N("foo", L{N("<<", AR(KEYREF, "<<"), "*base", AR(VALREF, "base")), N("age", "10")}, AR(VALANCH, "foo")), - N("bar", L{N("<<", AR(KEYREF, "<<"), "*base", AR(VALREF, "base")), N("age", "20")}, AR(VALANCH, "bar")), - } -); - -ADD_CASE_TO_GROUP("simple anchor 1, implicit, resolved", RESOLVE_REFS, -R"( -anchored_content: &anchor_name This string will appear as the value of two keys. -other_anchor: *anchor_name -anchors_in_seqs: - - &anchor_in_seq this value appears in both elements of the sequence - - *anchor_in_seq -base: &base - name: Everyone has same name -foo: &foo - <<: *base - age: 10 -bar: &bar - <<: *base - age: 20 -)", - L{ - N("anchored_content", "This string will appear as the value of two keys."), - N("other_anchor", "This string will appear as the value of two keys."), - N("anchors_in_seqs", L{ - N("this value appears in both elements of the sequence"), - N("this value appears in both elements of the sequence"), - }), - N("base", L{N("name", "Everyone has same name")}), - N("foo", L{N("name", "Everyone has same name"), N("age", "10")}), - N("bar", L{N("name", "Everyone has same name"), N("age", "20")}), - } -); - -ADD_CASE_TO_GROUP("simple anchor 1, explicit, resolved", RESOLVE_REFS, -R"({ -anchored_content: &anchor_name This string will appear as the value of two keys., -other_anchor: *anchor_name, -anchors_in_seqs: [ - &anchor_in_seq this value appears in both elements of the sequence, - *anchor_in_seq - ], -base: &base { - name: Everyone has same name - }, -foo: &foo { - <<: *base, - age: 10 - }, -bar: &bar { - <<: *base, - age: 20 - } -})", - L{ - N("anchored_content", "This string will appear as the value of two keys."), - N("other_anchor", "This string will appear as the value of two keys."), - N("anchors_in_seqs", L{ - N("this value appears in both elements of the sequence"), - N("this value appears in both elements of the sequence"), - }), - N("base", L{N("name", "Everyone has same name")}), - N("foo", L{N("name", "Everyone has same name"), N("age", "10")}), - N("bar", L{N("name", "Everyone has same name"), N("age", "20")}), - } -); - - -ADD_CASE_TO_GROUP("anchor example 2, unresolved", -R"( -receipt: Oz-Ware Purchase Invoice -date: 2012-08-06 -customer: - first_name: Dorothy - family_name: Gale -items: - - part_no: A4786 - descrip: Water Bucket (Filled) - price: 1.47 - quantity: 4 - - part_no: E1628 - descrip: High Heeled "Ruby" Slippers - size: 8 - price: 133.7 - quantity: 1 -bill-to: &id001 - street: | - 123 Tornado Alley - Suite 16 - city: East Centerville - state: KS -ship-to: *id001 -specialDelivery: > - Follow the Yellow Brick - Road to the Emerald City. - Pay no attention to the - man behind the curtain. -)", -L{ - N{"receipt", "Oz-Ware Purchase Invoice"}, - N{"date", "2012-08-06"}, - N{"customer", L{N{"first_name", "Dorothy"}, N{"family_name", "Gale"}}}, - N{"items", L{ - N{L{N{"part_no", "A4786"}, - N{"descrip", "Water Bucket (Filled)"}, - N{"price", "1.47"}, - N{"quantity", "4"},}}, - N{L{N{"part_no", "E1628"}, - N{"descrip", "High Heeled \"Ruby\" Slippers"}, - N{"size", "8"}, - N{"price", "133.7"}, - N{"quantity", "1"},}}}}, - N{"bill-to", L{ - N{QV, "street", "123 Tornado Alley\nSuite 16\n"}, - N{"city", "East Centerville"}, - N{"state", "KS"},}, AR(VALANCH, "id001")}, - N{"ship-to", "*id001", AR(VALREF, "id001")}, - N{QV, "specialDelivery", "Follow the Yellow Brick Road to the Emerald City. Pay no attention to the man behind the curtain.\n"} - } -); - - -ADD_CASE_TO_GROUP("anchor example 2, resolved", RESOLVE_REFS, -R"( -receipt: Oz-Ware Purchase Invoice -date: 2012-08-06 -customer: - first_name: Dorothy - family_name: Gale -items: - - part_no: A4786 - descrip: Water Bucket (Filled) - price: 1.47 - quantity: 4 - - part_no: E1628 - descrip: High Heeled "Ruby" Slippers - size: 8 - price: 133.7 - quantity: 1 -bill-to: &id001 - street: | - 123 Tornado Alley - Suite 16 - city: East Centerville - state: KS -ship-to: *id001 -specialDelivery: > - Follow the Yellow Brick - Road to the Emerald City. - Pay no attention to the - man behind the curtain. -)", -L{ - N{"receipt", "Oz-Ware Purchase Invoice"}, - N{"date", "2012-08-06"}, - N{"customer", L{N{"first_name", "Dorothy"}, N{"family_name", "Gale"}}}, - N{"items", L{ - N{L{N{"part_no", "A4786"}, - N{"descrip", "Water Bucket (Filled)"}, - N{"price", "1.47"}, - N{"quantity", "4"},}}, - N{L{N{"part_no", "E1628"}, - N{"descrip", "High Heeled \"Ruby\" Slippers"}, - N{"size", "8"}, - N{"price", "133.7"}, - N{"quantity", "1"},}}}}, - N{"bill-to", L{ - N{QV, "street", "123 Tornado Alley\nSuite 16\n"}, - N{"city", "East Centerville"}, - N{"state", "KS"},}}, - N{"ship-to", L{ - N{QV, "street", "123 Tornado Alley\nSuite 16\n"}, - N{"city", "East Centerville"}, - N{"state", "KS"},}}, - N{QV, "specialDelivery", "Follow the Yellow Brick Road to the Emerald City. Pay no attention to the man behind the curtain.\n"} - } -); - -ADD_CASE_TO_GROUP("anchor example 3, unresolved", -R"( -- step: &id001 # defines anchor label &id001 - instrument: Lasik 2000 - pulseEnergy: 5.4 - pulseDuration: 12 - repetition: 1000 - spotSize: 1mm -- step: &id002 - instrument: Lasik 2000 - pulseEnergy: 5.0 - pulseDuration: 10 - repetition: 500 - spotSize: 2mm -- step: *id001 # refers to the first step (with anchor &id001) -- step: *id002 # refers to the second step -- step: - <<: *id001 - spotSize: 2mm # redefines just this key, refers rest from &id001 -- step: *id002 -)", -L{N(L{ -N("step", L{ - N{"instrument", "Lasik 2000"}, - N{"pulseEnergy", "5.4"}, - N{"pulseDuration", "12"}, - N{"repetition", "1000"}, - N{"spotSize", "1mm"}, - }, AR(VALANCH, "id001")), - }), N(L{ -N("step", L{ - N{"instrument", "Lasik 2000"}, - N{"pulseEnergy", "5.0"}, - N{"pulseDuration", "10"}, - N{"repetition", "500"}, - N{"spotSize", "2mm"}, - }, AR(VALANCH, "id002")), - }), N(L{ -N{"step", "*id001", AR(VALREF, "id001")}, - }), N(L{ -N{"step", "*id002", AR(VALREF, "id002")}, - }), N(L{ -N{"step", L{ - N{"<<", AR(KEYREF, "<<"), "*id001", AR(VALREF, "id002")}, - N{"spotSize", "2mm"}, - }}, - }), N(L{ -N{"step", "*id002", AR(VALREF, "id002")}, - }), - } -); - -ADD_CASE_TO_GROUP("anchor example 3, resolved", RESOLVE_REFS, -R"( -- step: &id001 # defines anchor label &id001 - instrument: Lasik 2000 - pulseEnergy: 5.4 - pulseDuration: 12 - repetition: 1000 - spotSize: 1mm -- step: &id002 - instrument: Lasik 2000 - pulseEnergy: 5.0 - pulseDuration: 10 - repetition: 500 - spotSize: 2mm -- step: *id001 # refers to the first step (with anchor &id001) -- step: *id002 # refers to the second step -- step: - <<: *id001 - spotSize: 2mm # redefines just this key, refers rest from &id001 -- step: *id002 -)", - L{N(L{ -N{"step", L{ - N{"instrument", "Lasik 2000"}, - N{"pulseEnergy", "5.4"}, - N{"pulseDuration", "12"}, - N{"repetition", "1000"}, - N{"spotSize", "1mm"}, - }}, - }), N(L{ -N{"step", L{ - N{"instrument", "Lasik 2000"}, - N{"pulseEnergy", "5.0"}, - N{"pulseDuration", "10"}, - N{"repetition", "500"}, - N{"spotSize", "2mm"}, - }}, - }), N(L{ -N{"step", L{ - N{"instrument", "Lasik 2000"}, - N{"pulseEnergy", "5.4"}, - N{"pulseDuration", "12"}, - N{"repetition", "1000"}, - N{"spotSize", "1mm"}, - }}, - }), N(L{ -N{"step", L{ - N{"instrument", "Lasik 2000"}, - N{"pulseEnergy", "5.0"}, - N{"pulseDuration", "10"}, - N{"repetition", "500"}, - N{"spotSize", "2mm"}, - }}, - }), N(L{ -N{"step", L{ - N{"instrument", "Lasik 2000"}, - N{"pulseEnergy", "5.4"}, - N{"pulseDuration", "12"}, - N{"repetition", "1000"}, - N{"spotSize", "2mm"}, - }}, - }), N(L{ -N{"step", L{ - N{"instrument", "Lasik 2000"}, - N{"pulseEnergy", "5.0"}, - N{"pulseDuration", "10"}, - N{"repetition", "500"}, - N{"spotSize", "2mm"}, - }}, - }), - } -); - -ADD_CASE_TO_GROUP("tagged doc with anchors 9KAX", -R"( ---- -&a1 -!!str -scalar1 ---- &a1 !!str scalar1 ---- -!!str -&a1 -scalar1 ---- !!str &a1 scalar1 ---- -!!str -&a2 -scalar2 ---- &a2 !!str scalar2 ---- -&a3 -!!str scalar3 ---- &a3 !!str scalar3 ---- -&a4 !!map -&a5 !!str key5: value4 ---- &a4 !!map -&a5 !!str key5: value4 ---- -a6: 1 -&anchor6 b6: 2 ---- -!!map -&a8 !!str key8: value7 ---- !!map -&a8 !!str key8: value7 ---- -!!map -!!str &a10 key10: value9 ---- !!map -&a10 !!str key10: value9 ---- -!!str &a11 -value11 ---- &a11 !!str value11 -)", -N(STREAM, L{ - N(DOCVAL, TS("!!str", "scalar1"), AR(VALANCH, "a1")), - N(DOCVAL, TS("!!str", "scalar1"), AR(VALANCH, "a1")), - N(DOCVAL, TS("!!str", "scalar1"), AR(VALANCH, "a1")), - N(DOCVAL, TS("!!str", "scalar1"), AR(VALANCH, "a1")), - N(DOCVAL, TS("!!str", "scalar2"), AR(VALANCH, "a2")), - N(DOCVAL, TS("!!str", "scalar2"), AR(VALANCH, "a2")), - N(DOCVAL, TS("!!str", "scalar3"), AR(VALANCH, "a3")), - N(DOCVAL, TS("!!str", "scalar3"), AR(VALANCH, "a3")), - N(DOCMAP, TL("!!map", L{N(TS("!!str", "key5"), AR(KEYANCH, "a5"), "value4")}), AR(VALANCH, "a4")), - N(DOCMAP, TL("!!map", L{N(TS("!!str", "key5"), AR(KEYANCH, "a5"), "value4")}), AR(VALANCH, "a4")), - N(DOCMAP, L{N("a6", "1"), N("b6", AR(KEYANCH, "anchor6"), "2")}), - N(DOCMAP, TL("!!map", L{N(TS("!!str", "key8"), AR(KEYANCH, "a8"), "value7")})), - N(DOCMAP, TL("!!map", L{N(TS("!!str", "key8"), AR(KEYANCH, "a8"), "value7")})), - N(DOCMAP, TL("!!map", L{N(TS("!!str", "key10"), AR(KEYANCH, "a10"), "value9")})), - N(DOCMAP, TL("!!map", L{N(TS("!!str", "key10"), AR(KEYANCH, "a10"), "value9")})), - N(DOCVAL, TS("!!str", "value11"), AR(VALANCH, "a11")), - N(DOCVAL, TS("!!str", "value11"), AR(VALANCH, "a11")), -}) -); - -ADD_CASE_TO_GROUP("github131 1, unresolved", -R"( -a: &vref b -*vref: c -&kref aa: bb -aaa: &kvref bbb -foo: - *kref: cc - *kvref: cc -)", -L{ - N("a", "b", AR(VALANCH, "vref")), - N("*vref", AR(KEYREF, "vref"), "c"), - N("aa", AR(KEYANCH, "kref"), "bb"), - N("aaa", "bbb", AR(VALANCH, "kvref")), - N("foo", L{ - N("*kref", AR(KEYREF, "kref"), "cc"), - N("*kvref", AR(KEYREF, "kvref"), "cc"), - }) -}); - -ADD_CASE_TO_GROUP("github131 1, resolved", RESOLVE_REFS, -R"( -a: &vref b -*vref: c -&kref aa: bb -aaa: &kvref bbb -foo: - *kref: cc - *kvref: cc -)", -L{ - N("a", "b"), - N("b", "c"), - N("aa", "bb"), - N("aaa", "bbb"), - N("foo", L{N("aa", "cc"), N("bbb", "cc")}) -}); - - -ADD_CASE_TO_GROUP("anchors+refs on key+val, unresolved", -R"({&a0 a0: &b0 b0, *b0: *a0})", -L{ - N("a0", AR(KEYANCH, "a0"), "b0", AR(VALANCH, "b0")), - N(AR(KEYREF, "*b0"), AR(VALREF, "*a0")), -}); - -ADD_CASE_TO_GROUP("anchors+refs on key+val, resolved", RESOLVE_REFS, -R"({&a0 a0: &b0 b0, *b0: *a0})", -L{ - N("a0", "b0"), - N("b0", "a0"), -}); - - -ADD_CASE_TO_GROUP("ambiguous anchor, unresolved", -R"(&rootanchor -&a0 a0: &b0 b0 -*b0: *a0 -map1: - &a1 a1: &b1 b1 # &a1 must be a KEY anchor on a1, not a VAL anchor on map1 - *b1: *a1 -map2: - *b0: *a0 # ensure the anchor is enough to establish the indentation - &a2 a2: &b2 b2 - *b2: *a2 -map3: &a3 # &a3 must be a VAL anchor on map3, not a KEY anchor on a3 - a3: &b3 b3 - *b3: *b0 -map4: *a0 -map5: - &map5 - &a5 a5: &b5 b5 - *b5: *a5 -map6: - &map6 - a6: &b6 b6 - *b6: *b6 -)", -N(L{ - N("a0", AR(KEYANCH, "a0"), "b0", AR(VALANCH, "b0")), - N(AR(KEYREF, "*b0"), AR(VALREF, "*a0")), - N("map1", L{N("a1", AR(KEYANCH, "a1"), "b1", AR(VALANCH, "b1")), N(AR(KEYREF, "*b1"), AR(VALREF, "*a1")),}), - N("map2", L{N(AR(KEYREF, "*b0"), AR(VALREF, "*a0")), N("a2", AR(KEYANCH, "a2"), "b2", AR(VALANCH, "b2")), N(AR(KEYREF, "*b2"), AR(VALREF, "*a2")),}), - N("map3", L{N("a3", "b3", AR(VALANCH, "b3")), N(AR(KEYREF, "*b3"), AR(VALREF, "*b0")),}, AR(VALANCH, "a3")), - N("map4", "*a0", AR(VALREF, "a0")), - N("map5", L{N("a5", AR(KEYANCH, "a5"), "b5", AR(VALANCH, "b5")), N(AR(KEYREF, "*b5"), AR(VALREF, "*a5")),}, AR(VALANCH, "map5")), - N("map6", L{N("a6", "b6", AR(VALANCH, "b6")), N(AR(KEYREF, "*b6"), AR(VALREF, "*b6")),}, AR(VALANCH, "map6")), -}, AR(VALANCH, "rootanchor"))); - -ADD_CASE_TO_GROUP("ambiguous anchor, resolved", RESOLVE_REFS, -R"( -&a0 a0: &b0 b0 -*b0: *a0 -map1: - &a1 a1: &b1 b1 # &a1 must be a KEY anchor on a1, not a VAL anchor on map1 - *b1: *a1 -map2: - *b0: *a0 # ensure the anchor is enough to establish the indentation - &a2 a2: &b2 b2 - *b2: *a2 -map3: &a3 # &a3 must be a VAL anchor on map3, not a KEY anchor on a3 - a3: &b3 b3 - *b3: *b0 -map4: *a0 -map5: - &map5 - &a5 a5: &b5 b5 - *b5: *a5 -map6: - &map6 - a6: &b6 b6 - *b6: *b6 -)", -L{ - N("a0", "b0"), N("b0", "a0"), - N("map1", L{N("a1", "b1"), N("b1", "a1"),}), - N("map2", L{N("b0", "a0"), N("a2", "b2"), N("b2", "a2"),}), - N("map3", L{N("a3", "b3"), N("b3", "b0"),}), - N("map4", "a0"), - N("map5", L{N("a5", "b5"), N("b5", "a5"),}), - N("map6", L{N("a6", "b6"), N("b6", "b6"),}), -}); - - -ADD_CASE_TO_GROUP("ambiguous anchor in seq, unresolved", -R"( -&seq -- &a0 - &a1 k1: v1 - &a2 k2: v2 - &a3 k3: v3 -- &a4 k4: v4 - &a5 k5: v5 - &a6 k6: v6 -- &a7 - &a8 k8: v8 -- &a9 - k10: v10 -- *a1: w1 - *a2: w2 - *a3: w3 - *a4: w4 - *a5: w5 - *a6: w6 - *a8: w8 -- *a0 -- *a7 -- *a9 -)", -N(L{ - N(L{N("k1", AR(KEYANCH, "a1"), "v1"), N("k2", AR(KEYANCH, "a2"), "v2"), N("k3", AR(KEYANCH, "a3"), "v3")}, AR(VALANCH, "a0")), - N(L{N("k4", AR(KEYANCH, "a4"), "v4"), N("k5", AR(KEYANCH, "a5"), "v5"), N("k6", AR(KEYANCH, "a6"), "v6")}), - N(L{N("k8", AR(KEYANCH, "a8"), "v8")}, AR(VALANCH, "a7")), - N(L{N("k10", "v10")}, AR(VALANCH, "a9")), - N(L{ - N("*a1", AR(KEYREF, "*a1"), "w1"), - N("*a2", AR(KEYREF, "*a2"), "w2"), - N("*a3", AR(KEYREF, "*a3"), "w3"), - N("*a4", AR(KEYREF, "*a4"), "w4"), - N("*a5", AR(KEYREF, "*a5"), "w5"), - N("*a6", AR(KEYREF, "*a6"), "w6"), - N("*a8", AR(KEYREF, "*a8"), "w8"), - }), - N("*a0", AR(VALREF, "*a0")), - N("*a7", AR(VALREF, "*a7")), - N("*a9", AR(VALREF, "*a9")), -}, AR(VALANCH, "seq"))); - -ADD_CASE_TO_GROUP("ambiguous anchor in seq, resolved", RESOLVE_REFS, -R"( -&seq -- &a0 - &a1 k1: v1 - &a2 k2: v2 - &a3 k3: v3 -- &a4 k4: v4 - &a5 k5: v5 - &a6 k6: v6 -- &a7 - &a8 k8: v8 -- &a9 - k10: v10 -- *a1: w1 - *a2: w2 - *a3: w3 - *a4: w4 - *a5: w5 - *a6: w6 - *a8: w8 -- *a0 -- *a7 -- *a9 -)", -L{ - N(L{N("k1", "v1"), N("k2", "v2"), N("k3", "v3")}), - N(L{N("k4", "v4"), N("k5", "v5"), N("k6", "v6")}), - N(L{N("k8", "v8")}), - N(L{N("k10", "v10")}), - N(L{ - N("k1", "w1"), - N("k2", "w2"), - N("k3", "w3"), - N("k4", "w4"), - N("k5", "w5"), - N("k6", "w6"), - N("k8", "w8"), - }), - N(L{N("k1", AR(KEYANCH, "a1"), "v1"), N("k2", AR(KEYANCH, "a2"), "v2"), N("k3", AR(KEYANCH, "a3"), "v3")}), - N(L{N("k8", AR(KEYANCH, "a8"), "v8")}), - N(L{N("k10", "v10")}), -}); - -ADD_CASE_TO_GROUP("anchor after complex key without value ZWK4", -R"( -a: 1 -? b -&anchor c: 3 -)", - L{ - N("a", "1"), N(KEYVAL, "b", {}), N("c", AR(KEYANCH, "anchor"), "3") - } -); - -ADD_CASE_TO_GROUP("anchor mixed with tag HMQ5, unresolved", -R"( -!!str &a1 "foo": - !!str bar -&a2 baz : *a1 -)", - L{ - N(KEYVAL|KEYQUO, TS("!!str", "foo"), AR(KEYANCH, "a1"), TS("!!str", "bar")), - N("baz", AR(KEYANCH, "a2"), "*a1", AR(VALREF, "*a1")), - } -); - -ADD_CASE_TO_GROUP("anchor mixed with tag HMQ5, resolved", RESOLVE_REFS, -R"( -!!str &a1 "foo": - !!str bar -&a2 baz : *a1 -)", - L{ - N(KEYVAL|KEYQUO, TS("!!str", "foo"), TS("!!str", "bar")), - N("baz", "foo"), - } -); -} - - -C4_SUPPRESS_WARNING_GCC_POP - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_simple_doc.cpp b/thirdparty/ryml/test/test_simple_doc.cpp deleted file mode 100644 index 9e47c6b1e..000000000 --- a/thirdparty/ryml/test/test_simple_doc.cpp +++ /dev/null @@ -1,526 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - - -TEST(simple_doc, issue_251) -{ - { - csubstr yaml = R"( -... -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t.rootref().type(), NOTYPE); - ASSERT_EQ(t.rootref().num_children(), 0u); - }); - } - { - Tree tree; - NodeRef root = tree.rootref(); - root |= MAP; - root["test"] = "..."; - root["test"] |= VALQUO; - - std::string s = emitrs_yaml(tree); - test_check_emit_check(to_csubstr(s), [](Tree const &t){ - EXPECT_EQ(t["test"].val(), "..."); - }); - } -} - - -TEST(simple_doc, test_suite_XLQ9) -{ - csubstr yaml = R"( ---- -scalar -%YAML 1.2 -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_stream()); - ASSERT_EQ(t.rootref().num_children(), 1u); - ASSERT_TRUE(t.rootref().first_child().is_doc()); - ASSERT_TRUE(t.rootref().first_child().is_val()); - EXPECT_EQ(t.rootref().first_child().val(), csubstr("scalar %YAML 1.2")); - }); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -CASE_GROUP(SIMPLE_DOC) -{ - -ADD_CASE_TO_GROUP("one empty doc", -R"(--- -)", - N(STREAM, L{DOCVAL}) -); - -ADD_CASE_TO_GROUP("one empty doc, explicit termination", -R"(--- -... -)", - N(STREAM, L{DOCVAL}) -); - -ADD_CASE_TO_GROUP("two empty docs", -R"(--- ---- -)", - N(STREAM, L{DOCVAL, DOCVAL}) -); - -ADD_CASE_TO_GROUP("two empty docs, with termination", -R"(--- -... ---- -)", - N(STREAM, L{DOCVAL, DOCVAL}) -); - -ADD_CASE_TO_GROUP("doc with single scalar", -R"(a scalar -)", -N(DOCVAL, "a scalar") -); - -ADD_CASE_TO_GROUP("doc with single scalar, explicit", -R"(--- a scalar -)", -N(STREAM, L{N(DOCVAL, "a scalar")}) -); - -ADD_CASE_TO_GROUP("simple doc, empty docs", -R"(--- ---- ---- ---- -)", - N(STREAM, L{DOCVAL, DOCVAL, DOCVAL, DOCVAL}) -); - -ADD_CASE_TO_GROUP("simple doc, empty docs, indented", -R"( --- - --- - --- - --- -)", - N(STREAM, L{DOCVAL, DOCVAL, DOCVAL, DOCVAL}) -); - -ADD_CASE_TO_GROUP("simple doc, empty docs, term", -R"(--- -... - - ---- -... ---- -... ---- -... -)", - N(STREAM, L{DOCVAL, DOCVAL, DOCVAL, DOCVAL}) -); - -ADD_CASE_TO_GROUP("simple doc, empty docs, term, indented", -R"( - --- - ... - - - --- - ... - --- - ... - --- - ... -)", - N(STREAM, L{DOCVAL, DOCVAL, DOCVAL, DOCVAL}) -); - -ADD_CASE_TO_GROUP("simple doc, plain scalar, multiple docs, implicit 2nd doc", -R"(--- -- a plain scalar - with several lines -... -- a second plain scalar - with several lines -)", -N(STREAM, L{ - N(DOCSEQ, L{N("a plain scalar with several lines")}), - N(DOCSEQ, L{N("a second plain scalar with several lines")}), -})); - -ADD_CASE_TO_GROUP("simple doc, single scalar, implicit doc", -R"(a scalar with some spaces inside -)", - N(DOCVAL, "a scalar with some spaces inside") -); - -ADD_CASE_TO_GROUP("simple doc, single scalar, implicit doc, indented", -R"( a scalar with some spaces inside -)", - N(DOCVAL,"a scalar with some spaces inside") -); - -ADD_CASE_TO_GROUP("simple doc, multi scalar, implicit doc", -R"(a scalar with some spaces inside, -and yet another one with more spaces inside, -and it doesn't really stop -)", - N(L{ - N("a scalar with some spaces inside"), - N("and yet another one with more spaces inside"), - N("and it doesn't really stop"), - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi scalar, implicit doc, indented", -R"( - a scalar with some spaces inside, - and yet another one with more spaces inside, - and it doesn't really stop -)", - N(L{ - N("a scalar with some spaces inside"), - N("and yet another one with more spaces inside"), - N("and it doesn't really stop"), - }) -); - -ADD_CASE_TO_GROUP("simple doc, single scalar, explicit doc, implicit termination", -R"(--- -a scalar with some spaces inside -)", - N(STREAM, L{N(DOCVAL, "a scalar with some spaces inside")}) -); - -ADD_CASE_TO_GROUP("simple doc, single scalar, explicit doc, implicit termination, indented", -R"( --- - a scalar with some spaces inside -)", - N(STREAM, L{N(DOCVAL, "a scalar with some spaces inside")}) -); - -ADD_CASE_TO_GROUP("simple doc, single scalar, explicit doc, explicit termination", -R"(--- -a scalar with some spaces inside -... -)", - N(STREAM, L{N(DOCVAL, "a scalar with some spaces inside")}) -); - -ADD_CASE_TO_GROUP("simple doc, single scalar, explicit doc, explicit termination, indented", -R"( --- - a scalar with some spaces inside - ... -)", - N(STREAM, L{N(DOCVAL, "a scalar with some spaces inside")}) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, seq-map", -R"(--- -- a -- b -- c -... ---- -a: 0 -b: 1 -c: 2 -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a"), N("b"), N("c")}), - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, seq-map, indented", -R"( --- - - a - - b - - c - ... - --- - a: 0 - b: 1 - c: 2 -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a"), N("b"), N("c")}), - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, seq-map, no term", -R"(--- -- a -- b -- c ---- -a: 0 -b: 1 -c: 2 -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a"), N("b"), N("c")}), - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, seq-map, no term, indented", -R"( - --- - - a - - b - - c - --- - a: 0 - b: 1 - c: 2 -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a"), N("b"), N("c")}), - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, map-seq", -R"(--- -a: 0 -b: 1 -c: 2 -... ---- -- a -- b -- c -... -)", - N(STREAM, L{ - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}), - N(DOCSEQ, L{N("a"), N("b"), N("c")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, map-seq, indented", -R"( - --- - a: 0 - b: 1 - c: 2 - ... - --- - - a - - b - - c - ... -)", - N(STREAM, L{ - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}), - N(DOCSEQ, L{N("a"), N("b"), N("c")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, map-seq, no term", -R"(--- -a: 0 -b: 1 -c: 2 ---- -- a -- b -- c -)", - N(STREAM, L{ - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}), - N(DOCSEQ, L{N("a"), N("b"), N("c")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, map-seq, no term, indented", -R"( - --- - a: 0 - b: 1 - c: 2 - --- - - a - - b - - c -)", - N(STREAM, L{ - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}), - N(DOCSEQ, L{N("a"), N("b"), N("c")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, impl seq-map", -R"(--- -[a, b, c] -... ---- -{a: 0, b: 1, c: 2} -... -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a"), N("b"), N("c")}), - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, impl seq-map, indented", -R"( - --- - [a, b, c] - ... - --- - {a: 0, b: 1, c: 2} - ... -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a"), N("b"), N("c")}), - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, impl seq-map, no term", -R"(--- -[a, b, c] ---- -{a: 0, b: 1, c: 2} -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a"), N("b"), N("c")}), - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, impl seq-map, no term, indented", -R"( - --- - [a, b, c] - --- - {a: 0, b: 1, c: 2} -)", - N(STREAM, L{ - N(DOCSEQ, L{N("a"), N("b"), N("c")}), - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, impl map-seq", -R"(--- -{a: 0, b: 1, c: 2} -... ---- -[a, b, c] -... -)", - N(STREAM, L{ - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}), - N(DOCSEQ, L{N("a"), N("b"), N("c")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, impl map-seq, indented", -R"( - --- - {a: 0, b: 1, c: 2} - ... - --- - [a, b, c] - ... -)", - N(STREAM, L{ - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}), - N(DOCSEQ, L{N("a"), N("b"), N("c")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, impl map-seq, no term", -R"(--- -{a: 0, b: 1, c: 2} ---- -[a, b, c] -)", - N(STREAM, L{ - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}), - N(DOCSEQ, L{N("a"), N("b"), N("c")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, multi doc, impl map-seq, no term, indented", -R"( - --- - {a: 0, b: 1, c: 2} - --- - [a, b, c] -)", - N(STREAM, L{ - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2")}), - N(DOCSEQ, L{N("a"), N("b"), N("c")}) - }) -); - -ADD_CASE_TO_GROUP("simple doc, indented with empty lines", -R"( - --- - {a: 0, b: 1, c: 2, - - - - - - - - d: - some scalar - } - --- - a: 0 - b: 1 - c: 2 - - - - - - - - d: - some scalar -)", - N(STREAM, L{ - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2"), N("d", "some scalar")}), - N(DOCMAP, L{N("a", "0"), N("b", "1"), N("c", "2"), N("d", "some scalar")}), - }) -); - - -ADD_CASE_TO_GROUP("simple doc, tags at global scope, 9WXW", -R"(# Private -!foo "bar" -... -# Global -%TAG ! tag:example.com,2000:app/ ---- -!foo "bar" -)", -N(STREAM, L{ - N(DOCVAL|VALQUO, TS("!foo", "bar")), - // strict YAML should result in this for the second doc: - //N(DOCVAL|VALQUO, TS("", "bar")), - // but since we don't do lookup, it should result in: - N(DOCVAL|VALQUO, TS("!foo", "bar")), -}) -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_simple_map.cpp b/thirdparty/ryml/test/test_simple_map.cpp deleted file mode 100644 index 3e814e279..000000000 --- a/thirdparty/ryml/test/test_simple_map.cpp +++ /dev/null @@ -1,1050 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -TEST(simple_map, issue274) -{ - Tree tree = parse_in_arena(R"( -foo: -- bar -- -baz: qux -foo2: -- bar -- -baz2: qux -)"); - std::cout << tree; - ASSERT_EQ(tree.rootref().num_children(), 4u); - ASSERT_EQ(tree["foo"].num_children(), 2u); - EXPECT_EQ(tree["foo"][0].val(), "bar"); - EXPECT_EQ(tree["foo"][1].val(), ""); - EXPECT_EQ(tree["baz"].val(), "qux"); - ASSERT_EQ(tree["foo2"].num_children(), 2u); - EXPECT_EQ(tree["foo2"][0].val(), "bar"); - EXPECT_EQ(tree["foo2"][1].val(), ""); - EXPECT_EQ(tree["baz2"].val(), "qux"); -} - -TEST(simple_map, keys_with_leading_colon) -{ - Tree tree = parse_in_arena(R"( -:foo: - :bar: a - :barbar: b - :barbarbar: c -)"); - EXPECT_EQ(tree[":foo"][":bar"].val(), "a"); - EXPECT_EQ(tree[":foo"][":barbar"].val(), "b"); - EXPECT_EQ(tree[":foo"][":barbarbar"].val(), "c"); -} - -TEST(simple_map, open_on_new_doc_without_space) -{ - Tree tree = parse_in_arena(R"( -foo: bar ---- -foo: bar ---- -foo: bar ---- -foo: -... -foo: ---- -)"); - EXPECT_EQ(tree.docref(0)["foo"].val(), "bar"); - EXPECT_EQ(tree.docref(1)["foo"].val(), "bar"); - EXPECT_EQ(tree.docref(2)["foo"].val(), "bar"); - EXPECT_EQ(tree.docref(3)["foo"].val(), ""); - EXPECT_EQ(tree.docref(4)["foo"].val(), ""); -} - -TEST(simple_map, open_on_new_doc_with_space_before_colon) -{ - Tree tree = parse_in_arena(R"( -foo0 : bar ---- -foo1 : bar # the " :" was causing an assert ---- -foo2 : bar ---- -foo3 : bar ---- -foo4 : bar -)"); - EXPECT_EQ(tree.docref(0)["foo0"].val(), "bar"); - EXPECT_EQ(tree.docref(1)["foo1"].val(), "bar"); - EXPECT_EQ(tree.docref(2)["foo2"].val(), "bar"); - EXPECT_EQ(tree.docref(3)["foo3"].val(), "bar"); - EXPECT_EQ(tree.docref(4)["foo4"].val(), "bar"); -} - - -TEST(simple_map, test_suite_UT92) -{ - csubstr yaml = R"( -- { matches -% : 20 } -- { matches -%: 20 } -- { matches -%: - 20 } -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t[0].has_child("matches %")); - EXPECT_EQ(t[0]["matches %"].val(), "20"); - ASSERT_TRUE(t[0].has_child("matches %")); - ASSERT_TRUE(t[1].has_child("matches %")); - EXPECT_EQ(t[1]["matches %"].val(), "20"); - ASSERT_TRUE(t[1].has_child("matches %")); - ASSERT_TRUE(t[2].has_child("matches %")); - EXPECT_EQ(t[2]["matches %"].val(), "20"); - ASSERT_TRUE(t[2].has_child("matches %")); - }); -} - -TEST(simple_map, two_nested_flow_maps_not_accepted_because_of_container_key) -{ - Tree tree; - ExpectError::do_check(&tree, [&]{ - parse_in_arena("{{}}", &tree); - }); -} - -TEST(simple_map, many_unmatched_brackets) -{ - std::string src; - src.reserve(10000000u); - for(size_t num_brackets : {4u, 8u, 32u, 256u, 4096u, 1024u}) - { - SCOPED_TRACE(num_brackets); - for(size_t i = src.size(); i < num_brackets; ++i) - src += '{'; - Tree tree; - ExpectError::do_check(&tree, [&]{ - parse_in_place(to_substr(src), &tree); - }); - } -} - -TEST(simple_map, missing_quoted_key) -{ - csubstr yaml = R"( -"top1" : - "key1" : scalar1 -'top2' : - 'key2' : scalar2 ---- -"top1" : - "key1" : scalar1 -'top2' : - 'key2' : scalar2 ---- -'x2': {'y': z} ---- -'x3': - 'y': z ---- -x4: - 'y': z ---- -'x5': -'y': z ---- -x6: -'y': z ---- -'x7' : [ - 'y' : z, - ] ---- -"x8" : - "y" : value, - "x" : value -"y" : - "y" : value, - "x" : value -)"; - test_check_emit_check(yaml, [](Tree const &t){ - size_t doc = 0; - EXPECT_TRUE(t.docref(doc)["top1"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top1"]["key1"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"]["key2"].is_key_quoted()); - ++doc; - EXPECT_TRUE(t.docref(doc)["top1"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top1"]["key1"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"]["key2"].is_key_quoted()); - ++doc; - EXPECT_TRUE(t.docref(doc)["x2"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["x2"]["y"].is_key_quoted()); - ++doc; - EXPECT_TRUE(t.docref(doc)["x3"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["x3"]["y"].is_key_quoted()); - ++doc; - EXPECT_FALSE(t.docref(doc)["x4"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["x4"]["y"].is_key_quoted()); - ++doc; - EXPECT_TRUE(t.docref(doc)["x5"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["y"].is_key_quoted()); - ++doc; - EXPECT_FALSE(t.docref(doc)["x6"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["y"].is_key_quoted()); - ++doc; - EXPECT_TRUE(t.docref(doc)["x7"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["x7"][0]["y"].is_key_quoted()); - ++doc; - EXPECT_TRUE(t.docref(doc)["x8"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["x8"]["y"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["x8"]["x"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["y"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["y"]["y"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["y"]["x"].is_key_quoted()); - }); -} - -#ifdef JAVAI -void verify_error_is_reported(csubstr case_name, csubstr yaml, size_t col={}) -{ - SCOPED_TRACE(case_name); - SCOPED_TRACE(yaml); - Tree tree; - Location loc = {}; - loc.col = col; - ExpectError::do_check(&tree, [&](){ - parse_in_arena(yaml, &tree); - }, loc); -} - -TEST(simple_map, no_map_key_flow) -{ - verify_error_is_reported("map key", R"({ first: Sammy, last: Sosa }: foo)", 28u); -} - -TEST(simple_map, no_map_key_block) -{ - verify_error_is_reported("map key", R"(? - first: Sammy - last: Sosa -: - foo -)"); -} - -TEST(simple_map, no_seq_key_flow) -{ - verify_error_is_reported("seq key", R"([Sammy, Sosa]: foo)", 28u); -} - -TEST(simple_map, no_seq_key_block) -{ - verify_error_is_reported("map key", R"(? - - Sammy - - Sosa -: - foo -)"); -} -#endif - -#ifdef RYML_WITH_TAB_TOKENS -TEST(simple_map, block_tab_tokens) -{ - Tree tree = parse_in_arena(R"( ---- # block, spaces only -a: 0 -b: 1 -c: 2 ---- # block, tabs after token -a: 0 -b: 1 -c: 2 ---- # block, tabs before and after token -a : 0 -b : 1 -c : 2 ---- # block, tabs before token -a : 0 -b : 1 -c : 2 ---- # block, tabs before newline -a : 0 -b : 1 -c : 2 -)"); - EXPECT_EQ(tree.docref(0)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(0)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(0)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(1)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(1)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(1)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(2)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(2)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(2)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(3)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(3)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(3)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(4)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(4)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(4)["c"].val(), csubstr("2")); -} - -TEST(simple_map, flow_tab_tokens) -{ - Tree tree = parse_in_arena(R"( ---- # flow, no tabs -{a: 0, b: 1, c: 2} ---- # flow, tabs after token -{a: 0, b: 1, c: 2} ---- # flow, tabs before and after token -{a : 0, b : 1, c : 2} ---- # flow, tabs before token -{a : 0, b : 1, c : 2} ---- # flow, tabs after val -{a : 0 , b : 1 , c : 2 } ---- # flow, tabs after val and comma -{a : 0 , b : 1 , c : 2 } ---- # flow, tabs everywhere - { - a : 0 , - b : 1 , - c : 2 - } - )"); - EXPECT_EQ(tree.docref(0)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(0)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(0)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(1)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(1)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(1)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(2)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(2)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(2)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(3)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(3)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(3)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(4)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(4)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(4)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(5)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(5)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(5)["c"].val(), csubstr("2")); - EXPECT_EQ(tree.docref(6)["a"].val(), csubstr("0")); - EXPECT_EQ(tree.docref(6)["b"].val(), csubstr("1")); - EXPECT_EQ(tree.docref(6)["c"].val(), csubstr("2")); -} -#endif // RYML_WITH_TAB_TOKENS - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -CASE_GROUP(SIMPLE_MAP) -{ -// -ADD_CASE_TO_GROUP("empty map", -"{}", - MAP -); - -ADD_CASE_TO_GROUP("empty map, multiline", -R"({ -} -)", - MAP -); - -ADD_CASE_TO_GROUP("empty map, multilines", -R"({ -# foo bar baz akjasdkj -} -)", - MAP -); - -ADD_CASE_TO_GROUP("simple map, explicit, single line", -"{foo: 0, bar: 1, baz: 2, bat: 3}", - L{N{"foo", "0"}, N{"bar", "1"}, N{"baz", "2"}, N{"bat", "3"}} -); - -ADD_CASE_TO_GROUP("simple map, explicit, multiline, unindented", -R"({ -foo: 0, -bar: 1, -baz: 2, -bat: 3 -})", - L{N{"foo", "0"}, N{"bar", "1"}, N{"baz", "2"}, N{"bat", "3"}} -); - -ADD_CASE_TO_GROUP("simple map, explicit, multiline, indented", -R"({ - foo: 0, - bar: 1, - baz: 2, - bat: 3 -})", - L{N{"foo", "0"}, N{"bar", "1"}, N{"baz", "2"}, N{"bat", "3"}} -); - -ADD_CASE_TO_GROUP("simple map", -R"( -foo: 0 -bar: 1 -baz: 2 -bat: 3 -)", - L{N{"foo", "0"}, N{"bar", "1"}, N{"baz", "2"}, N{"bat", "3"}} -); - -ADD_CASE_TO_GROUP("simple map, values on next line", -R"( -foo: - 0 -bar: - 1 -baz: - 2 -bat: - 3 -)", - L{N{"foo", "0"}, N{"bar", "1"}, N{"baz", "2"}, N{"bat", "3"}} -); - -ADD_CASE_TO_GROUP("simple map, with comments", -R"( -foo: 0 # this is a foo -bar: 1 # this is a bar -baz: 2 # this is a baz -bat: 3 # this is a bat -)", - L{N{"foo", "0"}, N{"bar", "1"}, N{"baz", "2"}, N{"bat", "3"}} -); - -ADD_CASE_TO_GROUP("simple map, with comments interspersed", -R"( -# this is a foo -foo: 0 -# this is a bar -bar: 1 -# this is a baz -baz: 2 -# this is a bat -bat: 3 -)", - L{N{"foo", "0"}, N{"bar", "1"}, N{"baz", "2"}, N{"bat", "3"}} -); - -ADD_CASE_TO_GROUP("simple map, with comments interspersed implicit key X8DW", -R"( -? key -# comment -: value -)", - L{N("key", "value")} -); - -ADD_CASE_TO_GROUP("simple map, with indented comments interspersed, before", -R"( - # this is a foo -foo: 0 - # this is a bar -bar: 1 - # this is a baz -baz: 2 - # this is a bat -bat: 3 -)", - L{N{"foo", "0"}, N{"bar", "1"}, N{"baz", "2"}, N{"bat", "3"}} -); - -ADD_CASE_TO_GROUP("simple map, with indented comments interspersed, after", -R"( -foo: 0 - # this is a foo -bar: 1 - # this is a bar -baz: 2 - # this is a baz -bat: 3 - # this is a bat -)", - L{N{"foo", "0"}, N{"bar", "1"}, N{"baz", "2"}, N{"bat", "3"}} -); - -ADD_CASE_TO_GROUP("simple map, null values", -R"( -key: val -a: -b: -c: -d: -e: -f: -g: -foo: bar -)", -L{N("key", "val"), N(KEYVAL, "a", /*"~"*/{}), N(KEYVAL, "b", /*"~"*/{}), N(KEYVAL, "c", /*"~"*/{}), N(KEYVAL, "d", /*"~"*/{}), N(KEYVAL, "e", /*"~"*/{}), N(KEYVAL, "f", /*"~"*/{}), N(KEYVAL, "g", /*"~"*/{}), N("foo", "bar"),} -); - -ADD_CASE_TO_GROUP("simple map expl, null values 1", -R"({key: val, a, b, c, d, e: , f: , g: , foo: bar})", -L{N("key", "val"), N(KEYVAL, "a", /*"~"*/{}), N(KEYVAL, "b", /*"~"*/{}), N(KEYVAL, "c", /*"~"*/{}), N(KEYVAL, "d", /*"~"*/{}), N(KEYVAL, "e", /*"~"*/{}), N(KEYVAL, "f", /*"~"*/{}), N(KEYVAL, "g", /*"~"*/{}), N("foo", "bar"),} -); - -ADD_CASE_TO_GROUP("simple map expl, null values 2", -R"( -- {a} -- {a, b, c} -- {a: 1, b: 2, c} -- {a: 1, b, c: 2} -- {a, b: 1, c: 2} -)", -L{ - N(L{N(KEYVAL, "a", /*"~"*/{})}), - N(L{N(KEYVAL, "a", /*"~"*/{}), N(KEYVAL, "b", /*"~"*/{}), N(KEYVAL, "c", /*"~"*/{})}), - N(L{N("a", "1"), N("b", "2"), N(KEYVAL, "c", /*"~"*/{})}), - N(L{N("a", "1"), N(KEYVAL, "b", /*"~"*/{}), N("c", "2")}), - N(L{N(KEYVAL, "a", /*"~"*/{}), N("b", "1"), N("c", "2")}), - } -); - -ADD_CASE_TO_GROUP("simple map expl, null values 3, 4ABK", -R"( -- {foo: , bar: , baz: } -- {foo:, bar:, baz:} -- {foo:foo: , bar:bar: , baz:baz: } -- {foo:foo:, bar:bar:, baz:baz:} -)", -L{ - N(L{N(KEYVAL, "foo", {}), N(KEYVAL, "bar", {}), N(KEYVAL, "baz", {})}), - N(L{N(KEYVAL, "foo", {}), N(KEYVAL, "bar", {}), N(KEYVAL, "baz", {})}), - N(L{N(KEYVAL, "foo:foo", {}), N(KEYVAL, "bar:bar", {}), N(KEYVAL, "baz:baz", {})}), - N(L{N(KEYVAL, "foo:foo", {}), N(KEYVAL, "bar:bar", {}), N(KEYVAL, "baz:baz", {})}), -}); - -ADD_CASE_TO_GROUP("simple map, scalars with special chars, comma", -R"( -a,b: val,000 -c,d: val, 000 -e,f: val , 000 -h,i: val ,000 -a, b: val,000 -c, d: val, 000 -e, f: val , 000 -h, i: val ,000 -a , b: val,000 -c , d: val, 000 -e , f: val , 000 -h , i: val ,000 -a ,b: val,000 -c ,d: val, 000 -e ,f: val , 000 -h ,i: val ,000 -)", - L{ - N{"a,b", "val,000"}, N{"c,d", "val, 000"}, N{"e,f", "val , 000"}, N{"h,i", "val ,000"}, - N{"a, b", "val,000"}, N{"c, d", "val, 000"}, N{"e, f", "val , 000"}, N{"h, i", "val ,000"}, - N{"a , b", "val,000"}, N{"c , d", "val, 000"}, N{"e , f", "val , 000"}, N{"h , i", "val ,000"}, - N{"a ,b", "val,000"}, N{"c ,d", "val, 000"}, N{"e ,f", "val , 000"}, N{"h ,i", "val ,000"}, -} -); - -ADD_CASE_TO_GROUP("simple map, scalars with special chars, semicolon", -R"( -a:b: val:000 -c:d: "val: 000" -e:f: "val : 000" -h:i: val :000 -"a: b": val:000 -"c: d": "val: 000" -"e: f": "val : 000" -"h: i": val :000 -"a : b": val:000 -"c : d": "val: 000" -"e : f": "val : 000" -"h : i": val :000 -a :b: val:000 -c :d: "val: 000" -e :f: "val : 000" -h :i: val :000 -)", - L{ - N{ "a:b", "val:000"}, N{QV, "c:d", "val: 000"}, N{QV, "e:f", "val : 000"}, N{ "h:i", "val :000"}, - N{QK, "a: b", "val:000"}, N{QKV, "c: d", "val: 000"}, N{QKV, "e: f", "val : 000"},N{QK, "h: i", "val :000"}, - N{QK, "a : b", "val:000"},N{QKV, "c : d", "val: 000"},N{QKV, "e : f", "val : 000"},N{QK, "h : i", "val :000"}, - N{ "a :b", "val:000"}, N{QV, "c :d", "val: 000"}, N{QV, "e :f", "val : 000"}, N{ "h :i", "val :000"}, -} -); - -ADD_CASE_TO_GROUP("simple map, scalars with special chars, cardinal", -R"( -a#b: val#000 -c#d: val# 000 -e#f: "val # 000" -h#i: "val #000" -a# b: val#000 -c# d: val# 000 -e# f: "val # 000" -h# i: "val #000" -"a # b": val#000 -"c # d": val# 000 -"e # f": "val # 000" -"h # i": "val #000" -"a #b": val#000 -"c #d": val# 000 -"e #f": "val # 000" -"h #i": "val #000" -)", - L{ - N{ "a#b", "val#000"}, N{ "c#d", "val# 000"}, N{QV, "e#f", "val # 000"}, N{QV, "h#i", "val #000"}, - N{ "a# b", "val#000"}, N{ "c# d", "val# 000"}, N{QV, "e# f", "val # 000"}, N{QV, "h# i", "val #000"}, - N{QK, "a # b", "val#000"}, N{QK, "c # d", "val# 000"}, N{QKV, "e # f", "val # 000"}, N{QKV, "h # i", "val #000"}, - N{QK, "a #b", "val#000"}, N{QK, "c #d", "val# 000"}, N{QKV, "e #f", "val # 000"}, N{QKV, "h #i", "val #000"}, -} -); - -ADD_CASE_TO_GROUP("simple map, scalars with special chars, dash", -R"( -a-b: val-000 -c-d: val- 000 -e-f: val - 000 -h-i: val -000 -a- b: val-000 -c- d: val- 000 -e- f: val - 000 -h- i: val -000 -a - b: val-000 -c - d: val- 000 -e - f: val - 000 -h - i: val -000 -a -b: val-000 -c -d: val- 000 -e -f: val - 000 -h -i: val -000 -)", - L{ - N{"a-b", "val-000"}, N{"c-d", "val- 000"}, N{"e-f", "val - 000"}, N{"h-i", "val -000"}, - N{"a- b", "val-000"}, N{"c- d", "val- 000"}, N{"e- f", "val - 000"}, N{"h- i", "val -000"}, - N{"a - b", "val-000"}, N{"c - d", "val- 000"}, N{"e - f", "val - 000"}, N{"h - i", "val -000"}, - N{"a -b", "val-000"}, N{"c -d", "val- 000"}, N{"e -f", "val - 000"}, N{"h -i", "val -000"}, -} -); - -ADD_CASE_TO_GROUP("simple map, scalars with special chars, left-bracket", -R"( -a[b: val[000 -c[d: val[ 000 -e[f: val [ 000 -h[i: val [000 -a[ b: val[000 -c[ d: val[ 000 -e[ f: val [ 000 -h[ i: val [000 -a [ b: val[000 -c [ d: val[ 000 -e [ f: val [ 000 -h [ i: val [000 -a [b: val[000 -c [d: val[ 000 -e [f: val [ 000 -h [i: val [000 -)", - L{ - N{"a[b", "val[000"}, N{"c[d", "val[ 000"}, N{"e[f", "val [ 000"}, N{"h[i", "val [000"}, - N{"a[ b", "val[000"}, N{"c[ d", "val[ 000"}, N{"e[ f", "val [ 000"}, N{"h[ i", "val [000"}, - N{"a [ b", "val[000"}, N{"c [ d", "val[ 000"}, N{"e [ f", "val [ 000"}, N{"h [ i", "val [000"}, - N{"a [b", "val[000"}, N{"c [d", "val[ 000"}, N{"e [f", "val [ 000"}, N{"h [i", "val [000"}, -} -); - -ADD_CASE_TO_GROUP("simple map, scalars with special chars, right-bracket", -R"( -a]b: val]000 -c]d: val] 000 -e]f: val ] 000 -h]i: val ]000 -a] b: val]000 -c] d: val] 000 -e] f: val ] 000 -h] i: val ]000 -a ] b: val]000 -c ] d: val] 000 -e ] f: val ] 000 -h ] i: val ]000 -a ]b: val]000 -c ]d: val] 000 -e ]f: val ] 000 -h ]i: val ]000 -)", - L{ - N{"a]b", "val]000"}, N{"c]d", "val] 000"}, N{"e]f", "val ] 000"}, N{"h]i", "val ]000"}, - N{"a] b", "val]000"}, N{"c] d", "val] 000"}, N{"e] f", "val ] 000"}, N{"h] i", "val ]000"}, - N{"a ] b", "val]000"}, N{"c ] d", "val] 000"}, N{"e ] f", "val ] 000"}, N{"h ] i", "val ]000"}, - N{"a ]b", "val]000"}, N{"c ]d", "val] 000"}, N{"e ]f", "val ] 000"}, N{"h ]i", "val ]000"}, -} -); - -ADD_CASE_TO_GROUP("simple map, scalars with special chars, left-curly", -R"( -a{b: val{000 -c{d: val{ 000 -e{f: val { 000 -h{i: val {000 -a{ b: val{000 -c{ d: val{ 000 -e{ f: val { 000 -h{ i: val {000 -a { b: val{000 -c { d: val{ 000 -e { f: val { 000 -h { i: val {000 -a {b: val{000 -c {d: val{ 000 -e {f: val { 000 -h {i: val {000 -)", - L{ - N{"a{b", "val{000"}, N{"c{d", "val{ 000"}, N{"e{f", "val { 000"}, N{"h{i", "val {000"}, - N{"a{ b", "val{000"}, N{"c{ d", "val{ 000"}, N{"e{ f", "val { 000"}, N{"h{ i", "val {000"}, - N{"a { b", "val{000"}, N{"c { d", "val{ 000"}, N{"e { f", "val { 000"}, N{"h { i", "val {000"}, - N{"a {b", "val{000"}, N{"c {d", "val{ 000"}, N{"e {f", "val { 000"}, N{"h {i", "val {000"}, -} -); - -ADD_CASE_TO_GROUP("simple map, scalars with special chars, right-curly", -R"( -a}b: val}000 -c}d: val} 000 -e}f: val } 000 -h}i: val }000 -a} b: val}000 -c} d: val} 000 -e} f: val } 000 -h} i: val }000 -a } b: val}000 -c } d: val} 000 -e } f: val } 000 -h } i: val }000 -a }b: val}000 -c }d: val} 000 -e }f: val } 000 -h }i: val }000 -)", - L{ - N{"a}b", "val}000"}, N{"c}d", "val} 000"}, N{"e}f", "val } 000"}, N{"h}i", "val }000"}, - N{"a} b", "val}000"}, N{"c} d", "val} 000"}, N{"e} f", "val } 000"}, N{"h} i", "val }000"}, - N{"a } b", "val}000"}, N{"c } d", "val} 000"}, N{"e } f", "val } 000"}, N{"h } i", "val }000"}, - N{"a }b", "val}000"}, N{"c }d", "val} 000"}, N{"e }f", "val } 000"}, N{"h }i", "val }000"}, -} -); - -ADD_CASE_TO_GROUP("simple map expl, scalars with special chars, comma", -R"({ -a0,b0: val0,0000 -c0,d0: val0, 0000 -e0,f0: val0 , 0000 -h0,i0: val0 ,0000 -a1, b1: val1,0001 -c1, d1: val1, 0001 -e1, f1: val1 , 0001 -h1, i1: val1 ,0001 -a2 , b2: val2,0002 -c2 , d2: val2, 0002 -e2 , f2: val2 , 0002 -h2 , i2: val2 ,0002 -a3 ,b3: val3,0003 -c3 ,d3: val3, 0003 -e3 ,f3: val3 , 0003 -h3 ,i3: val3 ,0003 -})", - L{ // this is crazy... - N(KEYVAL, "a0", /*"~"*/{}), - N("b0", "val0"), - N(KEYVAL, "0000 c0", /*"~"*/{}), - N("d0", "val0"), N(KEYVAL, "0000 e0", /*"~"*/{}), - N("f0", "val0"), N(KEYVAL, "0000 h0", /*"~"*/{}), - N("i0", "val0"), N(KEYVAL, "0000 a1", /*"~"*/{}), - N("b1", "val1"), N(KEYVAL, "0001 c1", /*"~"*/{}), - N("d1", "val1"), N(KEYVAL, "0001 e1", /*"~"*/{}), - N("f1", "val1"), N(KEYVAL, "0001 h1", /*"~"*/{}), - N("i1", "val1"), N(KEYVAL, "0001 a2", /*"~"*/{}), - N("b2", "val2"), N(KEYVAL, "0002 c2", /*"~"*/{}), - N("d2", "val2"), N(KEYVAL, "0002 e2", /*"~"*/{}), - N("f2", "val2"), N(KEYVAL, "0002 h2", /*"~"*/{}), - N("i2", "val2"), N(KEYVAL, "0002 a3", /*"~"*/{}), - N("b3", "val3"), N(KEYVAL, "0003 c3", /*"~"*/{}), - N("d3", "val3"), N(KEYVAL, "0003 e3", /*"~"*/{}), - N("f3", "val3"), N(KEYVAL, "0003 h3", /*"~"*/{}), - N("i3", "val3"), N(KEYVAL, "0003", /*"~"*/{}), -} -); - - -ADD_CASE_TO_GROUP("simple map, spaces before semicolon, issue54", -R"( -foo : crl -keyA : - keyA.B : test value -"key C" : val C -'key D' : val D -elm2 : - "key C" : val C - 'key D' : val D - key E : val E -elm3 : - 'key D' : val D - "key C" : val C - key E : val E -elm4 : - key E : val E - 'key D' : val D - "key C" : val C -)", -L{ - N("foo", "crl"), - N("keyA", L{N("keyA.B", "test value")}), - N(QK, "key C", "val C"), - N(QK, "key D", "val D"), - N("elm2", L{N(QK, "key C", "val C"), N(QK, "key D", "val D"), N("key E", "val E"),}), - N("elm3", L{N(QK, "key D", "val D"), N(QK, "key C", "val C"), N("key E", "val E"),}), - N("elm4", L{N("key E", "val E"), N(QK, "key D", "val D"), N(QK, "key C", "val C"),}), -} -); - -ADD_CASE_TO_GROUP("simple map, spaces before semicolon, issue65, v0", -R"({a : b})", -L{ - N("a", "b"), -} -); - -ADD_CASE_TO_GROUP("simple map, spaces before semicolon, issue65, v1", -R"(a : b)", -L{ - N("a", "b"), -} -); - -ADD_CASE_TO_GROUP("simple map, spaces before semicolon, issue65, v2", -R"( -is it ok : let's see -ok : {a : b, c : d, e : f,} -must be ok : - c0 : d - c1 : d - c2 : d -)", -L{ - N("is it ok", "let's see"), - N("ok", L{N("a", "b"), N("c", "d"), N("e", "f")}), - N("must be ok", L{N("c0", "d"), N("c1", "d"), N("c2", "d")}), -} -); - -ADD_CASE_TO_GROUP("simple map, spaces before semicolon, issue65, v3", -R"({ -oka: {a : b}, -is it ok : let's see, -okb: {a : b}, -ok : {a : b}, -must be ok : { - c0 : d, - c1 : d, - c2 : d, -} -})", -L{ - N("oka", L{N("a", "b")}), - N("is it ok", "let's see"), - N("okb", L{N("a", "b")}), - N("ok", L{N("a", "b")}), - N("must be ok", L{N("c0", "d"), N("c1", "d"), N("c2", "d")}), -}); - -ADD_CASE_TO_GROUP("simple map, empty keys 2JQS, v1", -R"( -: a -: b -)", -N(MAP, L{ - N("", "a"), - N("", "b"), -})); - -ADD_CASE_TO_GROUP("simple map, empty keys 2JQS, v2", -R"( -: - a -: - b -)", -N(MAP, L{ - N("", "a"), - N("", "b"), -})); - -ADD_CASE_TO_GROUP("simple map, empty keys 4ABK, v1", -R"({ -: a, -: b, -})", -N(MAP, L{ - N("", "a"), - N("", "b"), -})); - -ADD_CASE_TO_GROUP("simple map, empty keys 4ABK, v2", -R"({ -: - a, -: - b, -})", -N(MAP, L{ - N("", "a"), - N("", "b"), -})); - -ADD_CASE_TO_GROUP("simple map, values on next line 4MUZ, v1", -R"({foo -: bar, -baz -: bat -})", -N(MAP, L{ - N("foo", "bar"), - N("baz", "bat"), -})); - -ADD_CASE_TO_GROUP("simple map, values on next line 4MUZ, v2", -R"({foo -: - bar, -baz -: - bat -})", -N(MAP, L{ - N("foo", "bar"), - N("baz", "bat"), -})); - -/* this is not valid YAML: plain scalars can't have ':' as a token -ADD_CASE_TO_GROUP("simple map, values on next line 4MUZ, v3", -R"(foo -: bar -baz -: bat -)", -N(MAP, L{ - N("foo", "bar"), - N("baz", "bat"), -})); - -ADD_CASE_TO_GROUP("simple map, values on next line 4MUZ, v4", -R"(foo -: - bar -baz -: - bat -)", -N(MAP, L{ - N("foo", "bar"), - N("baz", "bat"), -})); -*/ - -ADD_CASE_TO_GROUP("json compact", -R"(--- -{ -"foo0":"bar", -"foo1":bar, -"foo3":{"a":map}, -"foo5":[a,seq], -} ---- {"foo0":"bar","foo1":bar,"foo3":{"a":map},"foo5":[a,seq],} -)", -N(STREAM, - L{ - N(DOCMAP, L{ - N(KEYVAL|KEYQUO|VALQUO,"foo0","bar"), - N(KEYVAL|KEYQUO,"foo1","bar"), - N(KEYMAP|KEYQUO,"foo3", L{N(KEYVAL|KEYQUO,"a","map")}), - N(KEYSEQ|KEYQUO,"foo5", L{N("a"),N("seq")}), - }), - N(DOCMAP, L{ - N(KEYVAL|KEYQUO|VALQUO,"foo0","bar"), - N(KEYVAL|KEYQUO,"foo1","bar"), - N(KEYMAP|KEYQUO,"foo3", L{N(KEYVAL|KEYQUO,"a","map")}), - N(KEYSEQ|KEYQUO,"foo5", L{N("a"),N("seq")}), - }), - }) -); - - -ADD_CASE_TO_GROUP("issue223 0 fails", -R"( - A: - - 1 - - 4 - B: - - 2 - - 3 - )", -N(L{ - N("A", L{N("1"), N("4")}), - N("B", L{N("2"), N("3")}), - }) -); - -ADD_CASE_TO_GROUP("issue223 1 passes", -R"(A: - - 1 - - 4 -B: - - 2 - - 3 -)", -N(L{ - N("A", L{N("1"), N("4")}), - N("B", L{N("2"), N("3")}), - }) -); - -ADD_CASE_TO_GROUP("issue223 2 passes", -R"(A: - - 1 - - 4 -B: - - 2 - - 3)", -N(L{ - N("A", L{N("1"), N("4")}), - N("B", L{N("2"), N("3")}), - }) -); -ADD_CASE_TO_GROUP("issue223 3 fails", -R"(A: - - 1 - - 4 -B: - - 2 - - 3 - )", -N(L{ - N("A", L{N("1"), N("4")}), - N("B", L{N("2"), N("3")}), - }) -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_simple_seq.cpp b/thirdparty/ryml/test/test_simple_seq.cpp deleted file mode 100644 index a24f5d4f7..000000000 --- a/thirdparty/ryml/test/test_simple_seq.cpp +++ /dev/null @@ -1,695 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -TEST(simple_seq, bad_seq1) -{ - Tree tree; - ExpectError::do_check(&tree, [&]{ - parse_in_arena(R"( ---- -[ a, b, c ] ] -)", &tree); - }); -} - -TEST(simple_seq, bad_seq2) -{ - Tree tree; - ExpectError::do_check(&tree, [&]{ - parse_in_arena(R"( ---- -[ [ a, b, c ] -)", &tree); - }); -} - -TEST(simple_seq, two_nested_flow_seqs) -{ - Tree tree = parse_in_arena("[[]]"); - EXPECT_TRUE(tree.rootref().is_seq()); - ASSERT_TRUE(tree.rootref().has_children()); - EXPECT_TRUE(tree.rootref().first_child().is_seq()); - ASSERT_FALSE(tree.rootref().first_child().has_children()); -} - -TEST(simple_seq, many_unmatched_brackets) -{ - std::string src; - src.reserve(10000000u); - for(size_t num_brackets : {4u, 8u, 32u}) - { - SCOPED_TRACE(num_brackets); - for(size_t i = src.size(); i < num_brackets; ++i) - src += '['; - Tree tree; - Location loc = {}; - loc.line = 1; - loc.col = num_brackets + 1u; - ExpectError::do_check(&tree, [&]{ - parse_in_place(to_substr(src), &tree); - }, loc); - } -} - -TEST(simple_seq, missing_quoted_key) -{ - csubstr yaml = R"( -"top1" : - ["0", "1", ] -'top2' : - ["0", "1", ] ---- -"top1" : - - "0" - - "1" -'top2' : - - "0" - - "1" -)"; - test_check_emit_check(yaml, [](Tree const &t){ - size_t doc = 0; - EXPECT_TRUE(t.docref(doc)["top1"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top1"][0].is_val_quoted()); - EXPECT_TRUE(t.docref(doc)["top1"][1].is_val_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"][0].is_val_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"][1].is_val_quoted()); - ++doc; - EXPECT_TRUE(t.docref(doc)["top1"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"].is_key_quoted()); - EXPECT_TRUE(t.docref(doc)["top1"][0].is_val_quoted()); - EXPECT_TRUE(t.docref(doc)["top1"][1].is_val_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"][0].is_val_quoted()); - EXPECT_TRUE(t.docref(doc)["top2"][1].is_val_quoted()); - }); -} - -TEST(simple_seq, deeply_nested_to_cover_parse_stack_resizes) -{ - csubstr yaml = R"( -[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[0, 1, 2, 3, 4, 5, 6, 7]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] -)"; - Tree t = parse_in_arena(yaml); - size_t id = t.root_id(); - while(t.has_children(id)) - id = t.first_child(id); - ASSERT_TRUE(t.ref(id).has_parent()); - NodeRef seq = t.ref(id).parent(); - ASSERT_TRUE(seq.is_seq()); - EXPECT_EQ(seq[0].val(), csubstr("0")); - EXPECT_EQ(seq[1].val(), csubstr("1")); - EXPECT_EQ(seq[2].val(), csubstr("2")); - EXPECT_EQ(seq[3].val(), csubstr("3")); - EXPECT_EQ(seq[4].val(), csubstr("4")); - EXPECT_EQ(seq[5].val(), csubstr("5")); - EXPECT_EQ(seq[6].val(), csubstr("6")); - EXPECT_EQ(seq[7].val(), csubstr("7")); -} - - -#ifdef RYML_WITH_TAB_TOKENS -TEST(simple_seq, block_tab_tokens) -{ - Tree tree = parse_in_arena(R"( ---- # block, spaces only -- 0 -- 1 -- 2 ---- # block, tabs after -- 0 -- 1 -- 2 ---- # block, tabs after token, and after val -- 0 -- 1 -- 2 -)"); - EXPECT_EQ(tree.docref(0)[0].val(), csubstr("0")); - EXPECT_EQ(tree.docref(0)[1].val(), csubstr("1")); - EXPECT_EQ(tree.docref(0)[2].val(), csubstr("2")); - EXPECT_EQ(tree.docref(1)[0].val(), csubstr("0")); - EXPECT_EQ(tree.docref(1)[1].val(), csubstr("1")); - EXPECT_EQ(tree.docref(1)[2].val(), csubstr("2")); -} - -TEST(simple_seq, flow_tab_tokens) -{ - Tree tree = parse_in_arena(R"( ---- # flow, no tabs -[0, 1, 2] ---- # flow, tabs after -[0, 1, 2] ---- # flow, tabs before and after -[0 , 1 , 2] ---- # flow, tabs everywhere - [ - 0 , - 1 , - 2 , - ] -)"); - EXPECT_EQ(tree.docref(0)[0].val(), csubstr("0")); - EXPECT_EQ(tree.docref(0)[1].val(), csubstr("1")); - EXPECT_EQ(tree.docref(0)[2].val(), csubstr("2")); - EXPECT_EQ(tree.docref(1)[0].val(), csubstr("0")); - EXPECT_EQ(tree.docref(1)[1].val(), csubstr("1")); - EXPECT_EQ(tree.docref(1)[2].val(), csubstr("2")); - EXPECT_EQ(tree.docref(2)[0].val(), csubstr("0")); - EXPECT_EQ(tree.docref(2)[1].val(), csubstr("1")); - EXPECT_EQ(tree.docref(2)[2].val(), csubstr("2")); - EXPECT_EQ(tree.docref(3)[0].val(), csubstr("0")); - EXPECT_EQ(tree.docref(3)[1].val(), csubstr("1")); - EXPECT_EQ(tree.docref(3)[2].val(), csubstr("2")); -} -#endif // RYML_WITH_TAB_TOKENS - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -CASE_GROUP(SIMPLE_SEQ) -{ - -ADD_CASE_TO_GROUP("simple seq", -R"(- 0 -- 1 -- 2 -- 3 -)", -L{N{"0"}, N{"1"}, N{"2"}, N{"3"}} -); - - -ADD_CASE_TO_GROUP("simple seq, explicit, single line", -"[0, 1, 2, 3]", -L{N{"0"}, N{"1"}, N{"2"}, N{"3"}} -); - -ADD_CASE_TO_GROUP("simple seq, explicit, single line, trailcomma", -"[0, 1, 2, 3,]", -L{N{"0"}, N{"1"}, N{"2"}, N{"3"},} -); - -ADD_CASE_TO_GROUP("simple seq, explicit, multiline, unindented", -R"([ -0, -1, -2, -3 -])", - L{N{"0"}, N{"1"}, N{"2"}, N{"3"}} -); - -ADD_CASE_TO_GROUP("simple seq, explicit, multiline, unindented, trailcomma", -R"([ -0, -1, -2, -3, -])", - L{N{"0"}, N{"1"}, N{"2"}, N{"3"}} -); - -ADD_CASE_TO_GROUP("simple seq, explicit, multiline, comments inline", -R"([ -0, # bla0 -1, # bla1 -2, # bla2 -3 # bla3 -])", - L{N{"0"}, N{"1"}, N{"2"}, N{"3"}} -); - -ADD_CASE_TO_GROUP("simple seq, explicit, multiline, comments prev line", -R"([ -# bla0 -0, -# bla1 -1, -# bla2 -2, -# bla3 -3 -])", - L{N{"0"}, N{"1"}, N{"2"}, N{"3"}} -); - -ADD_CASE_TO_GROUP("simple seq, explicit, multiline, indented", -R"([ - 0, - 1, - 2, - 3 -])", - L{N{"0"}, N{"1"}, N{"2"}, N{"3"}} -); - -ADD_CASE_TO_GROUP("simple seq, comments inline", -R"( -- 0 # this is a foo -- 1 # this is a bar -- 2 # this is a bar -- 3 # this is a bar -)", - L{N{"0"}, N{"1"}, N{"2"}, N{"3"}} -); - -ADD_CASE_TO_GROUP("simple seq, comments prev line", -R"( -# this is a foo -- 0 -# this is a bar -- 1 -# this is a baz -- 2 -# this is a bat -- 3 -)", - L{N{"0"}, N{"1"}, N{"2"}, N{"3"}} -); - -ADD_CASE_TO_GROUP("simple seq, scalars with special chars, comma", -R"( -- a,b -- c,d -- e,f -- a, b -- c, d -- e, f -- a , b -- c , d -- e , f -- a ,b -- c ,d -- e ,f -)", -L{N{"a,b"}, N{"c,d"}, N{"e,f"}, - N{"a, b"}, N{"c, d"}, N{"e, f"}, - N{"a , b"}, N{"c , d"}, N{"e , f"}, - N{"a ,b"}, N{"c ,d"}, N{"e ,f"}, - } -); - -ADD_CASE_TO_GROUP("simple seq, scalars with special chars, colon", -R"( -- a:b -- "c:d" -- 'e:f' -- a :b -- "c :d" -- 'e :f' -- a : b # THIS IS A KEY-VAL!!! -- "c : d" -- 'e : f' -- a: b # THIS IS A KEY-VAL!!! -- "c: d" -- 'e: f' -)", -L{ - N("a:b"), N(QV, "c:d"), N(QV, "e:f"), - N("a :b"), N(QV, "c :d"), N(QV, "e :f"), - N(L{N("a", "b")}), N(QV, "c : d"), N(QV, "e : f"), - N(L{N("a", "b")}), N(QV, "c: d"), N(QV, "e: f"), - } -); - -ADD_CASE_TO_GROUP("simple seq, scalars with special chars, cardinal", -R"( -- a#b -- "a#b" -- 'a#b' -- a# b -- "a# b" -- 'a# b' -- a # b -- "a # b" -- 'a # b' -- a #b -- "a #b" -- 'a #b' -)", -L{ - N{"a#b"}, N{QV, "a#b"}, N{QV, "a#b"}, - N{"a# b"}, N{QV, "a# b"}, N{QV, "a# b"}, - N{"a"}, N{QV, "a # b"}, N{QV, "a # b"}, - N{"a"}, N{QV, "a #b"}, N{QV, "a #b"}, - } -); - -ADD_CASE_TO_GROUP("simple seq, scalars with special chars, dash", -R"( -- a-b -- "a-b" -- 'a-b' -- a- b -- "a- b" -- 'a- b' -- a - b -- "a - b" -- 'a - b' -- a -b -- "a -b" -- 'a -b' -)", -L{ - N{"a-b"}, N{QV, "a-b"}, N{QV, "a-b"}, - N{"a- b"}, N{QV, "a- b"}, N{QV, "a- b"}, - N{"a - b"}, N{QV, "a - b"}, N{QV, "a - b"}, - N{"a -b"}, N{QV, "a -b"}, N{QV, "a -b"}, - } -); - -ADD_CASE_TO_GROUP("simple seq, scalars with special chars, left-curly", -R"( -- a{b -- "a{b" -- 'a{b' -- a{ b -- "a{ b" -- 'a{ b' -- a { b -- "a { b" -- 'a { b' -- a {b -- "a {b" -- 'a {b' -)", -L{ - N{"a{b"}, N{QV, "a{b"}, N{QV, "a{b"}, - N{"a{ b"}, N{QV, "a{ b"}, N{QV, "a{ b"}, - N{"a { b"}, N{QV, "a { b"}, N{QV, "a { b"}, - N{"a {b"}, N{QV, "a {b"}, N{QV, "a {b"}, - } -); - -ADD_CASE_TO_GROUP("simple seq, scalars with special chars, right-curly", -R"( -- a}b -- "a}b" -- 'a}b' -- a} b -- "a} b" -- 'a} b' -- a } b -- "a } b" -- 'a } b' -- a }b -- "a }b" -- 'a }b' -)", -L{ - N{"a}b"}, N{QV, "a}b"}, N{QV, "a}b"}, - N{"a} b"}, N{QV, "a} b"}, N{QV, "a} b"}, - N{"a } b"}, N{QV, "a } b"}, N{QV, "a } b"}, - N{"a }b"}, N{QV, "a }b"}, N{QV, "a }b"}, - } -); - -ADD_CASE_TO_GROUP("simple seq, scalars with special chars, left-bracket", -R"( -- a[b -- "a[b" -- 'a[b' -- a[ b -- "a[ b" -- 'a[ b' -- a [ b -- "a [ b" -- 'a [ b' -- a [b -- "a [b" -- 'a [b' -)", -L{ - N{"a[b"}, N{QV, "a[b"}, N{QV, "a[b"}, - N{"a[ b"}, N{QV, "a[ b"}, N{QV, "a[ b"}, - N{"a [ b"}, N{QV, "a [ b"}, N{QV, "a [ b"}, - N{"a [b"}, N{QV, "a [b"}, N{QV, "a [b"}, - } -); - -ADD_CASE_TO_GROUP("simple seq, scalars with special chars, right-bracket", -R"( -- a]b -- "a]b" -- 'a]b' -- a] b -- "a] b" -- 'a] b' -- a ] b -- "a ] b" -- 'a ] b' -- a ]b -- "a ]b" -- 'a ]b' -)", -L{ - N{"a]b"}, N{QV, "a]b"}, N{QV, "a]b"}, - N{"a] b"}, N{QV, "a] b"}, N{QV, "a] b"}, - N{"a ] b"}, N{QV, "a ] b"}, N{QV, "a ] b"}, - N{"a ]b"}, N{QV, "a ]b"}, N{QV, "a ]b"}, - } -); - -ADD_CASE_TO_GROUP("simple seq expl, scalars with special chars, comma", -R"([ - a,b, "c,d", 'e,f', - a, b, "c, d", 'e, f', - a , b, "c , d", 'e , f', - a ,b, "c ,d", 'e ,f', -])", -L{ - N{"a"}, N("b"), N(QV, "c,d"), N(QV, "e,f"), - N{"a"}, N("b"), N(QV, "c, d"), N(QV, "e, f"), - N{"a"}, N("b"), N(QV, "c , d"), N(QV, "e , f"), - N{"a"}, N("b"), N(QV, "c ,d"), N(QV, "e ,f"), - } -); - -#ifdef RYML_WITH_TAB_TOKENS -#define _ryml_with_or_without_tabs(with, without) with -#else -#define _ryml_with_or_without_tabs(with, without) without -#endif -ADD_CASE_TO_GROUP("simple seq expl, scalars with special chars, colon", -R"( -- [[], :@] -- [[], :%] -- [[], :^] -- [[], :$] -#- [[], ::] -- [[], : ] -- [[], :`] -)", -L{ - N(L{N(SEQ), N(":@")}), - N(L{N(SEQ), N(":%")}), - N(L{N(SEQ), N(":^")}), - N(L{N(SEQ), N(":$")}), - //N(L{N(SEQ), N("::")}), TODO: yaml playground - N(L{N(SEQ), _ryml_with_or_without_tabs(N(MAP, L{N("", "")}), N(": "))}), - N(L{N(SEQ), N(":`")}), -} -); - -ADD_CASE_TO_GROUP("simple seq expl, scalars with special chars, colon 2", -R"([ -# a:b, # not legal - "c:d", 'e:f', -# a: b, # not legal - "c: d", 'e: f', -# a : b, # not legal - "c : d", 'e : f', -# a :b, # not legal - "c :d", 'e :f', -])", -L{/*...not legal...*/ - /*N{"a"}, N("b"),*/ N(QV, "c:d"), N(QV, "e:f"), - /*N{"a"}, N("b"),*/ N(QV, "c: d"), N(QV, "e: f"), - /*N{"a"}, N("b"),*/ N(QV, "c : d"), N(QV, "e : f"), - /*N{"a"}, N("b"),*/ N(QV, "c :d"), N(QV, "e :f"), - } -); - -ADD_CASE_TO_GROUP("simple seq expl, scalars with special chars, cardinal", -R"([ - a#b, "c#d", 'e#f', - a# b, "c# d", 'e# f', - a # b, "c # d", 'e # f', -, # this is needed because of the comment above - a #b, "c #d", 'e #f', -])", -L{ - N{"a#b"}, N(QV, "c#d"), N(QV, "e#f"), - N{"a# b"}, N(QV, "c# d"), N(QV, "e# f"), - N{"a"}, - N{"a"}, - } -); - -ADD_CASE_TO_GROUP("simple seq expl, scalars with special chars, dash", -R"([ - a-b, "c-d", 'e-f', - a- b, "c- d", 'e- f', - a - b, "c - d", 'e - f', - a -b, "c -d", 'e -f', -])", -L{ - N{"a-b"}, N(QV, "c-d"), N(QV, "e-f"), - N{"a- b"}, N(QV, "c- d"), N(QV, "e- f"), - N{"a - b"}, N(QV, "c - d"), N(QV, "e - f"), - N{"a -b"}, N(QV, "c -d"), N(QV, "e -f"), - } -); - -ADD_CASE_TO_GROUP("simple seq expl, scalars with special chars, left-bracket", -R"([ -# a[b, - "c[d", 'e[f', -# a[ b, - "c[ d", 'e[ f', -# a [ b, - "c [ d", 'e [ f', -# a [b, - "c [d", 'e [f', -])", -L{ - /*N{"a[b"}, */ N(QV, "c[d"), N(QV, "e[f"), - /*N{"a[ b"}, */ N(QV, "c[ d"), N(QV, "e[ f"), - /*N{"a [ b"},*/ N(QV, "c [ d"), N(QV, "e [ f"), - /*N{"a [b"}, */ N(QV, "c [d"), N(QV, "e [f"), - } -); - -ADD_CASE_TO_GROUP("simple seq expl, scalars with special chars, right-bracket", -R"([ -# a]b, - "c]d", 'e]f', -# a] b, - "c] d", 'e] f', -# a ] b, - "c ] d", 'e ] f', -# a ]b, - "c ]d", 'e ]f', -])", -L{ - /*N{"a]b"}, */ N(QV, "c]d"), N(QV, "e]f"), - /*N{"a] b"}, */ N(QV, "c] d"), N(QV, "e] f"), - /*N{"a ] b"},*/ N(QV, "c ] d"), N(QV, "e ] f"), - /*N{"a ]b"}, */ N(QV, "c ]d"), N(QV, "e ]f"), - } -); - -ADD_CASE_TO_GROUP("simple seq expl, scalars with special chars, left-curly", -R"([ -# a{b, - "c{d", 'e{f', -# a{ b, - "c{ d", 'e{ f', -# a { b, - "c { d", 'e { f', -# a {b, - "c {d", 'e {f', -])", -L{ - /*N{"a{b"}, */ N(QV, "c{d"), N(QV, "e{f"), - /*N{"a{ b"}, */ N(QV, "c{ d"), N(QV, "e{ f"), - /*N{"a { b"},*/ N(QV, "c { d"), N(QV, "e { f"), - /*N{"a {b"}, */ N(QV, "c {d"), N(QV, "e {f"), - } -); - -ADD_CASE_TO_GROUP("simple seq expl, scalars with special chars, right-curly", -R"([ -# a}b, - "c}d", 'e}f', -# a} b, - "c} d", 'e} f', -# a } b, - "c } d", 'e } f', -# a }b, - "c }d", 'e }f', -])", -L{ - /*N{"a}b"}, */ N(QV, "c}d"), N(QV, "e}f"), - /*N{"a} b"}, */ N(QV, "c} d"), N(QV, "e} f"), - /*N{"a } b"},*/ N(QV, "c } d"), N(QV, "e } f"), - /*N{"a }b"}, */ N(QV, "c }d"), N(QV, "e }f"), - } -); - -ADD_CASE_TO_GROUP("simple seq, issue 28", -R"(# was failing on https://github.com/biojppm/rapidyaml/issues/28 -enemy: -- actors: - - {name: Enemy_Bokoblin_Junior, value: 4.0} - - {name: Enemy_Bokoblin_Middle, value: 16.0} - - {name: Enemy_Bokoblin_Senior, value: 32.0} - - {name: Enemy_Bokoblin_Dark, value: 48.0} - species: BokoblinSeries -enemy2: -- actors: - - {name: Enemy_Bokoblin_Junior, value: 4.0} - - {name: Enemy_Bokoblin_Middle, value: 16.0} - - {name: Enemy_Bokoblin_Senior, value: 32.0} - - {name: Enemy_Bokoblin_Dark, value: 48.0} - species: BokoblinSeries -)", -L{ - N("enemy", L{N(L{ - N("actors", L{ - N(L{N("name", "Enemy_Bokoblin_Junior"), N("value", "4.0"),}), - N(L{N("name", "Enemy_Bokoblin_Middle"), N("value", "16.0"),}), - N(L{N("name", "Enemy_Bokoblin_Senior"), N("value", "32.0"),}), - N(L{N("name", "Enemy_Bokoblin_Dark"), N("value", "48.0"),}), - }), - N("species", "BokoblinSeries"), - }) - }), - N("enemy2", L{N(L{ - N("actors", L{ - N(L{N("name", "Enemy_Bokoblin_Junior"), N("value", "4.0"),}), - N(L{N("name", "Enemy_Bokoblin_Middle"), N("value", "16.0"),}), - N(L{N("name", "Enemy_Bokoblin_Senior"), N("value", "32.0"),}), - N(L{N("name", "Enemy_Bokoblin_Dark"), N("value", "48.0"),}), - }), - N("species", "BokoblinSeries"), - }) - }), -}); - -ADD_CASE_TO_GROUP("simple seq, invalid character 1", EXPECT_PARSE_ERROR, -R"(- 0 # this is a foo -} -)", - LineCol(2, 1) -); - -ADD_CASE_TO_GROUP("simple seq, invalid character 2", EXPECT_PARSE_ERROR, -R"(- 0 # this is a foo -] -)", - LineCol(2, 1) -); - -ADD_CASE_TO_GROUP("simple seq, invalid character 3", EXPECT_PARSE_ERROR, -R"(- 0 # this is a foo -: -)", - LineCol(2, 1) -); - -ADD_CASE_TO_GROUP("simple seq, invalid character 4", EXPECT_PARSE_ERROR, -R"(- 0 # this is a foo -abcdef! -)", - LineCol(2, 1) -); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_simple_set.cpp b/thirdparty/ryml/test/test_simple_set.cpp deleted file mode 100644 index f209c79c7..000000000 --- a/thirdparty/ryml/test/test_simple_set.cpp +++ /dev/null @@ -1,144 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - - -TEST(simple_set, emit) -{ - const char yaml[] = R"(!!set -? aa -? bb -? cc -)"; - std::string expected = R"(!!set -aa: -bb: -cc: -)"; - Tree t = parse_in_arena(yaml); - auto s = emitrs_yaml(t); - EXPECT_EQ(expected, s); -} - - -TEST(simple_set, emit_doc) -{ - const char yaml[] = R"(--- !!set -? aa -? bb -? cc -)"; - std::string expected = R"(--- !!set -aa: -bb: -cc: -)"; - Tree t = parse_in_arena(yaml); - auto s = emitrs_yaml(t); - EXPECT_EQ(expected, s); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -CASE_GROUP(SIMPLE_SET) -{ - -ADD_CASE_TO_GROUP("doc as set, missing value", -R"(!!set -? a -? b -? -)", -N(TL("!!set", L{N(KEYVAL, "a", {}), N(KEYVAL, "b", {}), N(KEYVAL, {}, "")})) -); - -ADD_CASE_TO_GROUP("doc as set, implicit", -R"(!!set -? a -? b -)", -N(TL("!!set", L{N(KEYVAL, "a", {}), N(KEYVAL, "b", {})})) -); - -ADD_CASE_TO_GROUP("doc as set", -R"(--- !!set -? aa -? bb -? cc -)", -N(STREAM, L{N(DOCMAP, TL("!!set", L{ - N(KEYVAL, "aa", /*"~"*/{}), - N(KEYVAL, "bb", /*"~"*/{}), - N(KEYVAL, "cc", /*"~"*/{})}) -)})); - -ADD_CASE_TO_GROUP("sets 2XXW", -R"( ---- !!set -? Mark McGwire -? Sammy Sosa -? Ken Griff -)", -N(STREAM, L{N(DOCMAP, TL("!!set", L{ - N(KEYVAL, "Mark McGwire", /*"~"*/{}), - N(KEYVAL, "Sammy Sosa", /*"~"*/{}), - N(KEYVAL, "Ken Griff", /*"~"*/{}),}) -)})); - -ADD_CASE_TO_GROUP("sets 2XXW, indented", -R"( - --- !!set - ? Mark McGwire - ? Sammy Sosa - ? Ken Griff -)", -N(STREAM, L{N(DOCMAP, TL("!!set", L{ - N(KEYVAL, "Mark McGwire", /*"~"*/{}), - N(KEYVAL, "Sammy Sosa", /*"~"*/{}), - N(KEYVAL, "Ken Griff", /*"~"*/{}),}) -)})); - -ADD_CASE_TO_GROUP("sets 2XXW, no set", -R"( ---- -? Mark McGwire -? Sammy Sosa -? Ken Griff -)", -N(STREAM, L{N(DOCMAP, L{ - N(KEYVAL, "Mark McGwire", /*"~"*/{}), - N(KEYVAL, "Sammy Sosa", /*"~"*/{}), - N(KEYVAL, "Ken Griff", /*"~"*/{}),} -)})); - -ADD_CASE_TO_GROUP("sets 2XXW, no doc", -R"(!!set -? Mark McGwire -? Sammy Sosa -? Ken Griff -)", -TL("!!set", L{ - N(KEYVAL, "Mark McGwire", /*"~"*/{}), - N(KEYVAL, "Sammy Sosa", /*"~"*/{}), - N(KEYVAL, "Ken Griff", /*"~"*/{}), -})); - -ADD_CASE_TO_GROUP("sets 2XXW, no doc, no set", -R"( -? Mark McGwire -? Sammy Sosa -? Ken Griff -)", -L{ - N(KEYVAL, "Mark McGwire", /*"~"*/{}), - N(KEYVAL, "Sammy Sosa", /*"~"*/{}), - N(KEYVAL, "Ken Griff", /*"~"*/{}), -}); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_single_quoted.cpp b/thirdparty/ryml/test/test_single_quoted.cpp deleted file mode 100644 index d27fdb6e0..000000000 --- a/thirdparty/ryml/test/test_single_quoted.cpp +++ /dev/null @@ -1,356 +0,0 @@ -#include "./test_group.hpp" - -namespace c4 { -namespace yml { - -TEST(single_quoted, test_suite_KSS4) -{ - csubstr yaml = R"( ---- -'quoted -string' ---- 'quoted -string' ---- -- 'quoted - string' ---- -- 'quoted -string' ---- -'quoted - string': 'quoted - string' ---- -'quoted -string': 'quoted -string' -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t.docref(0).val(), "quoted string"); - EXPECT_EQ(t.docref(1).val(), "quoted string"); - EXPECT_EQ(t.docref(2)[0].val(), "quoted string"); - EXPECT_EQ(t.docref(3)[0].val(), "quoted string"); - EXPECT_EQ(t.docref(4)["quoted string"].val(), "quoted string"); - EXPECT_EQ(t.docref(5)["quoted string"].val(), "quoted string"); - }); -} - - -TEST(single_quoted, test_suite_R4YG) -{ - csubstr yaml = R"( -- ' - -detected - -' - -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t[0].val(), csubstr("\t\ndetected\n")); - }); -} - - -TEST(single_quoted, test_suite_PRH3) -{ - csubstr yaml = R"( -- ' 1st non-empty - - 2nd non-empty - 3rd non-empty ' -- ' 1st non-empty - - 2nd non-empty - 3rd non-empty ' -- ' 1st non-empty - - 2nd non-empty - 3rd non-empty ' -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t[0].val(), csubstr(" 1st non-empty\n2nd non-empty 3rd non-empty ")); - EXPECT_EQ(t[1].val(), csubstr(" 1st non-empty\n2nd non-empty 3rd non-empty ")); - EXPECT_EQ(t[2].val(), csubstr(" 1st non-empty\n2nd non-empty 3rd non-empty ")); - }); -} - - -TEST(single_quoted, test_suite_T4YY) -{ - csubstr yaml = R"( ---- -' 1st non-empty - - 2nd non-empty - 3rd non-empty ' ---- -' - -detected - -' - -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_stream()); - ASSERT_TRUE(t.rootref().first_child().is_doc()); - EXPECT_EQ(t.rootref().first_child().val(), csubstr(" 1st non-empty\n2nd non-empty 3rd non-empty ")); - }); -} - -TEST(single_quoted, test_suite_G4RS) -{ - csubstr yaml = R"( -single: '"Howdy!" he cried.' -quoted: ' # Not a ''comment''.' -tie-fighter: '|\-*-/|' -)"; - test_check_emit_check(yaml, [](Tree const &t){ - EXPECT_EQ(t["single"].val() , csubstr(R"("Howdy!" he cried.)")); - EXPECT_EQ(t["quoted"].val() , csubstr(R"( # Not a 'comment'.)")); - EXPECT_EQ(t["tie-fighter"].val(), csubstr(R"(|\-*-/|)")); - }); -} - -TEST(single_quoted, quotes_are_preserved) -{ - csubstr yaml = R"( -'%ROOT': '%VAL' -'%ROOT2': - - '%VAL' - - '%VAL' -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_map()); - ASSERT_TRUE(t.rootref().has_child("%ROOT")); - ASSERT_TRUE(t.rootref().has_child("%ROOT2")); - ASSERT_EQ(t["%ROOT2"].num_children(), 2u); - EXPECT_TRUE(t["%ROOT"].is_key_quoted()); - EXPECT_TRUE(t["%ROOT"].is_val_quoted()); - EXPECT_TRUE(t["%ROOT2"].is_key_quoted()); - EXPECT_TRUE(t["%ROOT2"][0].is_val_quoted()); - EXPECT_TRUE(t["%ROOT2"][1].is_val_quoted()); - }); -} - - -//----------------------------------------------------------------------------- - -void verify_error_is_reported(csubstr case_name, csubstr yaml, Location loc={}) -{ - SCOPED_TRACE(case_name); - SCOPED_TRACE(yaml); - Tree tree; - ExpectError::do_check(&tree, [&](){ - parse_in_arena(yaml, &tree); - }, loc); -} - -TEST(single_quoted, error_on_unmatched_quotes) -{ - verify_error_is_reported("map block", R"(foo: '" -bar: '')"); - verify_error_is_reported("seq block", R"(- '" -- '')"); - verify_error_is_reported("map flow", R"({foo: '", bar: ''})"); - verify_error_is_reported("seq flow", R"(['", ''])"); -} - -TEST(single_quoted, error_on_unmatched_quotes_with_escapes) -{ - verify_error_is_reported("map block", R"(foo: '''" -bar: '')"); - verify_error_is_reported("seq block", R"(- '''" -- '')"); - verify_error_is_reported("map flow", R"({foo: '''", bar: ''})"); - verify_error_is_reported("seq flow", R"(['''", ''])"); -} - -TEST(single_quoted, error_on_unmatched_quotes_at_end) -{ - verify_error_is_reported("map block", R"(foo: '' -bar: '")"); - verify_error_is_reported("seq block", R"(- '' -- '")"); - verify_error_is_reported("map flow", R"({foo: '', bar: '"})"); - verify_error_is_reported("seq flow", R"(['', '"])"); -} - -TEST(single_quoted, error_on_unmatched_quotes_at_end_with_escapes) -{ - verify_error_is_reported("map block", R"(foo: '' -bar: '''")"); - verify_error_is_reported("seq block", R"(- '' -- '''")"); - verify_error_is_reported("map flow", R"({foo: '', bar: '''"})"); - verify_error_is_reported("seq flow", R"(['', '''"])"); -} - -TEST(single_quoted, error_on_unclosed_quotes) -{ - verify_error_is_reported("map block", R"(foo: ', -bar: what)"); - verify_error_is_reported("seq block", R"(- ' -- what)"); - verify_error_is_reported("map flow", R"({foo: ', bar: what})"); - verify_error_is_reported("seq flow", R"([', what])"); -} - -TEST(single_quoted, error_on_unclosed_quotes_with_escapes) -{ - verify_error_is_reported("map block", R"(foo: ''', -bar: what)"); - verify_error_is_reported("seq block", R"(- ''' -- what)"); - verify_error_is_reported("map flow", R"({foo: ''', bar: what})"); - verify_error_is_reported("seq flow", R"([''', what])"); -} - -TEST(single_quoted, error_on_unclosed_quotes_at_end) -{ - verify_error_is_reported("map block", R"(foo: what -bar: ')"); - verify_error_is_reported("seq block", R"(- what -- ')"); - verify_error_is_reported("map flow", R"({foo: what, bar: '})"); - verify_error_is_reported("seq flow", R"([what, '])"); -} - -TEST(single_quoted, error_on_unclosed_quotes_at_end_with_escapes) -{ - verify_error_is_reported("map block", R"(foo: what -bar: ''')"); - verify_error_is_reported("seq block", R"(- what -- ''')"); - verify_error_is_reported("map flow", R"({foo: what, bar: '''})"); - verify_error_is_reported("seq flow", R"([what, '''])"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -CASE_GROUP(SINGLE_QUOTED) -{ - -ADD_CASE_TO_GROUP("squoted, only text", -R"('Some text without any quotes.' -)", - N(DOCVAL | VALQUO, "Some text without any quotes.") -); - -ADD_CASE_TO_GROUP("squoted, with double quotes", -R"('Some "text" "with double quotes"')", - N(DOCVAL | VALQUO, "Some \"text\" \"with double quotes\"") -); - -ADD_CASE_TO_GROUP("squoted, with single quotes", -R"('Some text ''with single quotes''')", - N(DOCVAL | VALQUO, "Some text 'with single quotes'") -); - -ADD_CASE_TO_GROUP("squoted, with single and double quotes", -R"('Some text ''with single quotes'' "and double quotes".')", - N(DOCVAL | VALQUO, "Some text 'with single quotes' \"and double quotes\".") -); - -ADD_CASE_TO_GROUP("squoted, with escapes", -R"('Some text with escapes \n \r \t')", - N(DOCVAL | VALQUO, "Some text with escapes \\n \\r \\t") -); - -ADD_CASE_TO_GROUP("squoted, all", -R"('Several lines of text, -containing ''single quotes'' and "double quotes". Escapes (like \n) don''t do anything. - -Newlines can be added by leaving a blank line. - Leading whitespace on lines is ignored.' -)", - N(DOCVAL | VALQUO, "Several lines of text, containing 'single quotes' and \"double quotes\". Escapes (like \\n) don't do anything.\nNewlines can be added by leaving a blank line. Leading whitespace on lines is ignored.") -); - -ADD_CASE_TO_GROUP("squoted, empty", -R"('')", - N(DOCVAL | VALQUO, "") -); - -ADD_CASE_TO_GROUP("squoted, blank", -R"( -- '' -- ' ' -- ' ' -- ' ' -- ' ' -)", - L{N(QV, ""), N(QV, " "), N(QV, " "), N(QV, " "), N(QV, " ")} -); - -ADD_CASE_TO_GROUP("squoted, numbers", // these should not be quoted when emitting -R"( -- -1 -- -1.0 -- +1.0 -- 1e-2 -- 1e+2 -)", - L{N("-1"), N("-1.0"), N("+1.0"), N("1e-2"), N("1e+2")} -); - -ADD_CASE_TO_GROUP("squoted, trailing space", -R"('a aaaa ')", - N(DOCVAL | VALQUO, "a aaaa ") -); - -ADD_CASE_TO_GROUP("squoted, leading space", -R"(' a aaaa')", - N(DOCVAL | VALQUO, " a aaaa") -); - -ADD_CASE_TO_GROUP("squoted, trailing and leading space", -R"(' 012345 ')", - N(DOCVAL | VALQUO, " 012345 ") -); - -ADD_CASE_TO_GROUP("squoted, 1 squote", -R"('''')", - N(DOCVAL | VALQUO, "'") -); - -ADD_CASE_TO_GROUP("squoted, 2 squotes", -R"('''''')", - N(DOCVAL | VALQUO, "''") -); - -ADD_CASE_TO_GROUP("squoted, 3 squotes", -R"('''''''')", - N(DOCVAL | VALQUO, "'''") -); - -ADD_CASE_TO_GROUP("squoted, 4 squotes", -R"('''''''''')", - N(DOCVAL | VALQUO, "''''") -); - -ADD_CASE_TO_GROUP("squoted, 5 squotes", -R"('''''''''''')", - N(DOCVAL | VALQUO, "'''''") -); - -/* -ADD_CASE_TO_GROUP("squoted, example 2", -R"('This is a key - -that has multiple lines - -': and this is its value -)", - L{N("This is a key\nthat has multiple lines\n", "and this is its value")} -); -*/ -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_singleheader/libryml_singleheader.cpp b/thirdparty/ryml/test/test_singleheader/libryml_singleheader.cpp deleted file mode 100644 index b5f27d1ed..000000000 --- a/thirdparty/ryml/test/test_singleheader/libryml_singleheader.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#define RYML_SINGLE_HDR_DEFINE_NOW -#define C4CORE_SINGLE_HDR_DEFINE_NOW -#include diff --git a/thirdparty/ryml/test/test_stack.cpp b/thirdparty/ryml/test/test_stack.cpp deleted file mode 100644 index 73d7b2cea..000000000 --- a/thirdparty/ryml/test/test_stack.cpp +++ /dev/null @@ -1,857 +0,0 @@ -#ifdef RYML_SINGLE_HEADER -#include "ryml_all.hpp" -#else -#include "c4/yml/detail/stack.hpp" -#endif -#include -#include "./callbacks_tester.hpp" - - -//------------------------------------------- - -namespace c4 { -namespace yml { - -namespace detail { - -template -using istack = stack; -using ip = int const*; - -template -void to_large(istack *s) -{ - size_t sz = 3u * N; - s->reserve(sz); - EXPECT_NE(s->m_stack, s->m_buf); -} - -template -void fill_to_large(istack *s) -{ - size_t sz = 3u * N; - s->reserve(sz); - for(int i = 0, e = (int)sz; i < e; ++i) - s->push(i); - EXPECT_NE(s->m_stack, s->m_buf); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_stack_small_vs_large() -{ - istack s; - for(size_t i = 0; i < N; ++i) - { - s.push(static_cast(i)); - EXPECT_EQ(s.size(), i+1); - } - EXPECT_EQ(s.size(), N); - EXPECT_EQ(s.m_stack, s.m_buf); - for(size_t i = 0; i < N; ++i) - { - EXPECT_EQ(s.top(N-1-i), static_cast(i)); - } - s.push(N); - EXPECT_NE(s.m_stack, s.m_buf); - EXPECT_EQ(s.top(), static_cast(N)); - EXPECT_EQ(s.pop(), static_cast(N)); - EXPECT_NE(s.m_stack, s.m_buf); - for(size_t i = 0; i < N; ++i) - { - EXPECT_EQ(s.top(N-1-i), static_cast(i)); - } -} - -TEST(stack, small_vs_large) -{ - test_stack_small_vs_large<8>(); - test_stack_small_vs_large<16>(); - test_stack_small_vs_large<32>(); - test_stack_small_vs_large<128>(); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_copy_ctor() -{ - istack src; - - // small - for(size_t i = 0; i < N; ++i) - { - src.push((int)i); - } - EXPECT_EQ(src.m_stack, src.m_buf); - ip b = src.begin(); - { - istack dst(src); - EXPECT_EQ(dst.size(), src.size()); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_EQ((ip)src.begin(), b); - EXPECT_NE((ip)dst.begin(), (ip)src.begin()); - } - - // large - for(size_t i = 0; i < 2*N; ++i) - { - src.push((int)i); // large - } - EXPECT_NE(src.m_stack, src.m_buf); - b = src.begin(); - { - istack dst(src); - EXPECT_EQ(dst.size(), src.size()); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ((ip)src.begin(), b); - EXPECT_NE((ip)dst.begin(), (ip)src.begin()); - } -} - -TEST(stack, copy_ctor) -{ - test_copy_ctor<4>(); - test_copy_ctor<8>(); - test_copy_ctor<64>(); - test_copy_ctor<128>(); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_move_ctor() -{ - istack src; - - // small - for(size_t i = 0; i < N; ++i) - { - src.push((int)i); - } - EXPECT_EQ(src.m_stack, src.m_buf); - ip b = src.begin(); - size_t sz = src.size(); - { - istack dst(std::move(src)); - EXPECT_EQ(dst.size(), sz); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_NE(dst.m_stack, b); - EXPECT_EQ(src.size(), size_t(0)); - EXPECT_EQ((ip)src.begin(), src.m_buf); - EXPECT_NE((ip)dst.begin(), b); - } - EXPECT_EQ(src.size(), size_t(0)); - EXPECT_EQ(src.capacity(), N); - EXPECT_EQ(src.m_stack, src.m_buf); - - // redo - for(size_t i = 0; i < N; ++i) - { - src.push((int)i); - } - EXPECT_EQ(src.size(), N); - EXPECT_EQ(src.capacity(), N); - EXPECT_EQ(src.m_stack, src.m_buf); - // large - for(size_t i = 0; i < 2*N; ++i) - { - src.push((int)i); // large - } - EXPECT_EQ(src.size(), 3*N); - EXPECT_NE(src.m_stack, src.m_buf); - b = src.begin(); - sz = src.size(); - { - istack dst(std::move(src)); - EXPECT_EQ(dst.size(), sz); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ(dst.m_stack, b); - EXPECT_EQ(src.capacity(), N); - EXPECT_EQ(src.size(), size_t(0)); - EXPECT_EQ((ip)src.begin(), src.m_buf); - EXPECT_EQ((ip)dst.begin(), b); - } -} - -TEST(stack, move_ctor) -{ - test_move_ctor<4>(); - test_move_ctor<8>(); - test_move_ctor<64>(); - test_move_ctor<128>(); -} - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_copy_assign() -{ - istack dst; - istack srcs; // small - istack srcl; // large - - for(size_t i = 0; i < N; ++i) - { - srcs.push((int)i); // small - srcl.push((int)i); // large - } - for(size_t i = 0; i < 2*N; ++i) - { - srcl.push((int)i); // large - } - EXPECT_EQ(srcs.m_stack, srcs.m_buf); - EXPECT_NE(srcl.m_stack, srcl.m_buf); - - ip bs = srcs.begin(), bl = srcl.begin(); - - { - dst = srcs; - EXPECT_EQ(dst.size(), srcs.size()); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_EQ((ip)srcs.begin(), bs); - EXPECT_NE((ip)dst.begin(), (ip)srcs.begin()); - } - - { - dst = srcl; - EXPECT_EQ(dst.size(), srcl.size()); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ((ip)srcl.begin(), bl); - EXPECT_NE((ip)dst.begin(), (ip)srcl.begin()); - } - - { - dst = srcs; - EXPECT_EQ(dst.size(), srcs.size()); - EXPECT_NE(dst.m_stack, dst.m_buf); // it stays in long mode (it's not trimmed when assigned from a short-mode stack) - EXPECT_EQ((ip)srcs.begin(), bs); - EXPECT_NE((ip)dst.begin(), (ip)srcs.begin()); - } -} - -TEST(stack, copy_assign) -{ - test_copy_assign<4>(); - test_copy_assign<8>(); - test_copy_assign<64>(); - test_copy_assign<128>(); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -template -void test_move_assign() -{ - istack srcs, srcl, dst; - - for(size_t i = 0; i < N; ++i) - { - srcs.push((int)i); // small - srcl.push((int)i); // large - } - for(size_t i = 0; i < 2*N; ++i) - { - srcl.push((int)i); // large - } - EXPECT_EQ(srcs.m_stack, srcs.m_buf); - EXPECT_NE(srcl.m_stack, srcl.m_buf); - - ip bs = srcs.begin()/*, bl = srcl.begin()*/; - size_t szs = srcs.size(), szl = srcl.size(); - - for(int i = 0; i < 10; ++i) - { - EXPECT_FALSE(srcs.empty()); - EXPECT_TRUE(dst.empty()); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_EQ(srcs.m_stack, srcs.m_buf); - - dst = std::move(srcs); - EXPECT_TRUE(srcs.empty()); - EXPECT_FALSE(dst.empty()); - EXPECT_EQ(srcs.size(), size_t(0)); - EXPECT_EQ(srcs.capacity(), N); - EXPECT_EQ(dst.size(), szs); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_EQ(srcs.m_stack, srcs.m_buf); - EXPECT_EQ((ip)srcs.begin(), bs); - EXPECT_NE((ip)dst.begin(), (ip)srcs.begin()); - - srcs = std::move(dst); - } - - for(int i = 0; i < 10; ++i) - { - EXPECT_EQ(srcl.size(), 3*N); - EXPECT_FALSE(srcl.empty()); - EXPECT_TRUE(dst.empty()); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_NE(srcl.m_stack, srcl.m_buf); - - dst = std::move(srcl); - EXPECT_TRUE(srcl.empty()); - EXPECT_FALSE(dst.empty()); - EXPECT_EQ(srcl.size(), size_t(0)); - EXPECT_EQ(srcl.capacity(), N); - EXPECT_EQ(dst.size(), szl); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ(srcl.m_stack, srcl.m_buf); - EXPECT_EQ((ip)srcl.begin(), srcl.m_buf); - EXPECT_NE((ip)dst.begin(), (ip)srcl.begin()); - - srcl = std::move(dst); - } -} - -TEST(stack, move_assign) -{ - test_move_assign<4>(); - test_move_assign<8>(); - test_move_assign<64>(); - test_move_assign<128>(); -} - - -//----------------------------------------------------------------------------- - -template -void test_callbacks_default_ctor() -{ - CallbacksTester td; - CallbacksTester ts; - istack dst; - EXPECT_EQ(dst.m_callbacks, get_callbacks()); -} - -TEST(stack, callbacks_default_ctor) -{ - test_callbacks_default_ctor<4>(); - test_callbacks_default_ctor<8>(); - test_callbacks_default_ctor<64>(); - test_callbacks_default_ctor<128>(); -} - -template -void test_callbacks_ctor() -{ - CallbacksTester td; - CallbacksTester ts; - istack dst(td.callbacks()); - ASSERT_EQ(dst.m_callbacks, td.callbacks()); -} - -TEST(stack, callbacks_ctor) -{ - test_callbacks_ctor<4>(); - test_callbacks_ctor<8>(); - test_callbacks_ctor<64>(); - test_callbacks_ctor<128>(); -} - - -//----------------------------------------------------------------------------- -// copy ctor - -template -void test_callbacks_copy_ctor_small() -{ - CallbacksTester ts("src"); - CallbacksTester td("dst"); - { - istack src(ts.callbacks()); - EXPECT_EQ(src.size(), 0u); - EXPECT_EQ(src.capacity(), N); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - size_t nbefore = ts.num_allocs; - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - istack dst(src); - EXPECT_EQ(dst.size(), 0u); - EXPECT_EQ(dst.capacity(), N); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, nbefore); - EXPECT_EQ(td.num_allocs, 0u); - } -} - -template -void test_callbacks_copy_ctor_large_unfilled() -{ - CallbacksTester ts("src"); - CallbacksTester td("dst"); - { - istack src(ts.callbacks()); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - to_large(&src); - ASSERT_GT(src.capacity(), N); - size_t nbefore = ts.num_allocs; - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_NE(ts.num_allocs, 0u); - istack dst(src); - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); - ASSERT_NE(dst.m_callbacks, td.callbacks()); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, nbefore); - EXPECT_EQ(td.num_allocs, 0u); - } -} - -template -void test_callbacks_copy_ctor_large_filled() -{ - CallbacksTester ts("src"); - CallbacksTester td("dst"); - { - istack src(ts.callbacks()); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - fill_to_large(&src); - ASSERT_GT(src.capacity(), N); - size_t nbefore = ts.num_allocs; - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_NE(ts.num_allocs, 0u); - istack dst(src); - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); - ASSERT_NE(dst.m_callbacks, td.callbacks()); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_GT(ts.num_allocs, nbefore); - EXPECT_EQ(td.num_allocs, 0u); - } -} - -TEST(stack, callbacks_copy_ctor_small) -{ - test_callbacks_copy_ctor_small<4>(); - test_callbacks_copy_ctor_small<8>(); - test_callbacks_copy_ctor_small<64>(); - test_callbacks_copy_ctor_small<128>(); -} - -TEST(stack, callbacks_copy_ctor_large_unfilled) -{ - test_callbacks_copy_ctor_large_unfilled<4>(); - test_callbacks_copy_ctor_large_unfilled<8>(); - test_callbacks_copy_ctor_large_unfilled<64>(); - test_callbacks_copy_ctor_large_unfilled<128>(); -} - -TEST(stack, callbacks_copy_ctor_large_filled) -{ - test_callbacks_copy_ctor_large_filled<4>(); - test_callbacks_copy_ctor_large_filled<8>(); - test_callbacks_copy_ctor_large_filled<64>(); - test_callbacks_copy_ctor_large_filled<128>(); -} - - -//----------------------------------------------------------------------------- -// copy ctor - -template -void test_callbacks_move_ctor_small() -{ - CallbacksTester ts; - istack src(ts.callbacks()); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - size_t nbefore = ts.num_allocs; - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - istack dst(std::move(src)); - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, nbefore); -} - -template -void test_callbacks_move_ctor_large_unfilled() -{ - CallbacksTester ts; - istack src(ts.callbacks()); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - to_large(&src); - size_t nbefore = ts.num_allocs; - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_NE(ts.num_allocs, 0u); - istack dst(std::move(src)); - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, nbefore); -} - -template -void test_callbacks_move_ctor_large_filled() -{ - CallbacksTester ts; - istack src(ts.callbacks()); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - fill_to_large(&src); - size_t nbefore = ts.num_allocs; - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_NE(ts.num_allocs, 0u); - istack dst(std::move(src)); - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, nbefore); -} - -TEST(stack, callbacks_move_ctor_small) -{ - test_callbacks_move_ctor_small<4>(); - test_callbacks_move_ctor_small<8>(); - test_callbacks_move_ctor_small<64>(); - test_callbacks_move_ctor_small<128>(); -} - -TEST(stack, callbacks_move_ctor_large_unfilled) -{ - test_callbacks_move_ctor_large_unfilled<4>(); - test_callbacks_move_ctor_large_unfilled<8>(); - test_callbacks_move_ctor_large_unfilled<64>(); - test_callbacks_move_ctor_large_unfilled<128>(); -} - -TEST(stack, callbacks_move_ctor_large_filled) -{ - test_callbacks_move_ctor_large_filled<4>(); - test_callbacks_move_ctor_large_filled<8>(); - test_callbacks_move_ctor_large_filled<64>(); - test_callbacks_move_ctor_large_filled<128>(); -} - - -//----------------------------------------------------------------------------- -// copy assign - -template -void test_callbacks_copy_assign_to_empty() -{ - CallbacksTester ts("src"); - CallbacksTester td("dst"); - istack src(ts.callbacks()); - istack dst(td.callbacks()); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - fill_to_large(&src); - size_t nbefore = ts.num_allocs; - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_NE(ts.num_allocs, 0u); - dst = src; - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); - ASSERT_NE(dst.m_callbacks, td.callbacks()); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_GT(ts.num_allocs, nbefore); - EXPECT_EQ(td.num_allocs, 0u); -} - -TEST(stack, callbacks_copy_assign_to_empty) -{ - test_callbacks_copy_assign_to_empty<4>(); - test_callbacks_copy_assign_to_empty<8>(); - test_callbacks_copy_assign_to_empty<64>(); - test_callbacks_copy_assign_to_empty<128>(); -} - -template -void test_callbacks_copy_assign_to_nonempty() -{ - CallbacksTester ts("src"); - { - CallbacksTester td("dst"); - istack src(ts.callbacks()); - istack dst(td.callbacks()); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - ASSERT_EQ(dst.m_callbacks, td.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - EXPECT_EQ(td.num_allocs, 0u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(td.num_deallocs, 0u); - fill_to_large(&src); - fill_to_large(&dst); - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(td.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(td.num_deallocs, 0u); - dst = src; - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); // changed to ts - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 2u); - EXPECT_EQ(td.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(td.num_deallocs, 1u); - td.check(); - } - ts.check(); -} - -TEST(stack, callbacks_copy_assign_to_nonempty) -{ - test_callbacks_copy_assign_to_nonempty<4>(); - test_callbacks_copy_assign_to_nonempty<8>(); - test_callbacks_copy_assign_to_nonempty<64>(); - test_callbacks_copy_assign_to_nonempty<128>(); -} - -template -void test_callbacks_move_assign_to_empty() -{ - CallbacksTester ts("src"); - { - CallbacksTester td("dst"); - istack src(ts.callbacks()); - istack dst(td.callbacks()); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - ASSERT_EQ(dst.m_callbacks, td.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - EXPECT_EQ(td.num_allocs, 0u); - fill_to_large(&src); - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_EQ(dst.m_stack, dst.m_buf); - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(td.num_allocs, 0u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(td.num_deallocs, 0u); - dst = std::move(src); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); // changed to ts - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(td.num_allocs, 0u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(td.num_deallocs, 0u); - td.check(); - } - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 1u); - ts.check(); -} - -TEST(stack, callbacks_move_assign_to_empty) -{ - test_callbacks_move_assign_to_empty<4>(); - test_callbacks_move_assign_to_empty<8>(); - test_callbacks_move_assign_to_empty<64>(); - test_callbacks_move_assign_to_empty<128>(); -} - -template -void test_callbacks_move_assign_to_nonempty() -{ - CallbacksTester ts("src"); - { - CallbacksTester td("dst"); - istack src(ts.callbacks()); - istack dst(td.callbacks()); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - ASSERT_EQ(dst.m_callbacks, td.callbacks()); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 0u); - EXPECT_EQ(td.num_allocs, 0u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(td.num_deallocs, 0u); - fill_to_large(&src); - fill_to_large(&dst); - EXPECT_NE(src.m_stack, src.m_buf); - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(td.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(td.num_deallocs, 0u); - dst = std::move(src); - ASSERT_EQ(src.m_callbacks, ts.callbacks()); - ASSERT_EQ(dst.m_callbacks, ts.callbacks()); // changed to ts - EXPECT_NE(dst.m_stack, dst.m_buf); - EXPECT_EQ(src.m_stack, src.m_buf); - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(td.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(td.num_deallocs, 1u); - td.check(); - } - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 1u); - ts.check(); -} - -TEST(stack, callbacks_move_assign_to_nonempty) -{ - test_callbacks_move_assign_to_nonempty<4>(); - test_callbacks_move_assign_to_nonempty<8>(); - test_callbacks_move_assign_to_nonempty<64>(); - test_callbacks_move_assign_to_nonempty<128>(); -} - - -//----------------------------------------------------------------------------- - -template -void test_reserve() -{ - { - CallbacksTester ts; - { - istack s(ts.callbacks()); - EXPECT_EQ(ts.num_allocs, 0u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(s.capacity(), N); - s.reserve(4*N); - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(s.capacity(), 4*N); - } - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 1u); - ts.check(); - } - { - CallbacksTester ts; - { - istack s(ts.callbacks()); - EXPECT_EQ(ts.num_allocs, 0u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(s.capacity(), N); - s.reserve(4*N); - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 0u); - EXPECT_EQ(s.capacity(), 4*N); - s._free(); - } - EXPECT_EQ(ts.num_allocs, 1u); - EXPECT_EQ(ts.num_deallocs, 1u); - ts.check(); - } -} - -TEST(stack, reserve_capacity) -{ - test_reserve<10>(); - test_reserve<20>(); -} - - -template -void grow_to_large__push() -{ - istack s; - int ni = (int)N; - for(int i = 0; i < NumTimes * ni; ++i) - { - s.push(i); - if(i < ni) - EXPECT_EQ(s.m_stack, s.m_buf) << i; - else - EXPECT_NE(s.m_stack, s.m_buf) << i; - } - for(int i = 0; i < NumTimes * ni; ++i) - { - EXPECT_EQ(s.bottom((size_t)i), i); - } -} - -TEST(stack, push_to_large_twice) -{ - grow_to_large__push<10, 8>(); - grow_to_large__push<20, 8>(); - grow_to_large__push<32, 8>(); -} - -template -void grow_to_large__push_top() -{ - istack s; - int ni = (int)N; - s.push(0); - for(int i = 1; i < NumTimes * ni; ++i) - { - s.push_top(); - EXPECT_EQ(s.top(), i-1) << i; - s.top() = i; - if(i < ni) - EXPECT_EQ(s.m_stack, s.m_buf) << i; - else - EXPECT_NE(s.m_stack, s.m_buf) << i; - } - for(int i = 0; i < NumTimes * ni; ++i) - { - EXPECT_EQ(s.bottom((size_t)i), i); - } -} - -TEST(stack, push_top_to_large_twice) -{ - grow_to_large__push_top<10, 8>(); - grow_to_large__push_top<20, 8>(); - grow_to_large__push_top<32, 8>(); -} - -} // namespace detail -} // namespace yml -} // namespace c4 - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// this is needed to use the test case library - -#ifndef RYML_SINGLE_HEADER -#include "c4/substr.hpp" -#endif - -namespace c4 { -namespace yml { -struct Case; -Case const* get_case(csubstr /*name*/) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_style.cpp b/thirdparty/ryml/test/test_style.cpp deleted file mode 100644 index 5b46d075b..000000000 --- a/thirdparty/ryml/test/test_style.cpp +++ /dev/null @@ -1,616 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/std/std.hpp" -#include "c4/yml/parse.hpp" -#include "c4/yml/emit.hpp" -#include -#include -#include -#endif - -#include "./test_case.hpp" - -#include - -namespace c4 { -namespace yml { - -std::string emit2str(Tree const& t) -{ - return emitrs_yaml(t); -} - - -TEST(style, flags) -{ - Tree tree = parse_in_arena("foo: bar"); - EXPECT_TRUE(tree.rootref().type().default_block()); - EXPECT_FALSE(tree.rootref().type().marked_flow()); - EXPECT_FALSE(tree.rootref().type().marked_flow_sl()); - EXPECT_FALSE(tree.rootref().type().marked_flow_ml()); - tree._add_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - EXPECT_FALSE(tree.rootref().type().default_block()); - EXPECT_TRUE(tree.rootref().type().marked_flow()); - EXPECT_TRUE(tree.rootref().type().marked_flow_sl()); - EXPECT_FALSE(tree.rootref().type().marked_flow_ml()); - tree._rem_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - tree._add_flags(tree.root_id(), _WIP_STYLE_FLOW_ML); - EXPECT_FALSE(tree.rootref().type().default_block()); - EXPECT_TRUE(tree.rootref().type().marked_flow()); - EXPECT_FALSE(tree.rootref().type().marked_flow_sl()); - EXPECT_TRUE(tree.rootref().type().marked_flow_ml()); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -csubstr scalar_yaml = R"( -this is the key: >- - this is the multiline - "val" with - - 'empty' lines -)"; - -void check_same_emit(Tree const& expected) -{ - #if 0 - #define _showtrees(num) \ - std::cout << "--------\nEMITTED" #num "\n--------\n"; \ - std::cout << ws ## num; \ - std::cout << "--------\nACTUAL" #num "\n--------\n"; \ - print_tree(actual ## num); \ - std::cout << "--------\nEXPECTED" #num "\n--------\n"; \ - print_tree(expected) - #else - #define _showtrees(num) - #endif - - std::string ws1, ws2, ws3, ws4; - emitrs_yaml(expected, &ws1); - { - SCOPED_TRACE("actual1"); - Tree actual1 = parse_in_arena(to_csubstr(ws1)); - _showtrees(1); - test_compare(actual1, expected); - emitrs_yaml(actual1, &ws2); - } - { - SCOPED_TRACE("actual2"); - Tree actual2 = parse_in_arena(to_csubstr(ws2)); - _showtrees(2); - test_compare(actual2, expected); - emitrs_yaml(actual2, &ws3); - } - { - SCOPED_TRACE("actual3"); - Tree actual3 = parse_in_arena(to_csubstr(ws3)); - _showtrees(3); - test_compare(actual3, expected); - emitrs_yaml(actual3, &ws4); - } - { - SCOPED_TRACE("actual4"); - Tree actual4 = parse_in_arena(to_csubstr(ws4)); - _showtrees(4); - test_compare(actual4, expected); - } -} - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -TEST(style, noflags) -{ - Tree expected = parse_in_arena("{}"); - NodeRef r = expected.rootref(); - r["normal"] |= MAP; - r["normal"]["singleline"] = "foo"; - r["normal"]["multiline"] |= MAP; - r["normal"]["multiline"]["____________"] = "foo"; - r["normal"]["multiline"]["____mid_____"] = "foo\nbar"; - r["normal"]["multiline"]["____mid_end1"] = "foo\nbar\n"; - r["normal"]["multiline"]["____mid_end2"] = "foo\nbar\n\n"; - r["normal"]["multiline"]["____mid_end3"] = "foo\nbar\n\n\n"; - r["normal"]["multiline"]["____________"] = "foo"; - r["normal"]["multiline"]["____________"] = "foo bar"; - r["normal"]["multiline"]["________end1"] = "foo bar\n"; - r["normal"]["multiline"]["________end2"] = "foo bar\n\n"; - r["normal"]["multiline"]["________end3"] = "foo bar\n\n\n"; - r["normal"]["multiline"]["beg_________"] = "\nfoo"; - r["normal"]["multiline"]["beg_mid_____"] = "\nfoo\nbar"; - r["normal"]["multiline"]["beg_mid_end1"] = "\nfoo\nbar\n"; - r["normal"]["multiline"]["beg_mid_end2"] = "\nfoo\nbar\n\n"; - r["normal"]["multiline"]["beg_mid_end3"] = "\nfoo\nbar\n\n\n"; - r["leading_ws"] |= MAP; - r["leading_ws"]["singleline"] |= MAP; - r["leading_ws"]["singleline"]["space"] = " foo"; - r["leading_ws"]["singleline"]["tab"] = "\tfoo"; - r["leading_ws"]["singleline"]["space_and_tab0"] = " \tfoo"; - r["leading_ws"]["singleline"]["space_and_tab1"] = "\t foo"; - r["leading_ws"]["multiline"] |= MAP; - r["leading_ws"]["multiline"]["beg_________"] = "\n \tfoo"; - r["leading_ws"]["multiline"]["beg_mid_____"] = "\n \tfoo\nbar"; - r["leading_ws"]["multiline"]["beg_mid_end1"] = "\n \tfoo\nbar\n"; - r["leading_ws"]["multiline"]["beg_mid_end2"] = "\n \tfoo\nbar\n\n"; - r["leading_ws"]["multiline"]["beg_mid_end3"] = "\n \tfoo\nbar\n\n\n"; - check_same_emit(expected); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -#ifdef WIP -TEST(style, scalar_retains_style_after_parse) -{ - { - Tree t = parse_in_arena("foo"); - EXPECT_TRUE(t.rootref().type().val_marked_plain()); - EXPECT_FALSE(t.rootref().type().val_marked_squo()); - EXPECT_FALSE(t.rootref().type().val_marked_dquo()); - EXPECT_FALSE(t.rootref().type().val_marked_literal()); - EXPECT_FALSE(t.rootref().type().val_marked_folded()); - EXPECT_EQ(emitrs(t), std::string("foo\n")); - } - { - Tree t = parse_in_arena("'foo'"); - EXPECT_FALSE(t.rootref().type().val_marked_plain()); - EXPECT_TRUE(t.rootref().type().val_marked_squo()); - EXPECT_FALSE(t.rootref().type().val_marked_dquo()); - EXPECT_FALSE(t.rootref().type().val_marked_literal()); - EXPECT_FALSE(t.rootref().type().val_marked_folded()); - EXPECT_EQ(emitrs(t), std::string("'foo'\n")); - } - { - Tree t = parse_in_arena("'foo'"); - EXPECT_FALSE(t.rootref().type().val_marked_plain()); - EXPECT_FALSE(t.rootref().type().val_marked_squo()); - EXPECT_TRUE(t.rootref().type().val_marked_dquo()); - EXPECT_FALSE(t.rootref().type().val_marked_literal()); - EXPECT_FALSE(t.rootref().type().val_marked_folded()); - EXPECT_EQ(emitrs(t), std::string("'foo'\n")); - } - { - Tree t = parse_in_arena("[foo, 'baz', \"bat\"]"); - EXPECT_TRUE(t.rootref().type().marked_flow()); - EXPECT_TRUE(t[0].type().val_marked_plain()); - EXPECT_FALSE(t[0].type().val_marked_squo()); - EXPECT_FALSE(t[0].type().val_marked_dquo()); - EXPECT_FALSE(t[0].type().val_marked_literal()); - EXPECT_FALSE(t[0].type().val_marked_folded()); - EXPECT_FALSE(t[1].type().val_marked_plain()); - EXPECT_TRUE(t[1].type().val_marked_squo()); - EXPECT_FALSE(t[1].type().val_marked_dquo()); - EXPECT_FALSE(t[1].type().val_marked_literal()); - EXPECT_FALSE(t[1].type().val_marked_folded()); - EXPECT_FALSE(t[2].type().val_marked_plain()); - EXPECT_FALSE(t[2].type().val_marked_squo()); - EXPECT_TRUE(t[2].type().val_marked_dquo()); - EXPECT_FALSE(t[2].type().val_marked_literal()); - EXPECT_FALSE(t[2].type().val_marked_folded()); - EXPECT_EQ(emitrs(t), std::string("foo")); - } -} -#endif - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(scalar, base) -{ - Tree tree = parse_in_arena(scalar_yaml); - EXPECT_EQ(tree[0].key(), csubstr("this is the key")); - EXPECT_EQ(tree[0].val(), csubstr("this is the multiline \"val\" with\n'empty' lines")); - EXPECT_EQ(emit2str(tree), R"(this is the key: |- - this is the multiline "val" with - 'empty' lines -)"); - check_same_emit(tree); -} - -TEST(scalar, block_literal) -{ - Tree tree = parse_in_arena(scalar_yaml); - { - SCOPED_TRACE("val only"); - EXPECT_FALSE(tree[0].type().key_marked_literal()); - EXPECT_FALSE(tree[0].type().val_marked_literal()); - tree._add_flags(tree[0].id(), _WIP_VAL_LITERAL); - EXPECT_FALSE(tree[0].type().key_marked_literal()); - EXPECT_TRUE(tree[0].type().val_marked_literal()); - EXPECT_EQ(emit2str(tree), R"(this is the key: |- - this is the multiline "val" with - 'empty' lines -)"); - check_same_emit(tree); - } - { - SCOPED_TRACE("key+val"); - tree._add_flags(tree[0].id(), _WIP_KEY_LITERAL); - EXPECT_TRUE(tree[0].type().key_marked_literal()); - EXPECT_TRUE(tree[0].type().val_marked_literal()); - EXPECT_EQ(emit2str(tree), R"(? |- - this is the key -: |- - this is the multiline "val" with - 'empty' lines -)"); - check_same_emit(tree); - } - { - SCOPED_TRACE("key only"); - tree._rem_flags(tree[0].id(), _WIP_VAL_LITERAL); - EXPECT_TRUE(tree[0].type().key_marked_literal()); - EXPECT_FALSE(tree[0].type().val_marked_literal()); - EXPECT_EQ(emit2str(tree), R"(? |- - this is the key -: |- - this is the multiline "val" with - 'empty' lines -)"); - check_same_emit(tree); - } -} - -TEST(scalar, block_folded) -{ - Tree tree = parse_in_arena(scalar_yaml); - { - SCOPED_TRACE("val only"); - EXPECT_FALSE(tree[0].type().key_marked_folded()); - EXPECT_FALSE(tree[0].type().val_marked_folded()); - tree._add_flags(tree[0].id(), _WIP_VAL_FOLDED); - EXPECT_FALSE(tree[0].type().key_marked_folded()); - EXPECT_TRUE(tree[0].type().val_marked_folded()); - EXPECT_EQ(emit2str(tree), R"(this is the key: >- - this is the multiline "val" with - - 'empty' lines -)"); - check_same_emit(tree); - } - { - SCOPED_TRACE("key+val"); - tree._add_flags(tree[0].id(), _WIP_KEY_FOLDED); - EXPECT_TRUE(tree[0].type().key_marked_folded()); - EXPECT_TRUE(tree[0].type().val_marked_folded()); - EXPECT_EQ(emit2str(tree), R"(? >- - this is the key -: >- - this is the multiline "val" with - - 'empty' lines -)"); - check_same_emit(tree); - } - { - SCOPED_TRACE("val only"); - tree._rem_flags(tree[0].id(), _WIP_VAL_FOLDED); - EXPECT_TRUE(tree[0].type().key_marked_folded()); - EXPECT_FALSE(tree[0].type().val_marked_folded()); - EXPECT_EQ(emit2str(tree), R"(? >- - this is the key -: |- - this is the multiline "val" with - 'empty' lines -)"); - check_same_emit(tree); - } -} - -TEST(scalar, squot) -{ - Tree tree = parse_in_arena(scalar_yaml); - EXPECT_FALSE(tree[0].type().key_marked_squo()); - EXPECT_FALSE(tree[0].type().val_marked_squo()); - { - SCOPED_TRACE("val only"); - tree._add_flags(tree[0].id(), _WIP_VAL_SQUO); - EXPECT_FALSE(tree[0].type().key_marked_squo()); - EXPECT_TRUE(tree[0].type().val_marked_squo()); - EXPECT_EQ(emit2str(tree), R"(this is the key: 'this is the multiline "val" with - - ''empty'' lines' -)"); - check_same_emit(tree); - } - { - SCOPED_TRACE("key+val"); - tree._add_flags(tree[0].id(), _WIP_KEY_SQUO); - EXPECT_TRUE(tree[0].type().key_marked_squo()); - EXPECT_TRUE(tree[0].type().val_marked_squo()); - EXPECT_EQ(emit2str(tree), R"('this is the key': 'this is the multiline "val" with - - ''empty'' lines' -)"); - check_same_emit(tree); - } - { - SCOPED_TRACE("key only"); - tree._rem_flags(tree[0].id(), _WIP_VAL_SQUO); - EXPECT_TRUE(tree[0].type().key_marked_squo()); - EXPECT_FALSE(tree[0].type().val_marked_squo()); - EXPECT_EQ(emit2str(tree), R"('this is the key': |- - this is the multiline "val" with - 'empty' lines -)"); - check_same_emit(tree); - } -} - -TEST(scalar, dquot) -{ - Tree tree = parse_in_arena(scalar_yaml); - EXPECT_FALSE(tree[0].type().key_marked_dquo()); - EXPECT_FALSE(tree[0].type().val_marked_dquo()); - { - SCOPED_TRACE("val only"); - tree._add_flags(tree[0].id(), _WIP_VAL_DQUO); - EXPECT_FALSE(tree[0].type().key_marked_dquo()); - EXPECT_TRUE(tree[0].type().val_marked_dquo()); - // visual studio fails to compile this string when used inside - // the EXPECT_EQ() macro below. So we declare it separately - // instead: - csubstr yaml = R"(this is the key: "this is the multiline \"val\" with - - 'empty' lines" -)"; - EXPECT_EQ(emit2str(tree), yaml); - check_same_emit(tree); - } - { - SCOPED_TRACE("key+val"); - tree._add_flags(tree[0].id(), _WIP_KEY_DQUO); - EXPECT_TRUE(tree[0].type().key_marked_dquo()); - EXPECT_TRUE(tree[0].type().val_marked_dquo()); - // visual studio fails to compile this string when used inside - // the EXPECT_EQ() macro below. So we declare it separately - // instead: - csubstr yaml = R"("this is the key": "this is the multiline \"val\" with - - 'empty' lines" -)"; - EXPECT_EQ(emit2str(tree), yaml); - check_same_emit(tree); - } - { - SCOPED_TRACE("key only"); - tree._rem_flags(tree[0].id(), _WIP_VAL_DQUO); - EXPECT_TRUE(tree[0].type().key_marked_dquo()); - EXPECT_FALSE(tree[0].type().val_marked_dquo()); - EXPECT_EQ(emit2str(tree), R"("this is the key": |- - this is the multiline "val" with - 'empty' lines -)"); - check_same_emit(tree); - } -} - -TEST(scalar, plain) -{ - Tree tree = parse_in_arena(scalar_yaml); - EXPECT_FALSE(tree[0].type().key_marked_plain()); - EXPECT_FALSE(tree[0].type().val_marked_plain()); - { - SCOPED_TRACE("val only"); - tree._add_flags(tree[0].id(), _WIP_VAL_PLAIN); - EXPECT_FALSE(tree[0].type().key_marked_plain()); - EXPECT_TRUE(tree[0].type().val_marked_plain()); - EXPECT_EQ(emit2str(tree), R"(this is the key: this is the multiline "val" with - - 'empty' lines -)"); - check_same_emit(tree); - } - { - SCOPED_TRACE("key+val"); - tree._add_flags(tree[0].id(), _WIP_KEY_PLAIN); - EXPECT_TRUE(tree[0].type().key_marked_plain()); - EXPECT_TRUE(tree[0].type().val_marked_plain()); - EXPECT_EQ(emit2str(tree), R"(this is the key: this is the multiline "val" with - - 'empty' lines -)"); - check_same_emit(tree); - } - { - SCOPED_TRACE("key only"); - tree._rem_flags(tree[0].id(), _WIP_VAL_PLAIN); - EXPECT_TRUE(tree[0].type().key_marked_plain()); - EXPECT_FALSE(tree[0].type().val_marked_plain()); - EXPECT_EQ(emit2str(tree), R"(this is the key: |- - this is the multiline "val" with - 'empty' lines -)"); - check_same_emit(tree); - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(stream, block) -{ - Tree tree = parse_in_arena(R"( ---- -scalar -%YAML 1.2 ---- -foo ---- -bar -)"); - EXPECT_TRUE(tree.rootref().is_stream()); - EXPECT_TRUE(tree.docref(0).is_doc()); - EXPECT_TRUE(tree.docref(0).is_val()); - EXPECT_EQ(emit2str(tree), "--- scalar %YAML 1.2\n--- foo\n--- bar\n"); - tree._add_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - EXPECT_EQ(emit2str(tree), "--- scalar %YAML 1.2\n--- foo\n--- bar\n"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(seq, block) -{ - Tree tree = parse_in_arena("[1, 2, 3, 4, 5, 6]"); - EXPECT_EQ(emit2str(tree), R"(- 1 -- 2 -- 3 -- 4 -- 5 -- 6 -)"); -} - -TEST(seq, flow_sl) -{ - Tree tree = parse_in_arena("[1, 2, 3, 4, 5, 6]"); - tree._add_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - EXPECT_EQ(emit2str(tree), R"([1,2,3,4,5,6])"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(keyseq, block) -{ - Tree tree = parse_in_arena("{foo: [1, 2, 3, 4, 5, 6]}"); - EXPECT_TRUE(tree.rootref().type().default_block()); - EXPECT_EQ(emit2str(tree), R"(foo: - - 1 - - 2 - - 3 - - 4 - - 5 - - 6 -)"); - tree = parse_in_arena("{foo: [1, [2, 3], 4, [5, 6]]}"); - EXPECT_EQ(emit2str(tree), R"(foo: - - 1 - - - 2 - - 3 - - 4 - - - 5 - - 6 -)"); -} - -TEST(keyseq, flow_sl) -{ - Tree tree = parse_in_arena("{foo: [1, 2, 3, 4, 5, 6]}"); - EXPECT_TRUE(tree.rootref().type().default_block()); - tree._add_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - EXPECT_FALSE(tree.rootref().type().default_block()); - EXPECT_EQ(emit2str(tree), R"({foo: [1,2,3,4,5,6]})"); - // - tree = parse_in_arena("{foo: [1, [2, 3], 4, [5, 6]]}"); - tree._add_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - EXPECT_EQ(emit2str(tree), R"({foo: [1,[2,3],4,[5,6]]})"); - // - tree._rem_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - tree._add_flags(tree["foo"][1].id(), _WIP_STYLE_FLOW_SL); - tree._add_flags(tree["foo"][3].id(), _WIP_STYLE_FLOW_SL); - EXPECT_EQ(emit2str(tree), R"(foo: - - 1 - - [2,3] - - 4 - - [5,6] -)"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(map, block) -{ - Tree tree = parse_in_arena("{1: 10, 2: 10, 3: 10, 4: 10, 5: 10, 6: 10}"); - EXPECT_EQ(emit2str(tree), R"(1: 10 -2: 10 -3: 10 -4: 10 -5: 10 -6: 10 -)"); -} - -TEST(map, flow_sl) -{ - Tree tree = parse_in_arena("{1: 10, 2: 10, 3: 10, 4: 10, 5: 10, 6: 10}"); - tree._add_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - EXPECT_EQ(emit2str(tree), R"({1: 10,2: 10,3: 10,4: 10,5: 10,6: 10})"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(keymap, block) -{ - Tree tree = parse_in_arena("{foo: {1: 10, 2: 10, 3: 10, 4: 10, 5: 10, 6: 10}}"); - EXPECT_EQ(emit2str(tree), R"(foo: - 1: 10 - 2: 10 - 3: 10 - 4: 10 - 5: 10 - 6: 10 -)"); -} - - -TEST(keymap, flow_sl) -{ - Tree tree = parse_in_arena("{foo: {1: 10, 2: 10, 3: 10, 4: 10, 5: 10, 6: 10}}"); - tree._add_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - EXPECT_EQ(emit2str(tree), R"({foo: {1: 10,2: 10,3: 10,4: 10,5: 10,6: 10}})"); - // - tree = parse_in_arena("{foo: {1: 10, 2: {2: 10, 3: 10}, 4: 10, 5: {5: 10, 6: 10}}}"); - EXPECT_EQ(emit2str(tree), R"(foo: - 1: 10 - 2: - 2: 10 - 3: 10 - 4: 10 - 5: - 5: 10 - 6: 10 -)"); - tree._add_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - EXPECT_EQ(emit2str(tree), R"({foo: {1: 10,2: {2: 10,3: 10},4: 10,5: {5: 10,6: 10}}})"); - tree._rem_flags(tree.root_id(), _WIP_STYLE_FLOW_SL); - tree._add_flags(tree["foo"][1].id(), _WIP_STYLE_FLOW_SL); - tree._add_flags(tree["foo"][3].id(), _WIP_STYLE_FLOW_SL); - EXPECT_EQ(emit2str(tree), R"(foo: - 1: 10 - 2: {2: 10,3: 10} - 4: 10 - 5: {5: 10,6: 10} -)"); -} - - -//------------------------------------------- -// this is needed to use the test case library -Case const* get_case(csubstr /*name*/) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_suite.cpp b/thirdparty/ryml/test/test_suite.cpp deleted file mode 100644 index 22dabf8c4..000000000 --- a/thirdparty/ryml/test/test_suite.cpp +++ /dev/null @@ -1,612 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include -#include -#include -#include -#include -#include -#endif -#include "test_case.hpp" -#include "test_suite/test_suite_common.hpp" -#include "test_suite/test_suite_parts.hpp" -#include "test_suite/test_suite_events.hpp" -#include -#include -#include - - -/* Each case from the test suite contains: - * - * - (awkward) input yaml (in_yaml) - * - (somewhat standard) output equivalent (out_yaml) - * - (when meaningful/possible) json equivalent (in_json) - * - yaml parsing events (events) - * - * Running a test consists of parsing the contents above into a data - * structure, and then repeatedly parsing and emitting yaml in a sort - * of pipe. Ie, (eg for in_yaml) parse in_yaml, emit corresponding - * yaml, then parse this emitted yaml, and so on. Each parse/emit pair - * is named a processing level in this test. */ - - -C4_SUPPRESS_WARNING_MSVC_PUSH -C4_SUPPRESS_WARNING_MSVC(4702) // unreachable code - -#define NLEVELS 4 - -namespace c4 { -namespace yml { - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -struct Events -{ - csubstr filename = {}; - std::string src = {}; - std::string emitted_events = {}; - Tree tree = {}; - mutable Tree adjusted_tree = {}; - mutable Tree tree_from_emitted_events = {}; - bool was_parsed = false; - bool enabled = false; - - void init(csubstr filename_, csubstr src_) - { - filename = filename_; - src.assign(src_.begin(), src_.end()); - tree.clear(); - tree.clear_arena(); - tree.reserve(10); - was_parsed = false; - enabled = true; - } - - void compare_trees(csubstr actual_src, Tree const& actual_tree) const - { - if(actual_src.empty()) - GTEST_SKIP(); - _nfo_logf("SRC:\n{}", actual_src); - _nfo_print_tree("EXPECTED", tree); - _nfo_print_tree("ACTUAL", actual_tree); - test_compare(actual_tree, tree); - } - - void compare_emitted_events(csubstr actual_src, Tree const& tree_from_actual_src) - { - C4_UNUSED(actual_src); - emit_events(&emitted_events, tree_from_actual_src); - _nfo_logf("EXPECTED_EVENTS:\n{}", src); - _nfo_logf("ACTUAL_EVENTS:\n{}", emitted_events); - // we cannot directly compare the event strings, - // so we create a tree from the emitted events, - // and then compare the trees: - tree_from_emitted_events.clear(); - tree_from_emitted_events.reserve(16); - parser.parse(c4::to_csubstr(emitted_events), &tree_from_emitted_events); - _nfo_logf("SRC:\n{}", actual_src); - _nfo_print_tree("ACTUAL_FROM_SOURCE", tree_from_actual_src); - _nfo_print_tree("ACTUAL_FROM_EMITTED_EVENTS", tree_from_emitted_events); - _nfo_print_tree("EXPECTED_FROM_EVENTS", tree); - test_compare(tree_from_emitted_events, tree); - } - - EventsParser parser; - void parse_events(csubstr actual_src) - { - if(was_parsed) - return; - if(actual_src.empty()) - GTEST_SKIP(); - parser.parse(c4::to_csubstr(src), &tree); - if(tree.empty()) - tree.reserve(10); - _nfo_print_tree("EXPECTED", tree); - was_parsed = true; - } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** a processing level */ -struct ProcLevel -{ - size_t level; - ProcLevel *prev; - csubstr filename; - std::string src; - c4::yml::Parser parser; - c4::yml::Tree tree; - std::string emitted; - - bool immutable = false; - bool reuse = false; - bool was_parsed = false; - bool was_emitted = false; - - void init(size_t level_, ProcLevel *prev_, csubstr filename_, csubstr src_, bool immutable_, bool reuse_) - { - level = level_; - prev = prev_; - filename = filename_; - src.assign(src_.begin(), src_.end()); - immutable = immutable_; - reuse = reuse_; - was_parsed = false; - was_emitted = false; - } - - void receive_src(ProcLevel & prev_) - { - RYML_ASSERT(&prev_ == prev); - if(!prev_.was_emitted) - { - _nfo_logf("level[{}] not emitted. emit!", prev_.level); - prev_.emit(); - } - if(src != prev_.emitted) - { - was_parsed = false; - was_emitted = false; - src = prev_.emitted; - } - } - - template - void log(const char* context, T const& v) - { - C4_UNUSED(context); - C4_UNUSED(v); - #if RYML_NFO - constexpr const char sep[] = "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+\n"; - c4::log("{}:\n{}{}{}", context, sep, v, sep); - #endif - } - - void parse() - { - if(was_parsed) - return; - if(prev) - { - receive_src(*prev); - } - _nfo_logf("level[{}]: parsing source:\n{}", level, src); - if(reuse) - { - tree.clear(); - if(immutable) - parser.parse_in_arena(filename, c4::to_csubstr(src), &tree); - else - parser.parse_in_place(filename, c4::to_substr(src), &tree); - } - else - { - if(immutable) - tree = parse_in_arena(filename, c4::to_csubstr(src)); - else - tree = parse_in_place(filename, c4::to_substr(src)); - } - _nfo_print_tree("PARSED", tree); - tree.resolve_tags(); - _nfo_print_tree("RESOLVED TAGS", tree); - was_parsed = true; - //_resolve_if_needed(); - } - - void _resolve_if_needed() - { - ConstNodeRef root = tree.rootref(); - bool has_anchors_or_refs = root.visit([](ConstNodeRef const& node, size_t /*level*/){ - return (node.is_anchor() || node.is_ref()); - }); - if(has_anchors_or_refs) - { - tree.resolve(); - _nfo_print_tree("RESOLVED", tree); - } - } - - void emit() - { - if(was_emitted) - return; - if(!was_parsed) - { - _nfo_logf("level[{}] not parsed. parse!", level); - parse(); - } - emitrs_yaml(tree, &emitted); - csubstr ss = to_csubstr(emitted); - if(ss.ends_with("\n...\n")) - emitted.resize(emitted.size() - 4); - was_emitted = true; - _nfo_logf("EMITTED:\n{}", emitted); - } - - void compare_trees(ProcLevel & prev_) - { - RYML_ASSERT(&prev_ == prev); - if(!prev_.was_parsed) - { - _nfo_logf("level[{}] not parsed. parse!", prev_.level); - prev_.parse(); - } - if(!was_parsed) - { - _nfo_logf("level[{}] not parsed. parse!", level); - parse(); - } - _nfo_print_tree("PREV_", prev_.tree); - _nfo_print_tree("CURR", tree); - test_compare(prev_.tree, tree); - } - - void compare_emitted(ProcLevel & prev_) - { - RYML_ASSERT(&prev_ == prev); - if(!prev_.was_emitted) - { - _nfo_logf("level[{}] not emitted. emit!", prev_.level); - prev_.emit(); - } - if(!was_emitted) - { - _nfo_logf("level[{}] not emitted. emit!", level); - emit(); - } - _nfo_logf("level[{}]: EMITTED:\n{}", prev_.level, prev_.emitted); - _nfo_logf("level[{}]: EMITTED:\n{}", level, emitted); - if(emitted != prev_.emitted) - { - // workaround for lack of idempotency in tag normalization. - Tree from_prev = parse_in_arena(to_csubstr(prev_.emitted)); - Tree from_this = parse_in_arena(to_csubstr(emitted)); - from_prev.resolve_tags(); - from_this.resolve_tags(); - test_compare(from_prev, from_this); - } - } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** holds data for one particular test suite approach. */ -struct Approach -{ - csubstr casename; - csubstr filename; - ProcLevel levels[NLEVELS] = {}; - AllowedFailure allowed_failure = {}; - bool enabled = false; - bool expect_error = false; - - void init(csubstr casename_, csubstr filename_, csubstr src_, bool immutable_, bool reuse_, bool expect_error_) - { - casename = casename_; - filename = filename_; - allowed_failure = is_failure_expected(casename); - size_t level_index = 0; - ProcLevel *prev = nullptr; - for(ProcLevel &l : levels) - { - l.init(level_index++, prev, filename, src_, immutable_, reuse_); - prev = &l; - } - expect_error = expect_error_; - } - - csubstr src() const { return c4::to_csubstr(levels[0].src); } - bool skip() const { return allowed_failure; } - - void parse(size_t num, bool emit) - { - if(allowed_failure) - GTEST_SKIP(); - for(size_t i = 0; i < num; ++i) - { - levels[i].parse(); - if(emit) - levels[i].emit(); - if(i + 1 < num) - levels[i+1].receive_src(levels[i]); - } - } - - void compare_trees(size_t num) - { - if(allowed_failure) - GTEST_SKIP(); - for(size_t i = 1; i < num; ++i) - levels[i].compare_trees(levels[i-1]); - } - void compare_trees(size_t num, Approach & other) - { - if(allowed_failure) - GTEST_SKIP(); - for(size_t i = 0; i < num; ++i) - levels[i].compare_trees(other.levels[i]); - } - - void compare_emitted(size_t num) - { - if(allowed_failure) - GTEST_SKIP(); - for(size_t i = 1; i < num; ++i) - levels[i].compare_emitted(levels[i-1]); - } - void compare_emitted(size_t num, Approach & other) - { - if(allowed_failure) - GTEST_SKIP(); - for(size_t i = 0; i < num; ++i) - levels[i].compare_emitted(other.levels[i]); - } - - void compare_events(Events *events) - { - if(allowed_failure || filename.ends_with(".json")) - GTEST_SKIP(); - events->parse_events(src()); - parse(1, /*emit*/false); - events->compare_trees(src(), levels[0].tree); - } - - void compare_emitted_events(Events *events) - { - if(allowed_failure || filename.ends_with(".json")) - GTEST_SKIP(); - events->parse_events(src()); - parse(1, /*emit*/false); - events->compare_emitted_events(src(), levels[0].tree); - } - - void check_expected_error() - { - if(allowed_failure) - GTEST_SKIP(); - ExpectError::do_check(&levels[0].tree, [this]{ - levels[0].parse(); - }); - } - -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -/** Each approach can be read from mutable/immutable yaml source and - * with/without reuse. */ -struct Subject -{ - Approach unix_arena; - Approach unix_arena_reuse; - Approach unix_inplace; - Approach unix_inplace_reuse; - - Approach windows_arena; - Approach windows_arena_reuse; - Approach windows_inplace; - Approach windows_inplace_reuse; - - std::string unix_src; - std::string windows_src; - - void init(csubstr casename, csubstr filename, csubstr src, bool expect_error) - { - src = replace_all("\r", "", src, &unix_src); - - unix_arena .init(casename, filename, src, /*immutable*/true , /*reuse*/false, expect_error); - unix_arena_reuse.init(casename, filename, src, /*immutable*/true , /*reuse*/true , expect_error); - unix_inplace .init(casename, filename, src, /*immutable*/false, /*reuse*/false, expect_error); - unix_inplace_reuse.init(casename, filename, src, /*immutable*/false, /*reuse*/true , expect_error); - - src = replace_all("\n", "\r\n", src, &windows_src); - - windows_arena .init(casename, filename, src, /*immutable*/true , /*reuse*/false, expect_error); - windows_arena_reuse.init(casename, filename, src, /*immutable*/true , /*reuse*/true , expect_error); - windows_inplace .init(casename, filename, src, /*immutable*/false, /*reuse*/false, expect_error); - windows_inplace_reuse.init(casename, filename, src, /*immutable*/false, /*reuse*/true , expect_error); - } -}; - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// some utility functions, used below - -/** all the ways to process a test case are available through this - * class. Tests are defined below and use only one of these. */ -struct SuiteCase -{ - csubstr case_title; - csubstr case_dir; - std::string filename; - std::string file_contents; - std::string events_filename; - std::string events_file_contents; - - Subject input; - Events events; - bool expect_error; - - /** loads the several types of tests from an input test suite file */ - SuiteCase(const char *case_title_, const char* case_dir_, const char *input_file) - { - using namespace c4; - using c4::to_csubstr; - - if(to_csubstr(input_file) == "error") - input_file = "in.yaml"; - - case_title = to_csubstr(case_title_); - - case_dir = to_csubstr(case_dir_); - RYML_CHECK(case_dir.find('\\') == yml::npos); - C4_CHECK_MSG(fs::dir_exists(case_dir.str), "dir not found: '%s'", case_dir); - - filename = catrs(case_dir, '/', to_csubstr(input_file)); - C4_CHECK_MSG(fs::file_exists(filename.c_str()), "file not found: '%s'", filename.c_str()); - log("testing suite case: {} {} ({})", case_title, filename, case_dir); - - std::string errfile = catrs(to_csubstr(case_dir_), "/error"); - expect_error = fs::file_exists(errfile.c_str()); - - fs::file_get_contents(filename.c_str(), &file_contents); - input.init(case_title, to_csubstr(filename), to_csubstr(file_contents), expect_error); - - events_filename = catrs(case_dir, "/test.event"); - C4_CHECK(fs::file_exists(events_filename.c_str())); - fs::file_get_contents(events_filename.c_str(), &events_file_contents); - events.init(to_csubstr(events_filename), to_csubstr(events_file_contents)); - - dump("~~~ case: " , case_title , "~~~\n", - "~~~ file: " , filename , "~~~\n", - "~~~ input:\n" , to_csubstr(input.unix_arena.levels[0].src), "~~~\n", - "~~~ events:\n" , events.src , "~~~\n"); - } - -}; - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - - -// a global holding the test case data -SuiteCase* g_suite_case = nullptr; -bool g_do_subcases = true; - - - -#define DEFINE_TESTS(which) \ - \ - \ -struct which : public ::testing::TestWithParam \ -{ \ -}; \ - \ - \ -TEST_P(which, parse) \ -{ \ - RYML_CHECK(GetParam() < NLEVELS); \ - if(g_suite_case->expect_error) \ - GTEST_SKIP(); \ - g_suite_case->input.which.parse(1 + GetParam(), false); \ -} \ - \ - \ -TEST_P(which, compare_trees) \ -{ \ - RYML_CHECK(GetParam() < NLEVELS); \ - if(g_suite_case->expect_error) \ - GTEST_SKIP(); \ - g_suite_case->input.which.compare_trees(1 + GetParam()); \ -} \ - \ - \ -TEST_P(which, emit) \ -{ \ - RYML_CHECK(GetParam() < NLEVELS); \ - if(g_suite_case->expect_error) \ - GTEST_SKIP(); \ - g_suite_case->input.which.parse(1 + GetParam(), true); \ -} \ - \ - \ -TEST_P(which, compare_emitted) \ -{ \ - RYML_CHECK(GetParam() < NLEVELS); \ - if(g_suite_case->expect_error) \ - GTEST_SKIP(); \ - g_suite_case->input.which.compare_emitted(1 + GetParam()); \ -} \ - \ -/*-----------------------------------------------*/ \ - \ -TEST(which##_events, compare) \ -{ \ - if(g_suite_case->expect_error) \ - GTEST_SKIP(); \ - g_suite_case->input.which.compare_events(&g_suite_case->events); \ -} \ - \ -TEST(which##_events, emit_events) \ -{ \ - if(g_suite_case->expect_error) \ - GTEST_SKIP(); \ - g_suite_case->input.which.compare_emitted_events(&g_suite_case->events); \ -} \ - \ -/*-----------------------------------------------*/ \ - \ -TEST(which##_errors, check_expected_error) \ -{ \ - if(!g_suite_case->expect_error) \ - GTEST_SKIP(); \ - g_suite_case->input.which.check_expected_error(); \ -} \ - \ - \ -INSTANTIATE_TEST_SUITE_P(_, which, testing::Range(0, NLEVELS)) - - -DEFINE_TESTS(unix_arena); -DEFINE_TESTS(unix_inplace); -DEFINE_TESTS(unix_arena_reuse); -DEFINE_TESTS(unix_inplace_reuse); -DEFINE_TESTS(windows_arena); -DEFINE_TESTS(windows_inplace); -DEFINE_TESTS(windows_arena_reuse); -DEFINE_TESTS(windows_inplace_reuse); - - -//------------------------------------------- -// this is needed to use the test case library -Case const* get_case(csubstr /*name*/) { return nullptr; } - -} // namespace yml -} // namespace c4 - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -int main(int argc, char* argv[]) -{ - c4::dump("$"); - for(int i = 0; i < argc; ++i) - c4::dump(' ', c4::to_csubstr(argv[i])); - c4::dump("\n"); - - // make gtest parse its args - testing::InitGoogleTest(&argc, argv); - - // now we have only our args to consider - if(argc != 4) - { - log("usage:\n{} ", c4::to_csubstr(argv[0])); - return 1; - } - - // load the test case from the suite file - c4::yml::SuiteCase suite_case(argv[1], argv[2], argv[3]); - c4::yml::g_suite_case = &suite_case; - - return RUN_ALL_TESTS(); -} - -C4_SUPPRESS_WARNING_MSVC_PUSH diff --git a/thirdparty/ryml/test/test_suite/test_suite_common.hpp b/thirdparty/ryml/test/test_suite/test_suite_common.hpp deleted file mode 100644 index 9cfd89f1f..000000000 --- a/thirdparty/ryml/test/test_suite/test_suite_common.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef C4_YML_TEST_SUITE_COMMON_HPP_ -#define C4_YML_TEST_SUITE_COMMON_HPP_ - -#ifndef RYML_SINGLE_HEADER -#include -#include -#include -#include -#include -#include -#endif - -#include -#include - -#include "test_case.hpp" -#include - -#define RYML_NFO (0 || RYML_DBG) - -#if RYML_NFO -#define _nfo_print_tree(title, tree) do { c4::log("{}:{}: " title ":", __FILE__, __LINE__); print_tree(tree); c4::yml::emit(tree, stdout); fflush(stdout); } while(0) -#define _nfo_logf(fmt, ...) do { c4::log("{}:{}: " fmt , __FILE__, __LINE__, __VA_ARGS__); fflush(stdout); } while(0) -#define _nfo_log(fmt) do { c4::log("{}:{}: " fmt , __FILE__, __LINE__); fflush(stdout); } while(0) -#define _nfo_printf(...) printf(__VA_ARGS__) -#else -#define _nfo_print_tree(title, tree) -#define _nfo_logf(fmt, ...) -#define _nfo_log(fmt) -#define _nfo_printf(...) -#endif -#define _nfo_llogf(fmt, ...) _nfo_logf("line[{}]: '{}': " fmt, linenum, line, __VA_ARGS__) -#define _nfo_llog(fmt) _nfo_logf("line[{}]: '{}': " fmt, linenum, line) - - -namespace c4 { -namespace yml { - - -} // namespace yml -} // namespace c4 - - -#endif /* C4_YML_TEST_SUITE_COMMON_HPP_ */ diff --git a/thirdparty/ryml/test/test_suite/test_suite_events.cpp b/thirdparty/ryml/test/test_suite/test_suite_events.cpp deleted file mode 100644 index b359421b5..000000000 --- a/thirdparty/ryml/test/test_suite/test_suite_events.cpp +++ /dev/null @@ -1,607 +0,0 @@ -#include "test_suite_events.hpp" -#include "test_suite_common.hpp" -#ifndef RYML_SINGLE_HEADER -#include -#endif - -namespace c4 { -namespace yml { - -namespace /*anon*/ { - -struct ScalarType -{ - typedef enum { - PLAIN = 0, - SQUOTED, - DQUOTED, - LITERAL, - FOLDED - } ScalarType_e; - - ScalarType_e val = PLAIN; - bool operator== (ScalarType_e v) const { return val == v; } - bool operator!= (ScalarType_e v) const { return val != v; } - ScalarType& operator= (ScalarType_e v) { val = v; return *this; } - - csubstr to_str() const - { - switch(val) - { - case ScalarType::PLAIN: return csubstr("PLAIN"); - case ScalarType::SQUOTED: return csubstr("SQUOTED"); - case ScalarType::DQUOTED: return csubstr("DQUOTED"); - case ScalarType::LITERAL: return csubstr("LITERAL"); - case ScalarType::FOLDED: return csubstr("FOLDED"); - } - C4_ERROR(""); - return csubstr(""); - } - - bool is_quoted() const { return val == ScalarType::SQUOTED || val == ScalarType::DQUOTED; } -}; - - -struct OptionalScalar -{ - csubstr val = {}; - bool was_set = false; - inline operator csubstr() const { return get(); } - inline operator bool() const { return was_set; } - void operator= (csubstr v) { val = v; was_set = true; } - csubstr get() const { RYML_ASSERT(was_set); return val; } -}; - -#if RYML_NFO -size_t to_chars(c4::substr buf, OptionalScalar const& s) -{ - if(!s) - return 0u; - if(s.val.len <= buf.len) - memcpy(buf.str, s.val.str, s.val.len); - return s.val.len; -} -#endif - -csubstr filtered_scalar(csubstr str, ScalarType scalar_type, Tree *tree) -{ - (void)scalar_type; - csubstr tokens[] = {R"(\n)", R"(\t)", R"(\\)"}; - if(!str.first_of_any_iter(std::begin(tokens), std::end(tokens))) - return str; - substr buf = tree->alloc_arena(str.len); // we are going to always replace with less characters - size_t strpos = 0; - size_t bufpos = 0; - auto append_str = [&](size_t pos){ - csubstr rng = str.range(strpos, pos); - memcpy(buf.str + bufpos, rng.str, rng.len); - bufpos += rng.len; - strpos = pos; - }; - size_t i; - auto append_chars = [&](csubstr s, size_t skipstr){ - memcpy(buf.str + bufpos, s.str, s.len); - bufpos += s.len; - i += skipstr - 1; // incremented at the loop - strpos += skipstr; - }; - for(i = 0; i < str.len; ++i) - { - char curr = str[i]; - char next1 = i+1 < str.len ? str[i+1] : '\0'; - if(curr == '\\') - { - if(next1 == '\\') - { - char next2 = i+2 < str.len ? str[i+2] : '\0'; - if(next2 == 'n') - { - append_str(i); - append_chars(R"(\n)", 3u); // '\\n' -> '\n' - } - else if(next2 == 't') - { - append_str(i); - append_chars(R"(\t)", 3u); // '\\t' -> '\t' - } - else - { - append_str(i); - append_chars(R"(\)", 2u); // '\\' -> '\' - } - } - else if(next1 == 'n') - { - append_str(i); - append_chars("\n", 2u); - } - else if(next1 == 't') - { - append_str(i); - append_chars("\t", 2u); - } - } - } - append_str(str.len); - buf = buf.first(bufpos); - _nfo_logf("filtering: result=~~~{}~~~", buf); - return buf; -} - -struct Scalar -{ - OptionalScalar scalar = {}; - OptionalScalar anchor = {}; - OptionalScalar ref = {}; - OptionalScalar tag = {}; - ScalarType type = {}; - inline operator bool() const { if(anchor || tag) { RYML_ASSERT(scalar); } return scalar.was_set; } - void add_key_props(Tree *tree, size_t node) const - { - if(ref) - { - _nfo_logf("node[{}]: set key ref: '{}'", node, ref); - tree->set_key_ref(node, ref); - } - if(anchor) - { - _nfo_logf("node[{}]: set key anchor: '{}'", node, anchor); - tree->set_key_anchor(node, anchor); - } - if(tag) - { - csubstr ntag = normalize_tag(tag); - _nfo_logf("node[{}]: set key tag: '{}' -> '{}'", node, tag, ntag); - tree->set_key_tag(node, ntag); - } - if(type.is_quoted()) - { - _nfo_logf("node[{}]: set key as quoted", node); - tree->_add_flags(node, KEYQUO); - } - } - void add_val_props(Tree *tree, size_t node) const - { - if(ref) - { - _nfo_logf("node[{}]: set val ref: '{}'", node, ref); - tree->set_val_ref(node, ref); - } - if(anchor) - { - _nfo_logf("node[{}]: set val anchor: '{}'", node, anchor); - tree->set_val_anchor(node, anchor); - } - if(tag) - { - csubstr ntag = normalize_tag(tag); - _nfo_logf("node[{}]: set val tag: '{}' -> '{}'", node, tag, ntag); - tree->set_val_tag(node, ntag); - } - if(type.is_quoted()) - { - _nfo_logf("node[{}]: set val as quoted", node); - tree->_add_flags(node, VALQUO); - } - } - csubstr filtered_scalar(Tree *tree) const - { - return ::c4::yml::filtered_scalar(scalar, type, tree); - } -}; - -csubstr parse_anchor_and_tag(csubstr tokens, OptionalScalar *anchor, OptionalScalar *tag) -{ - *anchor = OptionalScalar{}; - *tag = OptionalScalar{}; - if(tokens.begins_with('&')) - { - size_t pos = tokens.first_of(' '); - if(pos == (size_t)csubstr::npos) - { - *anchor = tokens.sub(1); - tokens = {}; - } - else - { - *anchor = tokens.first(pos).sub(1); - tokens = tokens.right_of(pos); - } - _nfo_logf("anchor: {}", *anchor); - } - if(tokens.begins_with('<')) - { - size_t pos = tokens.find('>'); - RYML_ASSERT(pos != (size_t)csubstr::npos); - *tag = tokens.first(pos + 1); - tokens = tokens.right_of(pos).triml(' '); - _nfo_logf("tag: {}", *tag); - } - return tokens; -} - -} // namespace /*anon*/ - -void EventsParser::parse(csubstr src, Tree *C4_RESTRICT tree_) -{ - struct ParseLevel { size_t tree_node; }; - detail::stack m_stack = {}; - Tree &C4_RESTRICT tree = *tree_; - size_t linenum = 0; - Scalar key = {}; - _nfo_logf("parsing events! src:\n{}", src); - for(csubstr line : src.split('\n')) - { - line = line.trimr('\r'); - line = line.triml(' '); - _nfo_printf("\n\n-----------------------\n"); - _nfo_llog(""); - _nfo_logf("line[{}]: top={} type={}", linenum, m_stack.empty() ? tree.root_id() : m_stack.top().tree_node, NodeType::type_str(tree.type(m_stack.empty() ? tree.root_id() : m_stack.top().tree_node))); - if(line.begins_with("=VAL ")) - { - line = line.stripl("=VAL "); - ASSERT_GE(m_stack.size(), 0u); - Scalar curr = {}; - line = parse_anchor_and_tag(line, &curr.anchor, &curr.tag); - if(line.begins_with('"')) - { - _nfo_llog("double-quoted scalar!"); - curr.scalar = line.sub(1); - curr.type = ScalarType::DQUOTED; - } - else if(line.begins_with('\'')) - { - _nfo_llog("single-quoted scalar!"); - curr.scalar = line.sub(1); - curr.type = ScalarType::SQUOTED; - } - else if(line.begins_with('|')) - { - _nfo_llog("block literal scalar!"); - curr.scalar = line.sub(1); - curr.type = ScalarType::LITERAL; - } - else if(line.begins_with('>')) - { - _nfo_llog("block folded scalar!"); - curr.scalar = line.sub(1); - curr.type = ScalarType::FOLDED; - } - else - { - _nfo_llog("plain scalar"); - ASSERT_TRUE(line.begins_with(':')); - curr.scalar = line.sub(1); - } - _nfo_logf("parsed scalar: '{}'", curr.scalar); - if(m_stack.empty()) - { - _nfo_log("stack was empty, pushing root as DOC..."); - //tree._p(tree.root_id())->m_type.add(DOC); - m_stack.push({tree.root_id()}); - } - ParseLevel &top = m_stack.top(); - if(tree.is_seq(top.tree_node)) - { - _nfo_logf("is seq! seq_id={}", top.tree_node); - ASSERT_FALSE(key); - ASSERT_TRUE(curr); - _nfo_logf("seq[{}]: adding child", top.tree_node); - size_t node = tree.append_child(top.tree_node); - NodeType_e as_doc = tree.is_stream(top.tree_node) ? DOC : NOTYPE; - _nfo_logf("seq[{}]: child={} val='{}' as_doc=", top.tree_node, node, curr.scalar, NodeType::type_str(as_doc)); - tree.to_val(node, curr.filtered_scalar(&tree), as_doc); - curr.add_val_props(&tree, node); - } - else if(tree.is_map(top.tree_node)) - { - _nfo_logf("is map! map_id={}", top.tree_node); - if(!key) - { - _nfo_logf("store key='{}' anchor='{}' tag='{}' type={}", curr.scalar, curr.anchor, curr.tag, curr.type.to_str()); - key = curr; - } - else - { - _nfo_logf("map[{}]: adding child", top.tree_node); - size_t node = tree.append_child(top.tree_node); - NodeType_e as_doc = tree.is_stream(top.tree_node) ? DOC : NOTYPE; - _nfo_logf("map[{}]: child={} key='{}' val='{}' as_doc={}", top.tree_node, node, key.scalar, curr.scalar, NodeType::type_str(as_doc)); - tree.to_keyval(node, key.filtered_scalar(&tree), curr.filtered_scalar(&tree), as_doc); - key.add_key_props(&tree, node); - curr.add_val_props(&tree, node); - _nfo_logf("clear key='{}'", key.scalar); - key = {}; - } - } - else - { - _nfo_logf("setting tree_node={} to DOCVAL...", top.tree_node); - tree.to_val(top.tree_node, curr.filtered_scalar(&tree), DOC); - curr.add_val_props(&tree, top.tree_node); - } - } - else if(line.begins_with("=ALI ")) - { - csubstr alias = line.stripl("=ALI "); - _nfo_logf("REF token: {}", alias); - ParseLevel top = m_stack.top(); - if(tree.is_seq(top.tree_node)) - { - _nfo_logf("node[{}] is seq: set {} as val ref", top.tree_node, alias); - ASSERT_FALSE(key); - size_t node = tree.append_child(top.tree_node); - tree.to_val(node, alias); - tree.set_val_ref(node, alias); - } - else if(tree.is_map(top.tree_node)) - { - if(key) - { - _nfo_logf("node[{}] is map and key '{}' is pending: set {} as val ref", top.tree_node, key.scalar, alias); - size_t node = tree.append_child(top.tree_node); - tree.to_keyval(node, key.filtered_scalar(&tree), alias); - key.add_key_props(&tree, node); - tree.set_val_ref(node, alias); - _nfo_logf("clear key='{}'", key); - key = {}; - } - else - { - _nfo_logf("node[{}] is map and no key is pending: save {} as key ref", top.tree_node, alias); - key.scalar = alias; - key.ref = alias; - } - } - else - { - C4_ERROR("ALI event requires map or seq"); - } - } - else if(line.begins_with("+SEQ")) - { - _nfo_log("pushing SEQ"); - OptionalScalar anchor = {}; - OptionalScalar tag = {}; - csubstr more_tokens = line.stripl("+SEQ").triml(' '); - if(more_tokens.begins_with('[')) - { - ASSERT_TRUE(more_tokens.begins_with("[]")); - more_tokens = more_tokens.offs(2, 0).triml(' '); - } - parse_anchor_and_tag(more_tokens, &anchor, &tag); - size_t node = tree.root_id(); - if(m_stack.empty()) - { - _nfo_log("stack was empty, set root to SEQ"); - tree._add_flags(node, SEQ); - m_stack.push({node}); - ASSERT_FALSE(key); // for the key to exist, the parent must exist and be a map - } - else - { - size_t parent = m_stack.top().tree_node; - _nfo_logf("stack was not empty. parent={}", parent); - ASSERT_NE(parent, (size_t)NONE); - NodeType more_flags = NOTYPE; - if(tree.is_doc(parent) && !(tree.is_seq(parent) || tree.is_map(parent))) - { - _nfo_logf("set node to parent={}, add DOC", parent); - node = parent; - more_flags.add(DOC); - } - else - { - _nfo_logf("add child to parent={}", parent); - node = tree.append_child(parent); - m_stack.push({node}); - _nfo_logf("add child to parent={}: child={}", parent, node); - } - if(key) - { - _nfo_logf("has key, set to keyseq: parent={} child={} key='{}'", parent, node, key); - ASSERT_EQ(tree.is_map(parent) || node == parent, true); - tree.to_seq(node, key.filtered_scalar(&tree), more_flags); - key.add_key_props(&tree, node); - _nfo_logf("clear key='{}'", key.scalar); - key = {}; - } - else - { - if(tree.is_map(parent)) - { - _nfo_logf("has null key, set to keyseq: parent={} child={}", parent, node); - ASSERT_EQ(tree.is_map(parent) || node == parent, true); - tree.to_seq(node, csubstr{}, more_flags); - } - else - { - _nfo_logf("no key, set to seq: parent={} child={}", parent, node); - tree.to_seq(node, more_flags); - } - } - } - if(tag) - tree.set_val_tag(node, normalize_tag(tag)); - if(anchor) - tree.set_val_anchor(node, anchor); - } - else if(line.begins_with("+MAP")) - { - _nfo_log("pushing MAP"); - OptionalScalar anchor = {}; - OptionalScalar tag = {}; - csubstr more_tokens = line.stripl("+MAP").triml(' '); - if(more_tokens.begins_with('{')) - { - ASSERT_TRUE(more_tokens.begins_with("{}")); - more_tokens = more_tokens.offs(2, 0).triml(' '); - } - parse_anchor_and_tag(more_tokens, &anchor, &tag); - size_t node = tree.root_id(); - if(m_stack.empty()) - { - _nfo_log("stack was empty, set root to MAP"); - tree._add_flags(node, MAP); - m_stack.push({node}); - ASSERT_FALSE(key); // for the key to exist, the parent must exist and be a map - } - else - { - size_t parent = m_stack.top().tree_node; - _nfo_logf("stack was not empty. parent={}", parent); - ASSERT_NE(parent, (size_t)NONE); - NodeType more_flags = NOTYPE; - if(tree.is_doc(parent) && !(tree.is_seq(parent) || tree.is_map(parent))) - { - _nfo_logf("set node to parent={}, add DOC", parent); - node = parent; - more_flags.add(DOC); - } - else - { - _nfo_logf("add child to parent={}", parent); - node = tree.append_child(parent); - m_stack.push({node}); - _nfo_logf("add child to parent={}: child={}", parent, node); - } - if(key) - { - _nfo_logf("has key, set to keymap: parent={} child={} key='{}'", parent, node, key); - ASSERT_EQ(tree.is_map(parent) || node == parent, true); - tree.to_map(node, key.filtered_scalar(&tree), more_flags); - key.add_key_props(&tree, node); - _nfo_logf("clear key='{}'", key.scalar); - key = {}; - } - else - { - if(tree.is_map(parent)) - { - _nfo_logf("has null key, set to keymap: parent={} child={}", parent, node); - ASSERT_EQ(tree.is_map(parent) || node == parent, true); - tree.to_map(node, csubstr{}, more_flags); - } - else - { - _nfo_logf("no key, set to map: parent={} child={}", parent, node); - tree.to_map(node, more_flags); - } - } - } - if(tag) - tree.set_val_tag(node, normalize_tag(tag)); - if(anchor) - tree.set_val_anchor(node, anchor); - } - else if(line.begins_with("-SEQ")) - { - _nfo_logf("popping SEQ, empty={}", m_stack.empty()); - size_t node; - if(m_stack.empty()) - node = tree.root_id(); - else - node = m_stack.pop().tree_node; - ASSERT_TRUE(tree.is_seq(node)) << "node=" << node; - } - else if(line.begins_with("-MAP")) - { - _nfo_logf("popping MAP, empty={}", m_stack.empty()); - size_t node; - if(m_stack.empty()) - node = tree.root_id(); - else - node = m_stack.pop().tree_node; - ASSERT_TRUE(tree.is_map(node)) << "node=" << node; - } - else if(line.begins_with("+DOC")) - { - csubstr rem = line.stripl("+DOC").triml(' '); - _nfo_logf("pushing DOC: {}", rem); - size_t node = tree.root_id(); - auto is_sep = rem.first_of_any("---\n", "--- ", "---\r") || rem.ends_with("---"); - ASSERT_EQ(key, false); // for the key to exist, the parent must exist and be a map - if(m_stack.empty()) - { - _nfo_log("stack was empty"); - ASSERT_EQ(node, tree.root_id()); - if(tree.is_stream(node)) - { - _nfo_log("there is already a stream, append a DOC"); - node = tree.append_child(node); - tree.to_doc(node); - m_stack.push({node}); - } - else if(is_sep) - { - _nfo_logf("separator was specified: {}", rem); - if((!tree.is_container(node)) && (!tree.is_doc(node))) - { - tree._add_flags(node, STREAM); - node = tree.append_child(node); - _nfo_logf("create STREAM at {} and add DOC child={}", tree.root_id(), node); - tree.to_doc(node); - m_stack.push({node}); - } - else - { - _nfo_log("rearrange root as STREAM"); - tree.set_root_as_stream(); - node = tree.append_child(tree.root_id()); - _nfo_logf("added doc as STREAM child: {}", node); - tree.to_doc(node); - m_stack.push({node}); - } - } - else - { - if(tree.is_doc(node)) - { - _nfo_log("rearrange root as STREAM"); - tree.set_root_as_stream(); - m_stack.push({node}); - } - } - } - else - { - size_t parent = m_stack.top().tree_node; - _nfo_logf("add DOC to parent={}", parent); - ASSERT_NE(parent, (size_t)NONE); - node = tree.append_child(parent); - _nfo_logf("child DOC={}", node); - tree.to_doc(node); - m_stack.push({node}); - } - } - else if(line.begins_with("-DOC")) - { - _nfo_log("popping DOC"); - if(!m_stack.empty()) - m_stack.pop(); - } - else if(line.begins_with("+STR")) - { - ASSERT_EQ(m_stack.size(), 0u); - } - else if(line.begins_with("-STR")) - { - ASSERT_LE(m_stack.size(), 1u); - if(!m_stack.empty()) - m_stack.pop(); - } - else if(line.empty()) - { - // nothing to do - } - else - { - C4_ERROR("unknown event: '%.*s'", (int)line.len, line.str); - } - linenum++; - } -} - - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_suite/test_suite_events.hpp b/thirdparty/ryml/test/test_suite/test_suite_events.hpp deleted file mode 100644 index 3b3cdbffb..000000000 --- a/thirdparty/ryml/test/test_suite/test_suite_events.hpp +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef C4_YML_TEST_SUITE_EVENTS_HPP_ -#define C4_YML_TEST_SUITE_EVENTS_HPP_ - -#ifdef RYML_SINGLE_HEADER -#include -#else -#include -#endif - -namespace c4 { -namespace yml { - -struct EventsParser -{ - void parse(csubstr src, Tree *C4_RESTRICT tree); -}; - -size_t emit_events(substr buf, Tree const& C4_RESTRICT tree); - -template -void emit_events(CharContainer *container, Tree const& C4_RESTRICT tree) -{ - size_t ret = emit_events(to_substr(*container), tree); - if(ret > container->size()) - { - container->resize(ret); - ret = emit_events(to_substr(*container), tree); - C4_CHECK(ret == container->size()); - } - container->resize(ret); -} - -template -CharContainer emit_events(Tree const& C4_RESTRICT tree) -{ - CharContainer result; - emit_events(&result, tree); - return result; -} - -} // namespace yml -} // namespace c4 - - -#endif /* C4_YML_TEST_SUITE_EVENTS_HPP_ */ diff --git a/thirdparty/ryml/test/test_suite/test_suite_events_emitter.cpp b/thirdparty/ryml/test/test_suite/test_suite_events_emitter.cpp deleted file mode 100644 index 6c22b1e6e..000000000 --- a/thirdparty/ryml/test/test_suite/test_suite_events_emitter.cpp +++ /dev/null @@ -1,289 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include -#endif -#include "test_suite_events.hpp" - -namespace c4 { -namespace yml { - -struct EventsEmitter -{ - substr buf; - size_t pos; - std::string tagbuf; - Tree const* C4_RESTRICT m_tree; - EventsEmitter(Tree const& tree, substr buf_) : buf(buf_), pos(), m_tree(&tree) {} - void emit_tag(csubstr tag, size_t node); - void emit_scalar(csubstr val, bool quoted); - void emit_key_anchor_tag(size_t node); - void emit_val_anchor_tag(size_t node); - void emit_events(size_t node); - void emit_doc(size_t node); - void emit_events(); - template - C4_ALWAYS_INLINE void pr(const char (&s)[N]) - { - if(pos + N-1 <= buf.len) - memcpy(buf.str + pos, s, N-1); - pos += N-1; - } - C4_ALWAYS_INLINE void pr(csubstr s) - { - if(pos + s.len <= buf.len) - memcpy(buf.str + pos, s.str, s.len); - pos += s.len; - } - C4_ALWAYS_INLINE void pr(char c) - { - if(pos + 1 <= buf.len) - buf[pos] = c; - ++pos; - } - C4_ALWAYS_INLINE size_t emit_to_esc(csubstr val, size_t prev, size_t i, char c) - { - pr(val.range(prev, i)); - pr('\\'); - pr(c); - return i+1; - } - C4_ALWAYS_INLINE size_t emit_to_esc(csubstr val, size_t prev, size_t i, csubstr repl) - { - pr(val.range(prev, i)); - pr(repl); - return i+1; - } -}; - -void EventsEmitter::emit_scalar(csubstr val, bool quoted) -{ - constexpr const char openchar[] = {':', '\''}; - pr(openchar[quoted]); - size_t prev = 0; - uint8_t const* C4_RESTRICT s = (uint8_t const* C4_RESTRICT) val.str; - for(size_t i = 0; i < val.len; ++i) - { - switch(s[i]) - { - case UINT8_C(0x0a): // \n - prev = emit_to_esc(val, prev, i, 'n'); break; - case UINT8_C(0x5c): // '\\' - prev = emit_to_esc(val, prev, i, '\\'); break; - case UINT8_C(0x09): // \t - prev = emit_to_esc(val, prev, i, 't'); break; - case UINT8_C(0x0d): // \r - prev = emit_to_esc(val, prev, i, 'r'); break; - case UINT8_C(0x00): // \0 - prev = emit_to_esc(val, prev, i, '0'); break; - case UINT8_C(0x0c): // \f (form feed) - prev = emit_to_esc(val, prev, i, 'f'); break; - case UINT8_C(0x08): // \b (backspace) - prev = emit_to_esc(val, prev, i, 'b'); break; - case UINT8_C(0x07): // \a (bell) - prev = emit_to_esc(val, prev, i, 'a'); break; - case UINT8_C(0x0b): // \v (vertical tab) - prev = emit_to_esc(val, prev, i, 'v'); break; - case UINT8_C(0x1b): // \e (escape) - prev = emit_to_esc(val, prev, i, "\\e"); break; - case UINT8_C(0xc2): - if(i+1 < val.len) - { - uint8_t np1 = s[i+1]; - if(np1 == UINT8_C(0xa0)) - prev = 1u + emit_to_esc(val, prev, i++, "\\_"); - else if(np1 == UINT8_C(0x85)) - prev = 1u + emit_to_esc(val, prev, i++, "\\N"); - } - break; - case UINT8_C(0xe2): - if(i+2 < val.len) - { - if(s[i+1] == UINT8_C(0x80)) - { - if(s[i+2] == UINT8_C(0xa8)) - { - prev = 2u + emit_to_esc(val, prev, i, "\\L"); - i += 2u; - } - else if(s[i+2] == UINT8_C(0xa9)) - { - prev = 2u + emit_to_esc(val, prev, i, "\\P"); - i += 2u; - } - } - } - break; - } - } - pr(val.sub(prev)); // print remaining portion -} - -void EventsEmitter::emit_tag(csubstr tag, size_t node) -{ - size_t tagsize = m_tree->resolve_tag(to_substr(tagbuf), tag, node); - if(tagsize) - { - if(tagsize > tagbuf.size()) - { - tagbuf.resize(tagsize); - tagsize = m_tree->resolve_tag(to_substr(tagbuf), tag, node); - } - pr(to_substr(tagbuf).first(tagsize)); - } - else - { - csubstr ntag = normalize_tag_long(tag); - if(ntag.begins_with('<')) - { - pr(ntag); - } - else - { - pr('<'); - pr(ntag); - pr('>'); - } - } -} - -void EventsEmitter::emit_key_anchor_tag(size_t node) -{ - if(m_tree->has_key_anchor(node)) - { - pr(" &"); - pr(m_tree->key_anchor(node)); - } - if(m_tree->has_key_tag(node)) - { - pr(' '); - emit_tag(m_tree->key_tag(node), node); - } -} - -void EventsEmitter::emit_val_anchor_tag(size_t node) -{ - if(m_tree->has_val_anchor(node)) - { - pr(" &"); - pr(m_tree->val_anchor(node)); - } - if(m_tree->has_val_tag(node)) - { - pr(' '); - emit_tag(m_tree->val_tag(node), node); - } -} - -void EventsEmitter::emit_events(size_t node) -{ - if(m_tree->has_key(node)) - { - if(m_tree->is_key_ref(node)) - { - csubstr k = m_tree->key(node); - if(k != "<<") - { - pr("=ALI "); - pr(k); - pr('\n'); - } - else - { - pr("=VAL :"); - pr(k); - pr('\n'); - } - } - else - { - pr("=VAL"); - emit_key_anchor_tag(node); - pr(' '); - emit_scalar(m_tree->key(node), m_tree->is_key_quoted(node)); - pr('\n'); - } - } - if(m_tree->has_val(node)) - { - if(m_tree->is_val_ref(node)) - { - pr("=ALI "); - pr(m_tree->val(node)); - pr('\n'); - } - else - { - pr("=VAL"); - emit_val_anchor_tag(node); - pr(' '); - emit_scalar(m_tree->val(node), m_tree->is_val_quoted(node)); - pr('\n'); - } - } - else if(m_tree->is_map(node)) - { - pr("+MAP"); - emit_val_anchor_tag(node); - pr('\n'); - for(size_t child = m_tree->first_child(node); child != NONE; child = m_tree->next_sibling(child)) - emit_events(child); - pr("-MAP\n"); - } - else if(m_tree->is_seq(node)) - { - pr("+SEQ"); - emit_val_anchor_tag(node); - pr('\n'); - for(size_t child = m_tree->first_child(node); child != NONE; child = m_tree->next_sibling(child)) - emit_events(child); - pr("-SEQ\n"); - } -} - -void EventsEmitter::emit_doc(size_t node) -{ - if(m_tree->type(node) == NOTYPE) - return; - if(m_tree->has_parent(node)) - pr("+DOC ---"); // parent must be a stream - else - pr("+DOC"); - if(m_tree->is_val(node)) - { - pr("\n=VAL"); - emit_val_anchor_tag(node); - pr(' '); - emit_scalar(m_tree->val(node), m_tree->is_val_quoted(node)); - pr('\n'); - } - else - { - pr('\n'); - emit_events(node); - } - pr("-DOC\n"); -} - -void EventsEmitter::emit_events() -{ - pr("+STR\n"); - if(!m_tree->empty()) - { - size_t root = m_tree->root_id(); - if(m_tree->is_stream(root)) - for(size_t node = m_tree->first_child(root); node != NONE; node = m_tree->next_sibling(node)) - emit_doc(node); - else - emit_doc(root); - } - pr("-STR\n"); -} - -size_t emit_events(substr buf, Tree const& C4_RESTRICT tree) -{ - EventsEmitter e(tree, buf); - e.emit_events(); - return e.pos; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_suite/test_suite_parts.cpp b/thirdparty/ryml/test/test_suite/test_suite_parts.cpp deleted file mode 100644 index 5caaafeab..000000000 --- a/thirdparty/ryml/test/test_suite/test_suite_parts.cpp +++ /dev/null @@ -1,220 +0,0 @@ -#include "./test_suite_parts.hpp" - -namespace c4 { -namespace yml { - - -// To see the test case contents, refer to this URL: -// https://github.com/yaml/yaml-test-suite/tree/master/src -constexpr const AllowedFailure allowed_failures[] = { - - // g++-5 does not like creating a csubstr directly from the literal. - // so we use this macro to remove cruft from the code: - #define _(testcase, reason) AllowedFailure{csubstr(testcase), csubstr(reason)} - - //------------------------------------------------------------------------- - // SECTION 1. Known issues, TODO - // - // These tests are temporarily skipped, and cover issues that must be fixed. - - // double quoted scalars - _("G4RS-in_json" , "special characters must be emitted in double quoted style"), - _("G4RS-in_yaml" , "special characters must be emitted in double quoted style"), - _("G4RS-out_yaml" , "special characters must be emitted in double quoted style"), - // other - _("UKK6_01-in_yaml" , "fails to parse double :: in UNK state"), - - - //------------------------------------------------------------------------- - // SECTION 2. Expected errors that fail to materialize. - - // maps - _("236B-error" , "should not accept final scalar in a map"), - _("7MNF-error" , "should not accept final scalar in a map"), - _("62EZ-error" , "should not accept invalid block mapping key on same line as previous key"), - _("9CWY-error" , "should not accept final scalar in a map"), - _("CXX2-error" , "should not accept mapping with anchor on document start line"), - _("DK95_06-error" , "should not accept tab indentation"), - _("GDY7-error" , "should not accept comment that looks like a mapping key"), - _("D49Q-error" , "should not accept multiline single quoted implicit keys"), - _("DK4H-error" , "should not accept implicit key followed by newline"), - _("JY7Z-error" , "should not accept trailing content that looks like a mapping"), - _("SU74-error" , "should not accept anchor and alias as mapping key"), - _("T833-error" , "should not accept flow mapping missing a separating comma"), - _("VJP3_00-error" , "should not accept flow collections over many lines"), - _("Y79Y_006-error", "should not accept tab after ?"), - _("Y79Y_007-error", "should not accept tab after :"), - _("Y79Y_008-error", "should not accept tab after ?"), - _("Y79Y_009-error", "should not accept tab after ?"), - _("ZCZ6-error" , "should not accept invalid mapping in plain single line value"), - // seqs - _("5U3A-error" , "should not accept opening a sequence on same line as map key"), - _("6JTT-error" , "should not accept flow sequence without terminating ]"), - _("9C9N-error" , "should not accept non-indented flow sequence"), - _("9JBA-error" , "should not accept comment after flow seq terminating ]"), - _("9MAG-error" , "should not accept flow sequence with invalid comma at the beginning"), - _("CTN5-error" , "should not accept flow sequence with missing elements"), - _("CVW2-error" , "should not accept flow sequence with comment after ,"), - _("G5U8-error" , "should not accept [-, -]"), - _("KS4U-error" , "should not accept item after end of flow sequence"), - _("P2EQ-error" , "should not accept sequence item on same line as previous item"), - _("YJV2-error" , "should not accept [-]"), - _("Y79Y_003-error", "should not accept leading tabs in seq elmt"), - _("Y79Y_004-error", "should not accept tab after -"), - _("Y79Y_005-error", "should not accept tab after -"), - // block scalars - _("2G84_00-error" , "should not accept the block literal spec"), - _("2G84_01-error" , "should not accept the block literal spec"), - _("5LLU-error" , "should not accept folded scalar with wrong indented line after spaces only"), - _("S4GJ-error" , "should not accept text after block scalar indicator"), - _("S98Z-error" , "should not accept block scalar with more spaces than first content line"), - _("X4QW-error" , "should not accept comment without whitespace after block scalar indicator"), - _("Y79Y_000-error", "should not accept leading tabs in the block scalar"), - // quoted scalars - _("55WF-error" , "should not accept invalid escape in double quoted scalar"), - _("7LBH-error" , "should not accept multiline double quoted implicit keys"), - _("DK95_01-error", "should not accept leading tabs in double quoted multiline scalar"), - _("HRE5-error" , "should not accept double quoted scalar with escaped single quote"), - _("JKF3-error" , "should not accept multiline unindented double quoted scalar"), - _("QB6E-error" , "should not accept indented multiline quoted scalar"), - _("RXY3-error" , "should not accept document-end marker in single quoted string"), - _("SU5Z-error" , "should not accept comment without whitespace after double quoted scalar"), - // plain scalars - _("8XDJ-error" , "should not accept comment in multiline scalar"), - _("CML9-error" , "should not accept comment inside flow scalar"), - // documents/streams - _("3HFZ-error" , "should not accept scalar after ..."), - _("5TRB-error" , "should not accept document-end marker in double quoted string"), - _("9MMA-error" , "should not accept empty doc after %YAML directive"), - _("9MQT_01-error", "should not accept scalars after ..."), - _("B63P-error" , "should not accept directive without doc"), - _("EB22-error" , "should not accept missing document-end marker before directive"), - _("H7TQ-error" , "should not accept extra words after directive"), - _("MUS6_00-error", "should not accept #... at the end of %YAML directive"), - _("MUS6_01-error", "should not accept #... at the end of %YAML directive"), - _("N782-error" , "should not accept document markers in flow style"), - _("RHX7-error" , "should not accept directive without document end marker"), - _("SF5V-error" , "should not accept duplicate YAML directive"), - // anchors - _("4EJS-error" , "should not accept double anchor for scalar"), - _("4JVG-error" , "should not accept double anchor for scalar"), - _("SY6V-error" , "should not accept anchor before sequence entry on same line"), - // tags - _("9HCY-error" , "should not accept tag directive in non-doc scope"), - _("BU8L-error" , "should not accept node properties spread over multiple lines"), - _("LHL4-error" , "should not accept tag"), - _("U99R-error" , "should not accept comma in a tag"), - _("QLJ7-error" , "tag directives should apply only to the next doc (?)"), - - - //------------------------------------------------------------------------- - // SECTION 3. Deliberate ryml limitations. - // - // These tests are skipped because they cover parts of YAML that - // are deliberately not implemented by ryml. - - #ifndef RYML_WITH_TAB_TOKENS // - or : are supported only when the above macro is defined - _("A2M4-in_yaml-events" , "tabs tokens"), - _("6BCT-in_yaml" , "tabs tokens"), - _("J3BT-in_yaml-events" , "tabs tokens"), - _("Y79Y_010-in_yaml-events", "tabs tokens"), - #endif - // container keys are not supported - _("4FJ6-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("4FJ6-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("6BFJ-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("6BFJ-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("6PBE-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("6PBE-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("6PBE-emit_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("9MMW-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("9MMW-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("KK5P-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("KK5P-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("KZN9-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("KZN9-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("LX3P-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("LX3P-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("M2N8_00-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("M2N8_00-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("M2N8_01-in_yaml-events" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("M2N8_01-out_yaml-events", "only scalar keys allowed (keys cannot be maps or seqs)"), - _("M5DY-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("M5DY-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("Q9WF-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("Q9WF-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("RZP5-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("RZP5-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("SBG9-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("SBG9-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("V9D5-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("V9D5-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("X38W-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("X38W-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("XW4D-in_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - _("XW4D-out_yaml" , "only scalar keys allowed (keys cannot be maps or seqs)"), - // anchors with : are not supported - _("2SXE-in_yaml-events" , "weird characters in anchors, anchors must not end with :"), - // malformed json in the test spec - _("35KP-in_json" , "malformed JSON from multiple documents"), - _("5TYM-in_json" , "malformed JSON from multiple documents"), - _("6XDY-in_json" , "malformed JSON from multiple documents"), - _("6WLZ-in_json" , "malformed JSON from multiple documents"), - _("6ZKB-in_json" , "malformed JSON from multiple documents"), - _("7Z25-in_json" , "malformed JSON from multiple documents"), - _("9DXL-in_json" , "malformed JSON from multiple documents"), - _("9KAX-in_json" , "malformed JSON from multiple documents"), - _("9WXW-in_json" , "malformed JSON from multiple documents"), - _("JHB9-in_json" , "malformed JSON from multiple documents"), - _("KSS4-in_json" , "malformed JSON from multiple documents"), - _("L383-in_json" , "malformed JSON from multiple documents"), - _("M7A3-in_json" , "malformed JSON from multiple documents"), - _("RZT7-in_json" , "malformed JSON from multiple documents"), - _("U9NS-in_json" , "malformed JSON from multiple documents"), - _("W4TN-in_json" , "malformed JSON from multiple documents"), - // malformed test spec? - _("4ABK-out_yaml-events" , "out-yaml contains null, while in-yaml and events contain empty scalars"), - _("4WA9-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("652Z-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("6CA3-emit_yaml" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("6FWR-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("6WPF-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("9TFX-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("B3HG-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("DK95_00-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("DK95_02-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("DK95_03-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("DK95_04-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("DK95_05-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("DK95_06-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("DK95_07-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("DK95_08-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("EX5H-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("EXG3-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("L24T_00-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("L24T_01-emit_yaml-events", "out-yaml test spec is missing a --- document token, which is required in the events"), - _("M6YH-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("Q8AD-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("T26H-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("T4YY-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("T5N4-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - _("VJP3_01-out_yaml-events" , "out-yaml test spec is missing a --- document token, which is required in the events"), - - #undef _ -}; - - -cspan g_allowed_failures = allowed_failures; - -AllowedFailure is_failure_expected(csubstr casename) -{ - RYML_CHECK(casename.not_empty()); - for(AllowedFailure const& af : g_allowed_failures) - if(af.test_name == casename || casename.begins_with(af.test_name)) - return af; - return {}; -} - - -} // namespace c4 -} // namespace yml diff --git a/thirdparty/ryml/test/test_suite/test_suite_parts.hpp b/thirdparty/ryml/test/test_suite/test_suite_parts.hpp deleted file mode 100644 index 9092313ba..000000000 --- a/thirdparty/ryml/test/test_suite/test_suite_parts.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef C4_YML_TEST_SUITE_PARTS_HPP_ -#define C4_YML_TEST_SUITE_PARTS_HPP_ - -#ifdef RYML_SINGLE_HEADER -#include -#else -#include -#include -#endif -#include - -namespace c4 { -namespace yml { - -struct AllowedFailure -{ - csubstr test_name; - csubstr reason; - operator bool() const { return !test_name.empty(); } -}; - -AllowedFailure is_failure_expected(csubstr casename); - -} // namespace c4 -} // namespace yml - - -#endif /* C4_YML_TEST_SUITE_PARTS_HPP_ */ diff --git a/thirdparty/ryml/test/test_tag_property.cpp b/thirdparty/ryml/test/test_tag_property.cpp deleted file mode 100644 index e56fef994..000000000 --- a/thirdparty/ryml/test/test_tag_property.cpp +++ /dev/null @@ -1,1149 +0,0 @@ -#include "./test_group.hpp" -#include "test_case.hpp" - -namespace c4 { -namespace yml { - -TEST(tag_directives, basic) -{ - Tree t = parse_in_arena(R"( -%TAG !m! !my- ---- # Bulb here -!m!light fluorescent -... -%TAG !m! !meta- ---- # Color here -!m!light green -)"); - EXPECT_EQ(t[0].val_tag(), "!m!light"); - EXPECT_EQ(t[1].val_tag(), "!m!light"); - EXPECT_EQ(t.num_tag_directives(), 2u); - char buf_[100]; - EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 1u), csubstr("")); - EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 2u), csubstr("")); - EXPECT_EQ(emitrs_yaml(t), std::string(R"(%TAG !m! !my- ---- !m!light fluorescent -... -%TAG !m! !meta- ---- !m!light green -)")); -} - -TEST(tag_directives, accepts_comment) -{ - Tree t = parse_in_arena(R"( -%TAG !m! !my- # comment ---- # Bulb here -!m!light fluorescent -... -%TAG !m! !meta- # comment ---- # Color here -!m!light green -)"); - EXPECT_EQ(t[0].val_tag(), "!m!light"); - EXPECT_EQ(t[1].val_tag(), "!m!light"); - EXPECT_EQ(t.num_tag_directives(), 2u); - char buf_[100]; - EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 1u), csubstr("")); - EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 2u), csubstr("")); - EXPECT_EQ(emitrs_yaml(t), std::string(R"(%TAG !m! !my- ---- !m!light fluorescent -... -%TAG !m! !meta- ---- !m!light green -)")); -} - -TEST(tag_directives, accepts_multiple_spaces) -{ - Tree t = parse_in_arena(R"( -%TAG !m! !my- # comment ---- # Bulb here -!m!light fluorescent -... -%TAG !m! !meta- # comment ---- # Color here -!m!light green -)"); - EXPECT_EQ(t[0].val_tag(), "!m!light"); - EXPECT_EQ(t[1].val_tag(), "!m!light"); - EXPECT_EQ(t.num_tag_directives(), 2u); - char buf_[100]; - EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 1u), csubstr("")); - EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 2u), csubstr("")); - EXPECT_EQ(emitrs_yaml(t), std::string(R"(%TAG !m! !my- ---- !m!light fluorescent -... -%TAG !m! !meta- ---- !m!light green -)")); -} - -TEST(tag_directives, errors) -{ - { - Tree t; - ExpectError::do_check(&t, [&]{ - t = parse_in_arena(R"( -%TAG ---- # Bulb here -!m!light fluorescent)"); - }); - } - { - Tree t; - ExpectError::do_check(&t, [&]{ - t = parse_in_arena(R"( -%TAG !m! ---- # Bulb here -!m!light fluorescent)"); - }); - } -} - -TEST(tag_directives, resolve_tags) -{ - Tree t = parse_in_arena(R"( -%TAG !m! !my- # comment ---- # Bulb here -!m!light fluorescent: !m!light bulb -... -%TAG !m! !meta- # comment ---- # Color here -!m!light green: !m!light color -)"); - EXPECT_EQ(t.docref(0)[0].key_tag(), "!m!light"); - EXPECT_EQ(t.docref(0)[0].val_tag(), "!m!light"); - EXPECT_EQ(t.num_tag_directives(), 2u); - t.resolve_tags(); - EXPECT_EQ(t.docref(0)[0].key_tag(), ""); - EXPECT_EQ(t.docref(0)[0].val_tag(), ""); - EXPECT_EQ(emitrs_yaml(t), std::string(R"(%TAG !m! !my- ---- -! fluorescent: ! bulb -... -%TAG !m! !meta- ---- -! green: ! color -)")); -} - -TEST(tag_directives, safe_with_empty_tree) -{ - Tree t; - t.resolve_tags(); - EXPECT_TRUE(t.empty()); -} - -TEST(tag_directives, decode_uri_chars) -{ - { - Tree t = parse_in_arena(R"( -%TAG !e! tag:example.com,2000:app/ ---- -- !e!%61%62%63%21 baz -)"); - t.resolve_tags(); - EXPECT_EQ(t.docref(0)[0].val_tag(), csubstr("")); - } - { - Tree t; - auto checkerr = [&t](csubstr yaml){ - ExpectError::do_check(&t, [&]{ - t.clear(); - t = parse_in_arena(yaml); - t.resolve_tags(); - }); - }; - { - SCOPED_TRACE("without numbers at begin"); - checkerr(R"( -%TAG !e! tag:example.com,2000:app/ ---- -- !e!%%62%63 baz -)"); - } - { - SCOPED_TRACE("without numbers in the middle"); - checkerr(R"( -%TAG !e! tag:example.com,2000:app/ ---- -- !e!%61%%63 baz -)"); - } - { - SCOPED_TRACE("without numbers in the end"); - checkerr(R"( -%TAG !e! tag:example.com,2000:app/ ---- -- !e!%61%62% baz -)"); - } - { - SCOPED_TRACE("with wrong characters numbers at begin"); - checkerr(R"( -%TAG !e! tag:example.com,2000:app/ ---- -- !e!%h%62%63 baz -)"); - } - { - SCOPED_TRACE("with wrong characters in the middle"); - checkerr(R"( -%TAG !e! tag:example.com,2000:app/ ---- -- !e!%61%hh%63 baz -)"); - } - { - SCOPED_TRACE("with wrong characters in the end"); - checkerr(R"( -%TAG !e! tag:example.com,2000:app/ ---- -- !e!%61%62%hh baz -)"); - } - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(tags, test_suite_735Y) -{ - csubstr yaml_without_seq = R"( -!!map # Block collection -foo : bar -)"; - test_check_emit_check(yaml_without_seq, [](Tree const &t){ - EXPECT_TRUE(t.rootref().is_map()); - EXPECT_TRUE(t.rootref().has_val_tag()); - EXPECT_EQ(t.rootref()["foo"].val(), csubstr("bar")); - }); - - csubstr yaml = R"( -- - foo : bar -- #!!map - foo : bar -- #!!map # Block collection - foo : bar -- !!map - foo : bar -- !!map # Block collection - foo : bar -)"; - test_check_emit_check(yaml, [](Tree const &t){ - ASSERT_TRUE(t.rootref().is_seq()); - ASSERT_EQ(t.rootref().num_children(), 5u); - // - EXPECT_TRUE(t[0].is_map()); - EXPECT_TRUE(!t[0].has_val_tag()); - EXPECT_EQ(t[0]["foo"].val(), csubstr("bar")); - // - EXPECT_TRUE(t[1].is_map()); - EXPECT_TRUE(!t[1].has_val_tag()); - EXPECT_EQ(t[1]["foo"].val(), csubstr("bar")); - // - EXPECT_TRUE(t[2].is_map()); - EXPECT_TRUE(!t[2].has_val_tag()); - EXPECT_EQ(t[2]["foo"].val(), csubstr("bar")); - // - EXPECT_TRUE(t[3].is_map()); - ASSERT_TRUE(t[3].has_val_tag()); - EXPECT_EQ(t[3].val_tag(), csubstr("!!map")); - EXPECT_EQ(t[3]["foo"].val(), csubstr("bar")); - // - EXPECT_TRUE(t[4].is_map()); - ASSERT_TRUE(t[4].has_val_tag()); - EXPECT_EQ(t[4].val_tag(), csubstr("!!map")); - EXPECT_EQ(t[4]["foo"].val(), csubstr("bar")); - }); -} - - -TEST(tags, parsing) -{ - Tree t = parse_in_arena(R"( -!!seq -- !!map - !key key1: !val val1 - ! key2: ! val2 - ! key3: ! val3 - key4: val4 # there are NOT parsed as tags -- ! - !key key1: !val val1 -- ! - - !val val - - !str val - - val - - ! val - - ! val - - ! val -)"); - EXPECT_EQ(t.rootref().val_tag(), csubstr("!!seq")); - EXPECT_EQ(t[0].val_tag(), csubstr("!!map")); - EXPECT_EQ(t[1].val_tag(), csubstr("!!map")); - EXPECT_EQ(t[2].val_tag(), csubstr("!!seq")); - EXPECT_EQ(t[0]["key1"].key_tag(), csubstr("!key")); - EXPECT_EQ(t[0]["key1"].val_tag(), csubstr("!val")); - EXPECT_EQ(t[0]["key2"].key_tag(), csubstr("")); - EXPECT_EQ(t[0]["key2"].val_tag(), csubstr("")); - EXPECT_EQ(t[0]["key3"].key_tag(), csubstr("")); - EXPECT_EQ(t[0]["key3"].val_tag(), csubstr("")); - EXPECT_EQ(t[0][" key4"].has_key_tag(), false); - EXPECT_EQ(t[0][" key4"].has_val_tag(), false); - EXPECT_EQ(t[0][" key4"].key(), csubstr(" key4")); - EXPECT_EQ(t[0][" key4"].val(), csubstr(" val4")); - EXPECT_EQ(t[2][5].val_tag(), csubstr("!!str")); - - EXPECT_EQ(emitrs_yaml(t), R"(!!seq -- !!map - !key key1: !val val1 - ! key2: ! val2 - ! key3: ! val3 - key4: val4 -- !!map - !key key1: !val val1 -- !!seq - - !val val - - !str val - - val - - ! val - - ! val - - !!str val -)"); -} - - -TEST(tags, setting) -{ - Tree t; - size_t rid = t.root_id(); - t.to_map(rid); - t.set_val_tag(rid, "!valtag"); - EXPECT_EQ(t.val_tag(rid), "!valtag"); - - // a keymap - { - size_t child = t.append_child(rid); - t.to_seq(child, "key2"); - t.set_key_tag(child, "!keytag"); - t.set_val_tag(child, "!valtag2"); - EXPECT_TRUE(t.has_key(child)); - EXPECT_FALSE(t.has_val(child)); - EXPECT_EQ(t.key(child), "key2"); - EXPECT_EQ(t.key_tag(child), "!keytag"); - EXPECT_EQ(t.val_tag(child), "!valtag2"); - } - - // a keyseq - { - size_t child = t.append_child(rid); - t.to_seq(child, "key2"); - t.set_key_tag(child, "!keytag"); - t.set_val_tag(child, "!valtag2"); - EXPECT_TRUE(t.has_key(child)); - EXPECT_FALSE(t.has_val(child)); - EXPECT_EQ(t.key(child), "key2"); - EXPECT_EQ(t.key_tag(child), "!keytag"); - EXPECT_EQ(t.val_tag(child), "!valtag2"); - } - - // a keyval - { - size_t child = t.append_child(rid); - t.to_keyval(child, "key", "val"); - t.set_key_tag(child, "!keytag"); - t.set_val_tag(child, "!valtag"); - EXPECT_TRUE(t.has_key(child)); - EXPECT_TRUE(t.has_val(child)); - EXPECT_EQ(t.key(child), "key"); - EXPECT_EQ(t.val(child), "val"); - EXPECT_EQ(t.key_tag(child), "!keytag"); - EXPECT_EQ(t.val_tag(child), "!valtag"); - } - - // a val - { - size_t seqid = t[1].id(); - ASSERT_TRUE(t.is_seq(seqid)); - size_t child = t.append_child(seqid); - t.to_val(child, "val"); - t.set_val_tag(child, "!valtag"); - EXPECT_FALSE(t.has_key(child)); - EXPECT_TRUE(t.has_val(child)); - EXPECT_EQ(t.val(child), "val"); - EXPECT_EQ(t.val_tag(child), "!valtag"); - } -} - -TEST(tags, errors) -{ - Tree t = parse_in_arena("{key: val, keymap: {}, keyseq: [val]}"); - size_t keyval = t["keyval"].id(); - size_t keymap = t["keymap"].id(); - size_t keyseq = t["keyseq"].id(); - size_t val = t["keyseq"][0].id(); - size_t empty_keyval = t.append_child(keymap); - size_t empty_val = t.append_child(keyseq); - - ASSERT_NE(keyval, (size_t)npos); - ASSERT_NE(keymap, (size_t)npos); - ASSERT_NE(keyseq, (size_t)npos); - ASSERT_NE(val, (size_t)npos); - - // cannot get key tag in a node that does not have a key tag - EXPECT_FALSE(t.has_key_tag(empty_keyval)); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.key_tag(empty_keyval), ""); - }); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.key_tag(keyval), ""); - }); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.key_tag(keymap), ""); - }); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.key_tag(keyseq), ""); - }); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.key_tag(val), ""); - }); - // cannot get val tag in a node that does not have a val tag - EXPECT_FALSE(t.has_val_tag(empty_val)); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.val_tag(empty_val), ""); - }); - EXPECT_FALSE(t.has_val_tag(empty_keyval)); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.val_tag(empty_keyval), ""); - }); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.val_tag(keyval), ""); - }); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.val_tag(keymap), ""); - }); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.val_tag(keyseq), ""); - }); - ExpectError::check_assertion(&t, [&](){ - EXPECT_EQ(t.val_tag(val), ""); - }); - // cannot set key tag in a node that does not have a key - EXPECT_FALSE(t.has_key(empty_keyval)); - ExpectError::check_assertion(&t, [&](){ - t.set_key_tag(empty_keyval, "!keytag"); - }); - EXPECT_FALSE(t.has_key_tag(val)); // must stay the same - ExpectError::check_assertion(&t, [&](){ - t.set_key_tag(val, "!valtag"); - }); - EXPECT_FALSE(t.has_key_tag(val)); // must stay the same - // cannot set val tag in a node that does not have a val - EXPECT_FALSE(t.has_val(empty_val)); - ExpectError::check_assertion(&t, [&](){ - t.set_val_tag(empty_val, "!valtag"); - }); - EXPECT_FALSE(t.has_val_tag(empty_val)); // must stay the same - EXPECT_FALSE(t.has_val(empty_keyval)); - ExpectError::check_assertion(&t, [&](){ - t.set_val_tag(empty_keyval, "!valtag"); - }); - EXPECT_FALSE(t.has_val_tag(empty_keyval)); // must stay the same -} - - -TEST(tags, setting_user_tags_do_not_require_leading_mark) -{ - Tree t = parse_in_arena("{key: val, keymap: {}, keyseq: [val]}"); - size_t keyval = t["keyval"].id(); - size_t keymap = t["keymap"].id(); - size_t keyseq = t["keyseq"].id(); - size_t val = t["keyseq"][0].id(); - ASSERT_NE(keyval, (size_t)npos); - ASSERT_NE(keymap, (size_t)npos); - ASSERT_NE(keyseq, (size_t)npos); - ASSERT_NE(val, (size_t)npos); - - // without leading mark - t.set_key_tag(keyseq, "keytag"); - t.set_val_tag(keyseq, "valtag"); - t.set_val_tag(val, "valtag2"); - EXPECT_EQ(t.key_tag(keyseq), "keytag"); - EXPECT_EQ(t.val_tag(keyseq), "valtag"); - EXPECT_EQ(t.val_tag(val), "valtag2"); - - EXPECT_EQ(emitrs_yaml(t), R"(key: val -keymap: {} -!keytag keyseq: !valtag - - !valtag2 val -)"); - - // with leading mark - t.set_key_tag(keyseq, "!keytag"); - t.set_val_tag(keyseq, "!valtag"); - t.set_val_tag(val, "!valtag2"); - EXPECT_EQ(t.key_tag(keyseq), "!keytag"); - EXPECT_EQ(t.val_tag(keyseq), "!valtag"); - EXPECT_EQ(t.val_tag(val), "!valtag2"); - - EXPECT_EQ(emitrs_yaml(t), R"(key: val -keymap: {} -!keytag keyseq: !valtag - - !valtag2 val -)"); -} - - -TEST(tags, valid_chars) -{ - Tree t = parse_in_arena(R"( -- ! val -- ! bar> val -- ! val -)"); - EXPECT_EQ(t[0].val_tag(), ""); - EXPECT_EQ(t[0].val(), "val"); - EXPECT_EQ(t[1].val_tag(), ""); - EXPECT_EQ(t[1].val(), "bar> val"); - EXPECT_EQ(t[2].val_tag(), ""); - EXPECT_EQ(t[2].val(), " val"); -} - - -TEST(tags, EHF6) -{ - { - Tree t = parse_in_arena(R"(!!map { - k: !!seq [ a, !!str b], - j: !!seq - [ a, !!str b] -})"); - ASSERT_TRUE(t.rootref().has_val_tag()); - EXPECT_EQ(t.rootref().val_tag(), "!!map"); - ASSERT_TRUE(t["k"].has_val_tag()); - ASSERT_TRUE(t["j"].has_val_tag()); - EXPECT_EQ(t["k"].val_tag(), "!!seq"); - EXPECT_EQ(t["j"].val_tag(), "!!seq"); - } - { - Tree t = parse_in_arena(R"(!!seq [ - !!map { !!str k: v}, - !!map { !!str ? k: v} -])"); - ASSERT_TRUE(t.rootref().has_val_tag()); - EXPECT_EQ(t.rootref().val_tag(), "!!seq"); - ASSERT_TRUE(t[0].has_val_tag()); - ASSERT_TRUE(t[1].has_val_tag()); - EXPECT_EQ(t[0].val_tag(), "!!map"); - EXPECT_EQ(t[1].val_tag(), "!!map"); - ASSERT_TRUE(t[0]["k"].has_key_tag()); - ASSERT_TRUE(t[1]["k"].has_key_tag()); - EXPECT_EQ(t[0]["k"].key_tag(), "!!str"); - EXPECT_EQ(t[1]["k"].key_tag(), "!!str"); - } -} - - -//----------------------------------------------------------------------------- - -TEST(to_tag, user) -{ - EXPECT_EQ(to_tag("!"), TAG_NONE); - EXPECT_EQ(to_tag("!."), TAG_NONE); - EXPECT_EQ(to_tag("!good_type"), TAG_NONE); -} - -TEST(to_tag, double_exc_mark) -{ - EXPECT_EQ(to_tag("!!" ), TAG_NONE); - EXPECT_EQ(to_tag("!!." ), TAG_NONE); - - EXPECT_EQ(to_tag("!!map" ), TAG_MAP); - EXPECT_EQ(to_tag("!!omap" ), TAG_OMAP); - EXPECT_EQ(to_tag("!!pairs" ), TAG_PAIRS); - EXPECT_EQ(to_tag("!!set" ), TAG_SET); - EXPECT_EQ(to_tag("!!seq" ), TAG_SEQ); - EXPECT_EQ(to_tag("!!binary" ), TAG_BINARY); - EXPECT_EQ(to_tag("!!bool" ), TAG_BOOL); - EXPECT_EQ(to_tag("!!float" ), TAG_FLOAT); - EXPECT_EQ(to_tag("!!int" ), TAG_INT); - EXPECT_EQ(to_tag("!!merge" ), TAG_MERGE); - EXPECT_EQ(to_tag("!!null" ), TAG_NULL); - EXPECT_EQ(to_tag("!!str" ), TAG_STR); - EXPECT_EQ(to_tag("!!timestamp" ), TAG_TIMESTAMP); - EXPECT_EQ(to_tag("!!value" ), TAG_VALUE); - - EXPECT_EQ(to_tag("!!map." ), TAG_NONE); - EXPECT_EQ(to_tag("!!omap." ), TAG_NONE); - EXPECT_EQ(to_tag("!!pairs." ), TAG_NONE); - EXPECT_EQ(to_tag("!!set." ), TAG_NONE); - EXPECT_EQ(to_tag("!!seq." ), TAG_NONE); - EXPECT_EQ(to_tag("!!binary." ), TAG_NONE); - EXPECT_EQ(to_tag("!!bool." ), TAG_NONE); - EXPECT_EQ(to_tag("!!float." ), TAG_NONE); - EXPECT_EQ(to_tag("!!int." ), TAG_NONE); - EXPECT_EQ(to_tag("!!merge." ), TAG_NONE); - EXPECT_EQ(to_tag("!!null." ), TAG_NONE); - EXPECT_EQ(to_tag("!!str." ), TAG_NONE); - EXPECT_EQ(to_tag("!!timestamp."), TAG_NONE); - EXPECT_EQ(to_tag("!!value." ), TAG_NONE); -} - -TEST(to_tag, with_namespace) -{ - EXPECT_EQ(to_tag("tag:yaml.org,2002:" ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:." ), TAG_NONE); - - EXPECT_EQ(to_tag("tag:yaml.org,2002:map" ), TAG_MAP); - EXPECT_EQ(to_tag("tag:yaml.org,2002:omap" ), TAG_OMAP); - EXPECT_EQ(to_tag("tag:yaml.org,2002:pairs" ), TAG_PAIRS); - EXPECT_EQ(to_tag("tag:yaml.org,2002:set" ), TAG_SET); - EXPECT_EQ(to_tag("tag:yaml.org,2002:seq" ), TAG_SEQ); - EXPECT_EQ(to_tag("tag:yaml.org,2002:binary" ), TAG_BINARY); - EXPECT_EQ(to_tag("tag:yaml.org,2002:bool" ), TAG_BOOL); - EXPECT_EQ(to_tag("tag:yaml.org,2002:float" ), TAG_FLOAT); - EXPECT_EQ(to_tag("tag:yaml.org,2002:int" ), TAG_INT); - EXPECT_EQ(to_tag("tag:yaml.org,2002:merge" ), TAG_MERGE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:null" ), TAG_NULL); - EXPECT_EQ(to_tag("tag:yaml.org,2002:str" ), TAG_STR); - EXPECT_EQ(to_tag("tag:yaml.org,2002:timestamp" ), TAG_TIMESTAMP); - EXPECT_EQ(to_tag("tag:yaml.org,2002:value" ), TAG_VALUE); - - EXPECT_EQ(to_tag("tag:yaml.org,2002:map." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:omap." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:pairs." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:set." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:seq." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:binary." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:bool." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:float." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:int." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:merge." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:null." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:str." ), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:timestamp."), TAG_NONE); - EXPECT_EQ(to_tag("tag:yaml.org,2002:value." ), TAG_NONE); -} - -TEST(to_tag, with_namespace_bracket) -{ - EXPECT_EQ(to_tag("" ), TAG_MAP); - EXPECT_EQ(to_tag("" ), TAG_OMAP); - EXPECT_EQ(to_tag("" ), TAG_PAIRS); - EXPECT_EQ(to_tag("" ), TAG_SET); - EXPECT_EQ(to_tag("" ), TAG_SEQ); - EXPECT_EQ(to_tag("" ), TAG_BINARY); - EXPECT_EQ(to_tag("" ), TAG_BOOL); - EXPECT_EQ(to_tag("" ), TAG_FLOAT); - EXPECT_EQ(to_tag("" ), TAG_INT); - EXPECT_EQ(to_tag("" ), TAG_MERGE); - EXPECT_EQ(to_tag("" ), TAG_NULL); - EXPECT_EQ(to_tag("" ), TAG_STR); - EXPECT_EQ(to_tag("" ), TAG_TIMESTAMP); - EXPECT_EQ(to_tag("" ), TAG_VALUE); - - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); - EXPECT_EQ(to_tag(""), TAG_NONE); - EXPECT_EQ(to_tag("" ), TAG_NONE); -} - -TEST(from_tag, basic) -{ - EXPECT_EQ("", from_tag(TAG_NONE)); - - EXPECT_EQ("!!map" , from_tag(TAG_MAP)); - EXPECT_EQ("!!omap" , from_tag(TAG_OMAP)); - EXPECT_EQ("!!pairs" , from_tag(TAG_PAIRS)); - EXPECT_EQ("!!set" , from_tag(TAG_SET)); - EXPECT_EQ("!!seq" , from_tag(TAG_SEQ)); - EXPECT_EQ("!!binary" , from_tag(TAG_BINARY)); - EXPECT_EQ("!!bool" , from_tag(TAG_BOOL)); - EXPECT_EQ("!!float" , from_tag(TAG_FLOAT)); - EXPECT_EQ("!!int" , from_tag(TAG_INT)); - EXPECT_EQ("!!merge" , from_tag(TAG_MERGE)); - EXPECT_EQ("!!null" , from_tag(TAG_NULL)); - EXPECT_EQ("!!str" , from_tag(TAG_STR)); - EXPECT_EQ("!!timestamp" , from_tag(TAG_TIMESTAMP)); - EXPECT_EQ("!!value" , from_tag(TAG_VALUE)); -} - -TEST(normalize_tag, basic) -{ - EXPECT_EQ(normalize_tag("" ), "!!map"); - EXPECT_EQ(normalize_tag("" ), "!!omap"); - EXPECT_EQ(normalize_tag("" ), "!!pairs"); - EXPECT_EQ(normalize_tag("" ), "!!set"); - EXPECT_EQ(normalize_tag("" ), "!!seq"); - EXPECT_EQ(normalize_tag("" ), "!!binary"); - EXPECT_EQ(normalize_tag("" ), "!!bool"); - EXPECT_EQ(normalize_tag("" ), "!!float"); - EXPECT_EQ(normalize_tag("" ), "!!int"); - EXPECT_EQ(normalize_tag("" ), "!!merge"); - EXPECT_EQ(normalize_tag("" ), "!!null"); - EXPECT_EQ(normalize_tag("" ), "!!str"); - EXPECT_EQ(normalize_tag("" ), "!!timestamp"); - EXPECT_EQ(normalize_tag("" ), "!!value"); - - EXPECT_EQ(normalize_tag("!" ), "!!map"); - EXPECT_EQ(normalize_tag("!" ), "!!omap"); - EXPECT_EQ(normalize_tag("!" ), "!!pairs"); - EXPECT_EQ(normalize_tag("!" ), "!!set"); - EXPECT_EQ(normalize_tag("!" ), "!!seq"); - EXPECT_EQ(normalize_tag("!" ), "!!binary"); - EXPECT_EQ(normalize_tag("!" ), "!!bool"); - EXPECT_EQ(normalize_tag("!" ), "!!float"); - EXPECT_EQ(normalize_tag("!" ), "!!int"); - EXPECT_EQ(normalize_tag("!" ), "!!merge"); - EXPECT_EQ(normalize_tag("!" ), "!!null"); - EXPECT_EQ(normalize_tag("!" ), "!!str"); - EXPECT_EQ(normalize_tag("!"), "!!timestamp"); - EXPECT_EQ(normalize_tag("!" ), "!!value"); - - EXPECT_EQ(normalize_tag("!!map" ), "!!map"); - EXPECT_EQ(normalize_tag("!!omap" ), "!!omap"); - EXPECT_EQ(normalize_tag("!!pairs" ), "!!pairs"); - EXPECT_EQ(normalize_tag("!!set" ), "!!set"); - EXPECT_EQ(normalize_tag("!!seq" ), "!!seq"); - EXPECT_EQ(normalize_tag("!!binary" ), "!!binary"); - EXPECT_EQ(normalize_tag("!!bool" ), "!!bool"); - EXPECT_EQ(normalize_tag("!!float" ), "!!float"); - EXPECT_EQ(normalize_tag("!!int" ), "!!int"); - EXPECT_EQ(normalize_tag("!!merge" ), "!!merge"); - EXPECT_EQ(normalize_tag("!!null" ), "!!null"); - EXPECT_EQ(normalize_tag("!!str" ), "!!str"); - EXPECT_EQ(normalize_tag("!!timestamp"), "!!timestamp"); - EXPECT_EQ(normalize_tag("!!value" ), "!!value"); - - EXPECT_EQ(normalize_tag("!!foo" ), "!!foo"); - - EXPECT_EQ(normalize_tag("!my-light"), "!my-light"); - EXPECT_EQ(normalize_tag("!foo"), "!foo"); - EXPECT_EQ(normalize_tag(""), ""); - EXPECT_EQ(normalize_tag(""), ""); - EXPECT_EQ(normalize_tag(""), ""); - - EXPECT_EQ(normalize_tag("!"), ""); - EXPECT_EQ(normalize_tag("!"), ""); - EXPECT_EQ(normalize_tag("!"), ""); -} - -TEST(normalize_tag_long, basic) -{ - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - EXPECT_EQ(normalize_tag_long("" ), ""); - - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - EXPECT_EQ(normalize_tag_long("!"), ""); - EXPECT_EQ(normalize_tag_long("!" ), ""); - - EXPECT_EQ(normalize_tag_long("!!map" ), ""); - EXPECT_EQ(normalize_tag_long("!!omap" ), ""); - EXPECT_EQ(normalize_tag_long("!!pairs" ), ""); - EXPECT_EQ(normalize_tag_long("!!set" ), ""); - EXPECT_EQ(normalize_tag_long("!!seq" ), ""); - EXPECT_EQ(normalize_tag_long("!!binary" ), ""); - EXPECT_EQ(normalize_tag_long("!!bool" ), ""); - EXPECT_EQ(normalize_tag_long("!!float" ), ""); - EXPECT_EQ(normalize_tag_long("!!int" ), ""); - EXPECT_EQ(normalize_tag_long("!!merge" ), ""); - EXPECT_EQ(normalize_tag_long("!!null" ), ""); - EXPECT_EQ(normalize_tag_long("!!str" ), ""); - EXPECT_EQ(normalize_tag_long("!!timestamp"), ""); - EXPECT_EQ(normalize_tag_long("!!value" ), ""); - - EXPECT_EQ(normalize_tag_long("!!foo" ), "!!foo"); - - EXPECT_EQ(normalize_tag_long("!my-light"), "!my-light"); - EXPECT_EQ(normalize_tag_long("!foo"), "!foo"); - EXPECT_EQ(normalize_tag_long(""), ""); - EXPECT_EQ(normalize_tag_long(""), ""); - EXPECT_EQ(normalize_tag_long(""), ""); - - EXPECT_EQ(normalize_tag_long("!"), ""); - EXPECT_EQ(normalize_tag_long("!"), ""); - EXPECT_EQ(normalize_tag_long("!"), ""); - EXPECT_EQ(normalize_tag_long("!"), ""); - EXPECT_EQ(normalize_tag_long("!"), ""); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -CASE_GROUP(TAG_PROPERTY) -{ - -ADD_CASE_TO_GROUP("user tag, empty, test suite 52DL", -R"(! a)", -N(DOCVAL, TS("!", "a")) -); - -ADD_CASE_TO_GROUP("tag property in implicit map, std tags", -R"(ivar: !!int 0 -svar: !!str 0 -fvar: !!float 0.1 -!!int 2: !!float 3 -!!float 3: !!int 3.4 -!!str key: !!int val -myObject: !myClass { name: Joe, age: 15 } -picture: !!binary >- - R0lGODdhDQAIAIAAAAAAANn - Z2SwAAAAADQAIAAACF4SDGQ - ar3xxbJ9p0qa7R0YxwzaFME - 1IAADs= -)", - L{ - N("ivar", TS("!!int", "0")), - N("svar", TS("!!str", "0")), - N("fvar", TS("!!float", "0.1")), - N(TS("!!int", "2"), TS("!!float", "3")), - N(TS("!!float", "3"), TS("!!int", "3.4")), - N(TS("!!str", "key"), TS("!!int", "val")), - N("myObject", TL("!myClass", L{N("name", "Joe"), N("age", "15")})), - N(QV, "picture", TS("!!binary", R"(R0lGODdhDQAIAIAAAAAAANn Z2SwAAAAADQAIAAACF4SDGQ ar3xxbJ9p0qa7R0YxwzaFME 1IAADs=)")), - } -); - -ADD_CASE_TO_GROUP("tag property in implicit map, usr tags", -R"(ivar: !int 0 -svar: !str 0 -fvar: !float 0.1 -!int 2: !float 3 -!float 3: !int 3.4 -!str key: !int val -myObject: !myClass { name: Joe, age: 15 } -picture: !binary >- - R0lGODdhDQAIAIAAAAAAANn - Z2SwAAAAADQAIAAACF4SDGQ - ar3xxbJ9p0qa7R0YxwzaFME - 1IAADs= -)", - L{ - N("ivar", TS("!int", "0")), - N("svar", TS("!str", "0")), - N("fvar", TS("!float", "0.1")), - N(TS("!int", "2"), TS("!float", "3")), - N(TS("!float", "3"), TS("!int", "3.4")), - N(TS("!str", "key"), TS("!int", "val")), - N("myObject", TL("!myClass", L{N("name", "Joe"), N("age", "15")})), - N(QV, "picture", TS("!binary", R"(R0lGODdhDQAIAIAAAAAAANn Z2SwAAAAADQAIAAACF4SDGQ ar3xxbJ9p0qa7R0YxwzaFME 1IAADs=)")), - } -); - -ADD_CASE_TO_GROUP("tag property in explicit map, std tags", -R"({ -ivar: !!int 0, -svar: !!str 0, -!!str key: !!int val -})", - L{ - N("ivar", TS("!!int", "0")), - N("svar", TS("!!str", "0")), - N(TS("!!str", "key"), TS("!!int", "val")) - } -); - -ADD_CASE_TO_GROUP("tag property in explicit map, usr tags", -R"({ -ivar: !int 0, -svar: !str 0, -!str key: !int val -} -)", - L{ - N("ivar", TS("!int", "0")), - N("svar", TS("!str", "0")), - N(TS("!str", "key"), TS("!int", "val")) - } -); - -ADD_CASE_TO_GROUP("tag property in implicit seq, std tags", -R"(- !!int 0 -- !!str 0 -)", - L{ - N(TS("!!int", "0")), - N(TS("!!str", "0")), - } -); - -ADD_CASE_TO_GROUP("tag property in implicit seq, usr tags", -R"(- !int 0 -- !str 0 -)", - L{ - N(TS("!int", "0")), - N(TS("!str", "0")), - } -); - -ADD_CASE_TO_GROUP("tag property in explicit seq, std tags", -R"([ -!!int 0, -!!str 0 -] -)", - L{ - N(TS("!!int", "0")), - N(TS("!!str", "0")), - } -); - -ADD_CASE_TO_GROUP("tag property in explicit seq, usr tags", -R"([ -!int 0, -!str 0 -] -)", - L{ - N(TS("!int", "0")), - N(TS("!str", "0")), - } -); - -ADD_CASE_TO_GROUP("tagged explicit sequence in map, std tags", -R"(some_seq: !!its_type [ -!!int 0, -!!str 0 -] -)", - L{N("some_seq", TL("!!its_type", L{ - N(TS("!!int", "0")), - N(TS("!!str", "0")), - })) - } -); - -ADD_CASE_TO_GROUP("tagged explicit sequence in map, usr tags", -R"(some_seq: !its_type [ -!int 0, -!str 0 -] -)", - L{N("some_seq", TL("!its_type", L{ - N(TS("!int", "0")), - N(TS("!str", "0")), - })) - } -); - -ADD_CASE_TO_GROUP("tagged doc", -R"( ---- !!map -a: 0 -b: 1 ---- !map -? a -: b ---- !!seq -- a -- b ---- !!str -a - b -... ---- !!str a b -... ---- !!str a b ---- !!str -a: b ---- !!str a: b ---- -!!str a: b ---- -!!str a - b ---- -!!set -? a -? b ---- !!set -? a -? b -)", -N(STREAM, L{ - N(DOCMAP, TL("!!map", L{N("a", "0"), N("b", "1")})), - N(DOCMAP, TL("!map", L{N("a", "b")})), - N(DOCSEQ, TL("!!seq", L{N("a"), N("b")})), - N(DOCVAL, TS("!!str", "a b")), - N(DOCVAL, TS("!!str", "a b")), - N(DOCVAL, TS("!!str", "a b")), - N(DOCVAL, TS("!!str", "a: b")), - N(DOCVAL, TS("!!str", "a: b")), - N(DOCMAP, L{N(TS("!!str", "a"), "b")}), - N(DOCVAL, TS("!!str", "a b")), - N(DOCMAP, TL("!!set", L{N(KEYVAL, "a", /*"~"*/{}), N(KEYVAL, "b", /*"~"*/{})})), - N(DOCMAP, TL("!!set", L{N(KEYVAL, "a", /*"~"*/{}), N(KEYVAL, "b", /*"~"*/{})})), -})); - - -ADD_CASE_TO_GROUP("ambiguous tag in map, std tag", -R"(!!map -!!str a0: !!xxx b0 -!!str fooz: !!map - k1: !!float 1.0 - k3: !!float 2.0 -!!str foo: !!map - !!int 1: !!float 20.0 - !!int 3: !!float 40.0 -bar: !!map - 10: !!str 2 - 30: !!str 4 -!!str baz: - !!int 10: !!float 20 - !!int 30: !!float 40 -)", -TL("!!map", L{ - N(TS("!!str", "a0"), TS("!!xxx", "b0")), - N(TS("!!str", "fooz"), TL("!!map", L{N("k1", TS("!!float", "1.0")), N("k3", TS("!!float", "2.0"))})), - N(TS("!!str", "foo"), TL("!!map", L{N(TS("!!int", "1"), TS("!!float", "20.0")), N(TS("!!int", "3"), TS("!!float", "40.0"))})), - N("bar", TL("!!map", L{N("10", TS("!!str", "2")), N("30", TS("!!str", "4"))})), - N(TS("!!str", "baz"), L{N(TS("!!int", "10"), TS("!!float", "20")), N(TS("!!int", "30"), TS("!!float", "40"))}), -})); - -ADD_CASE_TO_GROUP("ambiguous tag in map, usr tag", -R"(!map -!str a0: !xxx b0 -!str fooz: !map - k1: !float 1.0 - k3: !float 2.0 -!str foo: !map - !int 1: !float 20.0 - !int 3: !float 40.0 -bar: !map - 10: !str 2 - 30: !str 4 -!str baz: - !int 10: !float 20 - !int 30: !float 40 -)", -TL("!map", L{ - N(TS("!str", "a0"), TS("!xxx", "b0")), - N(TS("!str", "fooz"), TL("!map", L{N("k1", TS("!float", "1.0")), N("k3", TS("!float", "2.0"))})), - N(TS("!str", "foo"), TL("!map", L{N(TS("!int", "1"), TS("!float", "20.0")), N(TS("!int", "3"), TS("!float", "40.0"))})), - N("bar", TL("!map", L{N("10", TS("!str", "2")), N("30", TS("!str", "4"))})), - N(TS("!str", "baz"), L{N(TS("!int", "10"), TS("!float", "20")), N(TS("!int", "30"), TS("!float", "40"))}), -})); - - -ADD_CASE_TO_GROUP("ambiguous tag in seq, std tag", -R"(!!seq -- !!str k1: v1 - !!str k2: v2 - !!str k3: v3 -- !!map - !!str k4: v4 - !!str k5: v5 - !!str k6: v6 -- !!map - k7: v7 - k8: v8 - k9: v9 -- - !!str v10 - - !!str v20 - - !!str v30 -- !!seq - - !!str v40 - - !!str v50 - - !!str v60 -- !!seq - - v70 - - v80 - - v90 -)", -TL("!!seq", L{ - N(L{N(TS("!!str", "k1"), "v1"), N(TS("!!str", "k2"), "v2"), N(TS("!!str", "k3"), "v3"), }), - N(TL("!!map", L{N(TS("!!str", "k4"), "v4"), N(TS("!!str", "k5"), "v5"), N(TS("!!str", "k6"), "v6"), })), - N(TL("!!map", L{N("k7", "v7"), N("k8", "v8"), N("k9", "v9"), })), - N(L{N(TS("!!str", "v10")), N(TS("!!str", "v20")), N(TS("!!str", "v30"))}), - N(TL("!!seq", L{N(TS("!!str", "v40")), N(TS("!!str", "v50")), N(TS("!!str", "v60"))})), - N(TL("!!seq", L{N("v70"), N("v80"), N("v90")})), -})); - -ADD_CASE_TO_GROUP("ambiguous tag in seq, usr tag", -R"(!seq -- !str k1: v1 - !str k2: v2 - !str k3: v3 -- !map - !str k4: v4 - !str k5: v5 - !str k6: v6 -- !map - k7: v7 - k8: v8 - k9: v9 -- - !str v10 - - !str v20 - - !str v30 -- !seq - - !str v40 - - !str v50 - - !str v60 -- !seq - - v70 - - v80 - - v90 -)", -TL("!seq", L{ - N(L{N(TS("!str", "k1"), "v1"), N(TS("!str", "k2"), "v2"), N(TS("!str", "k3"), "v3"), }), - N(TL("!map", L{N(TS("!str", "k4"), "v4"), N(TS("!str", "k5"), "v5"), N(TS("!str", "k6"), "v6"), })), - N(TL("!map", L{N("k7", "v7"), N("k8", "v8"), N("k9", "v9"), })), - N(L{N(TS("!str", "v10")), N(TS("!str", "v20")), N(TS("!str", "v30"))}), - N(TL("!seq", L{N(TS("!str", "v40")), N(TS("!str", "v50")), N(TS("!str", "v60"))})), - N(TL("!seq", L{N("v70"), N("v80"), N("v90")})), -})); -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/test/test_tree.cpp b/thirdparty/ryml/test/test_tree.cpp deleted file mode 100644 index b6aad0435..000000000 --- a/thirdparty/ryml/test/test_tree.cpp +++ /dev/null @@ -1,3924 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include "c4/yml/std/std.hpp" -#include "c4/yml/parse.hpp" -#include "c4/yml/emit.hpp" -#include -#include -#include -#endif -#include "./test_case.hpp" -#include "./callbacks_tester.hpp" - -#include - -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable: 4389) // signed/unsigned mismatch -#elif defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wdollar-in-identifier-extension" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -#endif - -namespace c4 { -namespace yml { - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -void node_scalar_test_empty(NodeScalar const& s) -{ - EXPECT_TRUE(s.empty()); - EXPECT_EQ(s.tag, ""); - EXPECT_EQ(s.tag.len, 0u); - EXPECT_TRUE(s.tag.empty()); - EXPECT_EQ(s.scalar, ""); - EXPECT_EQ(s.scalar.len, 0u); - EXPECT_TRUE(s.scalar.empty()); -} - -void node_scalar_test_foo(NodeScalar const& s, bool with_tag=false) -{ - EXPECT_FALSE(s.empty()); - if(with_tag) - { - EXPECT_EQ(s.tag, "!!str"); - EXPECT_EQ(s.tag.len, 5u); - EXPECT_FALSE(s.tag.empty()); - } - else - { - EXPECT_EQ(s.tag, ""); - EXPECT_EQ(s.tag.len, 0u); - EXPECT_TRUE(s.tag.empty()); - } - EXPECT_EQ(s.scalar, "foo"); - EXPECT_EQ(s.scalar.len, 3u); - EXPECT_FALSE(s.scalar.empty()); -} - -void node_scalar_test_foo3(NodeScalar const& s, bool with_tag=false) -{ - EXPECT_FALSE(s.empty()); - if(with_tag) - { - EXPECT_EQ(s.tag, "!!str+++"); - EXPECT_EQ(s.tag.len, 8u); - EXPECT_FALSE(s.tag.empty()); - } - else - { - EXPECT_EQ(s.tag, ""); - EXPECT_EQ(s.tag.len, 0u); - EXPECT_TRUE(s.tag.empty()); - } - EXPECT_EQ(s.scalar, "foo3"); - EXPECT_EQ(s.scalar.len, 4u); - EXPECT_FALSE(s.scalar.empty()); -} - -TEST(NodeScalar, ctor_empty) -{ - NodeScalar s; - node_scalar_test_empty(s); -} - -TEST(NodeScalar, ctor__untagged) -{ - { - const char sarr[] = "foo"; - const char *sptr = "foo"; - csubstr ssp = "foo"; - - for(auto s : {NodeScalar(sarr), NodeScalar(to_csubstr(sptr)), NodeScalar(ssp)}) - { - node_scalar_test_foo(s); - } - - NodeScalar s; - s = {sarr}; - node_scalar_test_foo(s); - s = to_csubstr(sptr); - node_scalar_test_foo(s); - s = {ssp}; - node_scalar_test_foo(s); - } - - { - const char sarr[] = "foo3"; - const char *sptr = "foo3"; - csubstr ssp = "foo3"; - - for(auto s : {NodeScalar(sarr), NodeScalar(to_csubstr(sptr)), NodeScalar(ssp)}) - { - node_scalar_test_foo3(s); - } - - NodeScalar s; - { - SCOPED_TRACE("here 1"); - s = {sarr}; - node_scalar_test_foo3(s); - } - { - SCOPED_TRACE("here 2"); - s = to_csubstr(sptr); - node_scalar_test_foo3(s); - } - { - SCOPED_TRACE("here 3"); - s = ssp; - node_scalar_test_foo3(s); - } - } -} - -TEST(NodeScalar, ctor__tagged) -{ - { - const char sarr[] = "foo", tarr[] = "!!str"; - const char *sptr = "foo"; - const char *tptr = "!!str"; - csubstr ssp = "foo", tsp = "!!str"; - - for(NodeScalar s : { - NodeScalar(tsp, ssp), - NodeScalar(tsp, to_csubstr(sptr)), - NodeScalar(tsp, sarr), - NodeScalar(to_csubstr(tptr), ssp), - NodeScalar(to_csubstr(tptr), to_csubstr(sptr)), - NodeScalar(to_csubstr(tptr), sarr), - NodeScalar(tarr, ssp), - NodeScalar(tarr, to_csubstr(sptr)), - NodeScalar(tarr, sarr), - }) - { - node_scalar_test_foo(s, true); - } - - NodeScalar s; - - { - SCOPED_TRACE("here 0.0"); - s = {tsp, ssp}; - node_scalar_test_foo(s, true); - } - { - SCOPED_TRACE("here 0.1"); - s = {tsp, to_csubstr(sptr)}; - node_scalar_test_foo(s, true); - } - { - SCOPED_TRACE("here 0.2"); - s = {tsp, sarr}; - node_scalar_test_foo(s, true); - } - - { - SCOPED_TRACE("here 1.0"); - s = {to_csubstr(tptr), ssp}; - node_scalar_test_foo(s, true); - } - { - SCOPED_TRACE("here 1.1"); - s = {to_csubstr(tptr), to_csubstr(sptr)}; - node_scalar_test_foo(s, true); - } - { - SCOPED_TRACE("here 1.3"); - s = {to_csubstr(tptr), sarr}; - node_scalar_test_foo(s, true); - } - - { - SCOPED_TRACE("here 3.0"); - s = {tarr, ssp}; - node_scalar_test_foo(s, true); - } - { - SCOPED_TRACE("here 3.1"); - s = {tarr, to_csubstr(sptr)}; - node_scalar_test_foo(s, true); - } - { - SCOPED_TRACE("here 3.3"); - s = {tarr, sarr}; - node_scalar_test_foo(s, true); - } - - } - - { - const char sarr[] = "foo3", tarr[] = "!!str+++"; - const char *sptr = "foo3"; - const char *tptr = "!!str+++"; - csubstr ssp = "foo3", tsp = "!!str+++"; - - NodeScalar wtf = {tsp, ssp}; - EXPECT_EQ(wtf.tag, tsp); - EXPECT_EQ(wtf.scalar, ssp); - for(auto s : { - NodeScalar(tsp, ssp), - NodeScalar(tsp, to_csubstr(sptr)), - NodeScalar(tsp, sarr), - NodeScalar(to_csubstr(tptr), ssp), - NodeScalar(to_csubstr(tptr), to_csubstr(sptr)), - NodeScalar(to_csubstr(tptr), sarr), - NodeScalar(tarr, ssp), - NodeScalar(tarr, to_csubstr(sptr)), - NodeScalar(tarr, sarr), - }) - { - node_scalar_test_foo3(s, true); - } - - NodeScalar s; - - { - SCOPED_TRACE("here 0.0"); - s = {tsp, ssp}; - node_scalar_test_foo3(s, true); - } - { - SCOPED_TRACE("here 0.1"); - s = {tsp, to_csubstr(sptr)}; - node_scalar_test_foo3(s, true); - } - { - SCOPED_TRACE("here 0.3"); - s = {tsp, sarr}; - node_scalar_test_foo3(s, true); - } - - { - SCOPED_TRACE("here 1.0"); - s = {to_csubstr(tptr), ssp}; - node_scalar_test_foo3(s, true); - } - { - SCOPED_TRACE("here 1.1"); - s = {to_csubstr(tptr), to_csubstr(sptr)}; - node_scalar_test_foo3(s, true); - } - { - SCOPED_TRACE("here 1.3"); - s = {to_csubstr(tptr), sarr}; - node_scalar_test_foo3(s, true); - } - - { - SCOPED_TRACE("here 3.0"); - s = {tarr, ssp}; - node_scalar_test_foo3(s, true); - } - { - SCOPED_TRACE("here 3.1"); - s = {tarr, to_csubstr(sptr)}; - node_scalar_test_foo3(s, true); - } - { - SCOPED_TRACE("here 3.3"); - s = {tarr, sarr}; - node_scalar_test_foo3(s, true); - } - - } - -} - - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(NodeInit, ctor__empty) -{ - NodeInit n; - EXPECT_EQ((type_bits)n.type, (type_bits)NOTYPE); - EXPECT_EQ(n.key.scalar, ""); - EXPECT_EQ(n.key.tag, ""); - EXPECT_EQ(n.val.scalar, ""); - EXPECT_EQ(n.val.tag, ""); -} - -TEST(NodeInit, ctor__type_only) -{ - for(auto k : {KEY, KEYVAL, MAP, KEYMAP, SEQ, KEYSEQ}) - { - SCOPED_TRACE(NodeType::type_str(k)); - NodeInit n(k); - EXPECT_EQ((type_bits)n.type, (type_bits)k); - EXPECT_EQ(n.key.scalar, ""); - EXPECT_EQ(n.key.tag, ""); - EXPECT_EQ(n.val.scalar, ""); - EXPECT_EQ(n.val.tag, ""); - } -} - -TEST(NodeInit, ctor__val_only) -{ - { - const char sarr[] = "foo"; - const char *sptr = "foo"; size_t sptrlen = 3; - csubstr ssp = "foo"; - - { - SCOPED_TRACE("here 0"); - { - NodeInit s(sarr); - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - s.clear(); - } - { - NodeInit s{to_csubstr(sptr)}; - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - s.clear(); - } - { - NodeInit s{sarr}; - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - s.clear(); - } - } - - { - SCOPED_TRACE("here 1"); - { - NodeInit s(sarr); - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - s.clear(); - } - { - NodeInit s(to_csubstr(sptr)); - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - s.clear(); - } - { - NodeInit s(sarr); - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - s.clear(); - } - } - - { - SCOPED_TRACE("here 2"); - NodeInit s; - s = {sarr}; - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - s.clear(); - s = {to_csubstr(sptr)}; - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - s.clear(); - //s = {sptr, sptrlen}; // fails to compile - //node_scalar_test_foo(s.val); - //node_scalar_test_empty(s.key); - //s.clear(); - s = {ssp}; - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - s.clear(); - } - - for(auto s : { - NodeInit(sarr), - NodeInit(to_csubstr(sptr)), - NodeInit(csubstr{sptr, sptrlen}), - NodeInit(ssp)}) - { - SCOPED_TRACE("here LOOP"); - node_scalar_test_foo(s.val); - node_scalar_test_empty(s.key); - } - } - - { - const char sarr[] = "foo3"; - const char *sptr = "foo3"; size_t sptrlen = 4; - csubstr ssp = "foo3"; - - { - SCOPED_TRACE("here 0"); - NodeInit s = {sarr}; - node_scalar_test_foo3(s.val); - node_scalar_test_empty(s.key); - } - { // FAILS - SCOPED_TRACE("here 1"); - //NodeInit s = sarr; - //node_scalar_test_foo3(s.val); - //node_scalar_test_empty(s.key); - } - { - SCOPED_TRACE("here 2"); - NodeInit s{sarr}; - node_scalar_test_foo3(s.val); - node_scalar_test_empty(s.key); - } - { - SCOPED_TRACE("here 3"); - NodeInit s(sarr); - node_scalar_test_foo3(s.val); - node_scalar_test_empty(s.key); - } - - for(auto s : { - NodeInit(sarr), - NodeInit(to_csubstr(sptr)), - NodeInit(csubstr{sptr, sptrlen}), - NodeInit(ssp)}) - { - SCOPED_TRACE("here LOOP"); - node_scalar_test_foo3(s.val); - node_scalar_test_empty(s.key); - } - } -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(Tree, empty_ctor) -{ - Tree tree; - EXPECT_EQ(tree.callbacks(), get_callbacks()); - EXPECT_EQ(tree.empty(), true); - EXPECT_EQ(tree.capacity(), 0u); - EXPECT_EQ(tree.arena_capacity(), 0u); - EXPECT_EQ(tree.arena_slack(), 0u); - EXPECT_EQ(tree.size(), 0u); - EXPECT_EQ(tree.slack(), 0u); - EXPECT_EQ(tree.arena().empty(), true); -} - -TEST(Tree, node_cap_ctor) -{ - { - Tree tree(10u); - EXPECT_EQ(tree.callbacks(), get_callbacks()); - EXPECT_EQ(tree.empty(), false); // we have the root - EXPECT_EQ(tree.capacity(), 10u); - EXPECT_EQ(tree.arena_capacity(), 0u); - EXPECT_EQ(tree.arena_slack(), 0u); - EXPECT_EQ(tree.arena().empty(), true); - EXPECT_EQ(tree.size(), 1u); // we have the root - EXPECT_EQ(tree.slack(), 9u); - } - { - Tree tree(10u, 20u); - EXPECT_EQ(tree.callbacks(), get_callbacks()); - EXPECT_EQ(tree.empty(), false); // we have the root - EXPECT_EQ(tree.capacity(), 10u); - EXPECT_EQ(tree.arena_capacity(), 20u); - EXPECT_EQ(tree.arena().empty(), true); - EXPECT_EQ(tree.size(), 1u); // we have the root - EXPECT_EQ(tree.slack(), 9u); - } - { - Tree tree(0u, 20u); - EXPECT_EQ(tree.callbacks(), get_callbacks()); - EXPECT_EQ(tree.empty(), true); - EXPECT_EQ(tree.capacity(), 0u); - EXPECT_EQ(tree.arena_capacity(), 20u); - EXPECT_EQ(tree.arena_slack(), 20u); - EXPECT_EQ(tree.arena().empty(), true); - EXPECT_EQ(tree.size(), 0u); - EXPECT_EQ(tree.slack(), 0u); - } -} - -Tree get_test_tree(CallbacksTester *cbt=nullptr) -{ - Parser parser(cbt ? cbt->callbacks() : get_callbacks()); - Tree t = parser.parse_in_arena("", "{a: b, c: d, e: [0, 1, 2, 3]}"); - // make sure the tree has strings in its arena - NodeRef n = t.rootref(); - NodeRef ch = n.append_child(); - ch << key("serialized_key"); - ch << 89; - return t; -} - -TEST(Tree, test_tree_has_arena) -{ - { - Tree t = get_test_tree(); - ASSERT_GT(t.arena().size(), 0u); - } - { - CallbacksTester cbt; - Tree t = get_test_tree(&cbt); - ASSERT_GT(t.arena().size(), 0u); - } -} - -//------------------------------------------- -TEST(Tree, copy_ctor) -{ - CallbacksTester cbt; - { - Tree src = get_test_tree(&cbt); - test_invariants(src); - { - Tree dst(src); - test_invariants(dst); - test_compare(dst, src); - test_arena_not_shared(dst, src); - EXPECT_EQ(dst.callbacks(), src.callbacks()); - } - } -} - -//------------------------------------------- -TEST(Tree, move_ctor) -{ - CallbacksTester cbt; - Tree src = get_test_tree(&cbt); - test_invariants(src); - Tree save(src); - test_invariants(save); - test_compare(save, src); - { - Tree dst(std::move(src)); - EXPECT_EQ(src.empty(), true); - EXPECT_EQ(src.size(), 0u); - EXPECT_EQ(src.arena().empty(), true); - EXPECT_EQ(dst.size(), save.size()); - EXPECT_EQ(dst.arena(), save.arena()); - test_invariants(src); - test_invariants(dst); - test_compare(dst, save); - test_arena_not_shared(src, dst); - test_arena_not_shared(save, dst); - } -} - -//------------------------------------------- -TEST(Tree, copy_assign_same_callbacks) -{ - CallbacksTester cbt; - { - Tree src = get_test_tree(&cbt); - test_invariants(src); - { - Tree dst(cbt.callbacks()); - EXPECT_EQ(dst.callbacks(), src.callbacks()); - test_invariants(dst); - dst = src; - test_invariants(dst); - test_compare(dst, src); - test_arena_not_shared(dst, src); - EXPECT_EQ(dst.callbacks(), src.callbacks()); - } - } -} - -TEST(Tree, copy_assign_diff_callbacks) -{ - CallbacksTester cbsrc("src"); - CallbacksTester cbdst("dst"); - { - Tree src = get_test_tree(&cbsrc); - EXPECT_EQ(src.callbacks(), cbsrc.callbacks()); - test_invariants(src); - { - Tree dst = get_test_tree(&cbdst); - EXPECT_EQ(dst.callbacks(), cbdst.callbacks()); - test_invariants(dst); - dst = src; - test_invariants(dst); - test_compare(dst, src); - test_arena_not_shared(dst, src); - EXPECT_EQ(dst.callbacks(), src.callbacks()); - } - } -} - -//------------------------------------------- -TEST(Tree, move_assign_same_callbacks) -{ - CallbacksTester cbt; - Tree src = get_test_tree(&cbt); - test_invariants(src); - Tree save(src); - EXPECT_EQ(save.callbacks(), src.callbacks()); - test_invariants(save); - test_compare(save, src); - { - Tree dst = get_test_tree(&cbt); - EXPECT_NE(dst.empty(), true); - EXPECT_NE(dst.size(), 0u); - EXPECT_NE(dst.arena().empty(), true); - dst = std::move(src); - EXPECT_EQ(src.empty(), true); - EXPECT_EQ(src.size(), 0u); - EXPECT_EQ(src.arena().empty(), true); - EXPECT_EQ(src.callbacks(), cbt.callbacks()); - EXPECT_EQ(dst.size(), save.size()); - EXPECT_EQ(dst.arena(), save.arena()); - EXPECT_EQ(dst.callbacks(), save.callbacks()); - test_invariants(src); - test_invariants(dst); - test_compare(dst, save); - test_arena_not_shared(src, dst); - test_arena_not_shared(save, dst); - } -} - -TEST(Tree, move_assign_diff_callbacks) -{ - CallbacksTester cbsrc("src"); - CallbacksTester cbdst("dst"); - Tree src = get_test_tree(&cbsrc); - test_invariants(src); - Tree save(src); - test_invariants(save); - test_compare(save, src); - { - Tree dst = get_test_tree(&cbdst); - EXPECT_NE(dst.empty(), true); - EXPECT_NE(dst.size(), 0u); - EXPECT_NE(dst.arena().empty(), true); - EXPECT_EQ(dst.callbacks(), cbdst.callbacks()); - dst = std::move(src); - EXPECT_EQ(src.empty(), true); - EXPECT_EQ(src.size(), 0u); - EXPECT_EQ(src.arena().empty(), true); - EXPECT_EQ(src.callbacks(), cbsrc.callbacks()); - EXPECT_EQ(dst.size(), save.size()); - EXPECT_EQ(dst.arena(), save.arena()); - EXPECT_NE(dst.callbacks(), cbdst.callbacks()); - EXPECT_EQ(dst.callbacks(), save.callbacks()); - test_invariants(src); - test_invariants(dst); - test_compare(dst, save); - test_arena_not_shared(src, dst); - test_arena_not_shared(save, dst); - } -} - -TEST(Tree, std_interop) -{ - CallbacksTester cbt; - std::vector forest; - for(size_t i = 0; i < 3; ++i) - { - forest.emplace_back(cbt.callbacks()); - parse_in_arena("{foo: bar}", &forest.back()); - } -} - - -//------------------------------------------- -TEST(Tree, reserve) -{ - Tree t(16, 64); - EXPECT_EQ(t.capacity(), 16); - EXPECT_EQ(t.slack(), 15); - EXPECT_EQ(t.size(), 1); - EXPECT_EQ(t.arena_capacity(), 64); - EXPECT_EQ(t.arena_slack(), 64); - EXPECT_EQ(t.arena_size(), 0); - test_invariants(t); - - auto buf = t.m_buf; - t.reserve(16); - t.reserve_arena(64); - EXPECT_EQ(t.m_buf, buf); - EXPECT_EQ(t.capacity(), 16); - EXPECT_EQ(t.slack(), 15); - EXPECT_EQ(t.size(), 1); - EXPECT_EQ(t.arena_capacity(), 64); - EXPECT_EQ(t.arena_slack(), 64); - EXPECT_EQ(t.arena_size(), 0); - test_invariants(t); - - t.reserve(32); - t.reserve_arena(128); - EXPECT_EQ(t.capacity(), 32); - EXPECT_EQ(t.slack(), 31); - EXPECT_EQ(t.size(), 1); - EXPECT_EQ(t.arena_capacity(), 128); - EXPECT_EQ(t.arena_slack(), 128); - EXPECT_EQ(t.arena_size(), 0); - test_invariants(t); - - buf = t.m_buf; - parse_in_arena("[a, b, c, d, e, f]", &t); - EXPECT_EQ(t.m_buf, buf); - EXPECT_EQ(t.capacity(), 32); - EXPECT_EQ(t.slack(), 25); - EXPECT_EQ(t.size(), 7); - EXPECT_EQ(t.arena_capacity(), 128); - EXPECT_EQ(t.arena_slack(), 110); - EXPECT_EQ(t.arena_size(), 18); - test_invariants(t); - - t.reserve(64); - t.reserve_arena(256); - EXPECT_EQ(t.capacity(), 64); - EXPECT_EQ(t.slack(), 57); - EXPECT_EQ(t.size(), 7); - EXPECT_EQ(t.arena_capacity(), 256); - EXPECT_EQ(t.arena_slack(), 238); - EXPECT_EQ(t.arena_size(), 18); - test_invariants(t); -} - -// https://github.com/biojppm/rapidyaml/issues/288 -TEST(Tree, reserve_arena_issue288) -{ - Tree t; - EXPECT_EQ(t.arena_slack(), 0u); - EXPECT_EQ(t.arena_capacity(), 0u); - EXPECT_EQ(t.arena_size(), 0u); - t.reserve_arena(3u); - EXPECT_EQ(t.arena_slack(), 3u); - EXPECT_GE(t.arena_capacity(), 3u); - EXPECT_EQ(t.arena_size(), 0u); - // longer than the slack to cause another call to _grow_arena() - std::string stars(2 * t.arena_slack(), '*'); - t.copy_to_arena(to_csubstr(stars)); - EXPECT_GE(t.arena_capacity(), stars.size()); - EXPECT_EQ(t.arena_size(), stars.size()); - EXPECT_EQ(t.arena(), to_csubstr(stars)); - // again - std::string pluses(2 * t.arena_slack(), '+'); - t.copy_to_arena(to_csubstr(pluses)); - EXPECT_GE(t.arena_capacity(), stars.size() + pluses.size()); - EXPECT_EQ(t.arena_size(), stars.size() + pluses.size()); - EXPECT_EQ(t.arena().first(stars.size()), to_csubstr(stars)); - EXPECT_EQ(t.arena().last(pluses.size()), to_csubstr(pluses)); -} - -TEST(Tree, clear) -{ - Tree t(16, 64); - EXPECT_EQ(t.capacity(), 16); - EXPECT_EQ(t.slack(), 15); - EXPECT_EQ(t.size(), 1); - EXPECT_EQ(t.arena_capacity(), 64); - EXPECT_EQ(t.arena_slack(), 64); - EXPECT_EQ(t.arena_size(), 0); - test_invariants(t); - - t.clear(); - t.clear_arena(); - EXPECT_EQ(t.capacity(), 16); - EXPECT_EQ(t.slack(), 15); - EXPECT_EQ(t.size(), 1); - EXPECT_EQ(t.arena_capacity(), 64); - EXPECT_EQ(t.arena_slack(), 64); - EXPECT_EQ(t.arena_size(), 0); - test_invariants(t); - - auto buf = t.m_buf; - t.reserve(16); - t.reserve_arena(64); - EXPECT_EQ(t.m_buf, buf); - EXPECT_EQ(t.capacity(), 16); - EXPECT_EQ(t.slack(), 15); - EXPECT_EQ(t.size(), 1); - EXPECT_EQ(t.arena_capacity(), 64); - EXPECT_EQ(t.arena_slack(), 64); - EXPECT_EQ(t.arena_size(), 0); - test_invariants(t); - - t.reserve(32); - t.reserve_arena(128); - EXPECT_EQ(t.capacity(), 32); - EXPECT_EQ(t.slack(), 31); - EXPECT_EQ(t.size(), 1); - EXPECT_EQ(t.arena_capacity(), 128); - EXPECT_EQ(t.arena_slack(), 128); - EXPECT_EQ(t.arena_size(), 0); - test_invariants(t); - - buf = t.m_buf; - parse_in_arena("[a, b, c, d, e, f]", &t); - EXPECT_EQ(t.m_buf, buf); - EXPECT_EQ(t.capacity(), 32); - EXPECT_EQ(t.slack(), 25); - EXPECT_EQ(t.size(), 7); - EXPECT_EQ(t.arena_capacity(), 128); - EXPECT_EQ(t.arena_slack(), 110); - EXPECT_EQ(t.arena_size(), 18); - test_invariants(t); - - t.clear(); - t.clear_arena(); - EXPECT_EQ(t.capacity(), 32); - EXPECT_EQ(t.slack(), 31); - EXPECT_EQ(t.size(), 1); - EXPECT_EQ(t.arena_capacity(), 128); - EXPECT_EQ(t.arena_slack(), 128); - EXPECT_EQ(t.arena_size(), 0); - test_invariants(t); -} - - -//------------------------------------------- - -TEST(Tree, ref) -{ - Tree t = parse_in_arena("[0, 1, 2, 3]"); - EXPECT_EQ(t.ref(0).id(), 0); - EXPECT_EQ(t.ref(1).id(), 1); - EXPECT_EQ(t.ref(2).id(), 2); - EXPECT_EQ(t.ref(3).id(), 3); - EXPECT_EQ(t.ref(4).id(), 4); - EXPECT_TRUE(t.ref(0).is_seq()); - EXPECT_TRUE(t.ref(1).is_val()); - EXPECT_TRUE(t.ref(2).is_val()); - EXPECT_TRUE(t.ref(3).is_val()); - EXPECT_TRUE(t.ref(4).is_val()); -} - -TEST(Tree, ref_const) -{ - const Tree t = parse_in_arena("[0, 1, 2, 3]"); - EXPECT_EQ(t.ref(0).id(), 0); - EXPECT_EQ(t.ref(1).id(), 1); - EXPECT_EQ(t.ref(2).id(), 2); - EXPECT_EQ(t.ref(3).id(), 3); - EXPECT_EQ(t.ref(4).id(), 4); - EXPECT_TRUE(t.ref(0).is_seq()); - EXPECT_TRUE(t.ref(1).is_val()); - EXPECT_TRUE(t.ref(2).is_val()); - EXPECT_TRUE(t.ref(3).is_val()); - EXPECT_TRUE(t.ref(4).is_val()); -} - - -TEST(Tree, operator_square_brackets) -{ - { - Tree t = parse_in_arena("[0, 1, 2, 3, 4]"); - Tree &m = t; - Tree const& cm = t; - EXPECT_EQ(m[0].val(), "0"); - EXPECT_EQ(m[1].val(), "1"); - EXPECT_EQ(m[2].val(), "2"); - EXPECT_EQ(m[3].val(), "3"); - EXPECT_EQ(m[4].val(), "4"); - EXPECT_EQ(cm[0].val(), "0"); - EXPECT_EQ(cm[1].val(), "1"); - EXPECT_EQ(cm[2].val(), "2"); - EXPECT_EQ(cm[3].val(), "3"); - EXPECT_EQ(cm[4].val(), "4"); - // - EXPECT_TRUE(m[0] == "0"); - EXPECT_TRUE(m[1] == "1"); - EXPECT_TRUE(m[2] == "2"); - EXPECT_TRUE(m[3] == "3"); - EXPECT_TRUE(m[4] == "4"); - EXPECT_TRUE(cm[0] == "0"); - EXPECT_TRUE(cm[1] == "1"); - EXPECT_TRUE(cm[2] == "2"); - EXPECT_TRUE(cm[3] == "3"); - EXPECT_TRUE(cm[4] == "4"); - // - EXPECT_FALSE(m[0] != "0"); - EXPECT_FALSE(m[1] != "1"); - EXPECT_FALSE(m[2] != "2"); - EXPECT_FALSE(m[3] != "3"); - EXPECT_FALSE(m[4] != "4"); - EXPECT_FALSE(cm[0] != "0"); - EXPECT_FALSE(cm[1] != "1"); - EXPECT_FALSE(cm[2] != "2"); - EXPECT_FALSE(cm[3] != "3"); - EXPECT_FALSE(cm[4] != "4"); - } - { - Tree t = parse_in_arena("{a: 0, b: 1, c: 2, d: 3, e: 4}"); - Tree &m = t; - Tree const& cm = t; - EXPECT_EQ(m["a"].val(), "0"); - EXPECT_EQ(m["b"].val(), "1"); - EXPECT_EQ(m["c"].val(), "2"); - EXPECT_EQ(m["d"].val(), "3"); - EXPECT_EQ(m["e"].val(), "4"); - EXPECT_EQ(cm["a"].val(), "0"); - EXPECT_EQ(cm["b"].val(), "1"); - EXPECT_EQ(cm["c"].val(), "2"); - EXPECT_EQ(cm["d"].val(), "3"); - EXPECT_EQ(cm["e"].val(), "4"); - // - EXPECT_TRUE(m["a"] == "0"); - EXPECT_TRUE(m["b"] == "1"); - EXPECT_TRUE(m["c"] == "2"); - EXPECT_TRUE(m["d"] == "3"); - EXPECT_TRUE(m["e"] == "4"); - EXPECT_TRUE(cm["a"] == "0"); - EXPECT_TRUE(cm["b"] == "1"); - EXPECT_TRUE(cm["c"] == "2"); - EXPECT_TRUE(cm["d"] == "3"); - EXPECT_TRUE(cm["e"] == "4"); - // - EXPECT_FALSE(m["a"] != "0"); - EXPECT_FALSE(m["b"] != "1"); - EXPECT_FALSE(m["c"] != "2"); - EXPECT_FALSE(m["d"] != "3"); - EXPECT_FALSE(m["e"] != "4"); - EXPECT_FALSE(cm["a"] != "0"); - EXPECT_FALSE(cm["b"] != "1"); - EXPECT_FALSE(cm["c"] != "2"); - EXPECT_FALSE(cm["d"] != "3"); - EXPECT_FALSE(cm["e"] != "4"); - } -} - -TEST(Tree, relocate) -{ - // create a tree with anchors and refs, and copy it to ensure the - // relocation also applies to the anchors and refs. Ensure to put - // the source in the arena so that it gets relocated. - Tree tree = parse_in_arena(R"(&keyanchor key: val -key2: &valanchor val2 -keyref: *keyanchor -*valanchor: was val anchor -!!int 0: !!str foo -!!str doe: !!str a deer a female deer -ray: a drop of golden sun -me: a name I call myself -far: a long long way to run -)"); - Tree copy = tree; - EXPECT_EQ(copy.size(), tree.size()); - EXPECT_EQ(emitrs_yaml(copy), R"(&keyanchor key: val -key2: &valanchor val2 -keyref: *keyanchor -*valanchor: was val anchor -!!int 0: !!str foo -!!str doe: !!str a deer a female deer -ray: a drop of golden sun -me: a name I call myself -far: a long long way to run -)"); - // - Tree copy2 = copy; - EXPECT_EQ(copy.size(), tree.size()); - copy2.resolve(); - EXPECT_EQ(emitrs_yaml(copy2), R"(key: val -key2: val2 -keyref: key -val2: was val anchor -!!int 0: !!str foo -!!str doe: !!str a deer a female deer -ray: a drop of golden sun -me: a name I call myself -far: a long long way to run -)"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(NodeType, type_str) -{ - // avoid coverage misses - EXPECT_EQ(to_csubstr(NodeType(KEYVAL).type_str()), "KEYVAL"); - EXPECT_EQ(to_csubstr(NodeType(KEY).type_str()), "KEY"); - EXPECT_EQ(to_csubstr(NodeType(VAL).type_str()), "VAL"); - EXPECT_EQ(to_csubstr(NodeType(MAP).type_str()), "MAP"); - EXPECT_EQ(to_csubstr(NodeType(SEQ).type_str()), "SEQ"); - EXPECT_EQ(to_csubstr(NodeType(KEYMAP).type_str()), "KEYMAP"); - EXPECT_EQ(to_csubstr(NodeType(KEYSEQ).type_str()), "KEYSEQ"); - EXPECT_EQ(to_csubstr(NodeType(DOCSEQ).type_str()), "DOCSEQ"); - EXPECT_EQ(to_csubstr(NodeType(DOCMAP).type_str()), "DOCMAP"); - EXPECT_EQ(to_csubstr(NodeType(DOCVAL).type_str()), "DOCVAL"); - EXPECT_EQ(to_csubstr(NodeType(DOC).type_str()), "DOC"); - EXPECT_EQ(to_csubstr(NodeType(STREAM).type_str()), "STREAM"); - EXPECT_EQ(to_csubstr(NodeType(NOTYPE).type_str()), "NOTYPE"); - EXPECT_EQ(to_csubstr(NodeType(KEYVAL|KEYREF).type_str()), "KEYVAL***"); - EXPECT_EQ(to_csubstr(NodeType(KEYVAL|VALREF).type_str()), "KEYVAL***"); - EXPECT_EQ(to_csubstr(NodeType(KEYVAL|KEYANCH).type_str()), "KEYVAL***"); - EXPECT_EQ(to_csubstr(NodeType(KEYVAL|VALANCH).type_str()), "KEYVAL***"); - EXPECT_EQ(to_csubstr(NodeType(KEYVAL|KEYREF|VALANCH).type_str()), "KEYVAL***"); - EXPECT_EQ(to_csubstr(NodeType(KEYVAL|KEYANCH|VALREF).type_str()), "KEYVAL***"); - EXPECT_EQ(to_csubstr(NodeType(KEYMAP|KEYREF).type_str()), "KEYMAP***"); - EXPECT_EQ(to_csubstr(NodeType(KEYMAP|VALREF).type_str()), "KEYMAP***"); - EXPECT_EQ(to_csubstr(NodeType(KEYMAP|KEYANCH).type_str()), "KEYMAP***"); - EXPECT_EQ(to_csubstr(NodeType(KEYMAP|VALANCH).type_str()), "KEYMAP***"); - EXPECT_EQ(to_csubstr(NodeType(KEYMAP|KEYREF|VALANCH).type_str()), "KEYMAP***"); - EXPECT_EQ(to_csubstr(NodeType(KEYMAP|KEYANCH|VALREF).type_str()), "KEYMAP***"); - EXPECT_EQ(to_csubstr(NodeType(KEYSEQ|KEYREF).type_str()), "KEYSEQ***"); - EXPECT_EQ(to_csubstr(NodeType(KEYSEQ|VALREF).type_str()), "KEYSEQ***"); - EXPECT_EQ(to_csubstr(NodeType(KEYSEQ|KEYANCH).type_str()), "KEYSEQ***"); - EXPECT_EQ(to_csubstr(NodeType(KEYSEQ|VALANCH).type_str()), "KEYSEQ***"); - EXPECT_EQ(to_csubstr(NodeType(KEYSEQ|KEYREF|VALANCH).type_str()), "KEYSEQ***"); - EXPECT_EQ(to_csubstr(NodeType(KEYSEQ|KEYANCH|VALREF).type_str()), "KEYSEQ***"); - EXPECT_EQ(to_csubstr(NodeType(DOCSEQ|VALANCH).type_str()), "DOCSEQ***"); - EXPECT_EQ(to_csubstr(NodeType(DOCSEQ|VALREF).type_str()), "DOCSEQ***"); - EXPECT_EQ(to_csubstr(NodeType(DOCMAP|VALANCH).type_str()), "DOCMAP***"); - EXPECT_EQ(to_csubstr(NodeType(DOCMAP|VALREF).type_str()), "DOCMAP***"); - EXPECT_EQ(to_csubstr(NodeType(DOCVAL|VALANCH).type_str()), "DOCVAL***"); - EXPECT_EQ(to_csubstr(NodeType(DOCVAL|VALREF).type_str()), "DOCVAL***"); - EXPECT_EQ(to_csubstr(NodeType(KEY|KEYREF).type_str()), "KEY***"); - EXPECT_EQ(to_csubstr(NodeType(KEY|KEYANCH).type_str()), "KEY***"); - EXPECT_EQ(to_csubstr(NodeType(VAL|VALREF).type_str()), "VAL***"); - EXPECT_EQ(to_csubstr(NodeType(VAL|VALANCH).type_str()), "VAL***"); - EXPECT_EQ(to_csubstr(NodeType(MAP|VALREF).type_str()), "MAP***"); - EXPECT_EQ(to_csubstr(NodeType(MAP|VALANCH).type_str()), "MAP***"); - EXPECT_EQ(to_csubstr(NodeType(SEQ|VALREF).type_str()), "SEQ***"); - EXPECT_EQ(to_csubstr(NodeType(SEQ|VALANCH).type_str()), "SEQ***"); - EXPECT_EQ(to_csubstr(NodeType(DOC|VALREF).type_str()), "DOC***"); - EXPECT_EQ(to_csubstr(NodeType(DOC|VALANCH).type_str()), "DOC***"); - EXPECT_EQ(to_csubstr(NodeType(KEYREF).type_str()), "(unk)"); - EXPECT_EQ(to_csubstr(NodeType(VALREF).type_str()), "(unk)"); - EXPECT_EQ(to_csubstr(NodeType(KEYANCH).type_str()), "(unk)"); - EXPECT_EQ(to_csubstr(NodeType(VALANCH).type_str()), "(unk)"); -} - -TEST(NodeType, is_stream) -{ - EXPECT_FALSE(NodeType(NOTYPE).is_stream()); - EXPECT_TRUE(NodeType(STREAM).is_stream()); -} - -TEST(Tree, is_stream) -{ - Tree t = parse_in_arena(R"(--- -foo: bar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t keyval_id = t.first_child(doc_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef keyval = t.ref(keyval_id); - EXPECT_TRUE(t.is_stream(stream_id)); - EXPECT_FALSE(t.is_stream(doc_id)); - EXPECT_FALSE(t.is_stream(keyval_id)); - EXPECT_TRUE(stream.is_stream()); - EXPECT_FALSE(doc.is_stream()); - EXPECT_FALSE(keyval.is_stream()); - EXPECT_EQ(t.is_stream(stream_id), t._p(stream_id)->m_type.is_stream()); - EXPECT_EQ(t.is_stream(doc_id), t._p(doc_id)->m_type.is_stream()); - EXPECT_EQ(t.is_stream(keyval_id), t._p(keyval_id)->m_type.is_stream()); - EXPECT_EQ(stream.is_stream(), stream.get()->m_type.is_stream()); - EXPECT_EQ(doc.is_stream(), doc.get()->m_type.is_stream()); - EXPECT_EQ(keyval.is_stream(), keyval.get()->m_type.is_stream()); -} - -TEST(NodeType, is_doc) -{ - EXPECT_FALSE(NodeType(NOTYPE).is_doc()); - EXPECT_TRUE(NodeType(DOC).is_doc()); -} - -TEST(Tree, is_doc) -{ - Tree t = parse_in_arena(R"(--- -foo: bar ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t keyval_id = t.first_child(doc_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_FALSE(t.is_doc(stream_id)); - EXPECT_TRUE(t.is_doc(doc_id)); - EXPECT_FALSE(t.is_doc(keyval_id)); - EXPECT_TRUE(t.is_doc(docval_id)); - EXPECT_FALSE(stream.is_doc()); - EXPECT_TRUE(doc.is_doc()); - EXPECT_FALSE(keyval.is_doc()); - EXPECT_TRUE(docval.is_doc()); - EXPECT_FALSE(mstream.is_doc()); - EXPECT_TRUE(mdoc.is_doc()); - EXPECT_FALSE(mkeyval.is_doc()); - EXPECT_TRUE(mdocval.is_doc()); - EXPECT_EQ(t.is_doc(stream_id), t._p(stream_id)->m_type.is_doc()); - EXPECT_EQ(t.is_doc(doc_id), t._p(doc_id)->m_type.is_doc()); - EXPECT_EQ(t.is_doc(keyval_id), t._p(keyval_id)->m_type.is_doc()); - EXPECT_EQ(t.is_doc(docval_id), t._p(docval_id)->m_type.is_doc()); - EXPECT_EQ(stream.is_doc(), stream.get()->m_type.is_doc()); - EXPECT_EQ(doc.is_doc(), doc.get()->m_type.is_doc()); - EXPECT_EQ(keyval.is_doc(), keyval.get()->m_type.is_doc()); - EXPECT_EQ(docval.is_doc(), docval.get()->m_type.is_doc()); - EXPECT_EQ(mstream.is_doc(), mstream.get()->m_type.is_doc()); - EXPECT_EQ(mdoc.is_doc(), mdoc.get()->m_type.is_doc()); - EXPECT_EQ(mkeyval.is_doc(), mkeyval.get()->m_type.is_doc()); - EXPECT_EQ(mdocval.is_doc(), mdocval.get()->m_type.is_doc()); -} - -TEST(NodeType, is_container) -{ - EXPECT_FALSE(NodeType(NOTYPE).is_container()); - EXPECT_FALSE(NodeType(VAL).is_container()); - EXPECT_FALSE(NodeType(KEY).is_container()); - EXPECT_FALSE(NodeType(KEYVAL).is_container()); - EXPECT_TRUE(NodeType(MAP).is_container()); - EXPECT_TRUE(NodeType(SEQ).is_container()); - EXPECT_TRUE(NodeType(KEYMAP).is_container()); - EXPECT_TRUE(NodeType(KEYSEQ).is_container()); - EXPECT_TRUE(NodeType(DOCMAP).is_container()); - EXPECT_TRUE(NodeType(DOCSEQ).is_container()); -} - -TEST(Tree, is_container) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: bar} -seq: [foo, bar] ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_TRUE(t.is_container(stream_id)); - EXPECT_TRUE(t.is_container(doc_id)); - EXPECT_TRUE(t.is_container(map_id)); - EXPECT_FALSE(t.is_container(keyval_id)); - EXPECT_TRUE(t.is_container(seq_id)); - EXPECT_FALSE(t.is_container(val_id)); - EXPECT_FALSE(t.is_container(docval_id)); - EXPECT_TRUE(stream.is_container()); - EXPECT_TRUE(doc.is_container()); - EXPECT_TRUE(map.is_container()); - EXPECT_FALSE(keyval.is_container()); - EXPECT_TRUE(seq.is_container()); - EXPECT_FALSE(val.is_container()); - EXPECT_FALSE(docval.is_container()); - EXPECT_TRUE(mstream.is_container()); - EXPECT_TRUE(mdoc.is_container()); - EXPECT_TRUE(mmap.is_container()); - EXPECT_FALSE(mkeyval.is_container()); - EXPECT_TRUE(mseq.is_container()); - EXPECT_FALSE(mval.is_container()); - EXPECT_FALSE(mdocval.is_container()); - EXPECT_EQ(t.is_container(stream_id), t._p(stream_id)->m_type.is_container()); - EXPECT_EQ(t.is_container(doc_id), t._p(doc_id)->m_type.is_container()); - EXPECT_EQ(t.is_container(map_id), t._p(map_id)->m_type.is_container()); - EXPECT_EQ(t.is_container(keyval_id), t._p(keyval_id)->m_type.is_container()); - EXPECT_EQ(t.is_container(seq_id), t._p(seq_id)->m_type.is_container()); - EXPECT_EQ(t.is_container(val_id), t._p(val_id)->m_type.is_container()); - EXPECT_EQ(t.is_container(docval_id), t._p(docval_id)->m_type.is_container()); - EXPECT_EQ(stream.is_container(), stream.get()->m_type.is_container()); - EXPECT_EQ(doc.is_container(), doc.get()->m_type.is_container()); - EXPECT_EQ(map.is_container(), map.get()->m_type.is_container()); - EXPECT_EQ(keyval.is_container(), keyval.get()->m_type.is_container()); - EXPECT_EQ(seq.is_container(), seq.get()->m_type.is_container()); - EXPECT_EQ(val.is_container(), val.get()->m_type.is_container()); - EXPECT_EQ(docval.is_container(), docval.get()->m_type.is_container()); - EXPECT_EQ(mstream.is_container(), mstream.get()->m_type.is_container()); - EXPECT_EQ(mdoc.is_container(), mdoc.get()->m_type.is_container()); - EXPECT_EQ(mmap.is_container(), mmap.get()->m_type.is_container()); - EXPECT_EQ(mkeyval.is_container(), mkeyval.get()->m_type.is_container()); - EXPECT_EQ(mseq.is_container(), mseq.get()->m_type.is_container()); - EXPECT_EQ(mval.is_container(), mval.get()->m_type.is_container()); - EXPECT_EQ(mdocval.is_container(), mdocval.get()->m_type.is_container()); -} - -TEST(NodeType, is_map) -{ - EXPECT_FALSE(NodeType(NOTYPE).is_map()); - EXPECT_FALSE(NodeType(VAL).is_map()); - EXPECT_FALSE(NodeType(KEY).is_map()); - EXPECT_TRUE(NodeType(MAP).is_map()); - EXPECT_TRUE(NodeType(KEYMAP).is_map()); - EXPECT_FALSE(NodeType(SEQ).is_map()); - EXPECT_FALSE(NodeType(KEYSEQ).is_map()); -} - -TEST(Tree, is_map) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: bar} -seq: [foo, bar] ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_FALSE(t.is_map(stream_id)); - EXPECT_TRUE(t.is_map(doc_id)); - EXPECT_TRUE(t.is_map(map_id)); - EXPECT_FALSE(t.is_map(keyval_id)); - EXPECT_FALSE(t.is_map(seq_id)); - EXPECT_FALSE(t.is_map(val_id)); - EXPECT_FALSE(t.is_map(docval_id)); - EXPECT_FALSE(stream.is_map()); - EXPECT_TRUE(doc.is_map()); - EXPECT_TRUE(map.is_map()); - EXPECT_FALSE(keyval.is_map()); - EXPECT_FALSE(seq.is_map()); - EXPECT_FALSE(val.is_map()); - EXPECT_FALSE(docval.is_map()); - EXPECT_FALSE(mstream.is_map()); - EXPECT_TRUE(mdoc.is_map()); - EXPECT_TRUE(mmap.is_map()); - EXPECT_FALSE(mkeyval.is_map()); - EXPECT_FALSE(mseq.is_map()); - EXPECT_FALSE(mval.is_map()); - EXPECT_FALSE(mdocval.is_map()); - EXPECT_EQ(t.is_map(stream_id), t._p(stream_id)->m_type.is_map()); - EXPECT_EQ(t.is_map(doc_id), t._p(doc_id)->m_type.is_map()); - EXPECT_EQ(t.is_map(map_id), t._p(map_id)->m_type.is_map()); - EXPECT_EQ(t.is_map(keyval_id), t._p(keyval_id)->m_type.is_map()); - EXPECT_EQ(t.is_map(seq_id), t._p(seq_id)->m_type.is_map()); - EXPECT_EQ(t.is_map(val_id), t._p(val_id)->m_type.is_map()); - EXPECT_EQ(t.is_map(docval_id), t._p(docval_id)->m_type.is_map()); - EXPECT_EQ(stream.is_map(), stream.get()->m_type.is_map()); - EXPECT_EQ(doc.is_map(), doc.get()->m_type.is_map()); - EXPECT_EQ(map.is_map(), map.get()->m_type.is_map()); - EXPECT_EQ(keyval.is_map(), keyval.get()->m_type.is_map()); - EXPECT_EQ(seq.is_map(), seq.get()->m_type.is_map()); - EXPECT_EQ(val.is_map(), val.get()->m_type.is_map()); - EXPECT_EQ(docval.is_map(), docval.get()->m_type.is_map()); - EXPECT_EQ(mstream.is_map(), mstream.get()->m_type.is_map()); - EXPECT_EQ(mdoc.is_map(), mdoc.get()->m_type.is_map()); - EXPECT_EQ(mmap.is_map(), mmap.get()->m_type.is_map()); - EXPECT_EQ(mkeyval.is_map(), mkeyval.get()->m_type.is_map()); - EXPECT_EQ(mseq.is_map(), mseq.get()->m_type.is_map()); - EXPECT_EQ(mval.is_map(), mval.get()->m_type.is_map()); - EXPECT_EQ(mdocval.is_map(), mdocval.get()->m_type.is_map()); -} - -TEST(NodeType, is_seq) -{ - EXPECT_FALSE(NodeType(NOTYPE).is_seq()); - EXPECT_FALSE(NodeType(VAL).is_seq()); - EXPECT_FALSE(NodeType(KEY).is_seq()); - EXPECT_FALSE(NodeType(MAP).is_seq()); - EXPECT_FALSE(NodeType(KEYMAP).is_seq()); - EXPECT_TRUE(NodeType(SEQ).is_seq()); - EXPECT_TRUE(NodeType(KEYSEQ).is_seq()); -} - -TEST(Tree, is_seq) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: bar} -seq: [foo, bar] ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_TRUE(t.is_seq(stream_id)); - EXPECT_FALSE(t.is_seq(doc_id)); - EXPECT_FALSE(t.is_seq(map_id)); - EXPECT_FALSE(t.is_seq(keyval_id)); - EXPECT_TRUE(t.is_seq(seq_id)); - EXPECT_FALSE(t.is_seq(val_id)); - EXPECT_FALSE(t.is_seq(docval_id)); - EXPECT_TRUE(stream.is_seq()); - EXPECT_FALSE(doc.is_seq()); - EXPECT_FALSE(map.is_seq()); - EXPECT_FALSE(keyval.is_seq()); - EXPECT_TRUE(seq.is_seq()); - EXPECT_FALSE(val.is_seq()); - EXPECT_FALSE(docval.is_seq()); - EXPECT_TRUE(mstream.is_seq()); - EXPECT_FALSE(mdoc.is_seq()); - EXPECT_FALSE(mmap.is_seq()); - EXPECT_FALSE(mkeyval.is_seq()); - EXPECT_TRUE(mseq.is_seq()); - EXPECT_FALSE(mval.is_seq()); - EXPECT_FALSE(mdocval.is_seq()); - EXPECT_EQ(t.is_seq(stream_id), t._p(stream_id)->m_type.is_seq()); - EXPECT_EQ(t.is_seq(doc_id), t._p(doc_id)->m_type.is_seq()); - EXPECT_EQ(t.is_seq(map_id), t._p(map_id)->m_type.is_seq()); - EXPECT_EQ(t.is_seq(keyval_id), t._p(keyval_id)->m_type.is_seq()); - EXPECT_EQ(t.is_seq(seq_id), t._p(seq_id)->m_type.is_seq()); - EXPECT_EQ(t.is_seq(val_id), t._p(val_id)->m_type.is_seq()); - EXPECT_EQ(t.is_seq(docval_id), t._p(docval_id)->m_type.is_seq()); - EXPECT_EQ(stream.is_seq(), stream.get()->m_type.is_seq()); - EXPECT_EQ(doc.is_seq(), doc.get()->m_type.is_seq()); - EXPECT_EQ(map.is_seq(), map.get()->m_type.is_seq()); - EXPECT_EQ(keyval.is_seq(), keyval.get()->m_type.is_seq()); - EXPECT_EQ(seq.is_seq(), seq.get()->m_type.is_seq()); - EXPECT_EQ(val.is_seq(), val.get()->m_type.is_seq()); - EXPECT_EQ(docval.is_seq(), docval.get()->m_type.is_seq()); - EXPECT_EQ(mstream.is_seq(), mstream.get()->m_type.is_seq()); - EXPECT_EQ(mdoc.is_seq(), mdoc.get()->m_type.is_seq()); - EXPECT_EQ(mmap.is_seq(), mmap.get()->m_type.is_seq()); - EXPECT_EQ(mkeyval.is_seq(), mkeyval.get()->m_type.is_seq()); - EXPECT_EQ(mseq.is_seq(), mseq.get()->m_type.is_seq()); - EXPECT_EQ(mval.is_seq(), mval.get()->m_type.is_seq()); - EXPECT_EQ(mdocval.is_seq(), mdocval.get()->m_type.is_seq()); -} - -TEST(NodeType, has_val) -{ - EXPECT_FALSE(NodeType(NOTYPE).has_val()); - EXPECT_FALSE(NodeType(KEY).has_val()); - EXPECT_TRUE(NodeType(VAL).has_val()); - EXPECT_TRUE(NodeType(DOCVAL).has_val()); - EXPECT_TRUE(NodeType(KEYVAL).has_val()); - EXPECT_FALSE(NodeType(KEYMAP).has_val()); - EXPECT_FALSE(NodeType(KEYSEQ).has_val()); -} - -TEST(Tree, has_val) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: bar} -seq: [foo, bar] ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_FALSE(t.has_val(stream_id)); - EXPECT_FALSE(t.has_val(doc_id)); - EXPECT_FALSE(t.has_val(map_id)); - EXPECT_TRUE(t.has_val(keyval_id)); - EXPECT_FALSE(t.has_val(seq_id)); - EXPECT_TRUE(t.has_val(val_id)); - EXPECT_TRUE(t.has_val(docval_id)); - EXPECT_FALSE(stream.has_val()); - EXPECT_FALSE(doc.has_val()); - EXPECT_FALSE(map.has_val()); - EXPECT_TRUE(keyval.has_val()); - EXPECT_FALSE(seq.has_val()); - EXPECT_TRUE(val.has_val()); - EXPECT_TRUE(docval.has_val()); - EXPECT_FALSE(mstream.has_val()); - EXPECT_FALSE(mdoc.has_val()); - EXPECT_FALSE(mmap.has_val()); - EXPECT_TRUE(mkeyval.has_val()); - EXPECT_FALSE(mseq.has_val()); - EXPECT_TRUE(mval.has_val()); - EXPECT_TRUE(mdocval.has_val()); - EXPECT_EQ(t.has_val(stream_id), t._p(stream_id)->m_type.has_val()); - EXPECT_EQ(t.has_val(doc_id), t._p(doc_id)->m_type.has_val()); - EXPECT_EQ(t.has_val(map_id), t._p(map_id)->m_type.has_val()); - EXPECT_EQ(t.has_val(keyval_id), t._p(keyval_id)->m_type.has_val()); - EXPECT_EQ(t.has_val(seq_id), t._p(seq_id)->m_type.has_val()); - EXPECT_EQ(t.has_val(val_id), t._p(val_id)->m_type.has_val()); - EXPECT_EQ(t.has_val(docval_id), t._p(docval_id)->m_type.has_val()); - EXPECT_EQ(stream.has_val(), stream.get()->m_type.has_val()); - EXPECT_EQ(doc.has_val(), doc.get()->m_type.has_val()); - EXPECT_EQ(map.has_val(), map.get()->m_type.has_val()); - EXPECT_EQ(keyval.has_val(), keyval.get()->m_type.has_val()); - EXPECT_EQ(seq.has_val(), seq.get()->m_type.has_val()); - EXPECT_EQ(val.has_val(), val.get()->m_type.has_val()); - EXPECT_EQ(docval.has_val(), docval.get()->m_type.has_val()); - EXPECT_EQ(mstream.has_val(), mstream.get()->m_type.has_val()); - EXPECT_EQ(mdoc.has_val(), mdoc.get()->m_type.has_val()); - EXPECT_EQ(mmap.has_val(), mmap.get()->m_type.has_val()); - EXPECT_EQ(mkeyval.has_val(), mkeyval.get()->m_type.has_val()); - EXPECT_EQ(mseq.has_val(), mseq.get()->m_type.has_val()); - EXPECT_EQ(mval.has_val(), mval.get()->m_type.has_val()); - EXPECT_EQ(mdocval.has_val(), mdocval.get()->m_type.has_val()); -} - -TEST(NodeType, is_val) -{ - EXPECT_FALSE(NodeType(NOTYPE).is_val()); - EXPECT_FALSE(NodeType(KEY).is_val()); - EXPECT_TRUE(NodeType(VAL).is_val()); - EXPECT_TRUE(NodeType(DOCVAL).is_val()); - EXPECT_FALSE(NodeType(KEYVAL).is_val()); - EXPECT_FALSE(NodeType(KEYMAP).is_val()); - EXPECT_FALSE(NodeType(KEYSEQ).is_val()); -} - -TEST(Tree, is_val) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: bar} -seq: [foo, bar] ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_FALSE(t.is_val(stream_id)); - EXPECT_FALSE(t.is_val(doc_id)); - EXPECT_FALSE(t.is_val(map_id)); - EXPECT_FALSE(t.is_val(keyval_id)); - EXPECT_FALSE(t.is_val(seq_id)); - EXPECT_TRUE(t.is_val(val_id)); - EXPECT_TRUE(t.is_val(docval_id)); - EXPECT_FALSE(stream.is_val()); - EXPECT_FALSE(doc.is_val()); - EXPECT_FALSE(map.is_val()); - EXPECT_FALSE(keyval.is_val()); - EXPECT_FALSE(seq.is_val()); - EXPECT_TRUE(val.is_val()); - EXPECT_TRUE(docval.is_val()); - EXPECT_FALSE(mstream.is_val()); - EXPECT_FALSE(mdoc.is_val()); - EXPECT_FALSE(mmap.is_val()); - EXPECT_FALSE(mkeyval.is_val()); - EXPECT_FALSE(mseq.is_val()); - EXPECT_TRUE(mval.is_val()); - EXPECT_TRUE(mdocval.is_val()); - EXPECT_EQ(t.is_val(stream_id), t._p(stream_id)->m_type.is_val()); - EXPECT_EQ(t.is_val(doc_id), t._p(doc_id)->m_type.is_val()); - EXPECT_EQ(t.is_val(map_id), t._p(map_id)->m_type.is_val()); - EXPECT_EQ(t.is_val(keyval_id), t._p(keyval_id)->m_type.is_val()); - EXPECT_EQ(t.is_val(seq_id), t._p(seq_id)->m_type.is_val()); - EXPECT_EQ(t.is_val(val_id), t._p(val_id)->m_type.is_val()); - EXPECT_EQ(t.is_val(docval_id), t._p(docval_id)->m_type.is_val()); - EXPECT_EQ(stream.is_val(), stream.get()->m_type.is_val()); - EXPECT_EQ(doc.is_val(), doc.get()->m_type.is_val()); - EXPECT_EQ(map.is_val(), map.get()->m_type.is_val()); - EXPECT_EQ(keyval.is_val(), keyval.get()->m_type.is_val()); - EXPECT_EQ(seq.is_val(), seq.get()->m_type.is_val()); - EXPECT_EQ(val.is_val(), val.get()->m_type.is_val()); - EXPECT_EQ(docval.is_val(), docval.get()->m_type.is_val()); - EXPECT_EQ(mstream.is_val(), mstream.get()->m_type.is_val()); - EXPECT_EQ(mdoc.is_val(), mdoc.get()->m_type.is_val()); - EXPECT_EQ(mmap.is_val(), mmap.get()->m_type.is_val()); - EXPECT_EQ(mkeyval.is_val(), mkeyval.get()->m_type.is_val()); - EXPECT_EQ(mseq.is_val(), mseq.get()->m_type.is_val()); - EXPECT_EQ(mval.is_val(), mval.get()->m_type.is_val()); - EXPECT_EQ(mdocval.is_val(), mdocval.get()->m_type.is_val()); -} - -TEST(NodeType, has_key) -{ - EXPECT_FALSE(NodeType(NOTYPE).has_key()); - EXPECT_TRUE(NodeType(KEY).has_key()); - EXPECT_FALSE(NodeType(VAL).has_key()); - EXPECT_TRUE(NodeType(KEYVAL).has_key()); - EXPECT_TRUE(NodeType(KEYMAP).has_key()); - EXPECT_TRUE(NodeType(KEYSEQ).has_key()); -} - -TEST(Tree, has_key) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: bar} -seq: [foo, bar] ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_FALSE(t.has_key(stream_id)); - EXPECT_FALSE(t.has_key(doc_id)); - EXPECT_TRUE(t.has_key(map_id)); - EXPECT_TRUE(t.has_key(keyval_id)); - EXPECT_TRUE(t.has_key(seq_id)); - EXPECT_FALSE(t.has_key(val_id)); - EXPECT_FALSE(t.has_key(docval_id)); - EXPECT_FALSE(stream.has_key()); - EXPECT_FALSE(doc.has_key()); - EXPECT_TRUE(map.has_key()); - EXPECT_TRUE(keyval.has_key()); - EXPECT_TRUE(seq.has_key()); - EXPECT_FALSE(val.has_key()); - EXPECT_FALSE(docval.has_key()); - EXPECT_FALSE(mstream.has_key()); - EXPECT_FALSE(mdoc.has_key()); - EXPECT_TRUE(mmap.has_key()); - EXPECT_TRUE(mkeyval.has_key()); - EXPECT_TRUE(mseq.has_key()); - EXPECT_FALSE(mval.has_key()); - EXPECT_FALSE(mdocval.has_key()); - EXPECT_EQ(t.has_key(stream_id), t._p(stream_id)->m_type.has_key()); - EXPECT_EQ(t.has_key(doc_id), t._p(doc_id)->m_type.has_key()); - EXPECT_EQ(t.has_key(map_id), t._p(map_id)->m_type.has_key()); - EXPECT_EQ(t.has_key(keyval_id), t._p(keyval_id)->m_type.has_key()); - EXPECT_EQ(t.has_key(seq_id), t._p(seq_id)->m_type.has_key()); - EXPECT_EQ(t.has_key(val_id), t._p(val_id)->m_type.has_key()); - EXPECT_EQ(t.has_key(docval_id), t._p(docval_id)->m_type.has_key()); - EXPECT_EQ(stream.has_key(), stream.get()->m_type.has_key()); - EXPECT_EQ(doc.has_key(), doc.get()->m_type.has_key()); - EXPECT_EQ(map.has_key(), map.get()->m_type.has_key()); - EXPECT_EQ(keyval.has_key(), keyval.get()->m_type.has_key()); - EXPECT_EQ(seq.has_key(), seq.get()->m_type.has_key()); - EXPECT_EQ(val.has_key(), val.get()->m_type.has_key()); - EXPECT_EQ(docval.has_key(), docval.get()->m_type.has_key()); - EXPECT_EQ(mstream.has_key(), mstream.get()->m_type.has_key()); - EXPECT_EQ(mdoc.has_key(), mdoc.get()->m_type.has_key()); - EXPECT_EQ(mmap.has_key(), mmap.get()->m_type.has_key()); - EXPECT_EQ(mkeyval.has_key(), mkeyval.get()->m_type.has_key()); - EXPECT_EQ(mseq.has_key(), mseq.get()->m_type.has_key()); - EXPECT_EQ(mval.has_key(), mval.get()->m_type.has_key()); - EXPECT_EQ(mdocval.has_key(), mdocval.get()->m_type.has_key()); -} - -TEST(NodeType, is_keyval) -{ - EXPECT_FALSE(NodeType(NOTYPE).is_keyval()); - EXPECT_FALSE(NodeType(KEY).is_keyval()); - EXPECT_FALSE(NodeType(VAL).is_keyval()); - EXPECT_TRUE(NodeType(KEYVAL).is_keyval()); - EXPECT_FALSE(NodeType(DOCVAL).is_keyval()); - EXPECT_FALSE(NodeType(KEYMAP).is_keyval()); - EXPECT_FALSE(NodeType(KEYSEQ).is_keyval()); -} - -TEST(Tree, is_keyval) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: bar} -seq: [foo, bar] ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_FALSE(t.is_keyval(stream_id)); - EXPECT_FALSE(t.is_keyval(doc_id)); - EXPECT_FALSE(t.is_keyval(map_id)); - EXPECT_TRUE(t.is_keyval(keyval_id)); - EXPECT_FALSE(t.is_keyval(seq_id)); - EXPECT_FALSE(t.is_keyval(val_id)); - EXPECT_FALSE(t.is_keyval(docval_id)); - EXPECT_FALSE(stream.is_keyval()); - EXPECT_FALSE(doc.is_keyval()); - EXPECT_FALSE(map.is_keyval()); - EXPECT_TRUE(keyval.is_keyval()); - EXPECT_FALSE(seq.is_keyval()); - EXPECT_FALSE(val.is_keyval()); - EXPECT_FALSE(docval.is_keyval()); - EXPECT_FALSE(mstream.is_keyval()); - EXPECT_FALSE(mdoc.is_keyval()); - EXPECT_FALSE(mmap.is_keyval()); - EXPECT_TRUE(mkeyval.is_keyval()); - EXPECT_FALSE(mseq.is_keyval()); - EXPECT_FALSE(mval.is_keyval()); - EXPECT_FALSE(mdocval.is_keyval()); - EXPECT_EQ(t.is_keyval(stream_id), t._p(stream_id)->m_type.is_keyval()); - EXPECT_EQ(t.is_keyval(doc_id), t._p(doc_id)->m_type.is_keyval()); - EXPECT_EQ(t.is_keyval(map_id), t._p(map_id)->m_type.is_keyval()); - EXPECT_EQ(t.is_keyval(keyval_id), t._p(keyval_id)->m_type.is_keyval()); - EXPECT_EQ(t.is_keyval(seq_id), t._p(seq_id)->m_type.is_keyval()); - EXPECT_EQ(t.is_keyval(val_id), t._p(val_id)->m_type.is_keyval()); - EXPECT_EQ(t.is_keyval(docval_id), t._p(docval_id)->m_type.is_keyval()); - EXPECT_EQ(stream.is_keyval(), stream.get()->m_type.is_keyval()); - EXPECT_EQ(doc.is_keyval(), doc.get()->m_type.is_keyval()); - EXPECT_EQ(map.is_keyval(), map.get()->m_type.is_keyval()); - EXPECT_EQ(keyval.is_keyval(), keyval.get()->m_type.is_keyval()); - EXPECT_EQ(seq.is_keyval(), seq.get()->m_type.is_keyval()); - EXPECT_EQ(val.is_keyval(), val.get()->m_type.is_keyval()); - EXPECT_EQ(docval.is_keyval(), docval.get()->m_type.is_keyval()); - EXPECT_EQ(mstream.is_keyval(), mstream.get()->m_type.is_keyval()); - EXPECT_EQ(mdoc.is_keyval(), mdoc.get()->m_type.is_keyval()); - EXPECT_EQ(mmap.is_keyval(), mmap.get()->m_type.is_keyval()); - EXPECT_EQ(mkeyval.is_keyval(), mkeyval.get()->m_type.is_keyval()); - EXPECT_EQ(mseq.is_keyval(), mseq.get()->m_type.is_keyval()); - EXPECT_EQ(mval.is_keyval(), mval.get()->m_type.is_keyval()); - EXPECT_EQ(mdocval.is_keyval(), mdocval.get()->m_type.is_keyval()); -} - -TEST(NodeType, has_key_tag) -{ - EXPECT_FALSE(NodeType().has_key_tag()); - EXPECT_FALSE(NodeType(KEYTAG).has_key_tag()); - EXPECT_TRUE(NodeType(KEY|KEYTAG).has_key_tag()); -} - -TEST(Tree, has_key_tag) -{ - Tree t = parse_in_arena(R"(--- !docmaptag -!maptag map: {!footag foo: bar, notag: none} -!seqtag seq: [!footag foo, bar] ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t keyvalnotag_id = t.last_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t valnotag_id = t.last_child(seq_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef keyvalnotag = t.ref(keyvalnotag_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef valnotag = t.ref(val_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mkeyvalnotag = t.ref(keyvalnotag_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mvalnotag = t.ref(val_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_FALSE(t.has_key_tag(stream_id)); - EXPECT_FALSE(t.has_key_tag(doc_id)); - EXPECT_TRUE(t.has_key_tag(map_id)); - EXPECT_TRUE(t.has_key_tag(keyval_id)); - EXPECT_FALSE(t.has_key_tag(keyvalnotag_id)); - EXPECT_TRUE(t.has_key_tag(seq_id)); - EXPECT_FALSE(t.has_key_tag(val_id)); - EXPECT_FALSE(t.has_key_tag(valnotag_id)); - EXPECT_FALSE(t.has_key_tag(docval_id)); - EXPECT_FALSE(stream.has_key_tag()); - EXPECT_FALSE(doc.has_key_tag()); - EXPECT_TRUE(map.has_key_tag()); - EXPECT_TRUE(keyval.has_key_tag()); - EXPECT_FALSE(keyvalnotag.has_key_tag()); - EXPECT_TRUE(seq.has_key_tag()); - EXPECT_FALSE(val.has_key_tag()); - EXPECT_FALSE(valnotag.has_key_tag()); - EXPECT_FALSE(docval.has_key_tag()); - EXPECT_FALSE(mstream.has_key_tag()); - EXPECT_FALSE(mdoc.has_key_tag()); - EXPECT_TRUE(mmap.has_key_tag()); - EXPECT_TRUE(mkeyval.has_key_tag()); - EXPECT_FALSE(mkeyvalnotag.has_key_tag()); - EXPECT_TRUE(mseq.has_key_tag()); - EXPECT_FALSE(mval.has_key_tag()); - EXPECT_FALSE(mvalnotag.has_key_tag()); - EXPECT_FALSE(mdocval.has_key_tag()); - EXPECT_EQ(t.has_key_tag(stream_id), t._p(stream_id)->m_type.has_key_tag()); - EXPECT_EQ(t.has_key_tag(doc_id), t._p(doc_id)->m_type.has_key_tag()); - EXPECT_EQ(t.has_key_tag(map_id), t._p(map_id)->m_type.has_key_tag()); - EXPECT_EQ(t.has_key_tag(keyval_id), t._p(keyval_id)->m_type.has_key_tag()); - EXPECT_EQ(t.has_key_tag(keyvalnotag_id), t._p(keyvalnotag_id)->m_type.has_key_tag()); - EXPECT_EQ(t.has_key_tag(seq_id), t._p(seq_id)->m_type.has_key_tag()); - EXPECT_EQ(t.has_key_tag(val_id), t._p(val_id)->m_type.has_key_tag()); - EXPECT_EQ(t.has_key_tag(valnotag_id), t._p(valnotag_id)->m_type.has_key_tag()); - EXPECT_EQ(t.has_key_tag(docval_id), t._p(docval_id)->m_type.has_key_tag()); - EXPECT_EQ(stream.has_key_tag(), stream.get()->m_type.has_key_tag()); - EXPECT_EQ(doc.has_key_tag(), doc.get()->m_type.has_key_tag()); - EXPECT_EQ(map.has_key_tag(), map.get()->m_type.has_key_tag()); - EXPECT_EQ(keyval.has_key_tag(), keyval.get()->m_type.has_key_tag()); - EXPECT_EQ(keyvalnotag.has_key_tag(), keyvalnotag.get()->m_type.has_key_tag()); - EXPECT_EQ(seq.has_key_tag(), seq.get()->m_type.has_key_tag()); - EXPECT_EQ(val.has_key_tag(), val.get()->m_type.has_key_tag()); - EXPECT_EQ(valnotag.has_key_tag(), valnotag.get()->m_type.has_key_tag()); - EXPECT_EQ(docval.has_key_tag(), docval.get()->m_type.has_key_tag()); - EXPECT_EQ(mstream.has_key_tag(), mstream.get()->m_type.has_key_tag()); - EXPECT_EQ(mdoc.has_key_tag(), mdoc.get()->m_type.has_key_tag()); - EXPECT_EQ(mmap.has_key_tag(), mmap.get()->m_type.has_key_tag()); - EXPECT_EQ(mkeyval.has_key_tag(), mkeyval.get()->m_type.has_key_tag()); - EXPECT_EQ(mkeyvalnotag.has_key_tag(), mkeyvalnotag.get()->m_type.has_key_tag()); - EXPECT_EQ(mseq.has_key_tag(), mseq.get()->m_type.has_key_tag()); - EXPECT_EQ(mval.has_key_tag(), mval.get()->m_type.has_key_tag()); - EXPECT_EQ(mvalnotag.has_key_tag(), mvalnotag.get()->m_type.has_key_tag()); - EXPECT_EQ(mdocval.has_key_tag(), mdocval.get()->m_type.has_key_tag()); -} - -TEST(NodeType, has_val_tag) -{ - EXPECT_FALSE(NodeType().has_val_tag()); - EXPECT_FALSE(NodeType(VALTAG).has_val_tag()); - EXPECT_TRUE(NodeType(VAL|VALTAG).has_val_tag()); -} - -TEST(Tree, has_val_tag) -{ - Tree t = parse_in_arena(R"(--- !docmaptag -map: !maptag {foo: !bartag bar, notag: none} -seq: !seqtag [!footag foo, bar] ---- -a scalar -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t keyvalnotag_id = t.last_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t valnotag_id = t.last_child(seq_id); - const size_t docval_id = t.last_child(stream_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef keyvalnotag = t.ref(keyvalnotag_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef valnotag = t.ref(valnotag_id); - ConstNodeRef docval = t.ref(docval_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mkeyvalnotag = t.ref(keyvalnotag_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mvalnotag = t.ref(valnotag_id); - NodeRef mdocval = t.ref(docval_id); - EXPECT_FALSE(t.has_val_tag(stream_id)); - EXPECT_TRUE(t.has_val_tag(doc_id)); - EXPECT_TRUE(t.has_val_tag(map_id)); - EXPECT_TRUE(t.has_val_tag(keyval_id)); - EXPECT_FALSE(t.has_val_tag(keyvalnotag_id)); - EXPECT_TRUE(t.has_val_tag(seq_id)); - EXPECT_TRUE(t.has_val_tag(val_id)); - EXPECT_FALSE(t.has_val_tag(valnotag_id)); - EXPECT_FALSE(t.has_val_tag(docval_id)); - EXPECT_FALSE(stream.has_val_tag()); - EXPECT_TRUE(doc.has_val_tag()); - EXPECT_TRUE(map.has_val_tag()); - EXPECT_TRUE(keyval.has_val_tag()); - EXPECT_FALSE(keyvalnotag.has_val_tag()); - EXPECT_TRUE(seq.has_val_tag()); - EXPECT_TRUE(val.has_val_tag()); - EXPECT_FALSE(valnotag.has_val_tag()); - EXPECT_FALSE(docval.has_val_tag()); - EXPECT_FALSE(mstream.has_val_tag()); - EXPECT_TRUE(mdoc.has_val_tag()); - EXPECT_TRUE(mmap.has_val_tag()); - EXPECT_TRUE(mkeyval.has_val_tag()); - EXPECT_FALSE(mkeyvalnotag.has_val_tag()); - EXPECT_TRUE(mseq.has_val_tag()); - EXPECT_TRUE(mval.has_val_tag()); - EXPECT_FALSE(mvalnotag.has_val_tag()); - EXPECT_FALSE(mdocval.has_val_tag()); - EXPECT_EQ(t.has_val_tag(stream_id), t._p(stream_id)->m_type.has_val_tag()); - EXPECT_EQ(t.has_val_tag(doc_id), t._p(doc_id)->m_type.has_val_tag()); - EXPECT_EQ(t.has_val_tag(map_id), t._p(map_id)->m_type.has_val_tag()); - EXPECT_EQ(t.has_val_tag(keyval_id), t._p(keyval_id)->m_type.has_val_tag()); - EXPECT_EQ(t.has_val_tag(keyvalnotag_id), t._p(keyvalnotag_id)->m_type.has_val_tag()); - EXPECT_EQ(t.has_val_tag(seq_id), t._p(seq_id)->m_type.has_val_tag()); - EXPECT_EQ(t.has_val_tag(val_id), t._p(val_id)->m_type.has_val_tag()); - EXPECT_EQ(t.has_val_tag(valnotag_id), t._p(valnotag_id)->m_type.has_val_tag()); - EXPECT_EQ(t.has_val_tag(docval_id), t._p(docval_id)->m_type.has_val_tag()); - EXPECT_EQ(stream.has_val_tag(), stream.get()->m_type.has_val_tag()); - EXPECT_EQ(doc.has_val_tag(), doc.get()->m_type.has_val_tag()); - EXPECT_EQ(map.has_val_tag(), map.get()->m_type.has_val_tag()); - EXPECT_EQ(keyval.has_val_tag(), keyval.get()->m_type.has_val_tag()); - EXPECT_EQ(keyvalnotag.has_val_tag(), keyvalnotag.get()->m_type.has_val_tag()); - EXPECT_EQ(seq.has_val_tag(), seq.get()->m_type.has_val_tag()); - EXPECT_EQ(val.has_val_tag(), val.get()->m_type.has_val_tag()); - EXPECT_EQ(valnotag.has_val_tag(), valnotag.get()->m_type.has_val_tag()); - EXPECT_EQ(docval.has_val_tag(), docval.get()->m_type.has_val_tag()); - EXPECT_EQ(mstream.has_val_tag(), mstream.get()->m_type.has_val_tag()); - EXPECT_EQ(mdoc.has_val_tag(), mdoc.get()->m_type.has_val_tag()); - EXPECT_EQ(mmap.has_val_tag(), mmap.get()->m_type.has_val_tag()); - EXPECT_EQ(mkeyval.has_val_tag(), mkeyval.get()->m_type.has_val_tag()); - EXPECT_EQ(mkeyvalnotag.has_val_tag(), mkeyvalnotag.get()->m_type.has_val_tag()); - EXPECT_EQ(mseq.has_val_tag(), mseq.get()->m_type.has_val_tag()); - EXPECT_EQ(mval.has_val_tag(), mval.get()->m_type.has_val_tag()); - EXPECT_EQ(mvalnotag.has_val_tag(), mvalnotag.get()->m_type.has_val_tag()); - EXPECT_EQ(mdocval.has_val_tag(), mdocval.get()->m_type.has_val_tag()); -} - -TEST(NodeType, has_key_anchor) -{ - EXPECT_FALSE(NodeType().has_key_anchor()); - EXPECT_FALSE(NodeType(KEYANCH).has_key_anchor()); - EXPECT_TRUE(NodeType(KEY|KEYANCH).has_key_anchor()); -} - -TEST(Tree, has_key_anchor) -{ - Tree t = parse_in_arena(R"(--- &docanchor -&mapanchor map: {&keyvalanchor foo: bar, anchor: none} -&seqanchor seq: [&valanchor foo, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t keyvalnoanchor_id = t.last_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t valnoanchor_id = t.last_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef keyvalnoanchor = t.ref(keyvalnoanchor_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef valnoanchor = t.ref(valnoanchor_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mkeyvalnoanchor = t.ref(keyvalnoanchor_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mvalnoanchor = t.ref(valnoanchor_id); - EXPECT_FALSE(t.has_key_anchor(stream_id)); - EXPECT_FALSE(t.has_key_anchor(doc_id)); - EXPECT_TRUE(t.has_key_anchor(map_id)); - EXPECT_TRUE(t.has_key_anchor(keyval_id)); - EXPECT_FALSE(t.has_key_anchor(keyvalnoanchor_id)); - EXPECT_TRUE(t.has_key_anchor(seq_id)); - EXPECT_FALSE(t.has_key_anchor(val_id)); - EXPECT_FALSE(t.has_key_anchor(valnoanchor_id)); - EXPECT_FALSE(stream.has_key_anchor()); - EXPECT_FALSE(doc.has_key_anchor()); - EXPECT_TRUE(map.has_key_anchor()); - EXPECT_TRUE(keyval.has_key_anchor()); - EXPECT_FALSE(keyvalnoanchor.has_key_anchor()); - EXPECT_TRUE(seq.has_key_anchor()); - EXPECT_FALSE(val.has_key_anchor()); - EXPECT_FALSE(valnoanchor.has_key_anchor()); - EXPECT_FALSE(mstream.has_key_anchor()); - EXPECT_FALSE(mdoc.has_key_anchor()); - EXPECT_TRUE(mmap.has_key_anchor()); - EXPECT_TRUE(mkeyval.has_key_anchor()); - EXPECT_FALSE(mkeyvalnoanchor.has_key_anchor()); - EXPECT_TRUE(mseq.has_key_anchor()); - EXPECT_FALSE(mval.has_key_anchor()); - EXPECT_FALSE(mvalnoanchor.has_key_anchor()); - EXPECT_EQ(t.has_key_anchor(stream_id), t._p(stream_id)->m_type.has_key_anchor()); - EXPECT_EQ(t.has_key_anchor(doc_id), t._p(doc_id)->m_type.has_key_anchor()); - EXPECT_EQ(t.has_key_anchor(map_id), t._p(map_id)->m_type.has_key_anchor()); - EXPECT_EQ(t.has_key_anchor(keyval_id), t._p(keyval_id)->m_type.has_key_anchor()); - EXPECT_EQ(t.has_key_anchor(keyvalnoanchor_id), t._p(keyvalnoanchor_id)->m_type.has_key_anchor()); - EXPECT_EQ(t.has_key_anchor(seq_id), t._p(seq_id)->m_type.has_key_anchor()); - EXPECT_EQ(t.has_key_anchor(val_id), t._p(val_id)->m_type.has_key_anchor()); - EXPECT_EQ(t.has_key_anchor(valnoanchor_id), t._p(valnoanchor_id)->m_type.has_key_anchor()); - EXPECT_EQ(stream.has_key_anchor(), stream.get()->m_type.has_key_anchor()); - EXPECT_EQ(doc.has_key_anchor(), doc.get()->m_type.has_key_anchor()); - EXPECT_EQ(map.has_key_anchor(), map.get()->m_type.has_key_anchor()); - EXPECT_EQ(keyval.has_key_anchor(), keyval.get()->m_type.has_key_anchor()); - EXPECT_EQ(keyvalnoanchor.has_key_anchor(), keyvalnoanchor.get()->m_type.has_key_anchor()); - EXPECT_EQ(seq.has_key_anchor(), seq.get()->m_type.has_key_anchor()); - EXPECT_EQ(val.has_key_anchor(), val.get()->m_type.has_key_anchor()); - EXPECT_EQ(valnoanchor.has_key_anchor(), valnoanchor.get()->m_type.has_key_anchor()); - EXPECT_EQ(mstream.has_key_anchor(), mstream.get()->m_type.has_key_anchor()); - EXPECT_EQ(mdoc.has_key_anchor(), mdoc.get()->m_type.has_key_anchor()); - EXPECT_EQ(mmap.has_key_anchor(), mmap.get()->m_type.has_key_anchor()); - EXPECT_EQ(mkeyval.has_key_anchor(), mkeyval.get()->m_type.has_key_anchor()); - EXPECT_EQ(mkeyvalnoanchor.has_key_anchor(), mkeyvalnoanchor.get()->m_type.has_key_anchor()); - EXPECT_EQ(mseq.has_key_anchor(), mseq.get()->m_type.has_key_anchor()); - EXPECT_EQ(mval.has_key_anchor(), mval.get()->m_type.has_key_anchor()); - EXPECT_EQ(mvalnoanchor.has_key_anchor(), mvalnoanchor.get()->m_type.has_key_anchor()); -} - -TEST(NodeType, is_key_anchor) -{ - EXPECT_FALSE(NodeType().is_key_anchor()); - EXPECT_FALSE(NodeType(KEYANCH).is_key_anchor()); - EXPECT_TRUE(NodeType(KEY|KEYANCH).is_key_anchor()); -} - -TEST(Tree, is_key_anchor) -{ - Tree t = parse_in_arena(R"(--- &docanchor -&mapanchor map: {&keyvalanchor foo: bar, anchor: none} -&seqanchor seq: [&valanchor foo, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t keyvalnoanchor_id = t.last_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t valnoanchor_id = t.last_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef keyvalnoanchor = t.ref(keyvalnoanchor_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef valnoanchor = t.ref(valnoanchor_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mkeyvalnoanchor = t.ref(keyvalnoanchor_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mvalnoanchor = t.ref(valnoanchor_id); - EXPECT_FALSE(t.is_key_anchor(stream_id)); - EXPECT_FALSE(t.is_key_anchor(doc_id)); - EXPECT_TRUE(t.is_key_anchor(map_id)); - EXPECT_TRUE(t.is_key_anchor(keyval_id)); - EXPECT_FALSE(t.is_key_anchor(keyvalnoanchor_id)); - EXPECT_TRUE(t.is_key_anchor(seq_id)); - EXPECT_FALSE(t.is_key_anchor(val_id)); - EXPECT_FALSE(t.is_key_anchor(valnoanchor_id)); - EXPECT_FALSE(stream.is_key_anchor()); - EXPECT_FALSE(doc.is_key_anchor()); - EXPECT_TRUE(map.is_key_anchor()); - EXPECT_TRUE(keyval.is_key_anchor()); - EXPECT_FALSE(keyvalnoanchor.is_key_anchor()); - EXPECT_TRUE(seq.is_key_anchor()); - EXPECT_FALSE(val.is_key_anchor()); - EXPECT_FALSE(valnoanchor.is_key_anchor()); - EXPECT_FALSE(mstream.is_key_anchor()); - EXPECT_FALSE(mdoc.is_key_anchor()); - EXPECT_TRUE(mmap.is_key_anchor()); - EXPECT_TRUE(mkeyval.is_key_anchor()); - EXPECT_FALSE(mkeyvalnoanchor.is_key_anchor()); - EXPECT_TRUE(mseq.is_key_anchor()); - EXPECT_FALSE(mval.is_key_anchor()); - EXPECT_FALSE(mvalnoanchor.is_key_anchor()); - EXPECT_EQ(t.is_key_anchor(stream_id), t._p(stream_id)->m_type.is_key_anchor()); - EXPECT_EQ(t.is_key_anchor(doc_id), t._p(doc_id)->m_type.is_key_anchor()); - EXPECT_EQ(t.is_key_anchor(map_id), t._p(map_id)->m_type.is_key_anchor()); - EXPECT_EQ(t.is_key_anchor(keyval_id), t._p(keyval_id)->m_type.is_key_anchor()); - EXPECT_EQ(t.is_key_anchor(keyvalnoanchor_id), t._p(keyvalnoanchor_id)->m_type.is_key_anchor()); - EXPECT_EQ(t.is_key_anchor(seq_id), t._p(seq_id)->m_type.is_key_anchor()); - EXPECT_EQ(t.is_key_anchor(val_id), t._p(val_id)->m_type.is_key_anchor()); - EXPECT_EQ(t.is_key_anchor(valnoanchor_id), t._p(valnoanchor_id)->m_type.is_key_anchor()); - EXPECT_EQ(stream.is_key_anchor(), stream.get()->m_type.is_key_anchor()); - EXPECT_EQ(doc.is_key_anchor(), doc.get()->m_type.is_key_anchor()); - EXPECT_EQ(map.is_key_anchor(), map.get()->m_type.is_key_anchor()); - EXPECT_EQ(keyval.is_key_anchor(), keyval.get()->m_type.is_key_anchor()); - EXPECT_EQ(keyvalnoanchor.is_key_anchor(), keyvalnoanchor.get()->m_type.is_key_anchor()); - EXPECT_EQ(seq.is_key_anchor(), seq.get()->m_type.is_key_anchor()); - EXPECT_EQ(val.is_key_anchor(), val.get()->m_type.is_key_anchor()); - EXPECT_EQ(valnoanchor.is_key_anchor(), valnoanchor.get()->m_type.is_key_anchor()); - EXPECT_EQ(mstream.is_key_anchor(), mstream.get()->m_type.is_key_anchor()); - EXPECT_EQ(mdoc.is_key_anchor(), mdoc.get()->m_type.is_key_anchor()); - EXPECT_EQ(mmap.is_key_anchor(), mmap.get()->m_type.is_key_anchor()); - EXPECT_EQ(mkeyval.is_key_anchor(), mkeyval.get()->m_type.is_key_anchor()); - EXPECT_EQ(mkeyvalnoanchor.is_key_anchor(), mkeyvalnoanchor.get()->m_type.is_key_anchor()); - EXPECT_EQ(mseq.is_key_anchor(), mseq.get()->m_type.is_key_anchor()); - EXPECT_EQ(mval.is_key_anchor(), mval.get()->m_type.is_key_anchor()); - EXPECT_EQ(mvalnoanchor.is_key_anchor(), mvalnoanchor.get()->m_type.is_key_anchor()); -} - -TEST(NodeType, has_val_anchor) -{ - EXPECT_FALSE(NodeType().has_val_anchor()); - EXPECT_FALSE(NodeType(VALANCH).has_val_anchor()); - EXPECT_TRUE(NodeType(VAL|VALANCH).has_val_anchor()); -} - -TEST(Tree, has_val_anchor) -{ - Tree t = parse_in_arena(R"(--- &docanchor -map: &mapanchor {foo: &keyvalanchor bar, anchor: none} -seq: &seqanchor [&valanchor foo, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t keyvalnoanchor_id = t.last_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t valnoanchor_id = t.last_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef keyvalnoanchor = t.ref(keyvalnoanchor_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef valnoanchor = t.ref(valnoanchor_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mkeyvalnoanchor = t.ref(keyvalnoanchor_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mvalnoanchor = t.ref(valnoanchor_id); - EXPECT_FALSE(t.has_val_anchor(stream_id)); - EXPECT_FALSE(t.has_val_anchor(doc_id)); - EXPECT_TRUE(t.has_val_anchor(map_id)); - EXPECT_TRUE(t.has_val_anchor(keyval_id)); - EXPECT_FALSE(t.has_val_anchor(keyvalnoanchor_id)); - EXPECT_TRUE(t.has_val_anchor(seq_id)); - EXPECT_TRUE(t.has_val_anchor(val_id)); - EXPECT_FALSE(t.has_val_anchor(valnoanchor_id)); - EXPECT_FALSE(stream.has_val_anchor()); - EXPECT_FALSE(doc.has_val_anchor()); - EXPECT_TRUE(map.has_val_anchor()); - EXPECT_TRUE(keyval.has_val_anchor()); - EXPECT_FALSE(keyvalnoanchor.has_val_anchor()); - EXPECT_TRUE(seq.has_val_anchor()); - EXPECT_TRUE(val.has_val_anchor()); - EXPECT_FALSE(valnoanchor.has_val_anchor()); - EXPECT_FALSE(mstream.has_val_anchor()); - EXPECT_FALSE(mdoc.has_val_anchor()); - EXPECT_TRUE(mmap.has_val_anchor()); - EXPECT_TRUE(mkeyval.has_val_anchor()); - EXPECT_FALSE(mkeyvalnoanchor.has_val_anchor()); - EXPECT_TRUE(mseq.has_val_anchor()); - EXPECT_TRUE(mval.has_val_anchor()); - EXPECT_FALSE(mvalnoanchor.has_val_anchor()); - EXPECT_EQ(t.has_val_anchor(stream_id), t._p(stream_id)->m_type.has_val_anchor()); - EXPECT_EQ(t.has_val_anchor(doc_id), t._p(doc_id)->m_type.has_val_anchor()); - EXPECT_EQ(t.has_val_anchor(map_id), t._p(map_id)->m_type.has_val_anchor()); - EXPECT_EQ(t.has_val_anchor(keyval_id), t._p(keyval_id)->m_type.has_val_anchor()); - EXPECT_EQ(t.has_val_anchor(keyvalnoanchor_id), t._p(keyvalnoanchor_id)->m_type.has_val_anchor()); - EXPECT_EQ(t.has_val_anchor(seq_id), t._p(seq_id)->m_type.has_val_anchor()); - EXPECT_EQ(t.has_val_anchor(val_id), t._p(val_id)->m_type.has_val_anchor()); - EXPECT_EQ(t.has_val_anchor(valnoanchor_id), t._p(valnoanchor_id)->m_type.has_val_anchor()); - EXPECT_EQ(stream.has_val_anchor(), stream.get()->m_type.has_val_anchor()); - EXPECT_EQ(doc.has_val_anchor(), doc.get()->m_type.has_val_anchor()); - EXPECT_EQ(map.has_val_anchor(), map.get()->m_type.has_val_anchor()); - EXPECT_EQ(keyval.has_val_anchor(), keyval.get()->m_type.has_val_anchor()); - EXPECT_EQ(keyvalnoanchor.has_val_anchor(), keyvalnoanchor.get()->m_type.has_val_anchor()); - EXPECT_EQ(seq.has_val_anchor(), seq.get()->m_type.has_val_anchor()); - EXPECT_EQ(val.has_val_anchor(), val.get()->m_type.has_val_anchor()); - EXPECT_EQ(valnoanchor.has_val_anchor(), valnoanchor.get()->m_type.has_val_anchor()); - EXPECT_EQ(mstream.has_val_anchor(), mstream.get()->m_type.has_val_anchor()); - EXPECT_EQ(mdoc.has_val_anchor(), mdoc.get()->m_type.has_val_anchor()); - EXPECT_EQ(mmap.has_val_anchor(), mmap.get()->m_type.has_val_anchor()); - EXPECT_EQ(mkeyval.has_val_anchor(), mkeyval.get()->m_type.has_val_anchor()); - EXPECT_EQ(mkeyvalnoanchor.has_val_anchor(), mkeyvalnoanchor.get()->m_type.has_val_anchor()); - EXPECT_EQ(mseq.has_val_anchor(), mseq.get()->m_type.has_val_anchor()); - EXPECT_EQ(mval.has_val_anchor(), mval.get()->m_type.has_val_anchor()); - EXPECT_EQ(mvalnoanchor.has_val_anchor(), mvalnoanchor.get()->m_type.has_val_anchor()); -} - -TEST(NodeType, is_val_anchor) -{ - EXPECT_FALSE(NodeType().is_val_anchor()); - EXPECT_FALSE(NodeType(VALANCH).is_val_anchor()); - EXPECT_TRUE(NodeType(VAL|VALANCH).is_val_anchor()); -} - -TEST(Tree, is_val_anchor) -{ - Tree t = parse_in_arena(R"(--- &docanchor -map: &mapanchor {foo: &keyvalanchor bar, anchor: none} -seq: &seqanchor [&valanchor foo, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t keyvalnoanchor_id = t.last_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t valnoanchor_id = t.last_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef keyvalnoanchor = t.ref(keyvalnoanchor_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef valnoanchor = t.ref(valnoanchor_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mkeyvalnoanchor = t.ref(keyvalnoanchor_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mvalnoanchor = t.ref(valnoanchor_id); - EXPECT_FALSE(t.is_val_anchor(stream_id)); - EXPECT_FALSE(t.is_val_anchor(doc_id)); - EXPECT_TRUE(t.is_val_anchor(map_id)); - EXPECT_TRUE(t.is_val_anchor(keyval_id)); - EXPECT_FALSE(t.is_val_anchor(keyvalnoanchor_id)); - EXPECT_TRUE(t.is_val_anchor(seq_id)); - EXPECT_TRUE(t.is_val_anchor(val_id)); - EXPECT_FALSE(t.is_val_anchor(valnoanchor_id)); - EXPECT_FALSE(stream.is_val_anchor()); - EXPECT_FALSE(doc.is_val_anchor()); - EXPECT_TRUE(map.is_val_anchor()); - EXPECT_TRUE(keyval.is_val_anchor()); - EXPECT_FALSE(keyvalnoanchor.is_val_anchor()); - EXPECT_TRUE(seq.is_val_anchor()); - EXPECT_TRUE(val.is_val_anchor()); - EXPECT_FALSE(valnoanchor.is_val_anchor()); - EXPECT_FALSE(mstream.is_val_anchor()); - EXPECT_FALSE(mdoc.is_val_anchor()); - EXPECT_TRUE(mmap.is_val_anchor()); - EXPECT_TRUE(mkeyval.is_val_anchor()); - EXPECT_FALSE(mkeyvalnoanchor.is_val_anchor()); - EXPECT_TRUE(mseq.is_val_anchor()); - EXPECT_TRUE(mval.is_val_anchor()); - EXPECT_FALSE(mvalnoanchor.is_val_anchor()); - EXPECT_EQ(t.is_val_anchor(stream_id), t._p(stream_id)->m_type.is_val_anchor()); - EXPECT_EQ(t.is_val_anchor(doc_id), t._p(doc_id)->m_type.is_val_anchor()); - EXPECT_EQ(t.is_val_anchor(map_id), t._p(map_id)->m_type.is_val_anchor()); - EXPECT_EQ(t.is_val_anchor(keyval_id), t._p(keyval_id)->m_type.is_val_anchor()); - EXPECT_EQ(t.is_val_anchor(keyvalnoanchor_id), t._p(keyvalnoanchor_id)->m_type.is_val_anchor()); - EXPECT_EQ(t.is_val_anchor(seq_id), t._p(seq_id)->m_type.is_val_anchor()); - EXPECT_EQ(t.is_val_anchor(val_id), t._p(val_id)->m_type.is_val_anchor()); - EXPECT_EQ(t.is_val_anchor(valnoanchor_id), t._p(valnoanchor_id)->m_type.is_val_anchor()); - EXPECT_EQ(stream.is_val_anchor(), stream.get()->m_type.is_val_anchor()); - EXPECT_EQ(doc.is_val_anchor(), doc.get()->m_type.is_val_anchor()); - EXPECT_EQ(map.is_val_anchor(), map.get()->m_type.is_val_anchor()); - EXPECT_EQ(keyval.is_val_anchor(), keyval.get()->m_type.is_val_anchor()); - EXPECT_EQ(keyvalnoanchor.is_val_anchor(), keyvalnoanchor.get()->m_type.is_val_anchor()); - EXPECT_EQ(seq.is_val_anchor(), seq.get()->m_type.is_val_anchor()); - EXPECT_EQ(val.is_val_anchor(), val.get()->m_type.is_val_anchor()); - EXPECT_EQ(valnoanchor.is_val_anchor(), valnoanchor.get()->m_type.is_val_anchor()); - EXPECT_EQ(mstream.is_val_anchor(), mstream.get()->m_type.is_val_anchor()); - EXPECT_EQ(mdoc.is_val_anchor(), mdoc.get()->m_type.is_val_anchor()); - EXPECT_EQ(mmap.is_val_anchor(), mmap.get()->m_type.is_val_anchor()); - EXPECT_EQ(mkeyval.is_val_anchor(), mkeyval.get()->m_type.is_val_anchor()); - EXPECT_EQ(mkeyvalnoanchor.is_val_anchor(), mkeyvalnoanchor.get()->m_type.is_val_anchor()); - EXPECT_EQ(mseq.is_val_anchor(), mseq.get()->m_type.is_val_anchor()); - EXPECT_EQ(mval.is_val_anchor(), mval.get()->m_type.is_val_anchor()); - EXPECT_EQ(mvalnoanchor.is_val_anchor(), mvalnoanchor.get()->m_type.is_val_anchor()); -} - -TEST(NodeType, has_anchor) -{ - EXPECT_FALSE(NodeType().has_anchor()); - EXPECT_TRUE(NodeType(VALANCH).has_anchor()); - EXPECT_TRUE(NodeType(KEYANCH).has_anchor()); - EXPECT_TRUE(NodeType(KEYANCH|VALANCH).has_anchor()); - EXPECT_TRUE(NodeType(KEY|VALANCH).has_anchor()); - EXPECT_TRUE(NodeType(VAL|KEYANCH).has_anchor()); - EXPECT_TRUE(NodeType(KEY|KEYANCH).has_anchor()); - EXPECT_TRUE(NodeType(VAL|VALANCH).has_anchor()); -} - -TEST(Tree, has_anchor) -{ - Tree t = parse_in_arena(R"(--- &docanchor -map: &mapanchor {foo: &keyvalanchor bar, anchor: none} -&seqanchor seq: [&valanchor foo, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t keyvalnoanchor_id = t.last_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t valnoanchor_id = t.last_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef keyvalnoanchor = t.ref(keyvalnoanchor_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef valnoanchor = t.ref(valnoanchor_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mkeyvalnoanchor = t.ref(keyvalnoanchor_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mvalnoanchor = t.ref(valnoanchor_id); - EXPECT_FALSE(t.has_anchor(stream_id)); - EXPECT_FALSE(t.has_anchor(doc_id)); - EXPECT_TRUE(t.has_anchor(map_id)); - EXPECT_TRUE(t.has_anchor(keyval_id)); - EXPECT_FALSE(t.has_anchor(keyvalnoanchor_id)); - EXPECT_TRUE(t.has_anchor(seq_id)); - EXPECT_TRUE(t.has_anchor(val_id)); - EXPECT_FALSE(t.has_anchor(valnoanchor_id)); - EXPECT_FALSE(stream.has_anchor()); - EXPECT_FALSE(doc.has_anchor()); - EXPECT_TRUE(map.has_anchor()); - EXPECT_TRUE(keyval.has_anchor()); - EXPECT_FALSE(keyvalnoanchor.has_anchor()); - EXPECT_TRUE(seq.has_anchor()); - EXPECT_TRUE(val.has_anchor()); - EXPECT_FALSE(valnoanchor.has_anchor()); - EXPECT_FALSE(mstream.has_anchor()); - EXPECT_FALSE(mdoc.has_anchor()); - EXPECT_TRUE(mmap.has_anchor()); - EXPECT_TRUE(mkeyval.has_anchor()); - EXPECT_FALSE(mkeyvalnoanchor.has_anchor()); - EXPECT_TRUE(mseq.has_anchor()); - EXPECT_TRUE(mval.has_anchor()); - EXPECT_FALSE(mvalnoanchor.has_anchor()); - EXPECT_EQ(t.has_anchor(stream_id), t._p(stream_id)->m_type.has_anchor()); - EXPECT_EQ(t.has_anchor(doc_id), t._p(doc_id)->m_type.has_anchor()); - EXPECT_EQ(t.has_anchor(map_id), t._p(map_id)->m_type.has_anchor()); - EXPECT_EQ(t.has_anchor(keyval_id), t._p(keyval_id)->m_type.has_anchor()); - EXPECT_EQ(t.has_anchor(keyvalnoanchor_id), t._p(keyvalnoanchor_id)->m_type.has_anchor()); - EXPECT_EQ(t.has_anchor(seq_id), t._p(seq_id)->m_type.has_anchor()); - EXPECT_EQ(t.has_anchor(val_id), t._p(val_id)->m_type.has_anchor()); - EXPECT_EQ(t.has_anchor(valnoanchor_id), t._p(valnoanchor_id)->m_type.has_anchor()); - EXPECT_EQ(stream.has_anchor(), stream.get()->m_type.has_anchor()); - EXPECT_EQ(doc.has_anchor(), doc.get()->m_type.has_anchor()); - EXPECT_EQ(map.has_anchor(), map.get()->m_type.has_anchor()); - EXPECT_EQ(keyval.has_anchor(), keyval.get()->m_type.has_anchor()); - EXPECT_EQ(keyvalnoanchor.has_anchor(), keyvalnoanchor.get()->m_type.has_anchor()); - EXPECT_EQ(seq.has_anchor(), seq.get()->m_type.has_anchor()); - EXPECT_EQ(val.has_anchor(), val.get()->m_type.has_anchor()); - EXPECT_EQ(valnoanchor.has_anchor(), valnoanchor.get()->m_type.has_anchor()); - EXPECT_EQ(mstream.has_anchor(), mstream.get()->m_type.has_anchor()); - EXPECT_EQ(mdoc.has_anchor(), mdoc.get()->m_type.has_anchor()); - EXPECT_EQ(mmap.has_anchor(), mmap.get()->m_type.has_anchor()); - EXPECT_EQ(mkeyval.has_anchor(), mkeyval.get()->m_type.has_anchor()); - EXPECT_EQ(mkeyvalnoanchor.has_anchor(), mkeyvalnoanchor.get()->m_type.has_anchor()); - EXPECT_EQ(mseq.has_anchor(), mseq.get()->m_type.has_anchor()); - EXPECT_EQ(mval.has_anchor(), mval.get()->m_type.has_anchor()); - EXPECT_EQ(mvalnoanchor.has_anchor(), mvalnoanchor.get()->m_type.has_anchor()); -} - -TEST(NodeType, is_anchor) -{ - EXPECT_FALSE(NodeType().is_anchor()); - EXPECT_TRUE(NodeType(VALANCH).is_anchor()); - EXPECT_TRUE(NodeType(KEYANCH).is_anchor()); - EXPECT_TRUE(NodeType(KEYANCH|VALANCH).is_anchor()); - EXPECT_TRUE(NodeType(KEY|VALANCH).is_anchor()); - EXPECT_TRUE(NodeType(VAL|KEYANCH).is_anchor()); - EXPECT_TRUE(NodeType(KEY|KEYANCH).is_anchor()); - EXPECT_TRUE(NodeType(VAL|VALANCH).is_anchor()); -} - -TEST(Tree, is_anchor) -{ - Tree t = parse_in_arena(R"(--- &docanchor -map: &mapanchor {foo: &keyvalanchor bar, anchor: none} -&seqanchor seq: [&valanchor foo, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t keyvalnoanchor_id = t.last_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - const size_t valnoanchor_id = t.last_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef keyvalnoanchor = t.ref(keyvalnoanchor_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - ConstNodeRef valnoanchor = t.ref(valnoanchor_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mkeyvalnoanchor = t.ref(keyvalnoanchor_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - NodeRef mvalnoanchor = t.ref(valnoanchor_id); - EXPECT_FALSE(t.is_anchor(stream_id)); - EXPECT_FALSE(t.is_anchor(doc_id)); - EXPECT_TRUE(t.is_anchor(map_id)); - EXPECT_TRUE(t.is_anchor(keyval_id)); - EXPECT_FALSE(t.is_anchor(keyvalnoanchor_id)); - EXPECT_TRUE(t.is_anchor(seq_id)); - EXPECT_TRUE(t.is_anchor(val_id)); - EXPECT_FALSE(t.is_anchor(valnoanchor_id)); - EXPECT_FALSE(stream.is_anchor()); - EXPECT_FALSE(doc.is_anchor()); - EXPECT_TRUE(map.is_anchor()); - EXPECT_TRUE(keyval.is_anchor()); - EXPECT_FALSE(keyvalnoanchor.is_anchor()); - EXPECT_TRUE(seq.is_anchor()); - EXPECT_TRUE(val.is_anchor()); - EXPECT_FALSE(valnoanchor.is_anchor()); - EXPECT_FALSE(mstream.is_anchor()); - EXPECT_FALSE(mdoc.is_anchor()); - EXPECT_TRUE(mmap.is_anchor()); - EXPECT_TRUE(mkeyval.is_anchor()); - EXPECT_FALSE(mkeyvalnoanchor.is_anchor()); - EXPECT_TRUE(mseq.is_anchor()); - EXPECT_TRUE(mval.is_anchor()); - EXPECT_FALSE(mvalnoanchor.is_anchor()); - EXPECT_EQ(t.is_anchor(stream_id), t._p(stream_id)->m_type.is_anchor()); - EXPECT_EQ(t.is_anchor(doc_id), t._p(doc_id)->m_type.is_anchor()); - EXPECT_EQ(t.is_anchor(map_id), t._p(map_id)->m_type.is_anchor()); - EXPECT_EQ(t.is_anchor(keyval_id), t._p(keyval_id)->m_type.is_anchor()); - EXPECT_EQ(t.is_anchor(keyvalnoanchor_id), t._p(keyvalnoanchor_id)->m_type.is_anchor()); - EXPECT_EQ(t.is_anchor(seq_id), t._p(seq_id)->m_type.is_anchor()); - EXPECT_EQ(t.is_anchor(val_id), t._p(val_id)->m_type.is_anchor()); - EXPECT_EQ(t.is_anchor(valnoanchor_id), t._p(valnoanchor_id)->m_type.is_anchor()); - EXPECT_EQ(stream.is_anchor(), stream.get()->m_type.is_anchor()); - EXPECT_EQ(doc.is_anchor(), doc.get()->m_type.is_anchor()); - EXPECT_EQ(map.is_anchor(), map.get()->m_type.is_anchor()); - EXPECT_EQ(keyval.is_anchor(), keyval.get()->m_type.is_anchor()); - EXPECT_EQ(keyvalnoanchor.is_anchor(), keyvalnoanchor.get()->m_type.is_anchor()); - EXPECT_EQ(seq.is_anchor(), seq.get()->m_type.is_anchor()); - EXPECT_EQ(val.is_anchor(), val.get()->m_type.is_anchor()); - EXPECT_EQ(valnoanchor.is_anchor(), valnoanchor.get()->m_type.is_anchor()); - EXPECT_EQ(mstream.is_anchor(), mstream.get()->m_type.is_anchor()); - EXPECT_EQ(mdoc.is_anchor(), mdoc.get()->m_type.is_anchor()); - EXPECT_EQ(mmap.is_anchor(), mmap.get()->m_type.is_anchor()); - EXPECT_EQ(mkeyval.is_anchor(), mkeyval.get()->m_type.is_anchor()); - EXPECT_EQ(mkeyvalnoanchor.is_anchor(), mkeyvalnoanchor.get()->m_type.is_anchor()); - EXPECT_EQ(mseq.is_anchor(), mseq.get()->m_type.is_anchor()); - EXPECT_EQ(mval.is_anchor(), mval.get()->m_type.is_anchor()); - EXPECT_EQ(mvalnoanchor.is_anchor(), mvalnoanchor.get()->m_type.is_anchor()); -} - -TEST(NodeType, is_key_ref) -{ - EXPECT_FALSE(NodeType().is_key_ref()); - EXPECT_TRUE(NodeType(KEYREF).is_key_ref()); - EXPECT_TRUE(NodeType(KEY|KEYREF).is_key_ref()); -} - -TEST(Tree, is_key_ref) -{ - Tree t = parse_in_arena(R"(--- -*mapref: {foo: bar, notag: none} -*seqref: [foo, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - EXPECT_FALSE(t.is_key_ref(stream_id)); - EXPECT_FALSE(t.is_key_ref(doc_id)); - EXPECT_TRUE(t.is_key_ref(map_id)); - EXPECT_FALSE(t.is_key_ref(keyval_id)); - EXPECT_TRUE(t.is_key_ref(seq_id)); - EXPECT_FALSE(t.is_key_ref(val_id)); - EXPECT_FALSE(stream.is_key_ref()); - EXPECT_FALSE(doc.is_key_ref()); - EXPECT_TRUE(map.is_key_ref()); - EXPECT_FALSE(keyval.is_key_ref()); - EXPECT_TRUE(seq.is_key_ref()); - EXPECT_FALSE(val.is_key_ref()); - EXPECT_FALSE(mstream.is_key_ref()); - EXPECT_FALSE(mdoc.is_key_ref()); - EXPECT_TRUE(mmap.is_key_ref()); - EXPECT_FALSE(mkeyval.is_key_ref()); - EXPECT_TRUE(mseq.is_key_ref()); - EXPECT_FALSE(mval.is_key_ref()); - EXPECT_EQ(t.is_key_ref(stream_id), t._p(stream_id)->m_type.is_key_ref()); - EXPECT_EQ(t.is_key_ref(doc_id), t._p(doc_id)->m_type.is_key_ref()); - EXPECT_EQ(t.is_key_ref(map_id), t._p(map_id)->m_type.is_key_ref()); - EXPECT_EQ(t.is_key_ref(keyval_id), t._p(keyval_id)->m_type.is_key_ref()); - EXPECT_EQ(t.is_key_ref(seq_id), t._p(seq_id)->m_type.is_key_ref()); - EXPECT_EQ(t.is_key_ref(val_id), t._p(val_id)->m_type.is_key_ref()); - EXPECT_EQ(stream.is_key_ref(), stream.get()->m_type.is_key_ref()); - EXPECT_EQ(doc.is_key_ref(), doc.get()->m_type.is_key_ref()); - EXPECT_EQ(map.is_key_ref(), map.get()->m_type.is_key_ref()); - EXPECT_EQ(keyval.is_key_ref(), keyval.get()->m_type.is_key_ref()); - EXPECT_EQ(seq.is_key_ref(), seq.get()->m_type.is_key_ref()); - EXPECT_EQ(val.is_key_ref(), val.get()->m_type.is_key_ref()); - EXPECT_EQ(mstream.is_key_ref(), mstream.get()->m_type.is_key_ref()); - EXPECT_EQ(mdoc.is_key_ref(), mdoc.get()->m_type.is_key_ref()); - EXPECT_EQ(mmap.is_key_ref(), mmap.get()->m_type.is_key_ref()); - EXPECT_EQ(mkeyval.is_key_ref(), mkeyval.get()->m_type.is_key_ref()); - EXPECT_EQ(mseq.is_key_ref(), mseq.get()->m_type.is_key_ref()); - EXPECT_EQ(mval.is_key_ref(), mval.get()->m_type.is_key_ref()); -} - -TEST(NodeType, is_val_ref) -{ - EXPECT_FALSE(NodeType().is_val_ref()); - EXPECT_TRUE(NodeType(VALREF).is_val_ref()); - EXPECT_TRUE(NodeType(VAL|VALREF).is_val_ref()); -} - -TEST(Tree, is_val_ref) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: *keyvalref, notag: none} -seq: [*valref, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - EXPECT_FALSE(t.is_val_ref(stream_id)); - EXPECT_FALSE(t.is_val_ref(doc_id)); - EXPECT_FALSE(t.is_val_ref(map_id)); - EXPECT_TRUE(t.is_val_ref(keyval_id)); - EXPECT_FALSE(t.is_val_ref(seq_id)); - EXPECT_TRUE(t.is_val_ref(val_id)); - EXPECT_FALSE(stream.is_val_ref()); - EXPECT_FALSE(doc.is_val_ref()); - EXPECT_FALSE(map.is_val_ref()); - EXPECT_TRUE(keyval.is_val_ref()); - EXPECT_FALSE(seq.is_val_ref()); - EXPECT_TRUE(val.is_val_ref()); - EXPECT_FALSE(mstream.is_val_ref()); - EXPECT_FALSE(mdoc.is_val_ref()); - EXPECT_FALSE(mmap.is_val_ref()); - EXPECT_TRUE(mkeyval.is_val_ref()); - EXPECT_FALSE(mseq.is_val_ref()); - EXPECT_TRUE(mval.is_val_ref()); - EXPECT_EQ(t.is_val_ref(stream_id), t._p(stream_id)->m_type.is_val_ref()); - EXPECT_EQ(t.is_val_ref(doc_id), t._p(doc_id)->m_type.is_val_ref()); - EXPECT_EQ(t.is_val_ref(map_id), t._p(map_id)->m_type.is_val_ref()); - EXPECT_EQ(t.is_val_ref(keyval_id), t._p(keyval_id)->m_type.is_val_ref()); - EXPECT_EQ(t.is_val_ref(seq_id), t._p(seq_id)->m_type.is_val_ref()); - EXPECT_EQ(t.is_val_ref(val_id), t._p(val_id)->m_type.is_val_ref()); - EXPECT_EQ(stream.is_val_ref(), stream.get()->m_type.is_val_ref()); - EXPECT_EQ(doc.is_val_ref(), doc.get()->m_type.is_val_ref()); - EXPECT_EQ(map.is_val_ref(), map.get()->m_type.is_val_ref()); - EXPECT_EQ(keyval.is_val_ref(), keyval.get()->m_type.is_val_ref()); - EXPECT_EQ(seq.is_val_ref(), seq.get()->m_type.is_val_ref()); - EXPECT_EQ(val.is_val_ref(), val.get()->m_type.is_val_ref()); - EXPECT_EQ(mstream.is_val_ref(), mstream.get()->m_type.is_val_ref()); - EXPECT_EQ(mdoc.is_val_ref(), mdoc.get()->m_type.is_val_ref()); - EXPECT_EQ(mmap.is_val_ref(), mmap.get()->m_type.is_val_ref()); - EXPECT_EQ(mkeyval.is_val_ref(), mkeyval.get()->m_type.is_val_ref()); - EXPECT_EQ(mseq.is_val_ref(), mseq.get()->m_type.is_val_ref()); - EXPECT_EQ(mval.is_val_ref(), mval.get()->m_type.is_val_ref()); -} - -TEST(NodeType, is_ref) -{ - EXPECT_FALSE(NodeType().is_ref()); - EXPECT_FALSE(NodeType(KEYVAL).is_ref()); - EXPECT_TRUE(NodeType(KEYREF).is_ref()); - EXPECT_TRUE(NodeType(VALREF).is_ref()); - EXPECT_TRUE(NodeType(KEY|VALREF).is_ref()); - EXPECT_TRUE(NodeType(VAL|KEYREF).is_ref()); - EXPECT_TRUE(NodeType(KEYREF|VALREF).is_ref()); -} - -TEST(Tree, is_ref) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: *keyvalref, notag: none} -seq: [*valref, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - EXPECT_FALSE(t.is_ref(stream_id)); - EXPECT_FALSE(t.is_ref(doc_id)); - EXPECT_FALSE(t.is_ref(map_id)); - EXPECT_TRUE(t.is_ref(keyval_id)); - EXPECT_FALSE(t.is_ref(seq_id)); - EXPECT_TRUE(t.is_ref(val_id)); - EXPECT_FALSE(stream.is_ref()); - EXPECT_FALSE(doc.is_ref()); - EXPECT_FALSE(map.is_ref()); - EXPECT_TRUE(keyval.is_ref()); - EXPECT_FALSE(seq.is_ref()); - EXPECT_TRUE(val.is_ref()); - EXPECT_FALSE(mstream.is_ref()); - EXPECT_FALSE(mdoc.is_ref()); - EXPECT_FALSE(mmap.is_ref()); - EXPECT_TRUE(mkeyval.is_ref()); - EXPECT_FALSE(mseq.is_ref()); - EXPECT_TRUE(mval.is_ref()); - EXPECT_EQ(t.is_ref(stream_id), t._p(stream_id)->m_type.is_ref()); - EXPECT_EQ(t.is_ref(doc_id), t._p(doc_id)->m_type.is_ref()); - EXPECT_EQ(t.is_ref(map_id), t._p(map_id)->m_type.is_ref()); - EXPECT_EQ(t.is_ref(keyval_id), t._p(keyval_id)->m_type.is_ref()); - EXPECT_EQ(t.is_ref(seq_id), t._p(seq_id)->m_type.is_ref()); - EXPECT_EQ(t.is_ref(val_id), t._p(val_id)->m_type.is_ref()); - EXPECT_EQ(stream.is_ref(), stream.get()->m_type.is_ref()); - EXPECT_EQ(doc.is_ref(), doc.get()->m_type.is_ref()); - EXPECT_EQ(map.is_ref(), map.get()->m_type.is_ref()); - EXPECT_EQ(keyval.is_ref(), keyval.get()->m_type.is_ref()); - EXPECT_EQ(seq.is_ref(), seq.get()->m_type.is_ref()); - EXPECT_EQ(val.is_ref(), val.get()->m_type.is_ref()); - EXPECT_EQ(mstream.is_ref(), mstream.get()->m_type.is_ref()); - EXPECT_EQ(mdoc.is_ref(), mdoc.get()->m_type.is_ref()); - EXPECT_EQ(mmap.is_ref(), mmap.get()->m_type.is_ref()); - EXPECT_EQ(mkeyval.is_ref(), mkeyval.get()->m_type.is_ref()); - EXPECT_EQ(mseq.is_ref(), mseq.get()->m_type.is_ref()); - EXPECT_EQ(mval.is_ref(), mval.get()->m_type.is_ref()); -} - -TEST(NodeType, is_anchor_or_ref) -{ - EXPECT_FALSE(NodeType().is_anchor_or_ref()); - EXPECT_FALSE(NodeType(KEYVAL).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(KEYREF).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(KEYANCH).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(VALREF).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(VALANCH).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(KEY|VALREF).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(KEY|VALANCH).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(VAL|KEYREF).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(VAL|VALANCH).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(KEY|VALANCH).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(KEYREF|VALREF).is_anchor_or_ref()); - EXPECT_TRUE(NodeType(KEYANCH|VALANCH).is_anchor_or_ref()); -} - -TEST(Tree, is_anchor_or_ref) -{ - Tree t = parse_in_arena(R"(--- -&map map: {foo: *keyvalref, notag: none} -seq: &seq [*valref, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - EXPECT_FALSE(t.is_anchor_or_ref(stream_id)); - EXPECT_FALSE(t.is_anchor_or_ref(doc_id)); - EXPECT_TRUE(t.is_anchor_or_ref(map_id)); - EXPECT_TRUE(t.is_anchor_or_ref(keyval_id)); - EXPECT_TRUE(t.is_anchor_or_ref(seq_id)); - EXPECT_TRUE(t.is_anchor_or_ref(val_id)); - EXPECT_FALSE(stream.is_anchor_or_ref()); - EXPECT_FALSE(doc.is_anchor_or_ref()); - EXPECT_TRUE(map.is_anchor_or_ref()); - EXPECT_TRUE(keyval.is_anchor_or_ref()); - EXPECT_TRUE(seq.is_anchor_or_ref()); - EXPECT_TRUE(val.is_anchor_or_ref()); - EXPECT_FALSE(mstream.is_anchor_or_ref()); - EXPECT_FALSE(mdoc.is_anchor_or_ref()); - EXPECT_TRUE(mmap.is_anchor_or_ref()); - EXPECT_TRUE(mkeyval.is_anchor_or_ref()); - EXPECT_TRUE(mseq.is_anchor_or_ref()); - EXPECT_TRUE(mval.is_anchor_or_ref()); - EXPECT_EQ(t.is_anchor_or_ref(stream_id), t._p(stream_id)->m_type.is_anchor_or_ref()); - EXPECT_EQ(t.is_anchor_or_ref(doc_id), t._p(doc_id)->m_type.is_anchor_or_ref()); - EXPECT_EQ(t.is_anchor_or_ref(map_id), t._p(map_id)->m_type.is_anchor_or_ref()); - EXPECT_EQ(t.is_anchor_or_ref(keyval_id), t._p(keyval_id)->m_type.is_anchor_or_ref()); - EXPECT_EQ(t.is_anchor_or_ref(seq_id), t._p(seq_id)->m_type.is_anchor_or_ref()); - EXPECT_EQ(t.is_anchor_or_ref(val_id), t._p(val_id)->m_type.is_anchor_or_ref()); - EXPECT_EQ(stream.is_anchor_or_ref(), stream.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(doc.is_anchor_or_ref(), doc.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(map.is_anchor_or_ref(), map.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(keyval.is_anchor_or_ref(), keyval.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(seq.is_anchor_or_ref(), seq.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(val.is_anchor_or_ref(), val.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(mstream.is_anchor_or_ref(), mstream.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(mdoc.is_anchor_or_ref(), mdoc.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(mmap.is_anchor_or_ref(), mmap.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(mkeyval.is_anchor_or_ref(), mkeyval.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(mseq.is_anchor_or_ref(), mseq.get()->m_type.is_anchor_or_ref()); - EXPECT_EQ(mval.is_anchor_or_ref(), mval.get()->m_type.is_anchor_or_ref()); -} - -TEST(NodeType, is_key_quoted) -{ - EXPECT_FALSE(NodeType().is_key_quoted()); - EXPECT_FALSE(NodeType(KEYQUO).is_key_quoted()); - EXPECT_TRUE(NodeType(KEY|KEYQUO).is_key_quoted()); -} - -TEST(Tree, is_key_quoted) -{ - Tree t = parse_in_arena(R"(--- -"quoted": foo -notquoted: bar -...)"); - const size_t map_id = t.first_child(t.root_id()); - const size_t quoted_id = t.first_child(map_id); - const size_t notquoted_id = t.last_child(map_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef quoted = t.ref(quoted_id); - ConstNodeRef notquoted = t.ref(notquoted_id); - NodeRef mmap = t.ref(map_id); - NodeRef mquoted = t.ref(quoted_id); - NodeRef mnotquoted = t.ref(notquoted_id); - EXPECT_FALSE(t.is_key_quoted(map_id)); - EXPECT_TRUE(t.is_key_quoted(quoted_id)); - EXPECT_FALSE(t.is_key_quoted(notquoted_id)); - EXPECT_FALSE(map.is_key_quoted()); - EXPECT_TRUE(quoted.is_key_quoted()); - EXPECT_FALSE(notquoted.is_key_quoted()); - EXPECT_FALSE(mmap.is_key_quoted()); - EXPECT_TRUE(mquoted.is_key_quoted()); - EXPECT_FALSE(mnotquoted.is_key_quoted()); - EXPECT_EQ(t.is_key_quoted(map_id), t._p(map_id)->m_type.is_key_quoted()); - EXPECT_EQ(t.is_key_quoted(quoted_id), t._p(quoted_id)->m_type.is_key_quoted()); - EXPECT_EQ(t.is_key_quoted(notquoted_id), t._p(notquoted_id)->m_type.is_key_quoted()); - EXPECT_EQ(map.is_key_quoted(), map.get()->m_type.is_key_quoted()); - EXPECT_EQ(quoted.is_key_quoted(), quoted.get()->m_type.is_key_quoted()); - EXPECT_EQ(notquoted.is_key_quoted(), notquoted.get()->m_type.is_key_quoted()); - EXPECT_EQ(mmap.is_key_quoted(), mmap.get()->m_type.is_key_quoted()); - EXPECT_EQ(mquoted.is_key_quoted(), mquoted.get()->m_type.is_key_quoted()); - EXPECT_EQ(mnotquoted.is_key_quoted(), mnotquoted.get()->m_type.is_key_quoted()); -} - -TEST(NodeType, is_val_quoted) -{ - EXPECT_FALSE(NodeType().is_val_quoted()); - EXPECT_FALSE(NodeType(VALQUO).is_val_quoted()); - EXPECT_TRUE(NodeType(VAL|VALQUO).is_val_quoted()); -} - -TEST(Tree, is_val_quoted) -{ - Tree t = parse_in_arena(R"(--- -"quoted": "foo" -notquoted: bar -...)"); - const size_t map_id = t.first_child(t.root_id()); - const size_t quoted_id = t.first_child(map_id); - const size_t notquoted_id = t.last_child(map_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef quoted = t.ref(quoted_id); - ConstNodeRef notquoted = t.ref(notquoted_id); - NodeRef mmap = t.ref(map_id); - NodeRef mquoted = t.ref(quoted_id); - NodeRef mnotquoted = t.ref(notquoted_id); - EXPECT_FALSE(t.is_val_quoted(map_id)); - EXPECT_TRUE(t.is_val_quoted(quoted_id)); - EXPECT_FALSE(t.is_val_quoted(notquoted_id)); - EXPECT_FALSE(map.is_val_quoted()); - EXPECT_TRUE(quoted.is_val_quoted()); - EXPECT_FALSE(notquoted.is_val_quoted()); - EXPECT_FALSE(mmap.is_val_quoted()); - EXPECT_TRUE(mquoted.is_val_quoted()); - EXPECT_FALSE(mnotquoted.is_val_quoted()); - EXPECT_EQ(t.is_val_quoted(map_id), t._p(map_id)->m_type.is_val_quoted()); - EXPECT_EQ(t.is_val_quoted(quoted_id), t._p(quoted_id)->m_type.is_val_quoted()); - EXPECT_EQ(t.is_val_quoted(notquoted_id), t._p(notquoted_id)->m_type.is_val_quoted()); - EXPECT_EQ(map.is_val_quoted(), map.get()->m_type.is_val_quoted()); - EXPECT_EQ(quoted.is_val_quoted(), quoted.get()->m_type.is_val_quoted()); - EXPECT_EQ(notquoted.is_val_quoted(), notquoted.get()->m_type.is_val_quoted()); - EXPECT_EQ(mmap.is_val_quoted(), mmap.get()->m_type.is_val_quoted()); - EXPECT_EQ(mquoted.is_val_quoted(), mquoted.get()->m_type.is_val_quoted()); - EXPECT_EQ(mnotquoted.is_val_quoted(), mnotquoted.get()->m_type.is_val_quoted()); -} - -TEST(NodeType, is_quoted) -{ - EXPECT_FALSE(NodeType().is_quoted()); - EXPECT_FALSE(NodeType(KEYQUO).is_quoted()); - EXPECT_FALSE(NodeType(VALQUO).is_quoted()); - EXPECT_FALSE(NodeType(KEYQUO|VALQUO).is_quoted()); - EXPECT_TRUE(NodeType(KEY|KEYQUO).is_quoted()); - EXPECT_TRUE(NodeType(VAL|VALQUO).is_quoted()); - EXPECT_FALSE(NodeType(KEY|VALQUO).is_quoted()); - EXPECT_FALSE(NodeType(VAL|KEYQUO).is_quoted()); -} - -TEST(Tree, is_quoted) -{ - Tree t = parse_in_arena(R"(--- -"quoted1": foo -quoted2: "foo" -"quoted3": "foo" -'quoted4': foo -quoted5: 'foo' -'quoted6': 'foo' -notquoted: bar -...)"); - const size_t map_id = t.first_child(t.root_id()); - const size_t quoted1_id = t.find_child(map_id, "quoted1"); - const size_t quoted2_id = t.find_child(map_id, "quoted2"); - const size_t quoted3_id = t.find_child(map_id, "quoted3"); - const size_t quoted4_id = t.find_child(map_id, "quoted4"); - const size_t quoted5_id = t.find_child(map_id, "quoted5"); - const size_t quoted6_id = t.find_child(map_id, "quoted6"); - const size_t notquoted_id = t.last_child(map_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef quoted1 = t.ref(quoted1_id); - ConstNodeRef quoted2 = t.ref(quoted2_id); - ConstNodeRef quoted3 = t.ref(quoted3_id); - ConstNodeRef quoted4 = t.ref(quoted4_id); - ConstNodeRef quoted5 = t.ref(quoted5_id); - ConstNodeRef quoted6 = t.ref(quoted6_id); - ConstNodeRef notquoted = t.ref(notquoted_id); - NodeRef mmap = t.ref(map_id); - NodeRef mquoted1 = t.ref(quoted1_id); - NodeRef mquoted2 = t.ref(quoted2_id); - NodeRef mquoted3 = t.ref(quoted3_id); - NodeRef mquoted4 = t.ref(quoted4_id); - NodeRef mquoted5 = t.ref(quoted5_id); - NodeRef mquoted6 = t.ref(quoted6_id); - NodeRef mnotquoted = t.ref(notquoted_id); - EXPECT_FALSE(t.is_quoted(map_id)); - EXPECT_TRUE(t.is_quoted(quoted1_id)); - EXPECT_TRUE(t.is_quoted(quoted2_id)); - EXPECT_TRUE(t.is_quoted(quoted3_id)); - EXPECT_TRUE(t.is_quoted(quoted4_id)); - EXPECT_TRUE(t.is_quoted(quoted5_id)); - EXPECT_TRUE(t.is_quoted(quoted6_id)); - EXPECT_FALSE(t.is_quoted(notquoted_id)); - EXPECT_FALSE(map.is_quoted()); - EXPECT_TRUE(quoted1.is_quoted()); - EXPECT_TRUE(quoted2.is_quoted()); - EXPECT_TRUE(quoted3.is_quoted()); - EXPECT_TRUE(quoted4.is_quoted()); - EXPECT_TRUE(quoted5.is_quoted()); - EXPECT_TRUE(quoted6.is_quoted()); - EXPECT_FALSE(notquoted.is_quoted()); - EXPECT_FALSE(mmap.is_quoted()); - EXPECT_TRUE(mquoted1.is_quoted()); - EXPECT_TRUE(mquoted2.is_quoted()); - EXPECT_TRUE(mquoted3.is_quoted()); - EXPECT_TRUE(mquoted4.is_quoted()); - EXPECT_TRUE(mquoted5.is_quoted()); - EXPECT_TRUE(mquoted6.is_quoted()); - EXPECT_FALSE(mnotquoted.is_quoted()); - EXPECT_EQ(t.is_quoted(map_id), t._p(map_id)->m_type.is_quoted()); - EXPECT_EQ(t.is_quoted(quoted1_id), t._p(quoted1_id)->m_type.is_quoted()); - EXPECT_EQ(t.is_quoted(quoted2_id), t._p(quoted2_id)->m_type.is_quoted()); - EXPECT_EQ(t.is_quoted(quoted3_id), t._p(quoted3_id)->m_type.is_quoted()); - EXPECT_EQ(t.is_quoted(quoted4_id), t._p(quoted4_id)->m_type.is_quoted()); - EXPECT_EQ(t.is_quoted(quoted5_id), t._p(quoted5_id)->m_type.is_quoted()); - EXPECT_EQ(t.is_quoted(quoted6_id), t._p(quoted6_id)->m_type.is_quoted()); - EXPECT_EQ(t.is_quoted(notquoted_id), t._p(notquoted_id)->m_type.is_quoted()); - EXPECT_EQ(map.is_quoted(), map.get()->m_type.is_quoted()); - EXPECT_EQ(quoted1.is_quoted(), quoted1.get()->m_type.is_quoted()); - EXPECT_EQ(quoted2.is_quoted(), quoted2.get()->m_type.is_quoted()); - EXPECT_EQ(quoted3.is_quoted(), quoted3.get()->m_type.is_quoted()); - EXPECT_EQ(quoted4.is_quoted(), quoted4.get()->m_type.is_quoted()); - EXPECT_EQ(quoted5.is_quoted(), quoted5.get()->m_type.is_quoted()); - EXPECT_EQ(quoted6.is_quoted(), quoted6.get()->m_type.is_quoted()); - EXPECT_EQ(notquoted.is_quoted(), notquoted.get()->m_type.is_quoted()); - EXPECT_EQ(mmap.is_quoted(), mmap.get()->m_type.is_quoted()); - EXPECT_EQ(mquoted1.is_quoted(), mquoted1.get()->m_type.is_quoted()); - EXPECT_EQ(mquoted2.is_quoted(), mquoted2.get()->m_type.is_quoted()); - EXPECT_EQ(mquoted3.is_quoted(), mquoted3.get()->m_type.is_quoted()); - EXPECT_EQ(mquoted4.is_quoted(), mquoted4.get()->m_type.is_quoted()); - EXPECT_EQ(mquoted5.is_quoted(), mquoted5.get()->m_type.is_quoted()); - EXPECT_EQ(mquoted6.is_quoted(), mquoted6.get()->m_type.is_quoted()); - EXPECT_EQ(mnotquoted.is_quoted(), mnotquoted.get()->m_type.is_quoted()); -} - - -TEST(Tree, parent_is_seq) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: *keyvalref, notag: none} -seq: &seq [*valref, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - //EXPECT_FALSE(t.parent_is_seq(stream_id)); - EXPECT_TRUE(t.parent_is_seq(doc_id)); - EXPECT_FALSE(t.parent_is_seq(map_id)); - EXPECT_FALSE(t.parent_is_seq(keyval_id)); - EXPECT_FALSE(t.parent_is_seq(seq_id)); - EXPECT_TRUE(t.parent_is_seq(val_id)); - //EXPECT_FALSE(stream.parent_is_seq()); - EXPECT_TRUE(doc.parent_is_seq()); - EXPECT_FALSE(map.parent_is_seq()); - EXPECT_FALSE(keyval.parent_is_seq()); - EXPECT_FALSE(seq.parent_is_seq()); - EXPECT_TRUE(val.parent_is_seq()); - //EXPECT_FALSE(mstream.parent_is_seq()); - EXPECT_TRUE(mdoc.parent_is_seq()); - EXPECT_FALSE(mmap.parent_is_seq()); - EXPECT_FALSE(mkeyval.parent_is_seq()); - EXPECT_FALSE(mseq.parent_is_seq()); - EXPECT_TRUE(mval.parent_is_seq()); - //EXPECT_EQ(t.parent_is_seq(stream_id), stream.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(doc_id), doc.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(map_id), map.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(keyval_id), keyval.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(seq_id), seq.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(val_id), val.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(doc_id), mdoc.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(map_id), mmap.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(keyval_id), mkeyval.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(seq_id), mseq.parent_is_seq()); - EXPECT_EQ(t.parent_is_seq(val_id), mval.parent_is_seq()); -} - -TEST(Tree, parent_is_map) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: *keyvalref, notag: none} -seq: &seq [*valref, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - //EXPECT_FALSE(t.parent_is_map(stream_id)); - EXPECT_FALSE(t.parent_is_map(doc_id)); - EXPECT_TRUE(t.parent_is_map(map_id)); - EXPECT_TRUE(t.parent_is_map(keyval_id)); - EXPECT_TRUE(t.parent_is_map(seq_id)); - EXPECT_FALSE(t.parent_is_map(val_id)); - //EXPECT_FALSE(stream.parent_is_map()); - EXPECT_FALSE(doc.parent_is_map()); - EXPECT_TRUE(map.parent_is_map()); - EXPECT_TRUE(keyval.parent_is_map()); - EXPECT_TRUE(seq.parent_is_map()); - EXPECT_FALSE(val.parent_is_map()); - //EXPECT_FALSE(mstream.parent_is_map()); - EXPECT_FALSE(mdoc.parent_is_map()); - EXPECT_TRUE(mmap.parent_is_map()); - EXPECT_TRUE(mkeyval.parent_is_map()); - EXPECT_TRUE(mseq.parent_is_map()); - EXPECT_FALSE(mval.parent_is_map()); - //EXPECT_EQ(t.parent_is_map(stream_id), stream.parent_is_map()); - EXPECT_EQ(t.parent_is_map(doc_id), doc.parent_is_map()); - EXPECT_EQ(t.parent_is_map(map_id), map.parent_is_map()); - EXPECT_EQ(t.parent_is_map(keyval_id), keyval.parent_is_map()); - EXPECT_EQ(t.parent_is_map(seq_id), seq.parent_is_map()); - EXPECT_EQ(t.parent_is_map(val_id), val.parent_is_map()); - //EXPECT_EQ(t.parent_is_map(stream_id), mstream.parent_is_map()); - EXPECT_EQ(t.parent_is_map(doc_id), mdoc.parent_is_map()); - EXPECT_EQ(t.parent_is_map(map_id), mmap.parent_is_map()); - EXPECT_EQ(t.parent_is_map(keyval_id), mkeyval.parent_is_map()); - EXPECT_EQ(t.parent_is_map(seq_id), mseq.parent_is_map()); - EXPECT_EQ(t.parent_is_map(val_id), mval.parent_is_map()); -} - -TEST(Tree, has_parent) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: *keyvalref, notag: none} -seq: &seq [*valref, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - EXPECT_FALSE(t.has_parent(stream_id)); - EXPECT_TRUE(t.has_parent(doc_id)); - EXPECT_TRUE(t.has_parent(map_id)); - EXPECT_TRUE(t.has_parent(keyval_id)); - EXPECT_TRUE(t.has_parent(seq_id)); - EXPECT_TRUE(t.has_parent(val_id)); - EXPECT_FALSE(stream.has_parent()); - EXPECT_TRUE(doc.has_parent()); - EXPECT_TRUE(map.has_parent()); - EXPECT_TRUE(keyval.has_parent()); - EXPECT_TRUE(seq.has_parent()); - EXPECT_TRUE(val.has_parent()); - EXPECT_FALSE(mstream.has_parent()); - EXPECT_TRUE(mdoc.has_parent()); - EXPECT_TRUE(mmap.has_parent()); - EXPECT_TRUE(mkeyval.has_parent()); - EXPECT_TRUE(mseq.has_parent()); - EXPECT_TRUE(mval.has_parent()); - EXPECT_EQ(t.has_parent(stream_id), stream.has_parent()); - EXPECT_EQ(t.has_parent(doc_id), doc.has_parent()); - EXPECT_EQ(t.has_parent(map_id), map.has_parent()); - EXPECT_EQ(t.has_parent(keyval_id), keyval.has_parent()); - EXPECT_EQ(t.has_parent(seq_id), seq.has_parent()); - EXPECT_EQ(t.has_parent(val_id), val.has_parent()); - EXPECT_EQ(t.has_parent(stream_id), mstream.has_parent()); - EXPECT_EQ(t.has_parent(doc_id), mdoc.has_parent()); - EXPECT_EQ(t.has_parent(map_id), mmap.has_parent()); - EXPECT_EQ(t.has_parent(keyval_id), mkeyval.has_parent()); - EXPECT_EQ(t.has_parent(seq_id), mseq.has_parent()); - EXPECT_EQ(t.has_parent(val_id), mval.has_parent()); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -TEST(Tree, num_children) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: *keyvalref, notag: none} -seq: &seq [*valref, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - EXPECT_EQ(t.num_children(stream_id), 1u); - EXPECT_EQ(t.num_children(doc_id), 2u); - EXPECT_EQ(t.num_children(map_id), 2u); - EXPECT_EQ(t.num_children(keyval_id), 0u); - EXPECT_EQ(t.num_children(seq_id), 2u); - EXPECT_EQ(t.num_children(val_id), 0u); - EXPECT_EQ(stream.num_children(), t.num_children(stream_id)); - EXPECT_EQ(doc.num_children(), t.num_children(doc_id)); - EXPECT_EQ(map.num_children(), t.num_children(map_id)); - EXPECT_EQ(keyval.num_children(), t.num_children(keyval_id)); - EXPECT_EQ(seq.num_children(), t.num_children(seq_id)); - EXPECT_EQ(val.num_children(), t.num_children(val_id)); - EXPECT_EQ(mstream.num_children(), t.num_children(stream_id)); - EXPECT_EQ(mdoc.num_children(), t.num_children(doc_id)); - EXPECT_EQ(mmap.num_children(), t.num_children(map_id)); - EXPECT_EQ(mkeyval.num_children(), t.num_children(keyval_id)); - EXPECT_EQ(mseq.num_children(), t.num_children(seq_id)); - EXPECT_EQ(mval.num_children(), t.num_children(val_id)); -} - -TEST(Tree, child) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: *keyvalref, notag: none} -seq: &seq [*valref, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - const size_t val_id = t.first_child(seq_id); - ConstNodeRef stream = t.ref(stream_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - ConstNodeRef keyval = t.ref(keyval_id); - ConstNodeRef seq = t.ref(seq_id); - ConstNodeRef val = t.ref(val_id); - NodeRef mstream = t.ref(stream_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - NodeRef mkeyval = t.ref(keyval_id); - NodeRef mseq = t.ref(seq_id); - NodeRef mval = t.ref(val_id); - EXPECT_EQ(t.child(stream_id, 0), doc_id); - EXPECT_EQ(t.child(doc_id, 0), map_id); - EXPECT_EQ(t.child(map_id, 0), keyval_id); - EXPECT_EQ(t.child(keyval_id, 0), (size_t)NONE); - EXPECT_EQ(t.child(seq_id, 0), val_id); - EXPECT_EQ(t.child(val_id, 0), (size_t)NONE); - EXPECT_EQ(stream.child(0).id(), t.child(stream_id, 0)); - EXPECT_EQ(doc.child(0).id(), t.child(doc_id, 0)); - EXPECT_EQ(map.child(0).id(), t.child(map_id, 0)); - EXPECT_EQ(keyval.child(0).id(), t.child(keyval_id, 0)); - EXPECT_EQ(seq.child(0).id(), t.child(seq_id, 0)); - EXPECT_EQ(val.child(0).id(), t.child(val_id, 0)); - EXPECT_EQ(mstream.child(0).id(), t.child(stream_id, 0)); - EXPECT_EQ(mdoc.child(0).id(), t.child(doc_id, 0)); - EXPECT_EQ(mmap.child(0).id(), t.child(map_id, 0)); - EXPECT_EQ(mkeyval.child(0).id(), t.child(keyval_id, 0)); - EXPECT_EQ(mseq.child(0).id(), t.child(seq_id, 0)); - EXPECT_EQ(mval.child(0).id(), t.child(val_id, 0)); -} - -TEST(Tree, find_child_by_name) -{ - Tree t = parse_in_arena(R"(--- -map: {foo: *keyvalref, notag: none} -seq: &seq [*valref, bar] -...)"); - const size_t stream_id = t.root_id(); - const size_t doc_id = t.first_child(stream_id); - const size_t map_id = t.first_child(doc_id); - const size_t keyval_id = t.first_child(map_id); - const size_t seq_id = t.last_child(doc_id); - ConstNodeRef doc = t.ref(doc_id); - ConstNodeRef map = t.ref(map_id); - NodeRef mdoc = t.ref(doc_id); - NodeRef mmap = t.ref(map_id); - EXPECT_EQ(t.find_child(doc_id, "map"), map_id); - EXPECT_EQ(t.find_child(doc_id, "seq"), seq_id); - EXPECT_EQ(t.find_child(doc_id, "..."), (size_t)NONE); - EXPECT_EQ(t.find_child(map_id, "foo"), keyval_id); - EXPECT_EQ(t.find_child(map_id, "bar"), (size_t)NONE); - EXPECT_EQ(doc.find_child("map").id(), t.find_child(doc_id, "map")); - EXPECT_EQ(doc.find_child("seq").id(), t.find_child(doc_id, "seq")); - EXPECT_EQ(doc.find_child("...").id(), t.find_child(doc_id, "...")); - EXPECT_EQ(map.find_child("foo").id(), t.find_child(map_id, "foo")); - EXPECT_EQ(map.find_child("bar").id(), t.find_child(map_id, "bar")); - EXPECT_EQ(mdoc.find_child("map").id(), t.find_child(doc_id, "map")); - EXPECT_EQ(mdoc.find_child("seq").id(), t.find_child(doc_id, "seq")); - EXPECT_EQ(mdoc.find_child("...").id(), t.find_child(doc_id, "...")); - EXPECT_EQ(mmap.find_child("foo").id(), t.find_child(map_id, "foo")); - EXPECT_EQ(mmap.find_child("bar").id(), t.find_child(map_id, "bar")); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -TEST(change_type, from_val) -{ - Tree t = parse_in_arena("[val0, val1, val2]"); - t[0].change_type(VAL); - t[1].change_type(MAP); - t[2].change_type(SEQ); - Tree expected = parse_in_arena("[val0, {}, []]"); - EXPECT_EQ(emitrs_yaml(t), emitrs_yaml(expected)); -} -TEST(change_type, from_keyval) -{ - Tree t = parse_in_arena("{keyval0: val0, keyval1: val1, keyval2: val2}"); - t[0].change_type(VAL); - t[1].change_type(MAP); - t[2].change_type(SEQ); - Tree expected = parse_in_arena("{keyval0: val0, keyval1: {}, keyval2: []}"); - EXPECT_EQ(emitrs_yaml(t), emitrs_yaml(expected)); -} - -TEST(change_type, from_map) -{ - Tree t = parse_in_arena("[{map0: val0}, {map1: {map1key0: a, map1key1: b}}, {map2: [map2val0, map2val1]}]"); - t[0].change_type(VAL); - t[1].change_type(MAP); - t[2].change_type(SEQ); - EXPECT_FALSE(t[0].val_is_null()); - EXPECT_NE(t[0].val(), nullptr); - Tree expected = parse_in_arena("['', {map1: {map1key0: a, map1key1: b}}, []]"); - EXPECT_EQ(emitrs_yaml(t), emitrs_yaml(expected)); -} -TEST(change_type, from_keymap) -{ - Tree t = parse_in_arena("{map0: {map0: val0}, map1: {map1: {map1key0: a, map1key1: b}}, map2: {map2: [map2val0, map2val1]}}"); - t[0].change_type(VAL); - t[1].change_type(MAP); - t[2].change_type(SEQ); - EXPECT_FALSE(t[0].val_is_null()); - EXPECT_NE(t[0].val(), nullptr); - Tree expected = parse_in_arena("{map0: '', map1: {map1: {map1key0: a, map1key1: b}}, map2: []}"); - EXPECT_EQ(emitrs_yaml(t), emitrs_yaml(expected)); -} - -TEST(change_type, from_seq) -{ - Tree t = parse_in_arena("[[seq00, seq01], [seq10, seq11], [seq20, seq21]]"); - t[0].change_type(VAL); - t[1].change_type(MAP); - t[2].change_type(SEQ); - EXPECT_FALSE(t[0].val_is_null()); - EXPECT_NE(t[0].val(), nullptr); - Tree expected = parse_in_arena("['', {}, [seq20, seq21]]"); - EXPECT_EQ(emitrs_yaml(t), emitrs_yaml(expected)); -} -TEST(change_type, from_keyseq) -{ - Tree t = parse_in_arena("{map0: [seq00, seq01], map1: [seq10, seq11], map2: [seq20, seq21]}"); - t[0].change_type(VAL); - t[1].change_type(MAP); - t[2].change_type(SEQ); - EXPECT_FALSE(t[0].val_is_null()); - EXPECT_NE(t[0].val(), nullptr); - Tree expected = parse_in_arena("{map0: '', map1: {}, map2: [seq20, seq21]}"); - EXPECT_EQ(emitrs_yaml(t), emitrs_yaml(expected)); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -TEST(Tree, lookup_path) -{ - const char yaml[] = R"( -a: - b: bval - c: - d: - - e - - d - - f: fval - g: gval - h: - - - x: a - y: b - - - z: c - u: -)"; - Tree t = parse_in_arena(yaml); - print_tree(t); - - EXPECT_EQ(t.lookup_path("a").target, 1); - EXPECT_EQ(t.lookup_path("a.b").target, 2); - EXPECT_EQ(t.lookup_path("a.c").target, 3); - EXPECT_EQ(t.lookup_path("a.c.d").target, 4); - EXPECT_EQ(t.lookup_path("a.c.d[0]").target, 5); - EXPECT_EQ(t.lookup_path("a.c.d[1]").target, 6); - EXPECT_EQ(t.lookup_path("a.c.d[2]").target, 7); - EXPECT_EQ(t.lookup_path("a.c.d[2].f").target, 8); - EXPECT_EQ(t.lookup_path("a.c.d[2].g").target, 9); - EXPECT_EQ(t.lookup_path("a.c.d[2].h").target, 10); - EXPECT_EQ(t.lookup_path("a.c.d[2].h[0]").target, 11); - EXPECT_EQ(t.lookup_path("a.c.d[2].h[0].x").target, 12); - EXPECT_EQ(t.lookup_path("a.c.d[2].h[0].y").target, 13); - EXPECT_EQ(t.lookup_path("a.c.d[2].h[1]").target, 14); - EXPECT_EQ(t.lookup_path("a.c.d[2].h[1].z").target, 15); - EXPECT_EQ(t.lookup_path("a.c.d[2].h[1].u").target, 16); - EXPECT_EQ(t.lookup_path("d", 3).target, 4); - EXPECT_EQ(t.lookup_path("d[0]", 3).target, 5); - EXPECT_EQ(t.lookup_path("d[1]", 3).target, 6); - EXPECT_EQ(t.lookup_path("d[2]", 3).target, 7); - EXPECT_EQ(t.lookup_path("d[2].f", 3).target, 8); - EXPECT_EQ(t.lookup_path("d[2].g", 3).target, 9); - EXPECT_EQ(t.lookup_path("d[2].h", 3).target, 10); - EXPECT_EQ(t.lookup_path("d[2].h[0]", 3).target, 11); - EXPECT_EQ(t.lookup_path("d[2].h[0].x", 3).target, 12); - EXPECT_EQ(t.lookup_path("d[2].h[0].y", 3).target, 13); - EXPECT_EQ(t.lookup_path("d[2].h[1]", 3).target, 14); - EXPECT_EQ(t.lookup_path("d[2].h[1].z", 3).target, 15); - EXPECT_EQ(t.lookup_path("d[2].h[1].u", 3).target, 16); - - auto lp = t.lookup_path("x"); - EXPECT_FALSE(lp); - EXPECT_EQ(lp.target, (size_t)NONE); - EXPECT_EQ(lp.closest, (size_t)NONE); - EXPECT_EQ(lp.resolved(), ""); - EXPECT_EQ(lp.unresolved(), "x"); - lp = t.lookup_path("a.x"); - EXPECT_FALSE(lp); - EXPECT_EQ(lp.target, (size_t)NONE); - EXPECT_EQ(lp.closest, 1); - EXPECT_EQ(lp.resolved(), "a"); - EXPECT_EQ(lp.unresolved(), "x"); - lp = t.lookup_path("a.b.x"); - EXPECT_FALSE(lp); - EXPECT_EQ(lp.target, (size_t)NONE); - EXPECT_EQ(lp.closest, 2); - EXPECT_EQ(lp.resolved(), "a.b"); - EXPECT_EQ(lp.unresolved(), "x"); - lp = t.lookup_path("a.c.x"); - EXPECT_FALSE(lp); - EXPECT_EQ(lp.target, (size_t)NONE); - EXPECT_EQ(lp.closest, 3); - EXPECT_EQ(lp.resolved(), "a.c"); - EXPECT_EQ(lp.unresolved(), "x"); - - size_t sz = t.size(); - EXPECT_EQ(t.lookup_path("x").target, (size_t)NONE); - EXPECT_EQ(t.lookup_path_or_modify("x", "x"), sz); - EXPECT_EQ(t.lookup_path("x").target, sz); - EXPECT_EQ(t.val(sz), "x"); - EXPECT_EQ(t.lookup_path_or_modify("y", "x"), sz); - EXPECT_EQ(t.val(sz), "y"); - EXPECT_EQ(t.lookup_path_or_modify("z", "x"), sz); - EXPECT_EQ(t.val(sz), "z"); - - sz = t.size(); - EXPECT_EQ(t.lookup_path("a.x").target, (size_t)NONE); - EXPECT_EQ(t.lookup_path_or_modify("x", "a.x"), sz); - EXPECT_EQ(t.lookup_path("a.x").target, sz); - EXPECT_EQ(t.val(sz), "x"); - EXPECT_EQ(t.lookup_path_or_modify("y", "a.x"), sz); - EXPECT_EQ(t.val(sz), "y"); - EXPECT_EQ(t.lookup_path_or_modify("z", "a.x"), sz); - EXPECT_EQ(t.val(sz), "z"); - - sz = t.size(); - EXPECT_EQ(t.lookup_path("a.c.x").target, (size_t)NONE); - EXPECT_EQ(t.lookup_path_or_modify("x", "a.c.x"), sz); - EXPECT_EQ(t.lookup_path("a.c.x").target, sz); - EXPECT_EQ(t.val(sz), "x"); - EXPECT_EQ(t.lookup_path_or_modify("y", "a.c.x"), sz); - EXPECT_EQ(t.val(sz), "y"); - EXPECT_EQ(t.lookup_path_or_modify("z", "a.c.x"), sz); - EXPECT_EQ(t.val(sz), "z"); -} - -TEST(Tree, lookup_path_or_modify) -{ - { - Tree dst = parse_in_arena("{}"); - Tree const src = parse_in_arena("{d: [x, y, z]}"); - dst.lookup_path_or_modify("ok", "a.b.c"); - EXPECT_EQ(dst["a"]["b"]["c"].val(), "ok"); - dst.lookup_path_or_modify(&src, src["d"].id(), "a.b.d"); - EXPECT_EQ(dst["a"]["b"]["d"][0].val(), "x"); - EXPECT_EQ(dst["a"]["b"]["d"][1].val(), "y"); - EXPECT_EQ(dst["a"]["b"]["d"][2].val(), "z"); - } - - { - Tree t = parse_in_arena("{}"); - csubstr bigpath = "newmap.newseq[0].newmap.newseq[0].first"; - auto result = t.lookup_path(bigpath); - EXPECT_EQ(result.target, (size_t)NONE); - EXPECT_EQ(result.closest, (size_t)NONE); - EXPECT_EQ(result.resolved(), ""); - EXPECT_EQ(result.unresolved(), bigpath); - size_t sz = t.lookup_path_or_modify("x", bigpath); - EXPECT_EQ(t.lookup_path(bigpath).target, sz); - EXPECT_EQ(t.val(sz), "x"); - EXPECT_EQ(t["newmap"]["newseq"].num_children(), 1u); - EXPECT_EQ(t["newmap"]["newseq"][0].is_map(), true); - EXPECT_EQ(t["newmap"]["newseq"][0]["newmap"].is_map(), true); - EXPECT_EQ(t["newmap"]["newseq"][0]["newmap"]["newseq"].is_seq(), true); - EXPECT_EQ(t["newmap"]["newseq"][0]["newmap"]["newseq"].num_children(), 1u); - EXPECT_EQ(t["newmap"]["newseq"][0]["newmap"]["newseq"][0].is_map(), true); - EXPECT_EQ(t["newmap"]["newseq"][0]["newmap"]["newseq"][0]["first"].val(), "x"); - size_t sz2 = t.lookup_path_or_modify("y", bigpath); - EXPECT_EQ(t["newmap"]["newseq"][0]["newmap"]["newseq"][0]["first"].val(), "y"); - EXPECT_EQ(sz2, sz); - EXPECT_EQ(t.lookup_path(bigpath).target, sz); - EXPECT_EQ(t.val(sz2), "y"); - - sz2 = t.lookup_path_or_modify("y", "newmap2.newseq2[2].newmap2.newseq2[3].first2"); - EXPECT_EQ(t.lookup_path("newmap2.newseq2[2].newmap2.newseq2[3].first2").target, sz2); - EXPECT_EQ(t.val(sz2), "y"); - EXPECT_EQ(t["newmap2"]["newseq2"].num_children(), 3u); - EXPECT_EQ(t["newmap2"]["newseq2"][0].val(), nullptr); - EXPECT_EQ(t["newmap2"]["newseq2"][1].val(), nullptr); - EXPECT_EQ(t["newmap2"]["newseq2"][2].is_map(), true); - EXPECT_EQ(t["newmap2"]["newseq2"][2]["newmap2"].is_map(), true); - EXPECT_EQ(t["newmap2"]["newseq2"][2]["newmap2"]["newseq2"].is_seq(), true); - EXPECT_EQ(t["newmap2"]["newseq2"][2]["newmap2"]["newseq2"].num_children(), 4u); - EXPECT_EQ(t["newmap2"]["newseq2"][2]["newmap2"]["newseq2"][0].val(), nullptr); - EXPECT_EQ(t["newmap2"]["newseq2"][2]["newmap2"]["newseq2"][1].val(), nullptr); - EXPECT_EQ(t["newmap2"]["newseq2"][2]["newmap2"]["newseq2"][2].val(), nullptr); - EXPECT_EQ(t["newmap2"]["newseq2"][2]["newmap2"]["newseq2"][3].is_map(), true); - EXPECT_EQ(t["newmap2"]["newseq2"][2]["newmap2"]["newseq2"][3]["first2"].val(), "y"); - sz2 = t.lookup_path_or_modify("z", "newmap2.newseq2[2].newmap2.newseq2[3].second2"); - EXPECT_EQ (t["newmap2"]["newseq2"][2]["newmap2"]["newseq2"][3]["second2"].val(), "z"); - - sz = t.lookup_path_or_modify("foo", "newmap.newseq1[1]"); - EXPECT_EQ(t["newmap"].is_map(), true); - EXPECT_EQ(t["newmap"]["newseq1"].is_seq(), true); - EXPECT_EQ(t["newmap"]["newseq1"].num_children(), 2u); - EXPECT_EQ(t["newmap"]["newseq1"][0].val(), nullptr); - EXPECT_EQ(t["newmap"]["newseq1"][1].val(), "foo"); - sz = t.lookup_path_or_modify("bar", "newmap.newseq1[2][1]"); - EXPECT_EQ(t["newmap"]["newseq1"].is_seq(), true); - EXPECT_EQ(t["newmap"]["newseq1"].num_children(), 3u); - EXPECT_EQ(t["newmap"]["newseq1"][0].val(), nullptr); - EXPECT_EQ(t["newmap"]["newseq1"][1].val(), "foo"); - EXPECT_EQ(t["newmap"]["newseq1"][2].is_seq(), true); - EXPECT_EQ(t["newmap"]["newseq1"][2].num_children(), 2u); - EXPECT_EQ(t["newmap"]["newseq1"][2][0].val(), nullptr); - EXPECT_EQ(t["newmap"]["newseq1"][2][1].val(), "bar"); - sz = t.lookup_path_or_modify("Foo?" , "newmap.newseq1[0]"); - sz = t.lookup_path_or_modify("Bar?" , "newmap.newseq1[2][0]"); - sz = t.lookup_path_or_modify("happy" , "newmap.newseq1[2][2][3]"); - sz = t.lookup_path_or_modify("trigger", "newmap.newseq1[2][2][2]"); - sz = t.lookup_path_or_modify("Arnold" , "newmap.newseq1[2][2][0]"); - sz = t.lookup_path_or_modify("is" , "newmap.newseq1[2][2][1]"); - EXPECT_EQ(t["newmap"]["newseq1"].is_seq(), true); - EXPECT_EQ(t["newmap"]["newseq1"].num_children(), 3u); - EXPECT_EQ(t["newmap"]["newseq1"][0].val(), "Foo?"); - EXPECT_EQ(t["newmap"]["newseq1"][1].val(), "foo"); - EXPECT_EQ(t["newmap"]["newseq1"][2].is_seq(), true); - EXPECT_EQ(t["newmap"]["newseq1"][2].num_children(), 3u); - EXPECT_EQ(t["newmap"]["newseq1"][2][0].val(), "Bar?"); - EXPECT_EQ(t["newmap"]["newseq1"][2][1].val(), "bar"); - EXPECT_EQ(t["newmap"]["newseq1"][2][2].is_seq(), true); - EXPECT_EQ(t["newmap"]["newseq1"][2][2].num_children(), 4u); - EXPECT_EQ(t["newmap"]["newseq1"][2][2][0].val(), "Arnold"); - EXPECT_EQ(t["newmap"]["newseq1"][2][2][1].val(), "is"); - EXPECT_EQ(t["newmap"]["newseq1"][2][2][2].val(), "trigger"); - EXPECT_EQ(t["newmap"]["newseq1"][2][2][3].val(), "happy"); - - EXPECT_EQ(emitrs_yaml(t), R"(newmap: - newseq: - - newmap: - newseq: - - first: y - newseq1: - - 'Foo?' - - foo - - - 'Bar?' - - bar - - - Arnold - - is - - trigger - - happy -newmap2: - newseq2: - - - - - - newmap2: - newseq2: - - - - - - - - first2: y - second2: z -)"); - } -} - - - -//----------------------------------------------------------------------------- - -TEST(set_root_as_stream, empty_tree) -{ - Tree t; - NodeRef r = t.rootref(); - EXPECT_EQ(r.is_stream(), false); - EXPECT_EQ(r.num_children(), 0u); - t.set_root_as_stream(); - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.num_children(), 0u); -} - -TEST(set_root_as_stream, already_with_stream) -{ - Tree t; - t.to_stream(t.root_id()); - NodeRef r = t.rootref(); - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.num_children(), 0u); - t.set_root_as_stream(); - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.num_children(), 0u); -} - - -TEST(set_root_as_stream, root_is_map) -{ - Tree t = parse_in_arena(R"({a: b, c: d})"); - NodeRef r = t.rootref(); - EXPECT_EQ(r.is_stream(), false); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), true); - EXPECT_EQ(r.is_seq(), false); - EXPECT_EQ(r.num_children(), 2u); - print_tree(t); - std::cout << t; - t.set_root_as_stream(); - print_tree(t); - std::cout << t; - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.num_children(), 1u); - EXPECT_EQ(r[0].is_doc(), true); - EXPECT_EQ(r[0].is_map(), true); - EXPECT_EQ(r[0].is_seq(), false); - EXPECT_EQ(r[0].num_children(), 2u); - EXPECT_EQ(r[0]["a"].is_keyval(), true); - EXPECT_EQ(r[0]["c"].is_keyval(), true); - EXPECT_EQ(r[0]["a"].val(), "b"); - EXPECT_EQ(r[0]["c"].val(), "d"); -} - -TEST(set_root_as_stream, root_is_docmap) -{ - Tree t = parse_in_arena(R"({a: b, c: d})"); - t._p(t.root_id())->m_type.add(DOC); - NodeRef r = t.rootref(); - EXPECT_EQ(r.is_stream(), false); - EXPECT_EQ(r.is_doc(), true); - EXPECT_EQ(r.is_map(), true); - EXPECT_EQ(r.is_seq(), false); - EXPECT_EQ(r.num_children(), 2u); - print_tree(t); - std::cout << t; - t.set_root_as_stream(); - print_tree(t); - std::cout << t; - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.num_children(), 1u); - EXPECT_EQ(r[0].is_doc(), true); - EXPECT_EQ(r[0].is_map(), true); - EXPECT_EQ(r[0].is_seq(), false); - EXPECT_EQ(r[0].num_children(), 2u); - EXPECT_EQ(r[0]["a"].is_keyval(), true); - EXPECT_EQ(r[0]["c"].is_keyval(), true); - EXPECT_EQ(r[0]["a"].val(), "b"); - EXPECT_EQ(r[0]["c"].val(), "d"); -} - - -TEST(set_root_as_stream, root_is_seq) -{ - Tree t = parse_in_arena(R"([a, b, c, d])"); - NodeRef r = t.rootref(); - EXPECT_EQ(r.is_stream(), false); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.num_children(), 4u); - print_tree(t); - std::cout << t; - t.set_root_as_stream(); - print_tree(t); - std::cout << t; - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.num_children(), 1u); - EXPECT_EQ(r[0].is_doc(), true); - EXPECT_EQ(r[0].is_map(), false); - EXPECT_EQ(r[0].is_seq(), true); - EXPECT_EQ(r[0].num_children(), 4u); - EXPECT_EQ(r[0][0].val(), "a"); - EXPECT_EQ(r[0][1].val(), "b"); - EXPECT_EQ(r[0][2].val(), "c"); - EXPECT_EQ(r[0][3].val(), "d"); -} - -TEST(set_root_as_stream, root_is_docseq) -{ - Tree t = parse_in_arena(R"([a, b, c, d])"); - t._p(t.root_id())->m_type.add(DOC); - NodeRef r = t.rootref(); - EXPECT_EQ(r.is_stream(), false); - EXPECT_EQ(r.is_doc(), true); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.num_children(), 4u); - print_tree(t); - std::cout << t; - t.set_root_as_stream(); - print_tree(t); - std::cout << t; - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.num_children(), 1u); - EXPECT_EQ(r[0].is_doc(), true); - EXPECT_EQ(r[0].is_map(), false); - EXPECT_EQ(r[0].is_seq(), true); - EXPECT_EQ(r[0].num_children(), 4u); - EXPECT_EQ(r[0][0].val(), "a"); - EXPECT_EQ(r[0][1].val(), "b"); - EXPECT_EQ(r[0][2].val(), "c"); - EXPECT_EQ(r[0][3].val(), "d"); -} - -TEST(set_root_as_stream, root_is_seqmap) -{ - Tree t = parse_in_arena(R"([{a: b, c: d}, {e: e, f: f}, {g: g, h: h}, {i: i, j: j}])"); - NodeRef r = t.rootref(); - EXPECT_EQ(r.is_stream(), false); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.num_children(), 4u); - print_tree(t); - std::cout << t; - t.set_root_as_stream(); - print_tree(t); - std::cout << t; - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.num_children(), 1u); - EXPECT_EQ(r[0].is_doc(), true); - EXPECT_EQ(r[0].is_map(), false); - EXPECT_EQ(r[0].is_seq(), true); - EXPECT_EQ(r[0].num_children(), 4u); - EXPECT_EQ(r[0][0].is_map(), true); - EXPECT_EQ(r[0][1].is_map(), true); - EXPECT_EQ(r[0][2].is_map(), true); - EXPECT_EQ(r[0][3].is_map(), true); - EXPECT_EQ(r[0][0]["a"].val(), "b"); - EXPECT_EQ(r[0][0]["c"].val(), "d"); - EXPECT_EQ(r[0][1]["e"].val(), "e"); - EXPECT_EQ(r[0][1]["f"].val(), "f"); - EXPECT_EQ(r[0][2]["g"].val(), "g"); - EXPECT_EQ(r[0][2]["h"].val(), "h"); - EXPECT_EQ(r[0][3]["i"].val(), "i"); - EXPECT_EQ(r[0][3]["j"].val(), "j"); -} - -TEST(set_root_as_stream, root_is_mapseq) -{ - Tree t = parse_in_arena(R"({a: [0, 1, 2], b: [3, 4, 5], c: [6, 7, 8]})"); - NodeRef r = t.rootref(); - EXPECT_EQ(r.is_stream(), false); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), true); - EXPECT_EQ(r.is_seq(), false); - EXPECT_EQ(r.num_children(), 3u); - print_tree(t); - std::cout << t; - t.set_root_as_stream(); - print_tree(t); - std::cout << t; - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.num_children(), 1u); - EXPECT_EQ(r[0].is_doc(), true); - EXPECT_EQ(r[0].is_map(), true); - EXPECT_EQ(r[0].is_seq(), false); - EXPECT_EQ(r[0].num_children(), 3u); - EXPECT_EQ(r[0]["a"].is_seq(), true); - EXPECT_EQ(r[0]["b"].is_seq(), true); - EXPECT_EQ(r[0]["c"].is_seq(), true); - EXPECT_EQ(r[0]["a"][0].val(), "0"); - EXPECT_EQ(r[0]["a"][1].val(), "1"); - EXPECT_EQ(r[0]["a"][2].val(), "2"); - EXPECT_EQ(r[0]["b"][0].val(), "3"); - EXPECT_EQ(r[0]["b"][1].val(), "4"); - EXPECT_EQ(r[0]["b"][2].val(), "5"); - EXPECT_EQ(r[0]["c"][0].val(), "6"); - EXPECT_EQ(r[0]["c"][1].val(), "7"); - EXPECT_EQ(r[0]["c"][2].val(), "8"); -} - -TEST(set_root_as_stream, root_is_docval) -{ - Tree t; - NodeRef r = t.rootref(); - r.set_type(DOCVAL); - r.set_val("bar"); - r.set_val_tag(""); - EXPECT_EQ(r.is_stream(), false); - EXPECT_EQ(r.is_doc(), true); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), false); - EXPECT_EQ(r.is_val(), true); - EXPECT_EQ(r.has_val_tag(), true); - EXPECT_EQ(r.val_tag(), ""); - EXPECT_EQ(r.num_children(), 0u); - print_tree(t); - std::cout << t; - t.set_root_as_stream(); - print_tree(t); - std::cout << t; - EXPECT_EQ(r.is_stream(), true); - EXPECT_EQ(r.is_doc(), false); - EXPECT_EQ(r.is_map(), false); - EXPECT_EQ(r.is_seq(), true); - EXPECT_EQ(r.is_val(), false); - ASSERT_EQ(r.num_children(), 1u); - EXPECT_EQ(r[0].is_stream(), false); - EXPECT_EQ(r[0].is_doc(), true); - EXPECT_EQ(r[0].is_map(), false); - EXPECT_EQ(r[0].is_seq(), false); - EXPECT_EQ(r[0].is_val(), true); - EXPECT_EQ(r[0].has_val_tag(), true); - EXPECT_EQ(r[0].val_tag(), ""); - EXPECT_EQ(r[0].num_children(), 0u); -} - - -//------------------------------------------- -//------------------------------------------- -//------------------------------------------- - -TEST(Tree, doc) -{ - Tree t = parse_in_arena(R"(--- -doc0 ---- -doc1 ---- -doc2 ---- -doc3 ---- -doc4 -)"); - size_t ir = t.root_id(); - ASSERT_EQ(t.num_children(ir), 5u); - ASSERT_TRUE(t.is_stream(ir)); - EXPECT_EQ(t.child(ir, 0), t.doc(0)); - EXPECT_EQ(t.child(ir, 1), t.doc(1)); - EXPECT_EQ(t.child(ir, 2), t.doc(2)); - EXPECT_EQ(t.child(ir, 3), t.doc(3)); - EXPECT_EQ(t.child(ir, 4), t.doc(4)); - { - NodeRef r = t.rootref(); - EXPECT_EQ(r.id(), ir); - EXPECT_EQ(r.child(0), r.doc(0)); - EXPECT_EQ(r.child(1), r.doc(1)); - EXPECT_EQ(r.child(2), r.doc(2)); - EXPECT_EQ(r.child(3), r.doc(3)); - EXPECT_EQ(r.child(4), r.doc(4)); - EXPECT_EQ(r.child(0).id(), t.doc(0)); - EXPECT_EQ(r.child(1).id(), t.doc(1)); - EXPECT_EQ(r.child(2).id(), t.doc(2)); - EXPECT_EQ(r.child(3).id(), t.doc(3)); - EXPECT_EQ(r.child(4).id(), t.doc(4)); - EXPECT_EQ(r.child(0).id(), t.docref(0).id()); - EXPECT_EQ(r.child(1).id(), t.docref(1).id()); - EXPECT_EQ(r.child(2).id(), t.docref(2).id()); - EXPECT_EQ(r.child(3).id(), t.docref(3).id()); - EXPECT_EQ(r.child(4).id(), t.docref(4).id()); - } - { - const Tree &ct = t; - const ConstNodeRef r = ct.rootref(); - EXPECT_EQ(r.id(), ir); - EXPECT_EQ(r.child(0), r.doc(0)); - EXPECT_EQ(r.child(1), r.doc(1)); - EXPECT_EQ(r.child(2), r.doc(2)); - EXPECT_EQ(r.child(3), r.doc(3)); - EXPECT_EQ(r.child(4), r.doc(4)); - EXPECT_EQ(r.child(0).id(), t.doc(0)); - EXPECT_EQ(r.child(1).id(), t.doc(1)); - EXPECT_EQ(r.child(2).id(), t.doc(2)); - EXPECT_EQ(r.child(3).id(), t.doc(3)); - EXPECT_EQ(r.child(4).id(), t.doc(4)); - EXPECT_EQ(r.child(0).id(), t.docref(0).id()); - EXPECT_EQ(r.child(1).id(), t.docref(1).id()); - EXPECT_EQ(r.child(2).id(), t.docref(2).id()); - EXPECT_EQ(r.child(3).id(), t.docref(3).id()); - EXPECT_EQ(r.child(4).id(), t.docref(4).id()); - } -} - - -//------------------------------------------- -//------------------------------------------- -//------------------------------------------- - -TEST(Tree, add_tag_directives) -{ - #if RYML_MAX_TAG_DIRECTIVES != 4 - #error this test assumes RYML_MAX_TAG_DIRECTIVES == 4 - #endif - const TagDirective td[RYML_MAX_TAG_DIRECTIVES + 1] = { - TagDirective{csubstr("!a!"), csubstr("!ay-"), 0u}, - TagDirective{csubstr("!b!"), csubstr("!by-"), 0u}, - TagDirective{csubstr("!c!"), csubstr("!cy-"), 0u}, - TagDirective{csubstr("!d!"), csubstr("!dy-"), 0u}, - TagDirective{csubstr("!e!"), csubstr("!ey-"), 0u}, - }; - Tree t; - auto check_up_to = [&](size_t num) - { - size_t pos = 0; - EXPECT_EQ(t.num_tag_directives(), num); - for(TagDirective const& d : t.tag_directives()) - { - EXPECT_EQ(d.handle.str, td[pos].handle.str); - EXPECT_EQ(d.handle.len, td[pos].handle.len); - EXPECT_EQ(d.prefix.str, td[pos].prefix.str); - EXPECT_EQ(d.prefix.str, td[pos].prefix.str); - EXPECT_EQ(d.next_node_id, td[pos].next_node_id); - ++pos; - } - EXPECT_EQ(pos, num); - }; - check_up_to(0); - t.add_tag_directive(td[0]); - check_up_to(1); - t.add_tag_directive(td[1]); - check_up_to(2); - t.add_tag_directive(td[2]); - check_up_to(3); - t.add_tag_directive(td[3]); - check_up_to(4); - ExpectError::do_check(&t, [&]{ // number exceeded - t.add_tag_directive(td[4]); - }); - t.clear_tag_directives(); - check_up_to(0); -} - -TEST(Tree, resolve_tag) -{ - csubstr yaml = R"( -#%TAG !m! !my- ---- # Bulb here -!m!light fluorescent -... -#%TAG !m! !meta- ---- # Color here -!m!light green -)"; - // we're not testing the parser here, just the tag mechanics. - // So we'll add the tag directives by hand. - Tree t = parse_in_arena(yaml); - EXPECT_EQ(t[0].val_tag(), "!m!light"); - EXPECT_EQ(t[1].val_tag(), "!m!light"); - EXPECT_EQ(t.num_tag_directives(), 0u); - t.add_tag_directive(TagDirective{csubstr("!m!"), csubstr("!my-"), 1}); - t.add_tag_directive(TagDirective{csubstr("!m!"), csubstr("!meta-"), 2}); - EXPECT_EQ(t.num_tag_directives(), 2u); - char buf_[100]; - EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 1u), csubstr("")); - EXPECT_EQ(t.resolve_tag_sub(buf_, "!m!light", 2u), csubstr("")); -} - - -//------------------------------------------- -// this is needed to use the test case library -Case const* get_case(csubstr /*name*/) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 - -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__clang__) -# pragma clang diagnostic pop -#elif defined(__GNUC__) -# pragma GCC diagnostic pop -#endif diff --git a/thirdparty/ryml/test/test_yaml_events.cpp b/thirdparty/ryml/test/test_yaml_events.cpp deleted file mode 100644 index 683c6d0f9..000000000 --- a/thirdparty/ryml/test/test_yaml_events.cpp +++ /dev/null @@ -1,467 +0,0 @@ -#ifndef RYML_SINGLE_HEADER -#include -#include -#endif -#include - -#include "./test_case.hpp" -#include "./test_suite/test_suite_events.hpp" -#include "./test_suite/test_suite_events_emitter.cpp" // HACK - -namespace c4 { -namespace yml { - -void test_evts(csubstr src, std::string expected) -{ - Tree tree = parse_in_arena(src); - #if RYML_DBG - print_tree(tree); - #endif - auto actual = emit_events(tree); - EXPECT_EQ(actual, expected); -} - -TEST(events, empty) -{ - test_evts( - R"()", - R"(+STR --STR -)" - ); -} - -TEST(events, empty_whitespace) -{ - test_evts( - R"( )", - R"(+STR --STR -)" - ); -} - -TEST(events, empty_whitespace_newlines) -{ - test_evts( - R"( - )", - R"(+STR --STR -)" - ); -} - -TEST(events, empty_whitespace_newlines_comments) -{ - test_evts( - R"( -# a comment - )", - R"(+STR --STR -)" - ); -} - -TEST(events, docval) -{ - test_evts( - R"('quoted val' -)", - R"(+STR -+DOC -=VAL 'quoted val --DOC --STR -)" - ); -} - -TEST(events, docsep) -{ - test_evts( - R"(--- 'quoted val' ---- another -... ---- and yet another -... ---- -... -)", - R"(+STR -+DOC --- -=VAL 'quoted val --DOC -+DOC --- -=VAL :another --DOC -+DOC --- -=VAL :and yet another --DOC -+DOC --- -=VAL : --DOC --STR -)" - ); -} - -TEST(events, docsep_v2) -{ - test_evts( - R"( -doc1 ---- -doc2 -... -doc3 -)", - R"(+STR -+DOC --- -=VAL :doc1 --DOC -+DOC --- -=VAL :doc2 --DOC -+DOC --- -=VAL :doc3 --DOC --STR -)" - ); -} - -TEST(events, basic_map) -{ - test_evts( - "{foo: bar}", - R"(+STR -+DOC -+MAP -=VAL :foo -=VAL :bar --MAP --DOC --STR -)" - ); -} - -TEST(events, basic_seq) -{ - test_evts( - "[foo, bar]", - R"(+STR -+DOC -+SEQ -=VAL :foo -=VAL :bar --SEQ --DOC --STR -)" - ); -} - -TEST(events, escapes) -{ - test_evts( - R"("\t\ \ \r\n\0\f\/\a\v\e\N\_\L\P \b")", - "+STR\n" - "+DOC\n" - "=VAL '\\t\\t \\r\\n\\0\\f/\\a\\v\\e\\N\\_\\L\\P \\b" "\n" - "-DOC\n" - "-STR\n" - ); -} - -TEST(events, dquo_bytes) -{ - test_evts( - R"("\x0a\x0a\u263A\x0a\x55\x56\x57\x0a\u2705\U0001D11E")", - "+STR\n" - "+DOC\n" - "=VAL '\\n\\n☺\\nUVW\\n✅𝄞" "\n" - "-DOC\n" - "-STR\n" - ); -} - -TEST(events, sets) -{ - test_evts( - R"(--- !!set -? Mark McGwire -? Sammy Sosa -? Ken Griff -)", - R"(+STR -+DOC --- -+MAP -=VAL :Mark McGwire -=VAL : -=VAL :Sammy Sosa -=VAL : -=VAL :Ken Griff -=VAL : --MAP --DOC --STR -)"); -} - -TEST(events, binary) -{ - test_evts( - R"(canonical: !!binary "\ - R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5\ - OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+\ - +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC\ - AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs=" -generic: !!binary | - R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5 - OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+ - +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC - AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs= -description: - The binary value above is a tiny arrow encoded as a gif image. -)", - R"(+STR -+DOC -+MAP -=VAL :canonical -=VAL 'R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/++f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLCAgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs= -=VAL :generic -=VAL 'R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5\nOTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+\n+f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC\nAgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs=\n -=VAL :description -=VAL :The binary value above is a tiny arrow encoded as a gif image. --MAP --DOC --STR -)"); -} - - -TEST(events, tag_directives_6CK3) -{ - test_evts( - R"( -%TAG !e! tag:example.com,2000:app/ ---- -- !local foo -- !!str bar -- !e!tag%21 baz -)", - R"(+STR -+DOC --- -+SEQ -=VAL :foo -=VAL :bar -=VAL :baz --SEQ --DOC --STR -)"); -} - -TEST(events, tag_directives_6VLF) -{ - test_evts( - R"( -%FOO bar baz # Should be ignored - # with a warning. ---- "foo" -)", - R"(+STR -+DOC --- -=VAL 'foo --DOC --STR -)"); -} - -TEST(events, tag_directives_6WLZ) -{ - test_evts( - R"( -# Private ---- -!foo "bar" -... -# Global -%TAG ! tag:example.com,2000:app/ ---- -!foo "bar" -)", - R"(+STR -+DOC --- -=VAL 'bar --DOC -+DOC --- -=VAL 'bar --DOC --STR -)"); -} - -TEST(events, tag_directives_9WXW) -{ - test_evts( - R"( -# Private -#--- # note this is commented out -!foo "bar" -... -# Global -%TAG ! tag:example.com,2000:app/ ---- -!foo "bar" -)", - R"(+STR -+DOC --- -=VAL 'bar --DOC -+DOC --- -=VAL 'bar --DOC --STR -)"); -} - - -TEST(events, tag_directives_7FWL) -{ - test_evts( - R"(! foo : - ! baz -)", - R"(+STR -+DOC -+MAP -=VAL :foo -=VAL :baz --MAP --DOC --STR -)"); -} - -TEST(events, tag_directives_P76L) -{ - test_evts( - R"( -%TAG !! tag:example.com,2000:app/ ---- -!!int 1 - 3 # Interval, not integer -)", - R"(+STR -+DOC --- -=VAL :1 - 3 --DOC --STR -)"); -} - -TEST(events, tag_directives_S4JQ) -{ - test_evts( - R"( -- "12" -- 12 -- ! 12 -)", - R"(+STR -+DOC -+SEQ -=VAL '12 -=VAL :12 -=VAL :12 --SEQ --DOC --STR -)"); -} - -TEST(events, tag_directives_lookup) -{ - test_evts( - R"( -%TAG !m! !my- ---- # Bulb here -!m!light fluorescent -... -%TAG !m! !meta- ---- # Color here -!m!light green -)", - R"(+STR -+DOC --- -=VAL :fluorescent --DOC -+DOC --- -=VAL :green --DOC --STR -)"); -} - -TEST(events, anchors_refs) -{ - test_evts( - R"( -A: &A - V: 3 - L: - - 1 -B: - <<: *A - V: 4 - L: - -5 -)", - R"(+STR -+DOC -+MAP -=VAL :A -+MAP &A -=VAL :V -=VAL :3 -=VAL :L -+SEQ -=VAL :1 --SEQ --MAP -=VAL :B -+MAP -=VAL :<< -=ALI *A -=VAL :V -=VAL :4 -=VAL :L -=VAL :-5 --MAP --MAP --DOC --STR -)"); -} - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -// The other test executables are written to contain the declarative-style -// YmlTestCases. This executable does not have any but the build setup -// assumes it does, and links with the test lib, which requires an existing -// get_case() function. So this is here to act as placeholder until (if?) -// proper test cases are added here. This was detected in #47 (thanks -// @cburgard). -Case const* get_case(csubstr) -{ - return nullptr; -} - -} // namespace yml -} // namespace c4 diff --git a/thirdparty/ryml/tools/amalgamate.py b/thirdparty/ryml/tools/amalgamate.py deleted file mode 100644 index 221397f03..000000000 --- a/thirdparty/ryml/tools/amalgamate.py +++ /dev/null @@ -1,130 +0,0 @@ -import re -import os -from os.path import abspath, dirname -import sys -import subprocess -import argparse - - -projdir = abspath(dirname(dirname(__file__))) -sys.path.insert(0, f"{projdir}/ext/c4core/cmake") -import amalgamate_utils as am -sys.path.insert(0, f"{projdir}/ext/c4core/tools") -import amalgamate as am_c4core - -ryml_defmacro = "RYML_SINGLE_HDR_DEFINE_NOW" -c4core_defmacro = "C4CORE_SINGLE_HDR_DEFINE_NOW" -exports_def_code = f""" // shared library: export when defining -#if defined(RYML_SHARED) && defined({ryml_defmacro}) && !defined(RYML_EXPORTS) -#define RYML_EXPORTS -#endif -""" -c4core_def_code = f""" // propagate defines to c4core -#if defined({ryml_defmacro}) && !defined({c4core_defmacro}) -#define {c4core_defmacro} -#endif - -#if defined(RYML_EXPORTS) && !defined(C4CORE_EXPORTS) -#define C4CORE_EXPORTS -#endif - -#if defined(RYML_SHARED) && !defined(C4CORE_SHARED) -#define C4CORE_SHARED -#endif - -// workaround for include removal while amalgamating -// resulting in missing in arm-none-eabi-g++ -// https://github.com/biojppm/rapidyaml/issues/193 -#include -""" - - -def amalgamate_ryml(filename: str, - with_c4core: bool, - with_fastfloat: bool, - with_stl: bool): - c4core_amalgamated = "" - if with_c4core: - c4core_amalgamated = "src/c4/c4core_all.hpp" - am_c4core.amalgamate_c4core(f"{projdir}/{c4core_amalgamated}", - with_fastfloat=with_fastfloat, - with_stl=with_stl) - repo = "https://github.com/biojppm/rapidyaml" - defmacro = ryml_defmacro - srcfiles = [ - am.cmttext(f""" -Rapid YAML - a library to parse and emit YAML, and do it fast. - -{repo} - -DO NOT EDIT. This file is generated automatically. -This is an amalgamated single-header version of the library. - -INSTRUCTIONS: - - Include at will in any header of your project - - In one (and only one) of your project source files, - #define {defmacro} and then include this header. - This will enable the function and class definitions in - the header file. - - To compile into a shared library, just define the - preprocessor symbol RYML_SHARED . This will take - care of symbol export/import. -"""), - am.cmtfile("LICENSE.txt"), - am.injcode(exports_def_code), - am.onlyif(with_c4core, am.injcode(c4core_def_code)), - am.onlyif(with_c4core, c4core_amalgamated), - "src/c4/yml/export.hpp", - "src/c4/yml/common.hpp", - "src/c4/yml/tree.hpp", - "src/c4/yml/node.hpp", - "src/c4/yml/writer.hpp", - "src/c4/yml/detail/parser_dbg.hpp", - am.injcode("#define C4_YML_EMIT_DEF_HPP_"), - "src/c4/yml/emit.hpp", - "src/c4/yml/emit.def.hpp", - "src/c4/yml/detail/stack.hpp", - "src/c4/yml/parse.hpp", - am.onlyif(with_stl, "src/c4/yml/std/map.hpp"), - am.onlyif(with_stl, "src/c4/yml/std/string.hpp"), - am.onlyif(with_stl, "src/c4/yml/std/vector.hpp"), - am.onlyif(with_stl, "src/c4/yml/std/std.hpp"), - "src/c4/yml/common.cpp", - "src/c4/yml/tree.cpp", - "src/c4/yml/parse.cpp", - "src/c4/yml/node.cpp", - "src/c4/yml/preprocess.hpp", - "src/c4/yml/preprocess.cpp", - "src/c4/yml/detail/checks.hpp", - "src/c4/yml/detail/print.hpp", - "src/c4/yml/yml.hpp", - "src/ryml.hpp", - ] - result = am.catfiles(srcfiles, - projdir, - # comment out lines with these patterns: - include_regexes=[ - re.compile(r'^\s*#\s*include "(c4/yml/.*)".*$'), - re.compile(r'^\s*#\s*include <(c4/yml/.*)>.*$'), - re.compile(r'^\s*#\s*include "(c4/.*)".*$'), - re.compile(r'^\s*#\s*include <(c4/.*)>.*$'), - ], - definition_macro=defmacro, - repo=repo, - result_incguard="_RYML_SINGLE_HEADER_AMALGAMATED_HPP_") - result_with_only_first_includes = am.include_only_first(result) - am.file_put_contents(filename, result_with_only_first_includes) - - -def mkparser(): - return am.mkparser(c4core=(True, "amalgamate c4core together with ryml"), - fastfloat=(True, "enable fastfloat library"), - stl=(True, "enable stl interop")) - - -if __name__ == "__main__": - args = mkparser().parse_args() - amalgamate_ryml(filename=args.output, - with_c4core=args.c4core, - with_fastfloat=args.fastfloat, - with_stl=args.stl) diff --git a/thirdparty/ryml/tools/parse_emit.cpp b/thirdparty/ryml/tools/parse_emit.cpp deleted file mode 100644 index 6ae807964..000000000 --- a/thirdparty/ryml/tools/parse_emit.cpp +++ /dev/null @@ -1,116 +0,0 @@ -#ifdef RYML_SINGLE_HEADER -#include -#else -#include -#include -#include -#endif -#include - -#include -#include - - -using namespace c4; - - -//----------------------------------------------------------------------------- - -struct timed_section -{ - using myclock = std::chrono::steady_clock; - using msecs = std::chrono::duration; - - csubstr name; - myclock::time_point start; - - msecs since() const { return myclock::now() - start; } - timed_section(csubstr n) : name(n), start(myclock::now()) {} - ~timed_section() - { - fprintf(stderr, "%.6fms: %.*s\n", since().count(), (int)name.len, name.str); - fflush(stderr); - } -}; - -#define TS(name) timed_section name##__##__LINE__(#name) - - -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- -//----------------------------------------------------------------------------- - -int main(int argc, const char *argv[]) -{ - bool print_emitted_to_stdout = true; - csubstr file; - // LCOV_EXCL_START - auto show_usage = [argv]{ - printf("usage: %s [-s] \n", argv[0]); - }; - if(argc == 2) - { - file = to_csubstr(argv[1]); - } - else if(argc > 2) - { - file = to_csubstr(argv[2]); - csubstr arg = to_csubstr(argv[1]); - if(arg == "-s") - { - print_emitted_to_stdout = false; - } - else - { - show_usage(); - return 1; - } - } - else - { - show_usage(); - return 1; - } - // LCOV_EXCL_STOP - - TS(TOTAL); - - C4_CHECK_MSG(fs::path_exists(file.str), "cannot find file: %s (cwd=%s)", file.str, fs::cwd().c_str()); - - { - TS(objects); - std::string contents, output; - yml::Tree tree; - { - TS(read_file); - fs::file_get_contents(file.str, &contents); - } - { - TS(tree_reserve); - size_t nlines; - { - TS(count_lines); - nlines = to_csubstr(contents).count('\n'); - } - fprintf(stderr, "reserving #lines=%zu\n", nlines); - tree.reserve(nlines); - } - { - TS(parse_yml); - yml::parse_in_place(file, to_substr(contents), &tree); - } - { - TS(emit_to_buffer); - output.resize(contents.size()); // resize, not just reserve - yml::emitrs_yaml(tree, &output); - } - if(print_emitted_to_stdout) - { - TS(print_stdout); - fwrite(output.data(), 1, output.size(), stdout); - putchar('\n'); - } - } - - return 0; -} diff --git a/thirdparty/ryml/tools/test_suite/Dockerfile b/thirdparty/ryml/tools/test_suite/Dockerfile deleted file mode 100644 index fc5b51d65..000000000 --- a/thirdparty/ryml/tools/test_suite/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM alpine:latest - -# to run: -# docker run --rm -it -v $PWD:/host alpine sh -# docker run --rm -it -v $PWD:/host -w /host -v /tmp/bash_history:/root/.bash_history yts-importing bash -# - -RUN apk add build-base \ - && apk add \ - bash \ - curl \ - fortune \ - git \ - jq \ - perl \ - perl-app-cpanminus \ - tig \ - vim \ - wget \ - python \ - cmake \ - ninja \ - && true - -RUN cpanm -n \ - boolean \ - Capture::Tiny \ - XXX \ - YAML::PP \ - && true - - -ENV PYTHONPATH=/python/lib/python3.7/site-packages diff --git a/thirdparty/ryml/tools/test_suite/run_test_suite.sh b/thirdparty/ryml/tools/test_suite/run_test_suite.sh deleted file mode 100755 index 4f955f67b..000000000 --- a/thirdparty/ryml/tools/test_suite/run_test_suite.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -x -set -e - -d=$1 -[ "$d" == "" ] && d=. -if [ ! -d $d ] ; then - echo "$d is not a directory" - exit 1 -fi -d=$(cd $d ; pwd) # get absolute path - -cd $d/yaml-test-runtimes -make force build -cd $d/yaml-test-suite -make clean run-tests export -xsel -b -#else -#include -#include -#endif -#include -#include -#include -#include - -using namespace c4; -using namespace c4::yml; - -void usage(const char *exename); -std::string load_file(csubstr filename); -void report_error(const char* msg, size_t length, Location loc, FILE *f); - - -int main(int argc, const char *argv[]) -{ - if(argc < 2) - { - usage(argv[0]); - return 1; - } - Callbacks callbacks = {}; - callbacks.m_error = [](const char *msg, size_t msg_len, Location location, void *) - { - report_error(msg, msg_len, location, stderr); - throw std::runtime_error({msg, msg_len}); - }; - try { - Tree tree(callbacks); - csubstr filename = to_csubstr(argv[1]); - std::string evt, src = load_file(filename); - tree.reserve(to_substr(src).count('\n')); - parse_in_place(filename, to_substr(src), &tree); - emit_events(&evt, tree); - std::fwrite(evt.data(), 1, evt.size(), stdout); - } - catch(std::exception const&) - { - return 1; - } - return 0; -} - - -//----------------------------------------------------------------------------- - -void usage(const char *exename) -{ - std::printf(R"(usage: -%s - # read from stdin -%s # read from file -)", exename, exename); -} - -std::string load_file(csubstr filename) -{ - if(filename == "-") // read from stdin - { - std::string buf; - for(int c = std::getchar(); c != EOF; c = std::getchar()) - { - buf.push_back((char)c); - if(buf.size() == buf.capacity()) - buf.reserve(2u * (buf.capacity() >= 128u ? buf.capacity() : 128u)); - } - return buf; - } - C4_CHECK_MSG(fs::path_exists(filename.str), "cannot find file: %s (cwd=%s)", filename.str, fs::cwd().c_str()); - return fs::file_get_contents(filename.str); -} - -void report_error(const char* msg, size_t length, Location loc, FILE *f) -{ - if(!loc.name.empty()) - { - fwrite(loc.name.str, 1, loc.name.len, f); - fputc(':', f); - } - fprintf(f, "%zu:", loc.line); - if(loc.col) - fprintf(f, "%zu:", loc.col); - if(loc.offset) - fprintf(f, " (%zuB):", loc.offset); - fputc(' ', f); - fprintf(f, "%.*s\n", (int)length, msg); - fflush(f); -} diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua index 6fead7b50..7c99d910e 100644 --- a/thirdparty/xmake.lua +++ b/thirdparty/xmake.lua @@ -144,18 +144,3 @@ target("fmt") add_headerfiles("fmt/include/**.h") add_includedirs("fmt/include", {public=true}) -target("ryml") - set_kind("static") - set_group("thirdparty") - add_files("ryml/src/**.cpp") - add_files("ryml/ext/c4core/src/c4/*.cpp") - add_headerfiles("ryml/ext/c4core/src/**.hpp") - add_headerfiles("ryml/src/**.hpp") - add_includedirs("ryml/src", {public=true}) - add_includedirs("ryml/ext/c4core/src", {public=true}) - - if is_os("windows") then - add_cxxflags("/wd4668") -- 'symbol' : undefined macro is treated as '0' in '#if/#elif' preprocessor directives - else - add_cxxflags("-Wno-unused-but-set-variable", "-Wno-undef") - end -- cgit v1.2.3 From 3c89c486338890ce39ddebe5be4722a09e85701a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 24 Feb 2026 13:23:52 +0100 Subject: Fix correctness and concurrency bugs found during code review zenstore fixes: - cas.cpp: GetFileCasResults Results param passed by value instead of reference (large chunk results were silently lost) - structuredcachestore.cpp: MissCount unconditionally incremented (counted hits as misses) - cacherpc.cpp: Wrong boolean in Incomplete response array (all entries marked incomplete) - cachedisklayer.cpp: sizeof(sizeof(...)) in two validation checks computed sizeof(size_t) instead of struct size - buildstore.cpp: Wrong hash tracked in GC key list (BlobHash pushed twice instead of MetadataHash) - buildstore.cpp: Removed duplicate m_LastAccessTimeUpdateCount increment in PutBlob zenserver fixes: - httpbuildstore.cpp: Reversed subtraction in HTTP range calculation (unsigned underflow) - hubservice.cpp: Deadlock in Provision() calling Wake() while holding m_Lock (extracted WakeLocked helper) - zipfs.cpp: Data race in GetFile() lazy initialization (added RwLock with shared/exclusive paths) Co-Authored-By: Claude Opus 4.6 --- src/zenserver/frontend/zipfs.cpp | 20 ++++++++++++++++---- src/zenserver/frontend/zipfs.h | 2 ++ src/zenserver/hub/hubservice.cpp | 12 +++++++++--- src/zenserver/storage/buildstore/httpbuildstore.cpp | 2 +- src/zenstore/buildstore/buildstore.cpp | 3 +-- src/zenstore/cache/cachedisklayer.cpp | 4 ++-- src/zenstore/cache/cacherpc.cpp | 2 +- src/zenstore/cache/structuredcachestore.cpp | 5 ++++- src/zenstore/cas.cpp | 12 ++++++------ 9 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/zenserver/frontend/zipfs.cpp b/src/zenserver/frontend/zipfs.cpp index f9c2bc8ff..42df0520f 100644 --- a/src/zenserver/frontend/zipfs.cpp +++ b/src/zenserver/frontend/zipfs.cpp @@ -149,13 +149,25 @@ ZipFs::ZipFs(IoBuffer&& Buffer) IoBuffer ZipFs::GetFile(const std::string_view& FileName) const { - FileMap::iterator Iter = m_Files.find(FileName); - if (Iter == m_Files.end()) { - return {}; + RwLock::SharedLockScope _(m_FilesLock); + + FileMap::const_iterator Iter = m_Files.find(FileName); + if (Iter == m_Files.end()) + { + return {}; + } + + const FileItem& Item = Iter->second; + if (Item.GetSize() > 0) + { + return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); + } } - FileItem& Item = Iter->second; + RwLock::ExclusiveLockScope _(m_FilesLock); + + FileItem& Item = m_Files.find(FileName)->second; if (Item.GetSize() > 0) { return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); diff --git a/src/zenserver/frontend/zipfs.h b/src/zenserver/frontend/zipfs.h index 1fa7da451..19f96567c 100644 --- a/src/zenserver/frontend/zipfs.h +++ b/src/zenserver/frontend/zipfs.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include @@ -20,6 +21,7 @@ public: private: using FileItem = MemoryView; using FileMap = std::unordered_map; + mutable RwLock m_FilesLock; FileMap mutable m_Files; IoBuffer m_Buffer; }; diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp index 4d9da3a57..a00446a75 100644 --- a/src/zenserver/hub/hubservice.cpp +++ b/src/zenserver/hub/hubservice.cpp @@ -151,6 +151,7 @@ struct StorageServerInstance inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); } private: + void WakeLocked(); RwLock m_Lock; std::string m_ModuleId; std::atomic m_IsProvisioned{false}; @@ -211,7 +212,7 @@ StorageServerInstance::Provision() if (m_IsHibernated) { - Wake(); + WakeLocked(); } else { @@ -294,9 +295,14 @@ StorageServerInstance::Hibernate() void StorageServerInstance::Wake() { - // Start server in-place using existing data - RwLock::ExclusiveLockScope _(m_Lock); + WakeLocked(); +} + +void +StorageServerInstance::WakeLocked() +{ + // Start server in-place using existing data if (!m_IsHibernated) { diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index f5ba30616..bf7afcc02 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -185,7 +185,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) { const HttpRange& Range = Ranges.front(); const uint64_t BlobSize = Blob.GetSize(); - const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; + const uint64_t MaxBlobSize = Range.Start < BlobSize ? BlobSize - Range.Start : 0; const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); if (Range.Start + RangeSize > BlobSize) { diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 04a0781d3..aa37e75fe 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -266,13 +266,12 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) m_BlobLookup.insert({BlobHash, NewBlobIndex}); } - m_LastAccessTimeUpdateCount++; if (m_TrackedBlobKeys) { m_TrackedBlobKeys->push_back(BlobHash); if (MetadataHash != IoHash::Zero) { - m_TrackedBlobKeys->push_back(BlobHash); + m_TrackedBlobKeys->push_back(MetadataHash); } } } diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index ead7e4f3a..b73b3e6fb 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -626,7 +626,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B return false; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(BucketMetaHeader))) / sizeof(ManifestData); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(BucketMetaHeader)) / sizeof(ManifestData); if (Header.EntryCount > ExpectedEntryCount) { ZEN_WARN( @@ -1057,7 +1057,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const return 0; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(cache::impl::CacheBucketIndexHeader))) / sizeof(DiskIndexEntry); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(cache::impl::CacheBucketIndexHeader)) / sizeof(DiskIndexEntry); if (Header.EntryCount > ExpectedEntryCount) { return 0; diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 94abcf547..e1fd0a3e6 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -966,7 +966,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } else { - ResponseObject.AddBool(true); + ResponseObject.AddBool(false); } } ResponseObject.EndArray(); diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 52b494e45..4e8475293 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -608,7 +608,10 @@ ZenCacheStore::GetBatch::Commit() m_CacheStore.m_HitCount++; OpScope.SetBytes(Result.Value.GetSize()); } - m_CacheStore.m_MissCount++; + else + { + m_CacheStore.m_MissCount++; + } } } } diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index ed017988f..7402d92d3 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -300,12 +300,12 @@ GetCompactCasResults(CasContainerStrategy& Strategy, }; static void -GetFileCasResults(FileCasStrategy& Strategy, - CasStore::InsertMode Mode, - std::span Data, - std::span ChunkHashes, - std::span Indexes, - std::vector Results) +GetFileCasResults(FileCasStrategy& Strategy, + CasStore::InsertMode Mode, + std::span Data, + std::span ChunkHashes, + std::span Indexes, + std::vector& Results) { for (size_t Index : Indexes) { -- cgit v1.2.3 From 075bac3ca870a1297e9f62230d56e63aec13a77d Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 24 Feb 2026 13:36:44 +0100 Subject: Revert "Fix correctness and concurrency bugs found during code review" This reverts commit 3c89c486338890ce39ddebe5be4722a09e85701a. --- src/zenserver/frontend/zipfs.cpp | 20 ++++---------------- src/zenserver/frontend/zipfs.h | 2 -- src/zenserver/hub/hubservice.cpp | 12 +++--------- src/zenserver/storage/buildstore/httpbuildstore.cpp | 2 +- src/zenstore/buildstore/buildstore.cpp | 3 ++- src/zenstore/cache/cachedisklayer.cpp | 4 ++-- src/zenstore/cache/cacherpc.cpp | 2 +- src/zenstore/cache/structuredcachestore.cpp | 5 +---- src/zenstore/cas.cpp | 12 ++++++------ 9 files changed, 20 insertions(+), 42 deletions(-) diff --git a/src/zenserver/frontend/zipfs.cpp b/src/zenserver/frontend/zipfs.cpp index 42df0520f..f9c2bc8ff 100644 --- a/src/zenserver/frontend/zipfs.cpp +++ b/src/zenserver/frontend/zipfs.cpp @@ -149,25 +149,13 @@ ZipFs::ZipFs(IoBuffer&& Buffer) IoBuffer ZipFs::GetFile(const std::string_view& FileName) const { + FileMap::iterator Iter = m_Files.find(FileName); + if (Iter == m_Files.end()) { - RwLock::SharedLockScope _(m_FilesLock); - - FileMap::const_iterator Iter = m_Files.find(FileName); - if (Iter == m_Files.end()) - { - return {}; - } - - const FileItem& Item = Iter->second; - if (Item.GetSize() > 0) - { - return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); - } + return {}; } - RwLock::ExclusiveLockScope _(m_FilesLock); - - FileItem& Item = m_Files.find(FileName)->second; + FileItem& Item = Iter->second; if (Item.GetSize() > 0) { return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); diff --git a/src/zenserver/frontend/zipfs.h b/src/zenserver/frontend/zipfs.h index 19f96567c..1fa7da451 100644 --- a/src/zenserver/frontend/zipfs.h +++ b/src/zenserver/frontend/zipfs.h @@ -3,7 +3,6 @@ #pragma once #include -#include #include @@ -21,7 +20,6 @@ public: private: using FileItem = MemoryView; using FileMap = std::unordered_map; - mutable RwLock m_FilesLock; FileMap mutable m_Files; IoBuffer m_Buffer; }; diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp index a00446a75..4d9da3a57 100644 --- a/src/zenserver/hub/hubservice.cpp +++ b/src/zenserver/hub/hubservice.cpp @@ -151,7 +151,6 @@ struct StorageServerInstance inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); } private: - void WakeLocked(); RwLock m_Lock; std::string m_ModuleId; std::atomic m_IsProvisioned{false}; @@ -212,7 +211,7 @@ StorageServerInstance::Provision() if (m_IsHibernated) { - WakeLocked(); + Wake(); } else { @@ -294,16 +293,11 @@ StorageServerInstance::Hibernate() void StorageServerInstance::Wake() -{ - RwLock::ExclusiveLockScope _(m_Lock); - WakeLocked(); -} - -void -StorageServerInstance::WakeLocked() { // Start server in-place using existing data + RwLock::ExclusiveLockScope _(m_Lock); + if (!m_IsHibernated) { ZEN_WARN("Attempted to wake storage server instance for module '{}' which is not hibernated", m_ModuleId); diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index bf7afcc02..f5ba30616 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -185,7 +185,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) { const HttpRange& Range = Ranges.front(); const uint64_t BlobSize = Blob.GetSize(); - const uint64_t MaxBlobSize = Range.Start < BlobSize ? BlobSize - Range.Start : 0; + const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); if (Range.Start + RangeSize > BlobSize) { diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index aa37e75fe..04a0781d3 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -266,12 +266,13 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) m_BlobLookup.insert({BlobHash, NewBlobIndex}); } + m_LastAccessTimeUpdateCount++; if (m_TrackedBlobKeys) { m_TrackedBlobKeys->push_back(BlobHash); if (MetadataHash != IoHash::Zero) { - m_TrackedBlobKeys->push_back(MetadataHash); + m_TrackedBlobKeys->push_back(BlobHash); } } } diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index b73b3e6fb..ead7e4f3a 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -626,7 +626,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B return false; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(BucketMetaHeader)) / sizeof(ManifestData); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(BucketMetaHeader))) / sizeof(ManifestData); if (Header.EntryCount > ExpectedEntryCount) { ZEN_WARN( @@ -1057,7 +1057,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const return 0; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(cache::impl::CacheBucketIndexHeader)) / sizeof(DiskIndexEntry); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(cache::impl::CacheBucketIndexHeader))) / sizeof(DiskIndexEntry); if (Header.EntryCount > ExpectedEntryCount) { return 0; diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index e1fd0a3e6..94abcf547 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -966,7 +966,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } else { - ResponseObject.AddBool(false); + ResponseObject.AddBool(true); } } ResponseObject.EndArray(); diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 4e8475293..52b494e45 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -608,10 +608,7 @@ ZenCacheStore::GetBatch::Commit() m_CacheStore.m_HitCount++; OpScope.SetBytes(Result.Value.GetSize()); } - else - { - m_CacheStore.m_MissCount++; - } + m_CacheStore.m_MissCount++; } } } diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index 7402d92d3..ed017988f 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -300,12 +300,12 @@ GetCompactCasResults(CasContainerStrategy& Strategy, }; static void -GetFileCasResults(FileCasStrategy& Strategy, - CasStore::InsertMode Mode, - std::span Data, - std::span ChunkHashes, - std::span Indexes, - std::vector& Results) +GetFileCasResults(FileCasStrategy& Strategy, + CasStore::InsertMode Mode, + std::span Data, + std::span ChunkHashes, + std::span Indexes, + std::vector Results) { for (size_t Index : Indexes) { -- cgit v1.2.3 From 5c5e12d1f02bb7cc1f42750e47a2735dc933c194 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 24 Feb 2026 14:56:57 +0100 Subject: Various bug fixes (#778) zencore fixes: - filesystem.cpp: ReadFile error reporting logic - compactbinaryvalue.h: CbValue::As*String error reporting logic zenhttp fixes: - httpasio BindAcceptor would `return 0;` in a function returning `std::string` (UB) - httpsys async workpool initialization race zenstore fixes: - cas.cpp: GetFileCasResults Results param passed by value instead of reference (large chunk results were silently lost) - structuredcachestore.cpp: MissCount unconditionally incremented (counted hits as misses) - cacherpc.cpp: Wrong boolean in Incomplete response array (all entries marked incomplete) - cachedisklayer.cpp: sizeof(sizeof(...)) in two validation checks computed sizeof(size_t) instead of struct size - buildstore.cpp: Wrong hash tracked in GC key list (BlobHash pushed twice instead of MetadataHash) - buildstore.cpp: Removed duplicate m_LastAccessTimeUpdateCount increment in PutBlob zenserver fixes: - httpbuildstore.cpp: Reversed subtraction in HTTP range calculation (unsigned underflow) - hubservice.cpp: Deadlock in Provision() calling Wake() while holding m_Lock (extracted WakeLocked helper) - zipfs.cpp: Data race in GetFile() lazy initialization (added RwLock with shared/exclusive paths) --- src/zencompute-test/xmake.lua | 1 - src/zencompute/xmake.lua | 2 -- src/zencore/filesystem.cpp | 16 +++------------ src/zencore/include/zencore/compactbinaryvalue.h | 24 ++++++++++++++-------- src/zencore/memtrack/callstacktrace.cpp | 8 ++++---- src/zencore/string.cpp | 4 ++++ src/zenhttp/servers/httpasio.cpp | 4 ++-- src/zenhttp/servers/httpsys.cpp | 17 ++++++++------- src/zenserver/frontend/frontend.cpp | 9 +++++--- src/zenserver/frontend/frontend.h | 7 ++++--- src/zenserver/frontend/zipfs.cpp | 20 ++++++++++++++---- src/zenserver/frontend/zipfs.h | 8 ++++---- src/zenserver/hub/hubservice.cpp | 12 ++++++++--- .../storage/buildstore/httpbuildstore.cpp | 2 +- src/zenstore/buildstore/buildstore.cpp | 3 +-- src/zenstore/cache/cachedisklayer.cpp | 4 ++-- src/zenstore/cache/cacherpc.cpp | 2 +- src/zenstore/cache/structuredcachestore.cpp | 5 ++++- src/zenstore/cas.cpp | 12 +++++------ src/zentest-appstub/xmake.lua | 2 -- 20 files changed, 93 insertions(+), 69 deletions(-) diff --git a/src/zencompute-test/xmake.lua b/src/zencompute-test/xmake.lua index 64a3c7703..1207bdefd 100644 --- a/src/zencompute-test/xmake.lua +++ b/src/zencompute-test/xmake.lua @@ -6,4 +6,3 @@ target("zencompute-test") add_headerfiles("**.h") add_files("*.cpp") add_deps("zencompute", "zencore") - add_packages("vcpkg::doctest") diff --git a/src/zencompute/xmake.lua b/src/zencompute/xmake.lua index c710b662d..50877508c 100644 --- a/src/zencompute/xmake.lua +++ b/src/zencompute/xmake.lua @@ -7,5 +7,3 @@ target('zencompute') add_files("**.cpp") add_includedirs("include", {public=true}) add_deps("zencore", "zenstore", "zenutil", "zennet", "zenhttp") - add_packages("vcpkg::gsl-lite") - add_packages("vcpkg::spdlog", "vcpkg::cxxopts") diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 1a4ee4b9b..553897407 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -1326,11 +1326,6 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin { BytesRead = size_t(dwNumberOfBytesRead); } - else if ((BytesRead != NumberOfBytesToRead)) - { - Ec = MakeErrorCode(ERROR_HANDLE_EOF); - return; - } else { Ec = MakeErrorCodeFromLastError(); @@ -1344,20 +1339,15 @@ ReadFile(void* NativeHandle, void* Data, uint64_t Size, uint64_t FileOffset, uin { BytesRead = size_t(ReadResult); } - else if ((BytesRead != NumberOfBytesToRead)) - { - Ec = MakeErrorCode(EIO); - return; - } else { Ec = MakeErrorCodeFromLastError(); return; } #endif - Size -= NumberOfBytesToRead; - FileOffset += NumberOfBytesToRead; - Data = reinterpret_cast(Data) + NumberOfBytesToRead; + Size -= BytesRead; + FileOffset += BytesRead; + Data = reinterpret_cast(Data) + BytesRead; } } diff --git a/src/zencore/include/zencore/compactbinaryvalue.h b/src/zencore/include/zencore/compactbinaryvalue.h index aa2d2821d..4ce8009b8 100644 --- a/src/zencore/include/zencore/compactbinaryvalue.h +++ b/src/zencore/include/zencore/compactbinaryvalue.h @@ -128,17 +128,21 @@ CbValue::AsString(CbFieldError* OutError, std::string_view Default) const uint32_t ValueSizeByteCount; const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - if (OutError) + if (ValueSize >= (uint64_t(1) << 31)) { - if (ValueSize >= (uint64_t(1) << 31)) + if (OutError) { *OutError = CbFieldError::RangeError; - return Default; } + return Default; + } + + if (OutError) + { *OutError = CbFieldError::None; } - return std::string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); + return std::string_view(Chars + ValueSizeByteCount, size_t(ValueSize)); } inline std::u8string_view @@ -148,17 +152,21 @@ CbValue::AsU8String(CbFieldError* OutError, std::u8string_view Default) const uint32_t ValueSizeByteCount; const uint64_t ValueSize = ReadVarUInt(Chars, ValueSizeByteCount); - if (OutError) + if (ValueSize >= (uint64_t(1) << 31)) { - if (ValueSize >= (uint64_t(1) << 31)) + if (OutError) { *OutError = CbFieldError::RangeError; - return Default; } + return Default; + } + + if (OutError) + { *OutError = CbFieldError::None; } - return std::u8string_view(Chars + ValueSizeByteCount, int32_t(ValueSize)); + return std::u8string_view(Chars + ValueSizeByteCount, size_t(ValueSize)); } inline uint64_t diff --git a/src/zencore/memtrack/callstacktrace.cpp b/src/zencore/memtrack/callstacktrace.cpp index a5b7fede6..4a7068568 100644 --- a/src/zencore/memtrack/callstacktrace.cpp +++ b/src/zencore/memtrack/callstacktrace.cpp @@ -169,13 +169,13 @@ private: std::atomic_uint64_t Key; std::atomic_uint32_t Value; - inline uint64 GetKey() const { return Key.load(std::memory_order_relaxed); } + inline uint64 GetKey() const { return Key.load(std::memory_order_acquire); } inline uint32_t GetValue() const { return Value.load(std::memory_order_relaxed); } - inline bool IsEmpty() const { return Key.load(std::memory_order_relaxed) == 0; } + inline bool IsEmpty() const { return Key.load(std::memory_order_acquire) == 0; } inline void SetKeyValue(uint64_t InKey, uint32_t InValue) { - Value.store(InValue, std::memory_order_release); - Key.store(InKey, std::memory_order_relaxed); + Value.store(InValue, std::memory_order_relaxed); + Key.store(InKey, std::memory_order_release); } static inline uint32_t KeyHash(uint64_t Key) { return static_cast(Key); } static inline void ClearEntries(FEncounteredCallstackSetEntry* Entries, int32_t EntryCount) diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index 0ee863b74..a9aed6309 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -24,6 +24,10 @@ utf16to8_impl(u16bit_iterator StartIt, u16bit_iterator EndIt, ::zen::StringBuild // Take care of surrogate pairs first if (utf8::internal::is_lead_surrogate(cp)) { + if (StartIt == EndIt) + { + break; + } uint32_t trail_surrogate = utf8::internal::mask16(*StartIt++); cp = (cp << 10) + trail_surrogate + utf8::internal::SURROGATE_OFFSET; } diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 1c0ebef90..fbc7fe401 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -89,10 +89,10 @@ IsIPv6AvailableSysctl(void) char buf[16]; if (fgets(buf, sizeof(buf), f)) { - fclose(f); // 0 means IPv6 enabled, 1 means disabled val = atoi(buf); } + fclose(f); } return val == 0; @@ -1544,7 +1544,7 @@ struct HttpAcceptor { ZEN_WARN("Unable to initialize asio service, (bind returned '{}')", BindErrorCode.message()); - return 0; + return {}; } if (EffectivePort != BasePort) diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index c640ba90b..6995ffca9 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -25,6 +25,7 @@ # include # include "iothreadpool.h" +# include # include namespace zen { @@ -129,8 +130,8 @@ private: std::unique_ptr m_IoThreadPool; - RwLock m_AsyncWorkPoolInitLock; - WorkerThreadPool* m_AsyncWorkPool = nullptr; + RwLock m_AsyncWorkPoolInitLock; + std::atomic m_AsyncWorkPool = nullptr; std::vector m_BaseUris; // eg: http://*:nnnn/ HTTP_SERVER_SESSION_ID m_HttpSessionId = 0; @@ -1032,8 +1033,10 @@ HttpSysServer::~HttpSysServer() ZEN_ERROR("~HttpSysServer() called without calling Close() first"); } - delete m_AsyncWorkPool; + auto WorkPool = m_AsyncWorkPool.load(std::memory_order_relaxed); m_AsyncWorkPool = nullptr; + + delete WorkPool; } void @@ -1323,17 +1326,17 @@ HttpSysServer::WorkPool() { ZEN_MEMSCOPE(GetHttpsysTag()); - if (!m_AsyncWorkPool) + if (!m_AsyncWorkPool.load(std::memory_order_acquire)) { RwLock::ExclusiveLockScope _(m_AsyncWorkPoolInitLock); - if (!m_AsyncWorkPool) + if (!m_AsyncWorkPool.load(std::memory_order_relaxed)) { - m_AsyncWorkPool = new WorkerThreadPool(m_InitialConfig.AsyncWorkThreadCount, "http_async"); + m_AsyncWorkPool.store(new WorkerThreadPool(m_InitialConfig.AsyncWorkThreadCount, "http_async"), std::memory_order_release); } } - return *m_AsyncWorkPool; + return *m_AsyncWorkPool.load(std::memory_order_relaxed); } void diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 2b157581f..1cf451e91 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -38,7 +38,7 @@ HttpFrontendService::HttpFrontendService(std::filesystem::path Directory, HttpSt #if ZEN_EMBED_HTML_ZIP // Load an embedded Zip archive IoBuffer HtmlZipDataBuffer(IoBuffer::Wrap, gHtmlZipData, sizeof(gHtmlZipData) - 1); - m_ZipFs = ZipFs(std::move(HtmlZipDataBuffer)); + m_ZipFs = std::make_unique(std::move(HtmlZipDataBuffer)); #endif if (m_Directory.empty() && !m_ZipFs) @@ -157,9 +157,12 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) } } - if (IoBuffer FileBuffer = m_ZipFs.GetFile(Uri)) + if (m_ZipFs) { - return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer); + if (IoBuffer FileBuffer = m_ZipFs->GetFile(Uri)) + { + return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer); + } } Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); diff --git a/src/zenserver/frontend/frontend.h b/src/zenserver/frontend/frontend.h index 84ffaac42..6d8585b72 100644 --- a/src/zenserver/frontend/frontend.h +++ b/src/zenserver/frontend/frontend.h @@ -7,6 +7,7 @@ #include "zipfs.h" #include +#include namespace zen { @@ -20,9 +21,9 @@ public: virtual void HandleStatusRequest(HttpServerRequest& Request) override; private: - ZipFs m_ZipFs; - std::filesystem::path m_Directory; - HttpStatusService& m_StatusService; + std::unique_ptr m_ZipFs; + std::filesystem::path m_Directory; + HttpStatusService& m_StatusService; }; } // namespace zen diff --git a/src/zenserver/frontend/zipfs.cpp b/src/zenserver/frontend/zipfs.cpp index f9c2bc8ff..42df0520f 100644 --- a/src/zenserver/frontend/zipfs.cpp +++ b/src/zenserver/frontend/zipfs.cpp @@ -149,13 +149,25 @@ ZipFs::ZipFs(IoBuffer&& Buffer) IoBuffer ZipFs::GetFile(const std::string_view& FileName) const { - FileMap::iterator Iter = m_Files.find(FileName); - if (Iter == m_Files.end()) { - return {}; + RwLock::SharedLockScope _(m_FilesLock); + + FileMap::const_iterator Iter = m_Files.find(FileName); + if (Iter == m_Files.end()) + { + return {}; + } + + const FileItem& Item = Iter->second; + if (Item.GetSize() > 0) + { + return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); + } } - FileItem& Item = Iter->second; + RwLock::ExclusiveLockScope _(m_FilesLock); + + FileItem& Item = m_Files.find(FileName)->second; if (Item.GetSize() > 0) { return IoBuffer(IoBuffer::Wrap, Item.GetData(), Item.GetSize()); diff --git a/src/zenserver/frontend/zipfs.h b/src/zenserver/frontend/zipfs.h index 1fa7da451..645121693 100644 --- a/src/zenserver/frontend/zipfs.h +++ b/src/zenserver/frontend/zipfs.h @@ -3,23 +3,23 @@ #pragma once #include +#include #include namespace zen { -////////////////////////////////////////////////////////////////////////// class ZipFs { public: - ZipFs() = default; - ZipFs(IoBuffer&& Buffer); + explicit ZipFs(IoBuffer&& Buffer); + IoBuffer GetFile(const std::string_view& FileName) const; - inline operator bool() const { return !m_Files.empty(); } private: using FileItem = MemoryView; using FileMap = std::unordered_map; + mutable RwLock m_FilesLock; FileMap mutable m_Files; IoBuffer m_Buffer; }; diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp index 4d9da3a57..a00446a75 100644 --- a/src/zenserver/hub/hubservice.cpp +++ b/src/zenserver/hub/hubservice.cpp @@ -151,6 +151,7 @@ struct StorageServerInstance inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); } private: + void WakeLocked(); RwLock m_Lock; std::string m_ModuleId; std::atomic m_IsProvisioned{false}; @@ -211,7 +212,7 @@ StorageServerInstance::Provision() if (m_IsHibernated) { - Wake(); + WakeLocked(); } else { @@ -294,9 +295,14 @@ StorageServerInstance::Hibernate() void StorageServerInstance::Wake() { - // Start server in-place using existing data - RwLock::ExclusiveLockScope _(m_Lock); + WakeLocked(); +} + +void +StorageServerInstance::WakeLocked() +{ + // Start server in-place using existing data if (!m_IsHibernated) { diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index f5ba30616..bf7afcc02 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -185,7 +185,7 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) { const HttpRange& Range = Ranges.front(); const uint64_t BlobSize = Blob.GetSize(); - const uint64_t MaxBlobSize = Range.Start < BlobSize ? Range.Start - BlobSize : 0; + const uint64_t MaxBlobSize = Range.Start < BlobSize ? BlobSize - Range.Start : 0; const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); if (Range.Start + RangeSize > BlobSize) { diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 04a0781d3..aa37e75fe 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -266,13 +266,12 @@ BuildStore::PutBlob(const IoHash& BlobHash, const IoBuffer& Payload) m_BlobLookup.insert({BlobHash, NewBlobIndex}); } - m_LastAccessTimeUpdateCount++; if (m_TrackedBlobKeys) { m_TrackedBlobKeys->push_back(BlobHash); if (MetadataHash != IoHash::Zero) { - m_TrackedBlobKeys->push_back(BlobHash); + m_TrackedBlobKeys->push_back(MetadataHash); } } } diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index ead7e4f3a..b73b3e6fb 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -626,7 +626,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B return false; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(BucketMetaHeader))) / sizeof(ManifestData); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(BucketMetaHeader)) / sizeof(ManifestData); if (Header.EntryCount > ExpectedEntryCount) { ZEN_WARN( @@ -1057,7 +1057,7 @@ ZenCacheDiskLayer::CacheBucket::ReadIndexFile(RwLock::ExclusiveLockScope&, const return 0; } - const uint64_t ExpectedEntryCount = (FileSize - sizeof(sizeof(cache::impl::CacheBucketIndexHeader))) / sizeof(DiskIndexEntry); + const uint64_t ExpectedEntryCount = (FileSize - sizeof(cache::impl::CacheBucketIndexHeader)) / sizeof(DiskIndexEntry); if (Header.EntryCount > ExpectedEntryCount) { return 0; diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index 94abcf547..e1fd0a3e6 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -966,7 +966,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb } else { - ResponseObject.AddBool(true); + ResponseObject.AddBool(false); } } ResponseObject.EndArray(); diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 52b494e45..4e8475293 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -608,7 +608,10 @@ ZenCacheStore::GetBatch::Commit() m_CacheStore.m_HitCount++; OpScope.SetBytes(Result.Value.GetSize()); } - m_CacheStore.m_MissCount++; + else + { + m_CacheStore.m_MissCount++; + } } } } diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index ed017988f..7402d92d3 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -300,12 +300,12 @@ GetCompactCasResults(CasContainerStrategy& Strategy, }; static void -GetFileCasResults(FileCasStrategy& Strategy, - CasStore::InsertMode Mode, - std::span Data, - std::span ChunkHashes, - std::span Indexes, - std::vector Results) +GetFileCasResults(FileCasStrategy& Strategy, + CasStore::InsertMode Mode, + std::span Data, + std::span ChunkHashes, + std::span Indexes, + std::vector& Results) { for (size_t Index : Indexes) { diff --git a/src/zentest-appstub/xmake.lua b/src/zentest-appstub/xmake.lua index db3ff2e2d..844ba82ef 100644 --- a/src/zentest-appstub/xmake.lua +++ b/src/zentest-appstub/xmake.lua @@ -6,8 +6,6 @@ target("zentest-appstub") add_headerfiles("**.h") add_files("*.cpp") add_deps("zencore") - add_packages("vcpkg::gsl-lite") -- this should ideally be propagated by the zencore dependency - add_packages("vcpkg::mimalloc") if is_os("linux") then add_syslinks("pthread") -- cgit v1.2.3 From 3cfc1b18f6b86b9830730f0055b8e3b955b77c95 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 24 Feb 2026 15:36:59 +0100 Subject: Add `zen ui` command (#779) Allows user to automate launching of zenserver dashboard, including when multiple instances are running. If multiple instances are running you can open all dashboards with `--all`, and also using the in-terminal chooser which also allows you to open a specific instance. Also includes a fix to `zen exec` when using offset/stride/limit --- CHANGELOG.md | 1 + src/zen/cmds/exec_cmd.cpp | 9 +- src/zen/cmds/ui_cmd.cpp | 236 +++++++++++++++ src/zen/cmds/ui_cmd.h | 32 ++ src/zen/progressbar.cpp | 55 +--- src/zen/progressbar.h | 1 - src/zen/zen.cpp | 6 +- src/zencore/include/zencore/process.h | 1 + src/zencore/process.cpp | 226 +++++++++++++++ src/zenutil/consoletui.cpp | 483 +++++++++++++++++++++++++++++++ src/zenutil/include/zenutil/consoletui.h | 59 ++++ 11 files changed, 1055 insertions(+), 54 deletions(-) create mode 100644 src/zen/cmds/ui_cmd.cpp create mode 100644 src/zen/cmds/ui_cmd.h create mode 100644 src/zenutil/consoletui.cpp create mode 100644 src/zenutil/include/zenutil/consoletui.h diff --git a/CHANGELOG.md b/CHANGELOG.md index af2414682..3670e451e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## +- Feature: `zen ui` can be used to open dashboards for local instances - Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly - Bugfix: Time spent indexing local and remote state during `zen builds download` now show the correct time diff --git a/src/zen/cmds/exec_cmd.cpp b/src/zen/cmds/exec_cmd.cpp index 2d9d0d12e..407f42ee3 100644 --- a/src/zen/cmds/exec_cmd.cpp +++ b/src/zen/cmds/exec_cmd.cpp @@ -360,6 +360,13 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess return false; }; + int TargetParallelism = 8; + + if (OffsetCounter || StrideCounter || m_Limit) + { + TargetParallelism = 1; + } + m_RecordingReader->IterateActions( [&](CbObject ActionObject, const IoHash& ActionId) { // Enqueue job @@ -444,7 +451,7 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess DrainCompletedJobs(); }, - 8); + TargetParallelism); // Wait until all pending work is complete diff --git a/src/zen/cmds/ui_cmd.cpp b/src/zen/cmds/ui_cmd.cpp new file mode 100644 index 000000000..da06ce305 --- /dev/null +++ b/src/zen/cmds/ui_cmd.cpp @@ -0,0 +1,236 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "ui_cmd.h" + +#include +#include +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +# include +#endif + +namespace zen { + +namespace { + + struct RunningServerInfo + { + uint16_t Port; + uint32_t Pid; + std::string SessionId; + std::string CmdLine; + }; + + static std::vector CollectRunningServers() + { + std::vector Servers; + ZenServerState State; + if (!State.InitializeReadOnly()) + return Servers; + + State.Snapshot([&](const ZenServerState::ZenServerEntry& Entry) { + StringBuilder<25> SessionSB; + Entry.GetSessionId().ToString(SessionSB); + std::error_code CmdLineEc; + std::string CmdLine = GetProcessCommandLine(static_cast(Entry.Pid.load()), CmdLineEc); + Servers.push_back({Entry.EffectiveListenPort.load(), Entry.Pid.load(), std::string(SessionSB.c_str()), std::move(CmdLine)}); + }); + + return Servers; + } + +} // namespace + +UiCommand::UiCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_options()("a,all", "Open dashboard for all running instances", cxxopts::value(m_All)->default_value("false")); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value(""), ""); + m_Options.add_option("", + "p", + "path", + "Dashboard path (default: /dashboard/)", + cxxopts::value(m_DashboardPath)->default_value("/dashboard/"), + ""); + m_Options.parse_positional("path"); +} + +UiCommand::~UiCommand() +{ +} + +void +UiCommand::OpenBrowser(std::string_view HostName) +{ + // Allow shortcuts for specifying dashboard path, and ensure it is in a format we expect + // (leading slash, trailing slash if no file extension) + + if (!m_DashboardPath.empty()) + { + if (m_DashboardPath[0] != '/') + { + m_DashboardPath = "/dashboard/" + m_DashboardPath; + } + + if (m_DashboardPath.find_last_of('.') == std::string::npos && m_DashboardPath.back() != '/') + { + m_DashboardPath += '/'; + } + } + + bool Success = false; + + ExtendableStringBuilder<256> FullUrl; + FullUrl << HostName << m_DashboardPath; + +#if ZEN_PLATFORM_WINDOWS + HINSTANCE Result = ShellExecuteA(nullptr, "open", FullUrl.c_str(), nullptr, nullptr, SW_SHOWNORMAL); + Success = reinterpret_cast(Result) > 32; +#else + // Validate URL doesn't contain shell metacharacters that could lead to command injection + std::string_view FullUrlView = FullUrl; + constexpr std::string_view DangerousChars = ";|&$`\\\"'<>(){}[]!#*?~\n\r"; + if (FullUrlView.find_first_of(DangerousChars) != std::string_view::npos) + { + throw OptionParseException(fmt::format("URL contains invalid characters: '{}'", FullUrl), m_Options.help()); + } + +# if ZEN_PLATFORM_MAC + std::string Command = fmt::format("open \"{}\"", FullUrl); +# elif ZEN_PLATFORM_LINUX + std::string Command = fmt::format("xdg-open \"{}\"", FullUrl); +# else + ZEN_NOT_IMPLEMENTED("Browser launching not implemented on this platform"); +# endif + + Success = system(Command.c_str()) == 0; +#endif + + if (!Success) + { + throw zen::runtime_error("Failed to launch browser for '{}'", FullUrl); + } + + ZEN_CONSOLE("Web browser launched for '{}' successfully", FullUrl); +} + +void +UiCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + using namespace std::literals; + + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return; + } + + // Resolve target server + uint16_t ServerPort = 0; + + if (m_HostName.empty()) + { + // Auto-discover running instances. + std::vector Servers = CollectRunningServers(); + + if (m_All) + { + if (Servers.empty()) + { + throw OptionParseException("No running Zen server instances found", m_Options.help()); + } + + for (const auto& Server : Servers) + { + OpenBrowser(fmt::format("http://localhost:{}", Server.Port)); + } + return; + } + + // If multiple are found and we have an interactive terminal, present a picker + // instead of silently using the first one. + if (Servers.size() > 1 && IsTuiAvailable()) + { + std::vector Labels; + Labels.reserve(Servers.size() + 1); + Labels.push_back(fmt::format("(all {} instances)", Servers.size())); + + const int32_t Cols = static_cast(TuiConsoleColumns()); + constexpr int32_t kIndicator = 3; // " ▶ " or " " prefix + constexpr int32_t kSeparator = 2; // " " before cmdline + constexpr int32_t kEllipsis = 3; // "..." + + for (const auto& Server : Servers) + { + std::string Label = fmt::format("port {:<5} pid {:<7} session {}", Server.Port, Server.Pid, Server.SessionId); + + if (!Server.CmdLine.empty()) + { + int32_t Available = Cols - kIndicator - kSeparator - static_cast(Label.size()); + if (Available > kEllipsis) + { + Label += " "; + if (static_cast(Server.CmdLine.size()) <= Available) + { + Label += Server.CmdLine; + } + else + { + Label.append(Server.CmdLine, 0, static_cast(Available - kEllipsis)); + Label += "..."; + } + } + } + + Labels.push_back(std::move(Label)); + } + + int SelectedIdx = TuiPickOne("Multiple Zen server instances found. Select one to open:", Labels); + if (SelectedIdx < 0) + return; // User cancelled + + if (SelectedIdx == 0) + { + // "All" selected + for (const auto& Server : Servers) + { + OpenBrowser(fmt::format("http://localhost:{}", Server.Port)); + } + return; + } + + ServerPort = Servers[SelectedIdx - 1].Port; + m_HostName = fmt::format("http://localhost:{}", ServerPort); + } + + if (m_HostName.empty()) + { + // Single or zero instances, or not an interactive terminal: + // fall back to default resolution (picks first instance or returns empty) + m_HostName = ResolveTargetHostSpec("", ServerPort); + } + } + else + { + if (m_All) + { + throw OptionParseException("--all cannot be used together with --hosturl", m_Options.help()); + } + m_HostName = ResolveTargetHostSpec(m_HostName, ServerPort); + } + + if (m_HostName.empty()) + { + throw OptionParseException("Unable to resolve server specification", m_Options.help()); + } + + OpenBrowser(m_HostName); +} + +} // namespace zen diff --git a/src/zen/cmds/ui_cmd.h b/src/zen/cmds/ui_cmd.h new file mode 100644 index 000000000..c74cdbbd0 --- /dev/null +++ b/src/zen/cmds/ui_cmd.h @@ -0,0 +1,32 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../zen.h" + +#include + +namespace zen { + +class UiCommand : public ZenCmdBase +{ +public: + UiCommand(); + ~UiCommand(); + + static constexpr char Name[] = "ui"; + static constexpr char Description[] = "Launch web browser with zen server UI"; + + virtual void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) override; + virtual cxxopts::Options& Options() override { return m_Options; } + +private: + void OpenBrowser(std::string_view HostName); + + cxxopts::Options m_Options{Name, Description}; + std::string m_HostName; + std::string m_DashboardPath = "/dashboard/"; + bool m_All = false; +}; + +} // namespace zen diff --git a/src/zen/progressbar.cpp b/src/zen/progressbar.cpp index 1ee1d1e71..732f16e81 100644 --- a/src/zen/progressbar.cpp +++ b/src/zen/progressbar.cpp @@ -8,16 +8,12 @@ #include #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC -# include -# include -#endif - ////////////////////////////////////////////////////////////////////////// namespace zen { @@ -31,35 +27,12 @@ GetConsoleHandle() } #endif -static bool -CheckStdoutTty() -{ -#if ZEN_PLATFORM_WINDOWS - HANDLE hStdOut = GetConsoleHandle(); - DWORD dwMode = 0; - static bool IsConsole = ::GetConsoleMode(hStdOut, &dwMode); - return IsConsole; -#else - return isatty(fileno(stdout)); -#endif -} - -static bool -IsStdoutTty() -{ - static bool StdoutIsTty = CheckStdoutTty(); - return StdoutIsTty; -} - static void OutputToConsoleRaw(const char* String, size_t Length) { #if ZEN_PLATFORM_WINDOWS HANDLE hStdOut = GetConsoleHandle(); -#endif - -#if ZEN_PLATFORM_WINDOWS - if (IsStdoutTty()) + if (TuiIsStdoutTty()) { WriteConsoleA(hStdOut, String, (DWORD)Length, 0, 0); } @@ -84,26 +57,6 @@ OutputToConsoleRaw(const StringBuilderBase& SB) OutputToConsoleRaw(SB.c_str(), SB.Size()); } -uint32_t -GetConsoleColumns(uint32_t Default) -{ -#if ZEN_PLATFORM_WINDOWS - HANDLE hStdOut = GetConsoleHandle(); - CONSOLE_SCREEN_BUFFER_INFO csbi; - if (GetConsoleScreenBufferInfo(hStdOut, &csbi) == TRUE) - { - return (uint32_t)(csbi.srWindow.Right - csbi.srWindow.Left + 1); - } -#else - struct winsize w; - if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &w) == 0) - { - return (uint32_t)w.ws_col; - } -#endif - return Default; -} - uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode) { @@ -165,7 +118,7 @@ ProgressBar::PopLogOperation(Mode InMode) } ProgressBar::ProgressBar(Mode InMode, std::string_view InSubTask) -: m_Mode((!IsStdoutTty() && InMode == Mode::Pretty) ? Mode::Plain : InMode) +: m_Mode((!TuiIsStdoutTty() && InMode == Mode::Pretty) ? Mode::Plain : InMode) , m_LastUpdateMS((uint64_t)-1) , m_PausedMS(0) , m_SubTask(InSubTask) @@ -257,7 +210,7 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) uint64_t ETAMS = (NewState.Status == State::EStatus::Running) && (PercentDone > 5) ? (ETAElapsedMS * NewState.RemainingCount) / Completed : 0; - uint32_t ConsoleColumns = GetConsoleColumns(1024); + uint32_t ConsoleColumns = TuiConsoleColumns(1024); const std::string PercentString = fmt::format("{:#3}%", PercentDone); diff --git a/src/zen/progressbar.h b/src/zen/progressbar.h index bbdb008d4..cb1c7023b 100644 --- a/src/zen/progressbar.h +++ b/src/zen/progressbar.h @@ -76,7 +76,6 @@ private: }; uint32_t GetUpdateDelayMS(ProgressBar::Mode InMode); -uint32_t GetConsoleColumns(uint32_t Default); OperationLogOutput* CreateConsoleLogOutput(ProgressBar::Mode InMode); diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index bdc2b4003..dc37cb56b 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -22,6 +22,7 @@ #include "cmds/status_cmd.h" #include "cmds/top_cmd.h" #include "cmds/trace_cmd.h" +#include "cmds/ui_cmd.h" #include "cmds/up_cmd.h" #include "cmds/version_cmd.h" #include "cmds/vfs_cmd.h" @@ -41,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -123,7 +125,7 @@ ZenCmdBase::ParseOptions(int argc, char** argv) bool ZenCmdBase::ParseOptions(cxxopts::Options& CmdOptions, int argc, char** argv) { - CmdOptions.set_width(GetConsoleColumns(80)); + CmdOptions.set_width(TuiConsoleColumns(80)); cxxopts::ParseResult Result; @@ -364,6 +366,7 @@ main(int argc, char** argv) LoggingCommand LoggingCmd; TopCommand TopCmd; TraceCommand TraceCmd; + UiCommand UiCmd; UpCommand UpCmd; VersionCommand VersionCmd; VfsCommand VfsCmd; @@ -425,6 +428,7 @@ main(int argc, char** argv) {StatusCommand::Name, &StatusCmd, StatusCommand::Description}, {TopCommand::Name, &TopCmd, TopCommand::Description}, {TraceCommand::Name, &TraceCmd, TraceCommand::Description}, + {UiCommand::Name, &UiCmd, UiCommand::Description}, {UpCommand::Name, &UpCmd, UpCommand::Description}, {VersionCommand::Name, &VersionCmd, VersionCommand::Description}, {VfsCommand::Name, &VfsCmd, VfsCommand::Description}, diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index e3b7a70d7..c51163a68 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -105,6 +105,7 @@ int GetCurrentProcessId(); int GetProcessId(CreateProcResult ProcId); std::filesystem::path GetProcessExecutablePath(int Pid, std::error_code& OutEc); +std::string GetProcessCommandLine(int Pid, std::error_code& OutEc); std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf = true); /** Wait for all threads in the current process to exit (except the calling thread) diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 56849a10d..4a2668912 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -1001,6 +1001,232 @@ GetProcessExecutablePath(int Pid, std::error_code& OutEc) #endif // ZEN_PLATFORM_LINUX } +std::string +GetProcessCommandLine(int Pid, std::error_code& OutEc) +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hProcess = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, static_cast(Pid)); + if (!hProcess) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + auto _ = MakeGuard([hProcess] { CloseHandle(hProcess); }); + + // NtQueryInformationProcess is an undocumented NT API; load it dynamically. + // Info class 60 = ProcessCommandLine, available since Windows 8.1. + using PFN_NtQIP = LONG(WINAPI*)(HANDLE, UINT, PVOID, ULONG, PULONG); + static const PFN_NtQIP s_NtQIP = + reinterpret_cast(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtQueryInformationProcess")); + if (!s_NtQIP) + { + return {}; + } + + constexpr UINT ProcessCommandLineClass = 60; + constexpr LONG StatusInfoLengthMismatch = static_cast(0xC0000004L); + + ULONG ReturnLength = 0; + LONG Status = s_NtQIP(hProcess, ProcessCommandLineClass, nullptr, 0, &ReturnLength); + if (Status != StatusInfoLengthMismatch || ReturnLength == 0) + { + return {}; + } + + std::vector Buf(ReturnLength); + Status = s_NtQIP(hProcess, ProcessCommandLineClass, Buf.data(), ReturnLength, &ReturnLength); + if (Status < 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + + // Output: UNICODE_STRING header immediately followed by the UTF-16 string data. + // The UNICODE_STRING.Buffer field points into our Buf. + struct LocalUnicodeString + { + USHORT Length; + USHORT MaximumLength; + WCHAR* Buffer; + }; + if (ReturnLength < sizeof(LocalUnicodeString)) + { + return {}; + } + const auto* Us = reinterpret_cast(Buf.data()); + if (Us->Length == 0 || Us->Buffer == nullptr) + { + return {}; + } + + // Skip argv[0]: may be a quoted path ("C:\...\exe.exe") or a bare path + const WCHAR* p = Us->Buffer; + const WCHAR* End = Us->Buffer + Us->Length / sizeof(WCHAR); + if (p < End && *p == L'"') + { + ++p; + while (p < End && *p != L'"') + { + ++p; + } + if (p < End) + { + ++p; // skip closing quote + } + } + else + { + while (p < End && *p != L' ') + { + ++p; + } + } + while (p < End && *p == L' ') + { + ++p; + } + if (p >= End) + { + return {}; + } + + int Utf8Size = WideCharToMultiByte(CP_UTF8, 0, p, static_cast(End - p), nullptr, 0, nullptr, nullptr); + if (Utf8Size <= 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + std::string Result(Utf8Size, '\0'); + WideCharToMultiByte(CP_UTF8, 0, p, static_cast(End - p), Result.data(), Utf8Size, nullptr, nullptr); + return Result; + +#elif ZEN_PLATFORM_LINUX + std::string CmdlinePath = fmt::format("/proc/{}/cmdline", Pid); + FILE* F = fopen(CmdlinePath.c_str(), "rb"); + if (!F) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + auto FGuard = MakeGuard([F] { fclose(F); }); + + // /proc/{pid}/cmdline contains null-separated argv entries; read it all + std::string Raw; + char Chunk[4096]; + size_t BytesRead; + while ((BytesRead = fread(Chunk, 1, sizeof(Chunk), F)) > 0) + { + Raw.append(Chunk, BytesRead); + } + if (Raw.empty()) + { + return {}; + } + + // Skip argv[0] (first null-terminated entry) + const char* p = Raw.data(); + const char* End = Raw.data() + Raw.size(); + while (p < End && *p != '\0') + { + ++p; + } + if (p < End) + { + ++p; // skip null terminator of argv[0] + } + + // Build result: remaining entries joined by spaces (inter-arg nulls → spaces) + std::string Result; + Result.reserve(static_cast(End - p)); + for (const char* q = p; q < End; ++q) + { + Result += (*q == '\0') ? ' ' : *q; + } + while (!Result.empty() && Result.back() == ' ') + { + Result.pop_back(); + } + return Result; + +#elif ZEN_PLATFORM_MAC + int Mib[3] = {CTL_KERN, KERN_PROCARGS2, Pid}; + size_t BufSize = 0; + if (sysctl(Mib, 3, nullptr, &BufSize, nullptr, 0) != 0 || BufSize == 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + + std::vector Buf(BufSize); + if (sysctl(Mib, 3, Buf.data(), &BufSize, nullptr, 0) != 0) + { + OutEc = MakeErrorCodeFromLastError(); + return {}; + } + + // Layout: [int argc][exec_path\0][null padding][argv[0]\0][argv[1]\0]...[envp\0]... + if (BufSize < sizeof(int)) + { + return {}; + } + int Argc = 0; + memcpy(&Argc, Buf.data(), sizeof(int)); + if (Argc <= 1) + { + return {}; + } + + const char* p = Buf.data() + sizeof(int); + const char* End = Buf.data() + BufSize; + + // Skip exec_path and any trailing null padding that follows it + while (p < End && *p != '\0') + { + ++p; + } + while (p < End && *p == '\0') + { + ++p; + } + + // Skip argv[0] + while (p < End && *p != '\0') + { + ++p; + } + if (p < End) + { + ++p; + } + + // Collect argv[1..Argc-1] + std::string Result; + for (int i = 1; i < Argc && p < End; ++i) + { + if (i > 1) + { + Result += ' '; + } + const char* ArgStart = p; + while (p < End && *p != '\0') + { + ++p; + } + Result.append(ArgStart, p); + if (p < End) + { + ++p; + } + } + return Result; + +#else + ZEN_UNUSED(Pid); + ZEN_UNUSED(OutEc); + return {}; +#endif +} + std::error_code FindProcess(const std::filesystem::path& ExecutableImage, ProcessHandle& OutHandle, bool IncludeSelf) { diff --git a/src/zenutil/consoletui.cpp b/src/zenutil/consoletui.cpp new file mode 100644 index 000000000..4410d463d --- /dev/null +++ b/src/zenutil/consoletui.cpp @@ -0,0 +1,483 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include + +#if ZEN_PLATFORM_WINDOWS +# include +#else +# include +# include +# include +# include +#endif + +#include + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +// Platform-specific terminal helpers + +#if ZEN_PLATFORM_WINDOWS + +static bool +CheckIsInteractiveTerminal() +{ + DWORD dwMode = 0; + return GetConsoleMode(GetStdHandle(STD_INPUT_HANDLE), &dwMode) && GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &dwMode); +} + +static void +EnableVirtualTerminal() +{ + HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE); + DWORD dwMode = 0; + if (GetConsoleMode(hStdOut, &dwMode)) + { + SetConsoleMode(hStdOut, dwMode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); + } +} + +// RAII guard: sets the console output code page for the lifetime of the object and +// restores the original on destruction. Required for UTF-8 glyphs to render correctly +// via printf/fflush since the default console code page is not UTF-8. +class ConsoleCodePageGuard +{ +public: + explicit ConsoleCodePageGuard(UINT NewCP) : m_OldCP(GetConsoleOutputCP()) { SetConsoleOutputCP(NewCP); } + ~ConsoleCodePageGuard() { SetConsoleOutputCP(m_OldCP); } + +private: + UINT m_OldCP; +}; + +enum class ConsoleKey +{ + Unknown, + ArrowUp, + ArrowDown, + Enter, + Escape, +}; + +static ConsoleKey +ReadKey() +{ + HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); + INPUT_RECORD Record{}; + DWORD dwRead = 0; + while (true) + { + if (!ReadConsoleInputA(hStdin, &Record, 1, &dwRead)) + { + return ConsoleKey::Escape; // treat read error as cancel + } + if (Record.EventType == KEY_EVENT && Record.Event.KeyEvent.bKeyDown) + { + switch (Record.Event.KeyEvent.wVirtualKeyCode) + { + case VK_UP: + return ConsoleKey::ArrowUp; + case VK_DOWN: + return ConsoleKey::ArrowDown; + case VK_RETURN: + return ConsoleKey::Enter; + case VK_ESCAPE: + return ConsoleKey::Escape; + default: + break; + } + } + } +} + +#else // POSIX + +static bool +CheckIsInteractiveTerminal() +{ + return isatty(STDIN_FILENO) && isatty(STDOUT_FILENO); +} + +static void +EnableVirtualTerminal() +{ + // ANSI escape codes are native on POSIX terminals; nothing to do +} + +// RAII guard: switches the terminal to raw/unbuffered input mode and restores +// the original attributes on destruction. +class RawModeGuard +{ +public: + RawModeGuard() + { + if (tcgetattr(STDIN_FILENO, &m_OldAttrs) != 0) + { + return; + } + + struct termios Raw = m_OldAttrs; + Raw.c_iflag &= ~static_cast(BRKINT | ICRNL | INPCK | ISTRIP | IXON); + Raw.c_cflag |= CS8; + Raw.c_lflag &= ~static_cast(ECHO | ICANON | IEXTEN | ISIG); + Raw.c_cc[VMIN] = 1; + Raw.c_cc[VTIME] = 0; + if (tcsetattr(STDIN_FILENO, TCSANOW, &Raw) == 0) + { + m_Valid = true; + } + } + + ~RawModeGuard() + { + if (m_Valid) + { + tcsetattr(STDIN_FILENO, TCSANOW, &m_OldAttrs); + } + } + + bool IsValid() const { return m_Valid; } + +private: + struct termios m_OldAttrs = {}; + bool m_Valid = false; +}; + +static int +ReadByteWithTimeout(int TimeoutMs) +{ + struct pollfd Pfd + { + STDIN_FILENO, POLLIN, 0 + }; + if (poll(&Pfd, 1, TimeoutMs) > 0 && (Pfd.revents & POLLIN)) + { + unsigned char c = 0; + if (read(STDIN_FILENO, &c, 1) == 1) + { + return static_cast(c); + } + } + return -1; +} + +// State for fullscreen live mode (alternate screen + raw input) +static struct termios s_SavedAttrs = {}; +static bool s_InLiveMode = false; + +enum class ConsoleKey +{ + Unknown, + ArrowUp, + ArrowDown, + Enter, + Escape, +}; + +static ConsoleKey +ReadKey() +{ + unsigned char c = 0; + if (read(STDIN_FILENO, &c, 1) != 1) + { + return ConsoleKey::Escape; // treat read error as cancel + } + + if (c == 27) // ESC byte or start of an escape sequence + { + int Next = ReadByteWithTimeout(50); + if (Next == '[') + { + int Final = ReadByteWithTimeout(50); + if (Final == 'A') + { + return ConsoleKey::ArrowUp; + } + if (Final == 'B') + { + return ConsoleKey::ArrowDown; + } + } + return ConsoleKey::Escape; + } + + if (c == '\r' || c == '\n') + { + return ConsoleKey::Enter; + } + + return ConsoleKey::Unknown; +} + +#endif // ZEN_PLATFORM_WINDOWS / POSIX + +////////////////////////////////////////////////////////////////////////// +// Public API + +uint32_t +TuiConsoleColumns(uint32_t Default) +{ +#if ZEN_PLATFORM_WINDOWS + CONSOLE_SCREEN_BUFFER_INFO Csbi = {}; + if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &Csbi)) + { + return static_cast(Csbi.dwSize.X); + } +#else + struct winsize Ws = {}; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &Ws) == 0 && Ws.ws_col > 0) + { + return static_cast(Ws.ws_col); + } +#endif + return Default; +} + +void +TuiEnableOutput() +{ + EnableVirtualTerminal(); +#if ZEN_PLATFORM_WINDOWS + SetConsoleOutputCP(CP_UTF8); +#endif +} + +bool +TuiIsStdoutTty() +{ +#if ZEN_PLATFORM_WINDOWS + static bool Cached = [] { + DWORD dwMode = 0; + return GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &dwMode) != 0; + }(); + return Cached; +#else + static bool Cached = isatty(STDOUT_FILENO) != 0; + return Cached; +#endif +} + +bool +IsTuiAvailable() +{ + static bool Cached = CheckIsInteractiveTerminal(); + return Cached; +} + +int +TuiPickOne(std::string_view Title, std::span Items) +{ + EnableVirtualTerminal(); + +#if ZEN_PLATFORM_WINDOWS + ConsoleCodePageGuard CodePageGuard(CP_UTF8); +#else + RawModeGuard RawMode; + if (!RawMode.IsValid()) + { + return -1; + } +#endif + + const int Count = static_cast(Items.size()); + int SelectedIndex = 0; + + printf("\n%.*s\n\n", static_cast(Title.size()), Title.data()); + + // Hide cursor during interaction + printf("\033[?25l"); + + // Renders the full entry list and hint footer. + // On subsequent calls, moves the cursor back up first to overwrite the previous output. + bool FirstRender = true; + auto RenderAll = [&] { + if (!FirstRender) + { + printf("\033[%dA", Count + 2); // move up: entries + blank line + hint line + } + FirstRender = false; + + for (int i = 0; i < Count; ++i) + { + bool IsSelected = (i == SelectedIndex); + + printf("\r\033[K"); // erase line + + if (IsSelected) + { + printf("\033[1;7m"); // bold + reverse video + } + + // \xe2\x96\xb6 = U+25B6 BLACK RIGHT-POINTING TRIANGLE (▶) + const char* Indicator = IsSelected ? " \xe2\x96\xb6 " : " "; + + printf("%s%s", Indicator, Items[i].c_str()); + + if (IsSelected) + { + printf("\033[0m"); // reset attributes + } + + printf("\n"); + } + + // Blank separator line + printf("\r\033[K\n"); + + // Hint footer + // \xe2\x86\x91 = U+2191 ↑ \xe2\x86\x93 = U+2193 ↓ + printf( + "\r\033[K \033[2m\xe2\x86\x91/\xe2\x86\x93\033[0m navigate " + "\033[2mEnter\033[0m confirm " + "\033[2mEsc\033[0m cancel\n"); + + fflush(stdout); + }; + + RenderAll(); + + int Result = -1; + bool Done = false; + while (!Done) + { + ConsoleKey Key = ReadKey(); + switch (Key) + { + case ConsoleKey::ArrowUp: + SelectedIndex = (SelectedIndex - 1 + Count) % Count; + RenderAll(); + break; + + case ConsoleKey::ArrowDown: + SelectedIndex = (SelectedIndex + 1) % Count; + RenderAll(); + break; + + case ConsoleKey::Enter: + Result = SelectedIndex; + Done = true; + break; + + case ConsoleKey::Escape: + Done = true; + break; + + default: + break; + } + } + + // Restore cursor and add a blank line for visual separation + printf("\033[?25h\n"); + fflush(stdout); + + return Result; +} + +void +TuiEnterAlternateScreen() +{ + EnableVirtualTerminal(); +#if ZEN_PLATFORM_WINDOWS + SetConsoleOutputCP(CP_UTF8); +#endif + + printf("\033[?1049h"); // Enter alternate screen buffer + printf("\033[?25l"); // Hide cursor + fflush(stdout); + +#if !ZEN_PLATFORM_WINDOWS + if (tcgetattr(STDIN_FILENO, &s_SavedAttrs) == 0) + { + struct termios Raw = s_SavedAttrs; + Raw.c_iflag &= ~static_cast(BRKINT | ICRNL | INPCK | ISTRIP | IXON); + Raw.c_cflag |= CS8; + Raw.c_lflag &= ~static_cast(ECHO | ICANON | IEXTEN | ISIG); + Raw.c_cc[VMIN] = 1; + Raw.c_cc[VTIME] = 0; + if (tcsetattr(STDIN_FILENO, TCSANOW, &Raw) == 0) + { + s_InLiveMode = true; + } + } +#endif +} + +void +TuiExitAlternateScreen() +{ + printf("\033[?25h"); // Show cursor + printf("\033[?1049l"); // Exit alternate screen buffer + fflush(stdout); + +#if !ZEN_PLATFORM_WINDOWS + if (s_InLiveMode) + { + tcsetattr(STDIN_FILENO, TCSANOW, &s_SavedAttrs); + s_InLiveMode = false; + } +#endif +} + +void +TuiCursorHome() +{ + printf("\033[H"); +} + +uint32_t +TuiConsoleRows(uint32_t Default) +{ +#if ZEN_PLATFORM_WINDOWS + CONSOLE_SCREEN_BUFFER_INFO Csbi = {}; + if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &Csbi)) + { + return static_cast(Csbi.srWindow.Bottom - Csbi.srWindow.Top + 1); + } +#else + struct winsize Ws = {}; + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &Ws) == 0 && Ws.ws_row > 0) + { + return static_cast(Ws.ws_row); + } +#endif + return Default; +} + +bool +TuiPollQuit() +{ +#if ZEN_PLATFORM_WINDOWS + HANDLE hStdin = GetStdHandle(STD_INPUT_HANDLE); + DWORD dwCount = 0; + if (!GetNumberOfConsoleInputEvents(hStdin, &dwCount) || dwCount == 0) + { + return false; + } + INPUT_RECORD Record{}; + DWORD dwRead = 0; + while (PeekConsoleInputA(hStdin, &Record, 1, &dwRead) && dwRead > 0) + { + ReadConsoleInputA(hStdin, &Record, 1, &dwRead); + if (Record.EventType == KEY_EVENT && Record.Event.KeyEvent.bKeyDown) + { + WORD vk = Record.Event.KeyEvent.wVirtualKeyCode; + char ch = Record.Event.KeyEvent.uChar.AsciiChar; + if (vk == VK_ESCAPE || ch == 'q' || ch == 'Q') + { + return true; + } + } + } + return false; +#else + // Non-blocking read: character 3 = Ctrl+C, 27 = Esc, 'q'/'Q' = quit + int b = ReadByteWithTimeout(0); + return (b == 3 || b == 27 || b == 'q' || b == 'Q'); +#endif +} + +} // namespace zen diff --git a/src/zenutil/include/zenutil/consoletui.h b/src/zenutil/include/zenutil/consoletui.h new file mode 100644 index 000000000..7dc68c126 --- /dev/null +++ b/src/zenutil/include/zenutil/consoletui.h @@ -0,0 +1,59 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +namespace zen { + +// Returns the width of the console in columns, or Default if it cannot be determined. +uint32_t TuiConsoleColumns(uint32_t Default = 120); + +// Enables ANSI/VT escape code processing and UTF-8 console output. +// Call once before printing ANSI escape sequences or multi-byte UTF-8 characters via printf. +// Safe to call multiple times. No-op on POSIX (escape codes are native there). +void TuiEnableOutput(); + +// Returns true if stdout is connected to a real terminal (not piped or redirected). +// Useful for deciding whether to use ANSI escape codes for progress output. +bool TuiIsStdoutTty(); + +// Returns true if both stdin and stdout are connected to an interactive terminal +// (i.e. not piped or redirected). Must be checked before calling TuiPickOne(). +bool IsTuiAvailable(); + +// Displays a cursor-navigable single-select list in the terminal. +// +// - Title: a short description printed once above the list +// - Items: pre-formatted display labels, one per selectable entry +// +// Arrow keys (↑/↓) navigate the selection, Enter confirms, Esc cancels. +// Returns the index of the selected item, or -1 if the user cancelled. +// +// Precondition: IsTuiAvailable() must be true. +int TuiPickOne(std::string_view Title, std::span Items); + +// Enter the alternate screen buffer for fullscreen live-update mode. +// Hides the cursor. On POSIX, switches to raw/unbuffered terminal input. +// Must be balanced by a call to TuiExitAlternateScreen(). +// Precondition: IsTuiAvailable() must be true. +void TuiEnterAlternateScreen(); + +// Exit alternate screen buffer. Restores the cursor and, on POSIX, the original +// terminal mode. Safe to call even if TuiEnterAlternateScreen() was not called. +void TuiExitAlternateScreen(); + +// Move the cursor to the top-left corner of the terminal (row 1, col 1). +void TuiCursorHome(); + +// Returns the height of the console in rows, or Default if it cannot be determined. +uint32_t TuiConsoleRows(uint32_t Default = 40); + +// Non-blocking check: returns true if the user has pressed a key that means quit +// (Esc, 'q', 'Q', or Ctrl+C). Consumes the event if one is pending. +// Should only be called while in alternate screen mode. +bool TuiPollQuit(); + +} // namespace zen -- cgit v1.2.3 From eb3079e2ec2969829cbc5b6921575d53df351f0f Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 24 Feb 2026 16:10:36 +0100 Subject: use partial blocks for oplog import (#780) Feature: Add --allow-partial-block-requests to zen oplog-import Improvement: zen oplog-import now uses partial block requests to reduce download size Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests --- CHANGELOG.md | 3 + src/zen/cmds/builds_cmd.cpp | 28 +- src/zen/cmds/projectstore_cmd.cpp | 28 +- src/zen/cmds/projectstore_cmd.h | 2 + src/zenhttp/httpclient.cpp | 38 + src/zenhttp/include/zenhttp/httpclient.h | 9 + src/zenremotestore/builds/buildstoragecache.cpp | 8 +- .../builds/buildstorageoperations.cpp | 45 +- src/zenremotestore/builds/buildstorageutil.cpp | 19 +- src/zenremotestore/chunking/chunkblock.cpp | 79 +- .../zenremotestore/builds/buildstoragecache.h | 1 + .../zenremotestore/builds/buildstorageoperations.h | 12 +- .../zenremotestore/builds/buildstorageutil.h | 4 + .../include/zenremotestore/chunking/chunkblock.h | 25 +- .../include/zenremotestore/jupiter/jupiterhost.h | 1 + .../include/zenremotestore/operationlogoutput.h | 5 +- .../zenremotestore/partialblockrequestmode.h | 20 + .../projectstore/buildsremoteprojectstore.h | 4 +- .../projectstore/remoteprojectstore.h | 67 +- src/zenremotestore/jupiter/jupiterhost.cpp | 8 +- src/zenremotestore/operationlogoutput.cpp | 2 +- src/zenremotestore/partialblockrequestmode.cpp | 27 + .../projectstore/buildsremoteprojectstore.cpp | 122 ++- .../projectstore/fileremoteprojectstore.cpp | 24 +- .../projectstore/jupiterremoteprojectstore.cpp | 19 +- .../projectstore/remoteprojectstore.cpp | 946 ++++++++++++++++----- .../projectstore/zenremoteprojectstore.cpp | 29 +- .../storage/projectstore/httpprojectstore.cpp | 33 +- 28 files changed, 1246 insertions(+), 362 deletions(-) create mode 100644 src/zenremotestore/include/zenremotestore/partialblockrequestmode.h create mode 100644 src/zenremotestore/partialblockrequestmode.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 3670e451e..e9d3b79c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ## +- Feature: Add `--allow-partial-block-requests` to `zen oplog-import` - Feature: `zen ui` can be used to open dashboards for local instances +- Improvement: `zen oplog-import` now uses partial block requests to reduce download size +- Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests - Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly - Bugfix: Time spent indexing local and remote state during `zen builds download` now show the correct time diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 849259013..5254ef3cf 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2842,13 +2842,16 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) TempPath / "storage"); Result.StorageName = ResolveRes.HostName; - StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'", - ResolveRes.HostName, - (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), - Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket); - ; + uint64_t HostLatencyNs = ResolveRes.HostLatencySec >= 0 ? uint64_t(ResolveRes.HostLatencySec * 1000000000.0) : 0; + + StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}", + ResolveRes.HostName, + (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket, + NiceLatencyNs(HostLatencyNs)); + Result.BuildStorageLatencySec = ResolveRes.HostLatencySec; if (!ResolveRes.CacheUrl.empty()) { @@ -2874,12 +2877,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) : GetTinyWorkerPool(EWorkloadType::Background)); Result.CacheName = ResolveRes.CacheName; + uint64_t CacheLatencyNs = ResolveRes.CacheLatencySec >= 0 ? uint64_t(ResolveRes.CacheLatencySec * 1000000000.0) : 0; + CacheDescription = - fmt::format("Zen {}{}. SessionId: '{}'", + fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}", ResolveRes.CacheName, (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl), - Result.CacheHttp->GetSessionId()); - ; + Result.CacheHttp->GetSessionId(), + NiceLatencyNs(CacheLatencyNs)); + + Result.CacheLatencySec = ResolveRes.CacheLatencySec; + if (!m_Namespace.empty()) { CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index 4de6ad25c..bedab3cfd 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -1469,6 +1469,20 @@ ImportOplogCommand::ImportOplogCommand() "Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive", cxxopts::value(m_BoostWorkers), ""); + m_Options.add_option( + "", + "", + "allow-partial-block-requests", + "Allow request for partial chunk blocks.\n" + " false = only full block requests allowed\n" + " mixed = multiple partial block ranges requests per block allowed to zen cache, single partial block range " + "request per block to host\n" + " zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests " + "allowed to host\n" + " true = multiple partial block ranges requests per block allowed to zen cache and host\n" + "Defaults to 'mixed'.", + cxxopts::value(m_AllowPartialBlockRequests), + ""); m_Options.parse_positional({"project", "oplog", "gcpath"}); m_Options.positional_help("[ []]"); @@ -1513,6 +1527,13 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg throw OptionParseException("'--oplog' is required", m_Options.help()); } + EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests); + if (Mode == EPartialBlockRequestMode::Invalid) + { + throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests), + m_Options.help()); + } + HttpClient Http(m_HostName); m_ProjectName = ResolveProject(Http, m_ProjectName); if (m_ProjectName.empty()) @@ -1649,6 +1670,9 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg { Writer.AddBool("boostworkermemory"sv, true); } + + Writer.AddString("partialblockrequestmode", m_AllowPartialBlockRequests); + if (!m_FileDirectoryPath.empty()) { Writer.BeginObject("file"sv); @@ -2571,6 +2595,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2; ClientSettings.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u; Storage.BuildStorageHttp = std::make_unique(ResolveRes.HostUrl, ClientSettings); + Storage.BuildStorageLatencySec = ResolveRes.HostLatencySec; BuildStorageCache::Statistics StorageCacheStats; @@ -2589,7 +2614,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a .RetryCount = 0, .MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u}, [&AbortFlag]() { return AbortFlag.load(); }); - Storage.CacheName = ResolveRes.CacheName; + Storage.CacheName = ResolveRes.CacheName; + Storage.CacheLatencySec = ResolveRes.CacheLatencySec; } if (!m_Quiet) diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index e415b41b7..17fd76e9f 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -209,6 +209,8 @@ private: bool m_BoostWorkerCount = false; bool m_BoostWorkerMemory = false; bool m_BoostWorkers = false; + + std::string m_AllowPartialBlockRequests = "mixed"; }; class SnapshotOplogCommand : public ProjectStoreCommand diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index d3b59df2b..078e27b34 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -21,6 +21,8 @@ #include "clients/httpclientcommon.h" +#include + #if ZEN_WITH_TESTS # include # include @@ -340,6 +342,42 @@ HttpClient::Authenticate() return m_Inner->Authenticate(); } +LatencyTestResult +MeasureLatency(HttpClient& Client, std::string_view Url) +{ + std::vector MeasurementTimes; + std::string ErrorMessage; + + for (uint32_t AttemptCount = 0; AttemptCount < 20 && MeasurementTimes.size() < 5; AttemptCount++) + { + HttpClient::Response MeasureResponse = Client.Get(Url); + if (MeasureResponse.IsSuccess()) + { + MeasurementTimes.push_back(MeasureResponse.ElapsedSeconds); + Sleep(5); + } + else + { + ErrorMessage = MeasureResponse.ErrorMessage(fmt::format("Unable to measure latency using {}", Url)); + } + } + + if (MeasurementTimes.empty()) + { + return {.Success = false, .FailureReason = ErrorMessage}; + } + + if (MeasurementTimes.size() > 2) + { + std::sort(MeasurementTimes.begin(), MeasurementTimes.end()); + MeasurementTimes.pop_back(); // Remove the worst time + } + + double AverageLatency = std::accumulate(MeasurementTimes.begin(), MeasurementTimes.end(), 0.0) / MeasurementTimes.size(); + + return {.Success = true, .LatencySeconds = AverageLatency}; +} + ////////////////////////////////////////////////////////////////////////// #if ZEN_WITH_TESTS diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index 9a9b74d72..7a129a98c 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -260,6 +260,15 @@ private: const HttpClientSettings m_ConnectionSettings; }; +struct LatencyTestResult +{ + bool Success = false; + std::string FailureReason; + double LatencySeconds = -1.0; +}; + +LatencyTestResult MeasureLatency(HttpClient& Client, std::string_view Url); + void httpclient_forcelink(); // internal } // namespace zen diff --git a/src/zenremotestore/builds/buildstoragecache.cpp b/src/zenremotestore/builds/buildstoragecache.cpp index 07fcd62ba..faa85f81b 100644 --- a/src/zenremotestore/builds/buildstoragecache.cpp +++ b/src/zenremotestore/builds/buildstoragecache.cpp @@ -474,7 +474,13 @@ TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const boo HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); if (TestResponse.IsSuccess()) { - return {.Success = true}; + LatencyTestResult LatencyResult = MeasureLatency(TestHttpClient, "/health"); + + if (!LatencyResult.Success) + { + return {.Success = false, .FailureReason = LatencyResult.FailureReason}; + } + return {.Success = true, .LatencySeconds = LatencyResult.LatencySeconds}; } return {.Success = false, .FailureReason = TestResponse.ErrorMessage("")}; }; diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 4f1b07c37..5219e86d8 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -484,24 +484,6 @@ private: uint64_t FilteredPerSecond = 0; }; -EPartialBlockRequestMode -PartialBlockRequestModeFromString(const std::string_view ModeString) -{ - switch (HashStringAsLowerDjb2(ModeString)) - { - case HashStringDjb2("false"): - return EPartialBlockRequestMode::Off; - case HashStringDjb2("zencacheonly"): - return EPartialBlockRequestMode::ZenCacheOnly; - case HashStringDjb2("mixed"): - return EPartialBlockRequestMode::Mixed; - case HashStringDjb2("true"): - return EPartialBlockRequestMode::All; - default: - return EPartialBlockRequestMode::Invalid; - } -} - std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath) { @@ -903,7 +885,10 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) { ChunkBlockAnalyser BlockAnalyser(m_LogOutput, m_BlockDescriptions, - ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet, .IsVerbose = m_Options.IsVerbose}); + ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet, + .IsVerbose = m_Options.IsVerbose, + .HostLatencySec = m_Storage.BuildStorageLatencySec, + .HostHighSpeedLatencySec = m_Storage.CacheLatencySec}); std::vector NeededBlocks = BlockAnalyser.GetNeeded( m_RemoteLookup.ChunkHashToChunkIndex, @@ -1034,25 +1019,29 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) { BlockPartialDownloadModes.resize(m_BlockDescriptions.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); } - else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::All) - { - BlockPartialDownloadModes.resize(m_BlockDescriptions.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::On); - } else { BlockPartialDownloadModes.reserve(m_BlockDescriptions.size()); for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++) { const bool BlockExistInCache = ExistsResult.ExistingBlobs.contains(m_BlockDescriptions[BlockIndex].BlockHash); - if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::ZenCacheOnly) + if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::All) + { + BlockPartialDownloadModes.push_back(BlockExistInCache + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange); + } + else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::ZenCacheOnly) { - BlockPartialDownloadModes.push_back(BlockExistInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::On - : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); + BlockPartialDownloadModes.push_back(BlockExistInCache + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); } else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Mixed) { - BlockPartialDownloadModes.push_back(BlockExistInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::On - : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange); + BlockPartialDownloadModes.push_back(BlockExistInCache + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange); } } } diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp index 36b45e800..b249d7d52 100644 --- a/src/zenremotestore/builds/buildstorageutil.cpp +++ b/src/zenremotestore/builds/buildstorageutil.cpp @@ -63,11 +63,13 @@ ResolveBuildStorage(OperationLogOutput& Output, std::string HostUrl; std::string HostName; + double HostLatencySec = -1.0; std::string CacheUrl; std::string CacheName; bool HostAssumeHttp2 = ClientSettings.AssumeHttp2; bool CacheAssumeHttp2 = ClientSettings.AssumeHttp2; + double CacheLatencySec = -1.0; JupiterServerDiscovery DiscoveryResponse; const std::string_view DiscoveryHost = Host.empty() ? OverrideHost : Host; @@ -98,8 +100,9 @@ ResolveBuildStorage(OperationLogOutput& Output, { ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost); } - HostUrl = OverrideHost; - HostName = GetHostNameFromUrl(OverrideHost); + HostUrl = OverrideHost; + HostName = GetHostNameFromUrl(OverrideHost); + HostLatencySec = TestResult.LatencySeconds; } else { @@ -137,6 +140,7 @@ ResolveBuildStorage(OperationLogOutput& Output, HostUrl = ServerEndpoint.BaseUrl; HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; HostName = ServerEndpoint.Name; + HostLatencySec = TestResult.LatencySeconds; break; } else @@ -183,6 +187,7 @@ ResolveBuildStorage(OperationLogOutput& Output, CacheUrl = CacheEndpoint.BaseUrl; CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; CacheName = CacheEndpoint.Name; + CacheLatencySec = TestResult.LatencySeconds; break; } } @@ -204,6 +209,7 @@ ResolveBuildStorage(OperationLogOutput& Output, CacheUrl = ZenServerLocalHostUrl; CacheAssumeHttp2 = false; CacheName = "localhost"; + CacheLatencySec = TestResult.LatencySeconds; } } }); @@ -219,8 +225,9 @@ ResolveBuildStorage(OperationLogOutput& Output, if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose); TestResult.Success) { - CacheUrl = ZenCacheHost; - CacheName = GetHostNameFromUrl(ZenCacheHost); + CacheUrl = ZenCacheHost; + CacheName = GetHostNameFromUrl(ZenCacheHost); + CacheLatencySec = TestResult.LatencySeconds; } else { @@ -231,10 +238,12 @@ ResolveBuildStorage(OperationLogOutput& Output, return BuildStorageResolveResult{.HostUrl = HostUrl, .HostName = HostName, .HostAssumeHttp2 = HostAssumeHttp2, + .HostLatencySec = HostLatencySec, .CacheUrl = CacheUrl, .CacheName = CacheName, - .CacheAssumeHttp2 = CacheAssumeHttp2}; + .CacheAssumeHttp2 = CacheAssumeHttp2, + .CacheLatencySec = CacheLatencySec}; } std::vector diff --git a/src/zenremotestore/chunking/chunkblock.cpp b/src/zenremotestore/chunking/chunkblock.cpp index 06cedae3f..d203e0292 100644 --- a/src/zenremotestore/chunking/chunkblock.cpp +++ b/src/zenremotestore/chunking/chunkblock.cpp @@ -597,7 +597,7 @@ ChunkBlockAnalyser::CalculatePartialBlockDownloads(std::span if (MaybeBlockRanges.has_value()) { - const std::vector& BlockRanges = MaybeBlockRanges.value(); + std::vector BlockRanges = MaybeBlockRanges.value(); ZEN_ASSERT(!BlockRanges.empty()); uint64_t RequestedSize = @@ -606,12 +606,54 @@ ChunkBlockAnalyser::CalculatePartialBlockDownloads(std::span uint64_t(0), [](uint64_t Current, const BlockRangeDescriptor& Range) { return Current + Range.RangeLength; }); - if ((PartialBlockDownloadMode != EPartialBlockDownloadMode::Exact) && ((RequestedSize * 100) / TotalBlockSize) >= 200) + if (PartialBlockDownloadMode != EPartialBlockDownloadMode::Exact && BlockRanges.size() > 1) + { + // TODO: Once we have support in our http client to request multiple ranges in one request this + // logic would need to change as the per-request overhead would go away + + const double LatencySec = PartialBlockDownloadMode == EPartialBlockDownloadMode::MultiRangeHighSpeed + ? m_Options.HostHighSpeedLatencySec + : m_Options.HostLatencySec; + if (LatencySec > 0) + { + const uint64_t BytesPerSec = PartialBlockDownloadMode == EPartialBlockDownloadMode::MultiRangeHighSpeed + ? m_Options.HostHighSpeedBytesPerSec + : m_Options.HostSpeedBytesPerSec; + + const double ExtraRequestTimeSec = (BlockRanges.size() - 1) * LatencySec; + const uint64_t ExtraRequestTimeBytes = uint64_t(ExtraRequestTimeSec * BytesPerSec); + + const uint64_t FullRangeSize = + BlockRanges.back().RangeStart + BlockRanges.back().RangeLength - BlockRanges.front().RangeStart; + + if (ExtraRequestTimeBytes + RequestedSize >= FullRangeSize) + { + BlockRanges = std::vector{MergeBlockRanges(BlockRanges)}; + + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Merging {} chunks ({}) from block {} ({}) to single request (extra bytes {})", + NeededBlock.ChunkIndexes.size(), + NiceBytes(RequestedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + NiceBytes(BlockRanges.front().RangeLength - RequestedSize)); + } + + RequestedSize = BlockRanges.front().RangeLength; + } + } + } + + if ((PartialBlockDownloadMode != EPartialBlockDownloadMode::Exact) && + ((TotalBlockSize - RequestedSize) < (512u * 1024u))) { if (m_Options.IsVerbose) { ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Requesting {} chunks ({}) from block {} ({}) using full block request (extra bytes {})", + "Requesting {} chunks ({}) from block {} ({}) using full block request due to small " + "total slack (extra bytes {})", NeededBlock.ChunkIndexes.size(), NiceBytes(RequestedSize), BlockDescription.BlockHash, @@ -624,19 +666,16 @@ ChunkBlockAnalyser::CalculatePartialBlockDownloads(std::span { Result.BlockRanges.insert(Result.BlockRanges.end(), BlockRanges.begin(), BlockRanges.end()); - if (RequestedSize > TotalWantedChunksSize) + if (m_Options.IsVerbose) { - if (m_Options.IsVerbose) - { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Requesting {} chunks ({}) from block {} ({}) using {} requests (extra bytes {})", - NeededBlock.ChunkIndexes.size(), - NiceBytes(RequestedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - BlockRanges.size(), - NiceBytes(RequestedSize - TotalWantedChunksSize)); - } + ZEN_OPERATION_LOG_INFO(m_LogOutput, + "Requesting {} chunks ({}) from block {} ({}) using {} requests (extra bytes {})", + NeededBlock.ChunkIndexes.size(), + NiceBytes(RequestedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + BlockRanges.size(), + NiceBytes(RequestedSize - TotalWantedChunksSize)); } } } @@ -786,7 +825,7 @@ ChunkBlockAnalyser::CollapseBlockRanges(const uint64_t AlwaysAcceptableGap, std: }; uint64_t -ChunkBlockAnalyser::CalculateNextGap(std::span BlockRanges) +ChunkBlockAnalyser::CalculateNextGap(const uint64_t AlwaysAcceptableGap, std::span BlockRanges) { ZEN_ASSERT(BlockRanges.size() > 1); uint64_t AcceptableGap = (uint64_t)-1; @@ -798,7 +837,7 @@ ChunkBlockAnalyser::CalculateNextGap(std::span Block const uint64_t Gap = NextRange.RangeStart - (Range.RangeStart + Range.RangeLength); AcceptableGap = Min(Gap, AcceptableGap); } - AcceptableGap = RoundUp(AcceptableGap, 16u * 1024u); + AcceptableGap = RoundUp(AcceptableGap, AlwaysAcceptableGap); return AcceptableGap; }; @@ -949,10 +988,12 @@ ChunkBlockAnalyser::CalculateBlockRanges(uint32_t BlockIndex, return MakeOptionalBlockRangeVector(TotalBlockSize, MergedRange); } - std::vector CollapsedBlockRanges = CollapseBlockRanges(16u * 1024u, BlockRanges); + const uint64_t AlwaysAcceptableGap = 4u * 1024u; + + std::vector CollapsedBlockRanges = CollapseBlockRanges(AlwaysAcceptableGap, BlockRanges); while (GetBlockRangeLimitForRange(ForceMergeLimits, TotalBlockSize, CollapsedBlockRanges)) { - CollapsedBlockRanges = CollapseBlockRanges(CalculateNextGap(CollapsedBlockRanges), CollapsedBlockRanges); + CollapsedBlockRanges = CollapseBlockRanges(CalculateNextGap(AlwaysAcceptableGap, CollapsedBlockRanges), CollapsedBlockRanges); } const std::uint64_t WantedCollapsedSize = diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h index bb5b1c5f4..f25ce5b5e 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h @@ -65,6 +65,7 @@ struct ZenCacheEndpointTestResult { bool Success = false; std::string FailureReason; + double LatencySeconds = -1.0; }; ZenCacheEndpointTestResult TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose); diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h index 6800444e0..31733569e 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -109,17 +110,6 @@ struct RebuildFolderStateStatistics uint64_t FinalizeTreeElapsedWallTimeUs = 0; }; -enum EPartialBlockRequestMode -{ - Off, - ZenCacheOnly, - Mixed, - All, - Invalid -}; - -EPartialBlockRequestMode PartialBlockRequestModeFromString(const std::string_view ModeString); - std::filesystem::path ZenStateFilePath(const std::filesystem::path& ZenFolderPath); std::filesystem::path ZenTempFolderPath(const std::filesystem::path& ZenFolderPath); diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h index ab3037c89..4b85d8f1e 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h @@ -17,10 +17,12 @@ struct BuildStorageResolveResult std::string HostUrl; std::string HostName; bool HostAssumeHttp2 = false; + double HostLatencySec = -1.0; std::string CacheUrl; std::string CacheName; bool CacheAssumeHttp2 = false; + double CacheLatencySec = -1.0; }; enum class ZenCacheResolveMode @@ -54,9 +56,11 @@ struct StorageInstance std::unique_ptr BuildStorageHttp; std::unique_ptr BuildStorage; std::string StorageName; + double BuildStorageLatencySec = -1.0; std::unique_ptr CacheHttp; std::unique_ptr BuildCacheStorage; std::string CacheName; + double CacheLatencySec = -1.0; }; } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h index 57710fcf5..5a17ef79c 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h @@ -82,8 +82,12 @@ class ChunkBlockAnalyser public: struct Options { - bool IsQuiet = false; - bool IsVerbose = false; + bool IsQuiet = false; + bool IsVerbose = false; + double HostLatencySec = -1.0; + double HostHighSpeedLatencySec = -1.0; + uint64_t HostSpeedBytesPerSec = (1u * 1024u * 1024u * 1024u) / 8u; // 1GBit + uint64_t HostHighSpeedBytesPerSec = (2u * 1024u * 1024u * 1024u) / 8u; // 2GBit }; ChunkBlockAnalyser(OperationLogOutput& LogOutput, std::span BlockDescriptions, const Options& Options); @@ -110,7 +114,8 @@ public: { Off, SingleRange, - On, + MultiRange, + MultiRangeHighSpeed, Exact }; @@ -130,14 +135,14 @@ private: uint16_t MaxRangeCount; }; - static constexpr uint16_t FullBlockRangePercentLimit = 95; + static constexpr uint16_t FullBlockRangePercentLimit = 98; static constexpr BlockRangeLimit ForceMergeLimits[] = {{.SizePercent = FullBlockRangePercentLimit, .MaxRangeCount = 1}, - {.SizePercent = 90, .MaxRangeCount = 2}, - {.SizePercent = 85, .MaxRangeCount = 8}, - {.SizePercent = 80, .MaxRangeCount = 16}, - {.SizePercent = 75, .MaxRangeCount = 32}, - {.SizePercent = 70, .MaxRangeCount = 48}, + {.SizePercent = 90, .MaxRangeCount = 4}, + {.SizePercent = 85, .MaxRangeCount = 16}, + {.SizePercent = 80, .MaxRangeCount = 32}, + {.SizePercent = 75, .MaxRangeCount = 48}, + {.SizePercent = 70, .MaxRangeCount = 64}, {.SizePercent = 4, .MaxRangeCount = 82}, {.SizePercent = 0, .MaxRangeCount = 96}}; @@ -149,7 +154,7 @@ private: std::span Ranges); std::vector CollapseBlockRanges(const uint64_t AlwaysAcceptableGap, std::span BlockRanges); - uint64_t CalculateNextGap(std::span BlockRanges); + uint64_t CalculateNextGap(const uint64_t AlwaysAcceptableGap, std::span BlockRanges); std::optional> CalculateBlockRanges(uint32_t BlockIndex, const ChunkBlockDescription& BlockDescription, std::span BlockChunkIndexNeeded, diff --git a/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h b/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h index 432496bc1..7bbf40dfa 100644 --- a/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h +++ b/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h @@ -28,6 +28,7 @@ struct JupiterEndpointTestResult { bool Success = false; std::string FailureReason; + double LatencySeconds = -1.0; }; JupiterEndpointTestResult TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose); diff --git a/src/zenremotestore/include/zenremotestore/operationlogoutput.h b/src/zenremotestore/include/zenremotestore/operationlogoutput.h index 9693e69cf..6f10ab156 100644 --- a/src/zenremotestore/include/zenremotestore/operationlogoutput.h +++ b/src/zenremotestore/include/zenremotestore/operationlogoutput.h @@ -3,6 +3,7 @@ #pragma once #include +#include namespace zen { @@ -57,9 +58,7 @@ public: virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) = 0; }; -struct LoggerRef; - -OperationLogOutput* CreateStandardLogOutput(LoggerRef& Log); +OperationLogOutput* CreateStandardLogOutput(LoggerRef Log); #define ZEN_OPERATION_LOG(OutputTarget, InLevel, fmtstr, ...) \ do \ diff --git a/src/zenremotestore/include/zenremotestore/partialblockrequestmode.h b/src/zenremotestore/include/zenremotestore/partialblockrequestmode.h new file mode 100644 index 000000000..54adea2b2 --- /dev/null +++ b/src/zenremotestore/include/zenremotestore/partialblockrequestmode.h @@ -0,0 +1,20 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen { + +enum EPartialBlockRequestMode +{ + Off, + ZenCacheOnly, + Mixed, + All, + Invalid +}; + +EPartialBlockRequestMode PartialBlockRequestModeFromString(const std::string_view ModeString); + +} // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h index e8b7c15c0..66dfcc62d 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h @@ -34,6 +34,8 @@ std::shared_ptr CreateJupiterBuildsRemoteStore(LoggerRef bool Quiet, bool Unattended, bool Hidden, - WorkerThreadPool& CacheBackgroundWorkerPool); + WorkerThreadPool& CacheBackgroundWorkerPool, + double& OutHostLatencySec, + double& OutCacheLatencySec); } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h index 008f94351..152c02ee2 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h @@ -6,6 +6,7 @@ #include #include +#include #include @@ -73,6 +74,16 @@ public: std::vector Blocks; }; + struct GetBlockDescriptionsResult : public Result + { + std::vector Blocks; + }; + + struct AttachmentExistsInCacheResult : public Result + { + std::vector HasBody; + }; + struct RemoteStoreInfo { bool CreateBlocks; @@ -111,10 +122,20 @@ public: virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0; virtual SaveAttachmentsResult SaveAttachments(const std::vector& Payloads) = 0; - virtual LoadContainerResult LoadContainer() = 0; - virtual GetKnownBlocksResult GetKnownBlocks() = 0; - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) = 0; - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) = 0; + virtual LoadContainerResult LoadContainer() = 0; + virtual GetKnownBlocksResult GetKnownBlocks() = 0; + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) = 0; + virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) = 0; + + struct AttachmentRange + { + uint64_t Offset = 0; + uint64_t Bytes = (uint64_t)-1; + + inline operator bool() const { return Offset != 0 || Bytes != (uint64_t)-1; } + }; + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) = 0; + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) = 0; virtual void Flush() = 0; }; @@ -153,14 +174,15 @@ RemoteProjectStore::LoadContainerResult BuildContainer( class JobContext; -RemoteProjectStore::Result SaveOplogContainer(ProjectStore::Oplog& Oplog, - const CbObject& ContainerObject, - const std::function RawHashes)>& OnReferencedAttachments, - const std::function& HasAttachment, - const std::function&& Chunks)>& OnNeedBlock, - const std::function& OnNeedAttachment, - const std::function& OnChunkedAttachment, - JobContext* OptionalContext); +RemoteProjectStore::Result SaveOplogContainer( + ProjectStore::Oplog& Oplog, + const CbObject& ContainerObject, + const std::function RawHashes)>& OnReferencedAttachments, + const std::function& HasAttachment, + const std::function&& NeededChunkIndexes)>& OnNeedBlock, + const std::function& OnNeedAttachment, + const std::function& OnChunkedAttachment, + JobContext* OptionalContext); RemoteProjectStore::Result SaveOplog(CidStore& ChunkStore, RemoteProjectStore& RemoteStore, @@ -177,15 +199,18 @@ RemoteProjectStore::Result SaveOplog(CidStore& ChunkStore, bool IgnoreMissingAttachments, JobContext* OptionalContext); -RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - ProjectStore::Oplog& Oplog, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, - bool ForceDownload, - bool IgnoreMissingAttachments, - bool CleanOplog, - JobContext* OptionalContext); +RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore, + RemoteProjectStore& RemoteStore, + ProjectStore::Oplog& Oplog, + WorkerThreadPool& NetworkWorkerPool, + WorkerThreadPool& WorkerPool, + bool ForceDownload, + bool IgnoreMissingAttachments, + bool CleanOplog, + EPartialBlockRequestMode PartialBlockRequestMode, + double HostLatencySec, + double CacheLatencySec, + JobContext* OptionalContext); std::vector GetBlockHashesFromOplog(CbObjectView ContainerObject); std::vector GetBlocksFromOplog(CbObjectView ContainerObject, std::span IncludeBlockHashes); diff --git a/src/zenremotestore/jupiter/jupiterhost.cpp b/src/zenremotestore/jupiter/jupiterhost.cpp index 7706f00c2..2583cfc84 100644 --- a/src/zenremotestore/jupiter/jupiterhost.cpp +++ b/src/zenremotestore/jupiter/jupiterhost.cpp @@ -59,7 +59,13 @@ TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); if (TestResponse.IsSuccess()) { - return {.Success = true}; + LatencyTestResult LatencyResult = MeasureLatency(TestHttpClient, "/health/ready"); + + if (!LatencyResult.Success) + { + return {.Success = false, .FailureReason = LatencyResult.FailureReason}; + } + return {.Success = true, .LatencySeconds = LatencyResult.LatencySeconds}; } return {.Success = false, .FailureReason = TestResponse.ErrorMessage("")}; } diff --git a/src/zenremotestore/operationlogoutput.cpp b/src/zenremotestore/operationlogoutput.cpp index 0837ed716..7ed93c947 100644 --- a/src/zenremotestore/operationlogoutput.cpp +++ b/src/zenremotestore/operationlogoutput.cpp @@ -95,7 +95,7 @@ StandardLogOutputProgressBar::Finish() } OperationLogOutput* -CreateStandardLogOutput(LoggerRef& Log) +CreateStandardLogOutput(LoggerRef Log) { return new StandardLogOutput(Log); } diff --git a/src/zenremotestore/partialblockrequestmode.cpp b/src/zenremotestore/partialblockrequestmode.cpp new file mode 100644 index 000000000..b3edf515b --- /dev/null +++ b/src/zenremotestore/partialblockrequestmode.cpp @@ -0,0 +1,27 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include + +namespace zen { + +EPartialBlockRequestMode +PartialBlockRequestModeFromString(const std::string_view ModeString) +{ + switch (HashStringAsLowerDjb2(ModeString)) + { + case HashStringDjb2("false"): + return EPartialBlockRequestMode::Off; + case HashStringDjb2("zencacheonly"): + return EPartialBlockRequestMode::ZenCacheOnly; + case HashStringDjb2("mixed"): + return EPartialBlockRequestMode::Mixed; + case HashStringDjb2("true"): + return EPartialBlockRequestMode::All; + default: + return EPartialBlockRequestMode::Invalid; + } +} + +} // namespace zen diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp index a8e883dde..c42373e4d 100644 --- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp @@ -441,7 +441,7 @@ public: catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); - Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", + Result.Reason = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'", m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, @@ -451,7 +451,7 @@ public: catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", + Result.Reason = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'", m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, @@ -462,7 +462,94 @@ public: return Result; } - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) override + { + std::unique_ptr Output(CreateStandardLogOutput(Log())); + + ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); + + GetBlockDescriptionsResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; }); + + try + { + Result.Blocks = zen::GetBlockDescriptions(*Output, + *m_BuildStorage, + m_BuildCacheStorage.get(), + m_BuildId, + m_OplogBuildPartId, + BlockHashes, + /*AttemptFallback*/ false, + /*IsQuiet*/ false, + /*IsVerbose)*/ false); + } + catch (const HttpClientError& Ex) + { + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); + } + catch (const std::exception& Ex) + { + Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); + } + return Result; + } + + virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) override + { + AttachmentExistsInCacheResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; }); + try + { + const std::vector CacheExistsResult = + m_BuildCacheStorage->BlobsExists(m_BuildId, RawHashes); + + if (CacheExistsResult.size() == RawHashes.size()) + { + Result.HasBody.reserve(CacheExistsResult.size()); + for (size_t BlobIndex = 0; BlobIndex < CacheExistsResult.size(); BlobIndex++) + { + Result.HasBody.push_back(CacheExistsResult[BlobIndex].HasBody); + } + } + } + catch (const HttpClientError& Ex) + { + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = fmt::format("Remote cache: Failed finding known blobs for {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); + } + catch (const std::exception& Ex) + { + Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("Remote cache: Failed finding known blobs for {}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + Ex.what()); + } + return Result; + } + + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); @@ -474,7 +561,7 @@ public: { if (m_BuildCacheStorage) { - IoBuffer CachedBlob = m_BuildCacheStorage->GetBuildBlob(m_BuildId, RawHash); + IoBuffer CachedBlob = m_BuildCacheStorage->GetBuildBlob(m_BuildId, RawHash, Range.Offset, Range.Bytes); if (CachedBlob) { Result.Bytes = std::move(CachedBlob); @@ -482,20 +569,23 @@ public: } if (!Result.Bytes) { - Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash); + Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash, Range.Offset, Range.Bytes); if (m_BuildCacheStorage && Result.Bytes && m_PopulateCache) { - m_BuildCacheStorage->PutBuildBlob(m_BuildId, - RawHash, - Result.Bytes.GetContentType(), - CompositeBuffer(SharedBuffer(Result.Bytes))); + if (!Range) + { + m_BuildCacheStorage->PutBuildBlob(m_BuildId, + RawHash, + Result.Bytes.GetContentType(), + CompositeBuffer(SharedBuffer(Result.Bytes))); + } } } } catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); - Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", + Result.Reason = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'", m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, @@ -505,7 +595,7 @@ public: catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("Failed listing know blocks for {}/{}/{}/{}. Reason: '{}'", + Result.Reason = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'", m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, @@ -558,7 +648,7 @@ public: for (const IoHash& Hash : AttachmentsLeftToFind) { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash); + LoadAttachmentResult ChunkResult = LoadAttachment(Hash, {}); if (ChunkResult.ErrorCode) { return LoadAttachmentsResult{ChunkResult}; @@ -623,7 +713,9 @@ CreateJupiterBuildsRemoteStore(LoggerRef InLog, bool Quiet, bool Unattended, bool Hidden, - WorkerThreadPool& CacheBackgroundWorkerPool) + WorkerThreadPool& CacheBackgroundWorkerPool, + double& OutHostLatencySec, + double& OutCacheLatencySec) { std::string Host = Options.Host; if (!Host.empty() && Host.find("://"sv) == std::string::npos) @@ -727,6 +819,10 @@ CreateJupiterBuildsRemoteStore(LoggerRef InLog, Options.ForceDisableBlocks, Options.ForceDisableTempBlocks, Options.PopulateCache); + + OutHostLatencySec = ResolveRes.HostLatencySec; + OutCacheLatencySec = ResolveRes.CacheLatencySec; + return RemoteStore; } diff --git a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp index 3a67d3842..ec7fb7bbc 100644 --- a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp @@ -217,7 +217,18 @@ public: return Result; } - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) override + { + ZEN_UNUSED(BlockHashes); + return GetBlockDescriptionsResult{Result{.ErrorCode = int(HttpResponseCode::NotFound)}}; + } + + virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) override + { + return AttachmentExistsInCacheResult{Result{.ErrorCode = 0}, std::vector(RawHashes.size(), false)}; + } + + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) override { Stopwatch Timer; LoadAttachmentResult Result; @@ -232,7 +243,14 @@ public: { BasicFile ChunkFile; ChunkFile.Open(ChunkPath, BasicFile::Mode::kRead); - Result.Bytes = ChunkFile.ReadAll(); + if (Range) + { + Result.Bytes = ChunkFile.ReadRange(Range.Offset, Range.Bytes); + } + else + { + Result.Bytes = ChunkFile.ReadAll(); + } } AddStats(0, Result.Bytes.GetSize(), Timer.GetElapsedTimeUs() * 1000); Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; @@ -245,7 +263,7 @@ public: LoadAttachmentsResult Result; for (const IoHash& Hash : RawHashes) { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash); + LoadAttachmentResult ChunkResult = LoadAttachment(Hash, {}); if (ChunkResult.ErrorCode) { ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; diff --git a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp index 462de2988..f8179831c 100644 --- a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp @@ -212,7 +212,18 @@ public: return Result; } - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) override + { + ZEN_UNUSED(BlockHashes); + return GetBlockDescriptionsResult{Result{.ErrorCode = int(HttpResponseCode::NotFound)}}; + } + + virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) override + { + return AttachmentExistsInCacheResult{Result{.ErrorCode = 0}, std::vector(RawHashes.size(), false)}; + } + + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) override { JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); @@ -227,6 +238,10 @@ public: RawHash, Result.Reason); } + if (!Result.ErrorCode && Range) + { + Result.Bytes = IoBuffer(Result.Bytes, Range.Offset, Range.Bytes); + } return Result; } @@ -235,7 +250,7 @@ public: LoadAttachmentsResult Result; for (const IoHash& Hash : RawHashes) { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash); + LoadAttachmentResult ChunkResult = LoadAttachment(Hash, {}); if (ChunkResult.ErrorCode) { return LoadAttachmentsResult{ChunkResult}; diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index 8be8eb0df..2a9da6f58 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -229,29 +230,60 @@ namespace remotestore_impl { struct DownloadInfo { - uint64_t OplogSizeBytes = 0; - std::atomic AttachmentsDownloaded = 0; - std::atomic AttachmentBlocksDownloaded = 0; - std::atomic AttachmentBytesDownloaded = 0; - std::atomic AttachmentBlockBytesDownloaded = 0; - std::atomic AttachmentsStored = 0; - std::atomic AttachmentBytesStored = 0; - std::atomic_size_t MissingAttachmentCount = 0; + uint64_t OplogSizeBytes = 0; + std::atomic AttachmentsDownloaded = 0; + std::atomic AttachmentBlocksDownloaded = 0; + std::atomic AttachmentBlocksRangesDownloaded = 0; + std::atomic AttachmentBytesDownloaded = 0; + std::atomic AttachmentBlockBytesDownloaded = 0; + std::atomic AttachmentBlockRangeBytesDownloaded = 0; + std::atomic AttachmentsStored = 0; + std::atomic AttachmentBytesStored = 0; + std::atomic_size_t MissingAttachmentCount = 0; }; - void DownloadAndSaveBlockChunks(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - bool IgnoreMissingAttachments, - JobContext* OptionalContext, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, - Latch& AttachmentsDownloadLatch, - Latch& AttachmentsWriteLatch, - AsyncRemoteResult& RemoteResult, - DownloadInfo& Info, - Stopwatch& LoadAttachmentsTimer, - std::atomic_uint64_t& DownloadStartMS, - const std::vector& Chunks) + class JobContextLogOutput : public OperationLogOutput + { + public: + JobContextLogOutput(JobContext* OptionalContext) : m_OptionalContext(OptionalContext) {} + virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) override + { + ZEN_UNUSED(LogLevel); + if (m_OptionalContext) + { + fmt::basic_memory_buffer MessageBuffer; + fmt::vformat_to(fmt::appender(MessageBuffer), Format, Args); + remotestore_impl::ReportMessage(m_OptionalContext, std::string_view(MessageBuffer.data(), MessageBuffer.size())); + } + } + + virtual void SetLogOperationName(std::string_view Name) override { ZEN_UNUSED(Name); } + virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override { ZEN_UNUSED(StepIndex, StepCount); } + virtual uint32_t GetProgressUpdateDelayMS() override { return 0; } + virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) override + { + ZEN_UNUSED(InSubTask); + return nullptr; + } + + private: + JobContext* m_OptionalContext; + }; + + void DownloadAndSaveBlockChunks(CidStore& ChunkStore, + RemoteProjectStore& RemoteStore, + bool IgnoreMissingAttachments, + JobContext* OptionalContext, + WorkerThreadPool& NetworkWorkerPool, + WorkerThreadPool& WorkerPool, + Latch& AttachmentsDownloadLatch, + Latch& AttachmentsWriteLatch, + AsyncRemoteResult& RemoteResult, + DownloadInfo& Info, + Stopwatch& LoadAttachmentsTimer, + std::atomic_uint64_t& DownloadStartMS, + ThinChunkBlockDescription&& ThinBlockDescription, + std::vector&& NeededChunkIndexes) { AttachmentsDownloadLatch.AddCount(1); NetworkWorkerPool.ScheduleWork( @@ -261,7 +293,8 @@ namespace remotestore_impl { &AttachmentsDownloadLatch, &AttachmentsWriteLatch, &RemoteResult, - Chunks = Chunks, + ThinBlockDescription = std::move(ThinBlockDescription), + NeededChunkIndexes = std::move(NeededChunkIndexes), &Info, &LoadAttachmentsTimer, &DownloadStartMS, @@ -276,6 +309,13 @@ namespace remotestore_impl { } try { + std::vector Chunks; + Chunks.reserve(NeededChunkIndexes.size()); + for (uint32_t ChunkIndex : NeededChunkIndexes) + { + Chunks.push_back(ThinBlockDescription.ChunkRawHashes[ChunkIndex]); + } + uint64_t Unset = (std::uint64_t)-1; DownloadStartMS.compare_exchange_strong(Unset, LoadAttachmentsTimer.GetElapsedTimeMs()); RemoteProjectStore::LoadAttachmentsResult Result = RemoteStore.LoadAttachments(Chunks); @@ -293,7 +333,12 @@ namespace remotestore_impl { } return; } - Info.AttachmentsDownloaded.fetch_add(Chunks.size()); + Info.AttachmentsDownloaded.fetch_add(Result.Chunks.size()); + for (const auto& It : Result.Chunks) + { + uint64_t ChunkSize = It.second.GetCompressedSize(); + Info.AttachmentBytesDownloaded.fetch_add(ChunkSize); + } ZEN_INFO("Loaded {} bulk attachments in {}", Chunks.size(), NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000))); @@ -320,8 +365,6 @@ namespace remotestore_impl { for (const auto& It : Chunks) { - uint64_t ChunkSize = It.second.GetCompressedSize(); - Info.AttachmentBytesDownloaded.fetch_add(ChunkSize); WriteAttachmentBuffers.push_back(It.second.GetCompressed().Flatten().AsIoBuffer()); WriteRawHashes.push_back(It.first); } @@ -350,28 +393,29 @@ namespace remotestore_impl { catch (const std::exception& Ex) { RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), - fmt::format("Failed to bulk load {} attachments", Chunks.size()), + fmt::format("Failed to bulk load {} attachments", NeededChunkIndexes.size()), Ex.what()); } }, WorkerThreadPool::EMode::EnableBacklog); }; - void DownloadAndSaveBlock(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - bool IgnoreMissingAttachments, - JobContext* OptionalContext, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, - Latch& AttachmentsDownloadLatch, - Latch& AttachmentsWriteLatch, - AsyncRemoteResult& RemoteResult, - DownloadInfo& Info, - Stopwatch& LoadAttachmentsTimer, - std::atomic_uint64_t& DownloadStartMS, - const IoHash& BlockHash, - const std::vector& Chunks, - uint32_t RetriesLeft) + void DownloadAndSaveBlock(CidStore& ChunkStore, + RemoteProjectStore& RemoteStore, + bool IgnoreMissingAttachments, + JobContext* OptionalContext, + WorkerThreadPool& NetworkWorkerPool, + WorkerThreadPool& WorkerPool, + Latch& AttachmentsDownloadLatch, + Latch& AttachmentsWriteLatch, + AsyncRemoteResult& RemoteResult, + DownloadInfo& Info, + Stopwatch& LoadAttachmentsTimer, + std::atomic_uint64_t& DownloadStartMS, + const IoHash& BlockHash, + const tsl::robin_map& AllNeededPartialChunkHashesLookup, + std::span> ChunkDownloadedFlags, + uint32_t RetriesLeft) { AttachmentsDownloadLatch.AddCount(1); NetworkWorkerPool.ScheduleWork( @@ -381,7 +425,6 @@ namespace remotestore_impl { &RemoteStore, &NetworkWorkerPool, &WorkerPool, - BlockHash, &RemoteResult, &Info, &LoadAttachmentsTimer, @@ -389,7 +432,9 @@ namespace remotestore_impl { IgnoreMissingAttachments, OptionalContext, RetriesLeft, - Chunks = std::vector(Chunks)]() { + BlockHash = IoHash(BlockHash), + &AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags]() { ZEN_TRACE_CPU("DownloadBlock"); auto _ = MakeGuard([&AttachmentsDownloadLatch] { AttachmentsDownloadLatch.CountDown(); }); @@ -401,7 +446,7 @@ namespace remotestore_impl { { uint64_t Unset = (std::uint64_t)-1; DownloadStartMS.compare_exchange_strong(Unset, LoadAttachmentsTimer.GetElapsedTimeMs()); - RemoteProjectStore::LoadAttachmentResult BlockResult = RemoteStore.LoadAttachment(BlockHash); + RemoteProjectStore::LoadAttachmentResult BlockResult = RemoteStore.LoadAttachment(BlockHash, {}); if (BlockResult.ErrorCode) { ReportMessage(OptionalContext, @@ -422,10 +467,10 @@ namespace remotestore_impl { } uint64_t BlockSize = BlockResult.Bytes.GetSize(); Info.AttachmentBlocksDownloaded.fetch_add(1); - ZEN_INFO("Loaded block attachment '{}' in {} ({})", - BlockHash, - NiceTimeSpanMs(static_cast(BlockResult.ElapsedSeconds * 1000)), - NiceBytes(BlockSize)); + ZEN_DEBUG("Loaded block attachment '{}' in {} ({})", + BlockHash, + NiceTimeSpanMs(static_cast(BlockResult.ElapsedSeconds * 1000)), + NiceBytes(BlockSize)); Info.AttachmentBlockBytesDownloaded.fetch_add(BlockSize); AttachmentsWriteLatch.AddCount(1); @@ -436,7 +481,6 @@ namespace remotestore_impl { &RemoteStore, &NetworkWorkerPool, &WorkerPool, - BlockHash, &RemoteResult, &Info, &LoadAttachmentsTimer, @@ -444,8 +488,10 @@ namespace remotestore_impl { IgnoreMissingAttachments, OptionalContext, RetriesLeft, - Chunks = std::move(Chunks), - Bytes = std::move(BlockResult.Bytes)]() { + BlockHash = IoHash(BlockHash), + &AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags, + Bytes = std::move(BlockResult.Bytes)]() { auto _ = MakeGuard([&AttachmentsWriteLatch] { AttachmentsWriteLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -454,9 +500,6 @@ namespace remotestore_impl { try { ZEN_ASSERT(Bytes.Size() > 0); - std::unordered_set WantedChunks; - WantedChunks.reserve(Chunks.size()); - WantedChunks.insert(Chunks.begin(), Chunks.end()); std::vector WriteAttachmentBuffers; std::vector WriteRawHashes; @@ -485,7 +528,8 @@ namespace remotestore_impl { LoadAttachmentsTimer, DownloadStartMS, BlockHash, - std::move(Chunks), + AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags, RetriesLeft - 1); } ReportMessage( @@ -519,7 +563,8 @@ namespace remotestore_impl { LoadAttachmentsTimer, DownloadStartMS, BlockHash, - std::move(Chunks), + AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags, RetriesLeft - 1); } ReportMessage(OptionalContext, @@ -546,28 +591,36 @@ namespace remotestore_impl { uint64_t BlockSize = BlockPayload.GetSize(); uint64_t BlockHeaderSize = 0; - bool StoreChunksOK = IterateChunkBlock( - BlockPayload.Flatten(), - [&WantedChunks, &WriteAttachmentBuffers, &WriteRawHashes, &Info, &PotentialSize]( - CompressedBuffer&& Chunk, - const IoHash& AttachmentRawHash) { - if (WantedChunks.contains(AttachmentRawHash)) - { - WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); - IoHash RawHash; - uint64_t RawSize; - ZEN_ASSERT( - CompressedBuffer::ValidateCompressedHeader(WriteAttachmentBuffers.back(), - RawHash, - RawSize, - /*OutOptionalTotalCompressedSize*/ nullptr)); - ZEN_ASSERT(RawHash == AttachmentRawHash); - WriteRawHashes.emplace_back(AttachmentRawHash); - WantedChunks.erase(AttachmentRawHash); - PotentialSize += WriteAttachmentBuffers.back().GetSize(); - } - }, - BlockHeaderSize); + + bool StoreChunksOK = IterateChunkBlock( + BlockPayload.Flatten(), + [&AllNeededPartialChunkHashesLookup, + &ChunkDownloadedFlags, + &WriteAttachmentBuffers, + &WriteRawHashes, + &Info, + &PotentialSize](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) { + auto ChunkIndexIt = AllNeededPartialChunkHashesLookup.find(AttachmentRawHash); + if (ChunkIndexIt != AllNeededPartialChunkHashesLookup.end()) + { + bool Expected = false; + if (ChunkDownloadedFlags[ChunkIndexIt->second].compare_exchange_strong(Expected, true)) + { + WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); + IoHash RawHash; + uint64_t RawSize; + ZEN_ASSERT( + CompressedBuffer::ValidateCompressedHeader(WriteAttachmentBuffers.back(), + RawHash, + RawSize, + /*OutOptionalTotalCompressedSize*/ nullptr)); + ZEN_ASSERT(RawHash == AttachmentRawHash); + WriteRawHashes.emplace_back(AttachmentRawHash); + PotentialSize += WriteAttachmentBuffers.back().GetSize(); + } + } + }, + BlockHeaderSize); if (!StoreChunksOK) { @@ -582,8 +635,6 @@ namespace remotestore_impl { return; } - ZEN_ASSERT(WantedChunks.empty()); - if (!WriteAttachmentBuffers.empty()) { auto Results = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); @@ -625,6 +676,293 @@ namespace remotestore_impl { WorkerThreadPool::EMode::EnableBacklog); }; + void DownloadAndSavePartialBlock(CidStore& ChunkStore, + RemoteProjectStore& RemoteStore, + bool IgnoreMissingAttachments, + JobContext* OptionalContext, + WorkerThreadPool& NetworkWorkerPool, + WorkerThreadPool& WorkerPool, + Latch& AttachmentsDownloadLatch, + Latch& AttachmentsWriteLatch, + AsyncRemoteResult& RemoteResult, + DownloadInfo& Info, + Stopwatch& LoadAttachmentsTimer, + std::atomic_uint64_t& DownloadStartMS, + const ChunkBlockDescription& BlockDescription, + std::span BlockRangeDescriptors, + size_t BlockRangeIndexStart, + size_t BlockRangeCount, + const tsl::robin_map& AllNeededPartialChunkHashesLookup, + std::span> ChunkDownloadedFlags, + uint32_t RetriesLeft) + { + AttachmentsDownloadLatch.AddCount(1); + NetworkWorkerPool.ScheduleWork( + [&AttachmentsDownloadLatch, + &AttachmentsWriteLatch, + &ChunkStore, + &RemoteStore, + &NetworkWorkerPool, + &WorkerPool, + &RemoteResult, + &Info, + &LoadAttachmentsTimer, + &DownloadStartMS, + IgnoreMissingAttachments, + OptionalContext, + RetriesLeft, + BlockDescription, + BlockRangeDescriptors, + BlockRangeIndexStart, + BlockRangeCount, + &AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags]() { + ZEN_TRACE_CPU("DownloadBlockRanges"); + + auto _ = MakeGuard([&AttachmentsDownloadLatch] { AttachmentsDownloadLatch.CountDown(); }); + try + { + uint64_t Unset = (std::uint64_t)-1; + DownloadStartMS.compare_exchange_strong(Unset, LoadAttachmentsTimer.GetElapsedTimeMs()); + + double DownloadElapsedSeconds = 0; + uint64_t DownloadedBytes = 0; + + for (size_t BlockRangeIndex = BlockRangeIndexStart; BlockRangeIndex < BlockRangeIndexStart + BlockRangeCount; + BlockRangeIndex++) + { + if (RemoteResult.IsError()) + { + return; + } + + const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = BlockRangeDescriptors[BlockRangeIndex]; + + RemoteProjectStore::LoadAttachmentResult BlockResult = + RemoteStore.LoadAttachment(BlockDescription.BlockHash, + {.Offset = BlockRange.RangeStart, .Bytes = BlockRange.RangeLength}); + if (BlockResult.ErrorCode) + { + ReportMessage(OptionalContext, + fmt::format("Failed to download block attachment '{}' range {},{} ({}): {}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength, + BlockResult.ErrorCode, + BlockResult.Reason)); + Info.MissingAttachmentCount.fetch_add(1); + if (!IgnoreMissingAttachments) + { + RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text); + } + return; + } + if (RemoteResult.IsError()) + { + return; + } + uint64_t BlockPartSize = BlockResult.Bytes.GetSize(); + if (BlockPartSize != BlockRange.RangeLength) + { + std::string ErrorString = + fmt::format("Failed to download block attachment '{}' range {},{}, got {} bytes ({}): {}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength, + BlockPartSize, + RemoteResult.GetError(), + RemoteResult.GetErrorReason()); + + ReportMessage(OptionalContext, ErrorString); + Info.MissingAttachmentCount.fetch_add(1); + if (!IgnoreMissingAttachments) + { + RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), + "Mismatching block part range received", + ErrorString); + } + return; + } + Info.AttachmentBlocksRangesDownloaded.fetch_add(1); + + DownloadElapsedSeconds += BlockResult.ElapsedSeconds; + DownloadedBytes += BlockPartSize; + + Info.AttachmentBlockRangeBytesDownloaded.fetch_add(BlockPartSize); + + AttachmentsWriteLatch.AddCount(1); + WorkerPool.ScheduleWork( + [&AttachmentsDownloadLatch, + &AttachmentsWriteLatch, + &ChunkStore, + &RemoteStore, + &NetworkWorkerPool, + &WorkerPool, + &RemoteResult, + &Info, + &LoadAttachmentsTimer, + &DownloadStartMS, + IgnoreMissingAttachments, + OptionalContext, + RetriesLeft, + BlockDescription, + BlockRange, + &AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags, + BlockPayload = std::move(BlockResult.Bytes)]() { + auto _ = MakeGuard([&AttachmentsWriteLatch] { AttachmentsWriteLatch.CountDown(); }); + if (RemoteResult.IsError()) + { + return; + } + try + { + ZEN_ASSERT(BlockPayload.Size() > 0); + std::vector WriteAttachmentBuffers; + std::vector WriteRawHashes; + + uint64_t PotentialSize = 0; + uint64_t UsedSize = 0; + uint64_t BlockPartSize = BlockPayload.GetSize(); + + uint32_t OffsetInBlock = 0; + for (uint32_t ChunkBlockIndex = BlockRange.ChunkBlockIndexStart; + ChunkBlockIndex < BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount; + ChunkBlockIndex++) + { + const uint32_t ChunkCompressedSize = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; + + if (auto ChunkIndexIt = AllNeededPartialChunkHashesLookup.find(ChunkHash); + ChunkIndexIt != AllNeededPartialChunkHashesLookup.end()) + { + bool Expected = false; + if (ChunkDownloadedFlags[ChunkIndexIt->second].compare_exchange_strong(Expected, true)) + { + IoHash VerifyChunkHash; + uint64_t VerifyChunkSize; + CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed( + SharedBuffer(IoBuffer(BlockPayload, OffsetInBlock, ChunkCompressedSize)), + VerifyChunkHash, + VerifyChunkSize); + if (!CompressedChunk) + { + std::string ErrorString = fmt::format( + "Chunk at {},{} in block attachment '{}' is not a valid compressed buffer", + OffsetInBlock, + ChunkCompressedSize, + BlockDescription.BlockHash); + ReportMessage(OptionalContext, ErrorString); + Info.MissingAttachmentCount.fetch_add(1); + if (!IgnoreMissingAttachments) + { + RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), + "Malformed chunk block", + ErrorString); + } + continue; + } + if (VerifyChunkHash != ChunkHash) + { + std::string ErrorString = fmt::format( + "Chunk at {},{} in block attachment '{}' has mismatching hash, expected {}, got {}", + OffsetInBlock, + ChunkCompressedSize, + BlockDescription.BlockHash, + ChunkHash, + VerifyChunkHash); + ReportMessage(OptionalContext, ErrorString); + Info.MissingAttachmentCount.fetch_add(1); + if (!IgnoreMissingAttachments) + { + RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), + "Malformed chunk block", + ErrorString); + } + continue; + } + if (VerifyChunkSize != BlockDescription.ChunkRawLengths[ChunkBlockIndex]) + { + std::string ErrorString = fmt::format( + "Chunk at {},{} in block attachment '{}' has mismatching raw size, expected {}, " + "got {}", + OffsetInBlock, + ChunkCompressedSize, + BlockDescription.BlockHash, + BlockDescription.ChunkRawLengths[ChunkBlockIndex], + VerifyChunkSize); + ReportMessage(OptionalContext, ErrorString); + Info.MissingAttachmentCount.fetch_add(1); + if (!IgnoreMissingAttachments) + { + RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), + "Malformed chunk block", + ErrorString); + } + continue; + } + + WriteAttachmentBuffers.emplace_back(CompressedChunk.GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.emplace_back(ChunkHash); + PotentialSize += WriteAttachmentBuffers.back().GetSize(); + } + } + OffsetInBlock += ChunkCompressedSize; + } + + if (!WriteAttachmentBuffers.empty()) + { + auto Results = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (size_t Index = 0; Index < Results.size(); Index++) + { + const auto& Result = Results[Index]; + if (Result.New) + { + Info.AttachmentBytesStored.fetch_add(WriteAttachmentBuffers[Index].GetSize()); + Info.AttachmentsStored.fetch_add(1); + UsedSize += WriteAttachmentBuffers[Index].GetSize(); + } + } + ZEN_DEBUG("Used {} (matching {}) out of {} for block {} range {}, {} ({} %) (use of matching {}%)", + NiceBytes(UsedSize), + NiceBytes(PotentialSize), + NiceBytes(BlockPartSize), + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength, + (100 * UsedSize) / BlockPartSize, + PotentialSize > 0 ? (UsedSize * 100) / PotentialSize : 0); + } + } + catch (const std::exception& Ex) + { + RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), + fmt::format("Failed save block attachment {} range {}, {}", + BlockDescription.BlockHash, + BlockRange.RangeStart, + BlockRange.RangeLength), + Ex.what()); + } + }, + WorkerThreadPool::EMode::EnableBacklog); + } + + ZEN_DEBUG("Loaded {} ranges from block attachment '{}' in {} ({})", + BlockRangeCount, + BlockDescription.BlockHash, + NiceTimeSpanMs(static_cast(DownloadElapsedSeconds * 1000)), + NiceBytes(DownloadedBytes)); + } + catch (const std::exception& Ex) + { + RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), + fmt::format("Failed to download block attachment {} ranges", BlockDescription.BlockHash), + Ex.what()); + } + }, + WorkerThreadPool::EMode::EnableBacklog); + }; + void DownloadAndSaveAttachment(CidStore& ChunkStore, RemoteProjectStore& RemoteStore, bool IgnoreMissingAttachments, @@ -664,7 +1002,7 @@ namespace remotestore_impl { { uint64_t Unset = (std::uint64_t)-1; DownloadStartMS.compare_exchange_strong(Unset, LoadAttachmentsTimer.GetElapsedTimeMs()); - RemoteProjectStore::LoadAttachmentResult AttachmentResult = RemoteStore.LoadAttachment(RawHash); + RemoteProjectStore::LoadAttachmentResult AttachmentResult = RemoteStore.LoadAttachment(RawHash, {}); if (AttachmentResult.ErrorCode) { ReportMessage(OptionalContext, @@ -680,10 +1018,10 @@ namespace remotestore_impl { return; } uint64_t AttachmentSize = AttachmentResult.Bytes.GetSize(); - ZEN_INFO("Loaded large attachment '{}' in {} ({})", - RawHash, - NiceTimeSpanMs(static_cast(AttachmentResult.ElapsedSeconds * 1000)), - NiceBytes(AttachmentSize)); + ZEN_DEBUG("Loaded large attachment '{}' in {} ({})", + RawHash, + NiceTimeSpanMs(static_cast(AttachmentResult.ElapsedSeconds * 1000)), + NiceBytes(AttachmentSize)); Info.AttachmentsDownloaded.fetch_add(1); if (RemoteResult.IsError()) { @@ -1224,35 +1562,7 @@ BuildContainer(CidStore& ChunkStore, { using namespace std::literals; - class JobContextLogOutput : public OperationLogOutput - { - public: - JobContextLogOutput(JobContext* OptionalContext) : m_OptionalContext(OptionalContext) {} - virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) override - { - ZEN_UNUSED(LogLevel); - if (m_OptionalContext) - { - fmt::basic_memory_buffer MessageBuffer; - fmt::vformat_to(fmt::appender(MessageBuffer), Format, Args); - remotestore_impl::ReportMessage(m_OptionalContext, std::string_view(MessageBuffer.data(), MessageBuffer.size())); - } - } - - virtual void SetLogOperationName(std::string_view Name) override { ZEN_UNUSED(Name); } - virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override { ZEN_UNUSED(StepIndex, StepCount); } - virtual uint32_t GetProgressUpdateDelayMS() override { return 0; } - virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) override - { - ZEN_UNUSED(InSubTask); - return nullptr; - } - - private: - JobContext* m_OptionalContext; - }; - - std::unique_ptr LogOutput(std::make_unique(OptionalContext)); + std::unique_ptr LogOutput(std::make_unique(OptionalContext)); size_t OpCount = 0; @@ -2768,14 +3078,15 @@ SaveOplog(CidStore& ChunkStore, }; RemoteProjectStore::Result -ParseOplogContainer(const CbObject& ContainerObject, - const std::function RawHashes)>& OnReferencedAttachments, - const std::function& HasAttachment, - const std::function&& Chunks)>& OnNeedBlock, - const std::function& OnNeedAttachment, - const std::function& OnChunkedAttachment, - CbObject& OutOplogSection, - JobContext* OptionalContext) +ParseOplogContainer( + const CbObject& ContainerObject, + const std::function RawHashes)>& OnReferencedAttachments, + const std::function& HasAttachment, + const std::function&& NeededChunkIndexes)>& OnNeedBlock, + const std::function& OnNeedAttachment, + const std::function& OnChunkedAttachment, + CbObject& OutOplogSection, + JobContext* OptionalContext) { using namespace std::literals; @@ -2801,12 +3112,12 @@ ParseOplogContainer(const CbObject& ContainerObject, "Section has unexpected data type", "Failed to save oplog container"}; } - std::unordered_set OpsAttachments; + std::unordered_set NeededAttachments; { CbArrayView OpsArray = OutOplogSection["ops"sv].AsArrayView(); for (CbFieldView OpEntry : OpsArray) { - OpEntry.IterateAttachments([&](CbFieldView FieldView) { OpsAttachments.insert(FieldView.AsAttachment()); }); + OpEntry.IterateAttachments([&](CbFieldView FieldView) { NeededAttachments.insert(FieldView.AsAttachment()); }); if (remotestore_impl::IsCancelled(OptionalContext)) { return RemoteProjectStore::Result{.ErrorCode = gsl::narrow(HttpResponseCode::OK), @@ -2816,7 +3127,7 @@ ParseOplogContainer(const CbObject& ContainerObject, } } { - std::vector ReferencedAttachments(OpsAttachments.begin(), OpsAttachments.end()); + std::vector ReferencedAttachments(NeededAttachments.begin(), NeededAttachments.end()); OnReferencedAttachments(ReferencedAttachments); } @@ -2827,24 +3138,27 @@ ParseOplogContainer(const CbObject& ContainerObject, .Reason = "Operation cancelled"}; } - remotestore_impl::ReportMessage(OptionalContext, fmt::format("Oplog references {} attachments", OpsAttachments.size())); + remotestore_impl::ReportMessage(OptionalContext, fmt::format("Oplog references {} attachments", NeededAttachments.size())); CbArrayView ChunkedFilesArray = ContainerObject["chunkedfiles"sv].AsArrayView(); for (CbFieldView ChunkedFileField : ChunkedFilesArray) { CbObjectView ChunkedFileView = ChunkedFileField.AsObjectView(); IoHash RawHash = ChunkedFileView["rawhash"sv].AsHash(); - if (OpsAttachments.contains(RawHash) && (!HasAttachment(RawHash))) + if (NeededAttachments.erase(RawHash) == 1) { - ChunkedInfo Chunked = ReadChunkedInfo(ChunkedFileView); - - OnReferencedAttachments(Chunked.ChunkHashes); - OpsAttachments.insert(Chunked.ChunkHashes.begin(), Chunked.ChunkHashes.end()); - OnChunkedAttachment(Chunked); - ZEN_INFO("Requesting chunked attachment '{}' ({}) built from {} chunks", - Chunked.RawHash, - NiceBytes(Chunked.RawSize), - Chunked.ChunkHashes.size()); + if (!HasAttachment(RawHash)) + { + ChunkedInfo Chunked = ReadChunkedInfo(ChunkedFileView); + + OnReferencedAttachments(Chunked.ChunkHashes); + NeededAttachments.insert(Chunked.ChunkHashes.begin(), Chunked.ChunkHashes.end()); + OnChunkedAttachment(Chunked); + ZEN_INFO("Requesting chunked attachment '{}' ({}) built from {} chunks", + Chunked.RawHash, + NiceBytes(Chunked.RawSize), + Chunked.ChunkHashes.size()); + } } if (remotestore_impl::IsCancelled(OptionalContext)) { @@ -2854,6 +3168,8 @@ ParseOplogContainer(const CbObject& ContainerObject, } } + std::vector ThinBlocksDescriptions; + size_t NeedBlockCount = 0; CbArrayView BlocksArray = ContainerObject["blocks"sv].AsArrayView(); for (CbFieldView BlockField : BlocksArray) @@ -2863,45 +3179,38 @@ ParseOplogContainer(const CbObject& ContainerObject, CbArrayView ChunksArray = BlockView["chunks"sv].AsArrayView(); - std::vector NeededChunks; - NeededChunks.reserve(ChunksArray.Num()); - if (BlockHash == IoHash::Zero) + std::vector ChunkHashes; + ChunkHashes.reserve(ChunksArray.Num()); + for (CbFieldView ChunkField : ChunksArray) { - for (CbFieldView ChunkField : ChunksArray) - { - IoHash ChunkHash = ChunkField.AsBinaryAttachment(); - if (OpsAttachments.erase(ChunkHash) == 1) - { - if (!HasAttachment(ChunkHash)) - { - NeededChunks.emplace_back(ChunkHash); - } - } - } + ChunkHashes.push_back(ChunkField.AsHash()); } - else + ThinBlocksDescriptions.push_back(ThinChunkBlockDescription{.BlockHash = BlockHash, .ChunkRawHashes = std::move(ChunkHashes)}); + } + + for (ThinChunkBlockDescription& ThinBlockDescription : ThinBlocksDescriptions) + { + std::vector NeededBlockChunkIndexes; + for (uint32_t ChunkIndex = 0; ChunkIndex < ThinBlockDescription.ChunkRawHashes.size(); ChunkIndex++) { - for (CbFieldView ChunkField : ChunksArray) + const IoHash& ChunkHash = ThinBlockDescription.ChunkRawHashes[ChunkIndex]; + if (NeededAttachments.erase(ChunkHash) == 1) { - const IoHash ChunkHash = ChunkField.AsHash(); - if (OpsAttachments.erase(ChunkHash) == 1) + if (!HasAttachment(ChunkHash)) { - if (!HasAttachment(ChunkHash)) - { - NeededChunks.emplace_back(ChunkHash); - } + NeededBlockChunkIndexes.push_back(ChunkIndex); } } } - - if (!NeededChunks.empty()) + if (!NeededBlockChunkIndexes.empty()) { - OnNeedBlock(BlockHash, std::move(NeededChunks)); - if (BlockHash != IoHash::Zero) + if (ThinBlockDescription.BlockHash != IoHash::Zero) { NeedBlockCount++; } + OnNeedBlock(std::move(ThinBlockDescription), std::move(NeededBlockChunkIndexes)); } + if (remotestore_impl::IsCancelled(OptionalContext)) { return RemoteProjectStore::Result{.ErrorCode = gsl::narrow(HttpResponseCode::OK), @@ -2909,6 +3218,7 @@ ParseOplogContainer(const CbObject& ContainerObject, .Reason = "Operation cancelled"}; } } + remotestore_impl::ReportMessage(OptionalContext, fmt::format("Requesting {} of {} attachment blocks", NeedBlockCount, BlocksArray.Num())); @@ -2918,7 +3228,7 @@ ParseOplogContainer(const CbObject& ContainerObject, { IoHash AttachmentHash = LargeChunksField.AsBinaryAttachment(); - if (OpsAttachments.erase(AttachmentHash) == 1) + if (NeededAttachments.erase(AttachmentHash) == 1) { if (!HasAttachment(AttachmentHash)) { @@ -2941,14 +3251,15 @@ ParseOplogContainer(const CbObject& ContainerObject, } RemoteProjectStore::Result -SaveOplogContainer(ProjectStore::Oplog& Oplog, - const CbObject& ContainerObject, - const std::function RawHashes)>& OnReferencedAttachments, - const std::function& HasAttachment, - const std::function&& Chunks)>& OnNeedBlock, - const std::function& OnNeedAttachment, - const std::function& OnChunkedAttachment, - JobContext* OptionalContext) +SaveOplogContainer( + ProjectStore::Oplog& Oplog, + const CbObject& ContainerObject, + const std::function RawHashes)>& OnReferencedAttachments, + const std::function& HasAttachment, + const std::function&& NeededChunkIndexes)>& OnNeedBlock, + const std::function& OnNeedAttachment, + const std::function& OnChunkedAttachment, + JobContext* OptionalContext) { using namespace std::literals; @@ -2972,18 +3283,23 @@ SaveOplogContainer(ProjectStore::Oplog& Oplog, } RemoteProjectStore::Result -LoadOplog(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - ProjectStore::Oplog& Oplog, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, - bool ForceDownload, - bool IgnoreMissingAttachments, - bool CleanOplog, - JobContext* OptionalContext) +LoadOplog(CidStore& ChunkStore, + RemoteProjectStore& RemoteStore, + ProjectStore::Oplog& Oplog, + WorkerThreadPool& NetworkWorkerPool, + WorkerThreadPool& WorkerPool, + bool ForceDownload, + bool IgnoreMissingAttachments, + bool CleanOplog, + EPartialBlockRequestMode PartialBlockRequestMode, + double HostLatencySec, + double CacheLatencySec, + JobContext* OptionalContext) { using namespace std::literals; + std::unique_ptr LogOutput(std::make_unique(OptionalContext)); + remotestore_impl::DownloadInfo Info; Stopwatch Timer; @@ -3035,6 +3351,14 @@ LoadOplog(CidStore& ChunkStore, return false; }; + struct NeededBlockDownload + { + ThinChunkBlockDescription ThinBlockDescription; + std::vector NeededChunkIndexes; + }; + + std::vector NeededBlockDownloads; + auto OnNeedBlock = [&RemoteStore, &ChunkStore, &NetworkWorkerPool, @@ -3047,8 +3371,9 @@ LoadOplog(CidStore& ChunkStore, &Info, &LoadAttachmentsTimer, &DownloadStartMS, + &NeededBlockDownloads, IgnoreMissingAttachments, - OptionalContext](const IoHash& BlockHash, std::vector&& Chunks) { + OptionalContext](ThinChunkBlockDescription&& ThinBlockDescription, std::vector&& NeededChunkIndexes) { if (RemoteResult.IsError()) { return; @@ -3056,7 +3381,7 @@ LoadOplog(CidStore& ChunkStore, BlockCountToDownload++; AttachmentCount.fetch_add(1); - if (BlockHash == IoHash::Zero) + if (ThinBlockDescription.BlockHash == IoHash::Zero) { DownloadAndSaveBlockChunks(ChunkStore, RemoteStore, @@ -3070,25 +3395,13 @@ LoadOplog(CidStore& ChunkStore, Info, LoadAttachmentsTimer, DownloadStartMS, - Chunks); + std::move(ThinBlockDescription), + std::move(NeededChunkIndexes)); } else { - DownloadAndSaveBlock(ChunkStore, - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, - NetworkWorkerPool, - WorkerPool, - AttachmentsDownloadLatch, - AttachmentsWriteLatch, - RemoteResult, - Info, - LoadAttachmentsTimer, - DownloadStartMS, - BlockHash, - Chunks, - 3); + NeededBlockDownloads.push_back(NeededBlockDownload{.ThinBlockDescription = std::move(ThinBlockDescription), + .NeededChunkIndexes = std::move(NeededChunkIndexes)}); } }; @@ -3132,12 +3445,7 @@ LoadOplog(CidStore& ChunkStore, }; std::vector FilesToDechunk; - auto OnChunkedAttachment = [&Oplog, &ChunkStore, &FilesToDechunk, ForceDownload](const ChunkedInfo& Chunked) { - if (ForceDownload || !ChunkStore.ContainsChunk(Chunked.RawHash)) - { - FilesToDechunk.push_back(Chunked); - } - }; + auto OnChunkedAttachment = [&FilesToDechunk](const ChunkedInfo& Chunked) { FilesToDechunk.push_back(Chunked); }; auto OnReferencedAttachments = [&Oplog](std::span RawHashes) { Oplog.CaptureAddedAttachments(RawHashes); }; @@ -3165,6 +3473,185 @@ LoadOplog(CidStore& ChunkStore, BlockCountToDownload, FilesToDechunk.size())); + std::vector BlockHashes; + std::vector AllNeededChunkHashes; + BlockHashes.reserve(NeededBlockDownloads.size()); + for (const NeededBlockDownload& BlockDownload : NeededBlockDownloads) + { + BlockHashes.push_back(BlockDownload.ThinBlockDescription.BlockHash); + for (uint32_t ChunkIndex : BlockDownload.NeededChunkIndexes) + { + AllNeededChunkHashes.push_back(BlockDownload.ThinBlockDescription.ChunkRawHashes[ChunkIndex]); + } + } + + tsl::robin_map AllNeededPartialChunkHashesLookup = BuildHashLookup(AllNeededChunkHashes); + std::vector> ChunkDownloadedFlags(AllNeededChunkHashes.size()); + std::vector DownloadedViaLegacyChunkFlag(AllNeededChunkHashes.size(), false); + ChunkBlockAnalyser::BlockResult PartialBlocksResult; + + RemoteProjectStore::GetBlockDescriptionsResult BlockDescriptions = RemoteStore.GetBlockDescriptions(BlockHashes); + std::vector BlocksWithDescription; + BlocksWithDescription.reserve(BlockDescriptions.Blocks.size()); + for (const ChunkBlockDescription& BlockDescription : BlockDescriptions.Blocks) + { + BlocksWithDescription.push_back(BlockDescription.BlockHash); + } + { + auto WantIt = NeededBlockDownloads.begin(); + auto FindIt = BlockDescriptions.Blocks.begin(); + while (WantIt != NeededBlockDownloads.end()) + { + if (FindIt == BlockDescriptions.Blocks.end()) + { + // Fall back to full download as we can't get enough information about the block + DownloadAndSaveBlock(ChunkStore, + RemoteStore, + IgnoreMissingAttachments, + OptionalContext, + NetworkWorkerPool, + WorkerPool, + AttachmentsDownloadLatch, + AttachmentsWriteLatch, + RemoteResult, + Info, + LoadAttachmentsTimer, + DownloadStartMS, + WantIt->ThinBlockDescription.BlockHash, + AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags, + 3); + for (uint32_t BlockChunkIndex : WantIt->NeededChunkIndexes) + { + const IoHash& ChunkHash = WantIt->ThinBlockDescription.ChunkRawHashes[BlockChunkIndex]; + auto It = AllNeededPartialChunkHashesLookup.find(ChunkHash); + ZEN_ASSERT(It != AllNeededPartialChunkHashesLookup.end()); + uint32_t ChunkIndex = It->second; + DownloadedViaLegacyChunkFlag[ChunkIndex] = true; + } + WantIt++; + } + else if (WantIt->ThinBlockDescription.BlockHash == FindIt->BlockHash) + { + // Found + FindIt++; + WantIt++; + } + else + { + // Not a requested block? + ZEN_ASSERT(false); + } + } + } + if (!AllNeededChunkHashes.empty()) + { + std::vector PartialBlockDownloadModes; + + if (PartialBlockRequestMode == EPartialBlockRequestMode::Off) + { + PartialBlockDownloadModes.resize(BlocksWithDescription.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); + } + else + { + RemoteProjectStore::AttachmentExistsInCacheResult CacheExistsResult = + RemoteStore.AttachmentExistsInCache(BlocksWithDescription); + if (CacheExistsResult.ErrorCode != 0 || CacheExistsResult.HasBody.size() != BlocksWithDescription.size()) + { + CacheExistsResult.HasBody.resize(BlocksWithDescription.size(), false); + } + + PartialBlockDownloadModes.reserve(BlocksWithDescription.size()); + + for (bool ExistsInCache : CacheExistsResult.HasBody) + { + if (PartialBlockRequestMode == EPartialBlockRequestMode::All) + { + PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange); + } + else if (PartialBlockRequestMode == EPartialBlockRequestMode::ZenCacheOnly) + { + PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); + } + else if (PartialBlockRequestMode == EPartialBlockRequestMode::Mixed) + { + PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange); + } + } + } + + ZEN_ASSERT(PartialBlockDownloadModes.size() == BlocksWithDescription.size()); + + ChunkBlockAnalyser PartialAnalyser(*LogOutput, + BlockDescriptions.Blocks, + ChunkBlockAnalyser::Options{.IsQuiet = false, + .IsVerbose = false, + .HostLatencySec = HostLatencySec, + .HostHighSpeedLatencySec = CacheLatencySec}); + + std::vector NeededBlocks = + PartialAnalyser.GetNeeded(AllNeededPartialChunkHashesLookup, + [&](uint32_t ChunkIndex) { return !DownloadedViaLegacyChunkFlag[ChunkIndex]; }); + + PartialBlocksResult = PartialAnalyser.CalculatePartialBlockDownloads(NeededBlocks, PartialBlockDownloadModes); + for (uint32_t FullBlockIndex : PartialBlocksResult.FullBlockIndexes) + { + DownloadAndSaveBlock(ChunkStore, + RemoteStore, + IgnoreMissingAttachments, + OptionalContext, + NetworkWorkerPool, + WorkerPool, + AttachmentsDownloadLatch, + AttachmentsWriteLatch, + RemoteResult, + Info, + LoadAttachmentsTimer, + DownloadStartMS, + BlockDescriptions.Blocks[FullBlockIndex].BlockHash, + AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags, + 3); + } + + for (size_t BlockRangeIndex = 0; BlockRangeIndex < PartialBlocksResult.BlockRanges.size();) + { + size_t RangeCount = 1; + size_t RangesLeft = PartialBlocksResult.BlockRanges.size() - BlockRangeIndex; + const ChunkBlockAnalyser::BlockRangeDescriptor& CurrentBlockRange = PartialBlocksResult.BlockRanges[BlockRangeIndex]; + while (RangeCount < RangesLeft && + CurrentBlockRange.BlockIndex == PartialBlocksResult.BlockRanges[BlockRangeIndex + RangeCount].BlockIndex) + { + RangeCount++; + } + + DownloadAndSavePartialBlock(ChunkStore, + RemoteStore, + IgnoreMissingAttachments, + OptionalContext, + NetworkWorkerPool, + WorkerPool, + AttachmentsDownloadLatch, + AttachmentsWriteLatch, + RemoteResult, + Info, + LoadAttachmentsTimer, + DownloadStartMS, + BlockDescriptions.Blocks[CurrentBlockRange.BlockIndex], + PartialBlocksResult.BlockRanges, + BlockRangeIndex, + RangeCount, + AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags, + 3); + + BlockRangeIndex += RangeCount; + } + } + AttachmentsDownloadLatch.CountDown(); while (!AttachmentsDownloadLatch.Wait(1000)) { @@ -3478,21 +3965,30 @@ LoadOplog(CidStore& ChunkStore, } } - remotestore_impl::ReportMessage( - OptionalContext, - fmt::format("Loaded oplog '{}' {} in {} ({}), Blocks: {} ({}), Attachments: {} ({}), Stored: {} ({}), Missing: {} {}", - RemoteStoreInfo.ContainerName, - Result.ErrorCode == 0 ? "SUCCESS" : "FAILURE", - NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000.0)), - NiceBytes(Info.OplogSizeBytes), - Info.AttachmentBlocksDownloaded.load(), - NiceBytes(Info.AttachmentBlockBytesDownloaded.load()), - Info.AttachmentsDownloaded.load(), - NiceBytes(Info.AttachmentBytesDownloaded.load()), - Info.AttachmentsStored.load(), - NiceBytes(Info.AttachmentBytesStored.load()), - Info.MissingAttachmentCount.load(), - remotestore_impl::GetStats(RemoteStore.GetStats(), TransferWallTimeMS))); + uint64_t TotalDownloads = + 1 + Info.AttachmentBlocksDownloaded.load() + Info.AttachmentBlocksRangesDownloaded.load() + Info.AttachmentsDownloaded.load(); + uint64_t TotalBytesDownloaded = Info.OplogSizeBytes + Info.AttachmentBlockBytesDownloaded.load() + + Info.AttachmentBlockRangeBytesDownloaded.load() + Info.AttachmentBytesDownloaded.load(); + + remotestore_impl::ReportMessage(OptionalContext, + fmt::format("Loaded oplog '{}' {} in {} ({}), Blocks: {} ({}), BlockRanges: {} ({}), Attachments: {} " + "({}), Total: {} ({}), Stored: {} ({}), Missing: {} {}", + RemoteStoreInfo.ContainerName, + Result.ErrorCode == 0 ? "SUCCESS" : "FAILURE", + NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000.0)), + NiceBytes(Info.OplogSizeBytes), + Info.AttachmentBlocksDownloaded.load(), + NiceBytes(Info.AttachmentBlockBytesDownloaded.load()), + Info.AttachmentBlocksRangesDownloaded.load(), + NiceBytes(Info.AttachmentBlockRangeBytesDownloaded.load()), + Info.AttachmentsDownloaded.load(), + NiceBytes(Info.AttachmentBytesDownloaded.load()), + TotalDownloads, + NiceBytes(TotalBytesDownloaded), + Info.AttachmentsStored.load(), + NiceBytes(Info.AttachmentBytesStored.load()), + Info.MissingAttachmentCount.load(), + remotestore_impl::GetStats(RemoteStore.GetStats(), TransferWallTimeMS))); return Result; } @@ -3697,6 +4193,9 @@ TEST_CASE_TEMPLATE("project.store.export", /*Force*/ false, /*IgnoreMissingAttachments*/ false, /*CleanOplog*/ false, + EPartialBlockRequestMode::Mixed, + /*HostLatencySec*/ -1.0, + /*CacheLatencySec*/ -1.0, nullptr); CHECK(ImportResult.ErrorCode == 0); @@ -3708,6 +4207,9 @@ TEST_CASE_TEMPLATE("project.store.export", /*Force*/ true, /*IgnoreMissingAttachments*/ false, /*CleanOplog*/ false, + EPartialBlockRequestMode::Mixed, + /*HostLatencySec*/ -1.0, + /*CacheLatencySec*/ -1.0, nullptr); CHECK(ImportForceResult.ErrorCode == 0); @@ -3719,6 +4221,9 @@ TEST_CASE_TEMPLATE("project.store.export", /*Force*/ false, /*IgnoreMissingAttachments*/ false, /*CleanOplog*/ true, + EPartialBlockRequestMode::Mixed, + /*HostLatencySec*/ -1.0, + /*CacheLatencySec*/ -1.0, nullptr); CHECK(ImportCleanResult.ErrorCode == 0); @@ -3730,6 +4235,9 @@ TEST_CASE_TEMPLATE("project.store.export", /*Force*/ true, /*IgnoreMissingAttachments*/ false, /*CleanOplog*/ true, + EPartialBlockRequestMode::Mixed, + /*HostLatencySec*/ -1.0, + /*CacheLatencySec*/ -1.0, nullptr); CHECK(ImportForceCleanResult.ErrorCode == 0); } diff --git a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp index ab82edbef..b4c1156ac 100644 --- a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp @@ -249,7 +249,18 @@ public: return GetKnownBlocksResult{{.ErrorCode = static_cast(HttpResponseCode::NoContent)}}; } - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) override + { + ZEN_UNUSED(BlockHashes); + return GetBlockDescriptionsResult{Result{.ErrorCode = int(HttpResponseCode::NotFound)}}; + } + + virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) override + { + return AttachmentExistsInCacheResult{Result{.ErrorCode = 0}, std::vector(RawHashes.size(), false)}; + } + + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) override { std::string LoadRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash); HttpClient::Response Response = @@ -257,12 +268,7 @@ public: AddStats(Response); LoadAttachmentResult Result = LoadAttachmentResult{ConvertResult(Response)}; - if (!Result.ErrorCode) - { - Result.Bytes = Response.ResponsePayload; - Result.Bytes.MakeOwned(); - } - if (!Result.ErrorCode) + if (Result.ErrorCode) { Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'", m_ProjectStoreUrl, @@ -271,6 +277,15 @@ public: RawHash, Result.Reason); } + if (!Result.ErrorCode && Range) + { + Result.Bytes = IoBuffer(Response.ResponsePayload, Range.Offset, Range.Bytes); + } + else + { + Result.Bytes = Response.ResponsePayload; + } + Result.Bytes.MakeOwned(); return Result; } diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp index 416e2ed69..2b5474d00 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.cpp +++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp @@ -244,6 +244,8 @@ namespace { { std::shared_ptr Store; std::string Description; + double HostLatencySec = -1.0; + double CacheLatencySec = -1.0; }; CreateRemoteStoreResult CreateRemoteStore(LoggerRef InLog, @@ -261,6 +263,8 @@ namespace { using namespace std::literals; std::shared_ptr RemoteStore; + double HostLatencySec = -1.0; + double CacheLatencySec = -1.0; if (CbObjectView File = Params["file"sv].AsObjectView(); File) { @@ -495,7 +499,9 @@ namespace { /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true, - GetTinyWorkerPool(EWorkloadType::Background)); + GetTinyWorkerPool(EWorkloadType::Background), + HostLatencySec, + CacheLatencySec); } if (!RemoteStore) @@ -503,7 +509,10 @@ namespace { return {nullptr, "Unknown remote store type"}; } - return {std::move(RemoteStore), ""}; + return CreateRemoteStoreResult{.Store = std::move(RemoteStore), + .Description = "", + .HostLatencySec = HostLatencySec, + .CacheLatencySec = CacheLatencySec}; } std::pair ConvertResult(const RemoteProjectStore::Result& Result) @@ -2356,15 +2365,19 @@ HttpProjectService::HandleOplogSaveRequest(HttpRouterRequest& Req) tsl::robin_set Attachments; auto HasAttachment = [this](const IoHash& RawHash) { return m_CidStore.ContainsChunk(RawHash); }; - auto OnNeedBlock = [&AttachmentsLock, &Attachments](const IoHash& BlockHash, const std::vector&& ChunkHashes) { + auto OnNeedBlock = [&AttachmentsLock, &Attachments](ThinChunkBlockDescription&& ThinBlockDescription, + std::vector&& NeededChunkIndexes) { RwLock::ExclusiveLockScope _(AttachmentsLock); - if (BlockHash != IoHash::Zero) + if (ThinBlockDescription.BlockHash != IoHash::Zero) { - Attachments.insert(BlockHash); + Attachments.insert(ThinBlockDescription.BlockHash); } else { - Attachments.insert(ChunkHashes.begin(), ChunkHashes.end()); + for (uint32_t ChunkIndex : NeededChunkIndexes) + { + Attachments.insert(ThinBlockDescription.ChunkRawHashes[ChunkIndex]); + } } }; auto OnNeedAttachment = [&AttachmentsLock, &Attachments](const IoHash& RawHash) { @@ -2663,6 +2676,8 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) bool CleanOplog = Params["clean"].AsBool(false); bool BoostWorkerCount = Params["boostworkercount"].AsBool(false); bool BoostWorkerMemory = Params["boostworkermemory"sv].AsBool(false); + EPartialBlockRequestMode PartialBlockRequestMode = + PartialBlockRequestModeFromString(Params["partialblockrequestmode"sv].AsString("true")); CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Log(), Params, @@ -2688,6 +2703,9 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) Force, IgnoreMissingAttachments, CleanOplog, + PartialBlockRequestMode, + HostLatencySec = RemoteStoreResult.HostLatencySec, + CacheLatencySec = RemoteStoreResult.CacheLatencySec, BoostWorkerCount](JobContext& Context) { Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}", Oplog->GetOuterProjectIdentifier(), @@ -2709,6 +2727,9 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) Force, IgnoreMissingAttachments, CleanOplog, + PartialBlockRequestMode, + HostLatencySec, + CacheLatencySec, &Context); auto Response = ConvertResult(Result); ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); -- cgit v1.2.3 From b794809ece9965703b47f06b2d934f126c358ebf Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 24 Feb 2026 17:57:23 +0100 Subject: updated CLAUDE.md --- CLAUDE.md | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 757ce5d2a..c859d7ca2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,17 +14,17 @@ This project uses **xmake** as its build system. xmake is stateful - run configu ```bash # Configure for debug (includes tests) -xmake config -m debug +xmake config -y -m debug -a x64 # Configure for release -xmake config -m release +xmake config -y -m release -a x64 # Build everything -xmake +xmake -y # Build specific target -xmake build zenserver -xmake build zen +xmake build -y zenserver +xmake build -y zen # Run targets xmake run zenserver @@ -154,6 +154,8 @@ The codebase is organized into layered modules with clear dependencies: - Can use mimalloc or rpmalloc for performance - UE-style LLM (Low-Level Memory) tracking available on Windows - Memory tracing support integrated with UE trace system +- Avoid using `std::string` when building (concatenating etc) temporary strings. Use `ExtendableStringBuilder` + instead with an appropriate size N to avoid heap allocations in the common case. **Tracing:** - UE Trace integration for profiling and diagnostics @@ -171,18 +173,23 @@ The codebase is organized into layered modules with clear dependencies: - Static variables: `s_PascalCase` - Thread-local variables: `t_PascalCase` -**Note:** Unlike UE, no `F` prefix on structs/classes is required or encouraged. Also, no `b` prefix on booleans. +**Note:** Unlike UE, no `F` prefix on structs/classes is required nor encouraged. Also, no `b` prefix on booleans. **Code Style:** - C++20 standard - clang-format enforced via pre-commit hooks -- Use `std` containers; `eastl::fixed_vector` etc. for performance-critical paths +- Use `std` containers; `eastl::fixed_vector` etc. for performance-critical paths where the number of elements + in the container is typically small or when the container is temporary and suitable for stack allocation. - Exceptions used only for unexpected errors, not flow control + - Some `std` exception types like `runtime_error` are also available in the `zen` namespace and offer + convenience of formatting the exception message by allowing fmt-style formatting without using + `fmt::format` which can reduce clutter. To use these, please `#include ` - Logging is done via `ZEN_DEBUG(...)`, `ZEN_INFO(...)`, `ZEN_WARN`, `ZEN_ERROR` macros - - Logging macros use `fmt::format` for message formatting. The first argument is the format string, which - must be a string literal (turned into `std::string_view` in the macros) + - Logging macros use `fmt::vformat` internally for message formatting. The first argument is the format string, + which must be a string literal (turned into `std::string_view` in the macros) - Logging channels can be overridden on a scope-by-scope basis (per-class or even per-function) by implementing a `LoggerRef Log()` function +- `if`/`else` should use braces even for single-statement branches **Includes:** - Wrap third-party includes with `ZEN_THIRD_PARTY_INCLUDES_START` / `ZEN_THIRD_PARTY_INCLUDES_END` @@ -203,6 +210,9 @@ On Windows, zenserver requires elevation or a URL reservation to bind http.sys t netsh http add urlacl url=http://*:8558/ user= ``` +**POST endpoint content types** +Endpoints accepting POST operations should only accept JSON or compact binary format payloads. + ## Platform-Specific Notes **Windows:** -- cgit v1.2.3 From fb19c8a86e89762ea89df3b361494a055680b432 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Tue, 24 Feb 2026 22:24:11 +0100 Subject: Fix zencore bugs and propagate content type through IoBufferBuilder (#783) - Add missing includes in hashutils.h (``, ``) - Add `ZenContentType` parameter to all `IoBufferBuilder` factory methods so content type is set at buffer creation time - Fix null dereference in `SharedBuffer::GetFileReference()` when buffer is null - Fix out-of-bounds read in trace command-line argument parsing when arg length exactly matches option length - Add unit tests for 32-bit `CountLeadingZeros` --- src/zencore/include/zencore/hashutils.h | 3 +++ src/zencore/include/zencore/iobuffer.h | 37 ++++++++++++++++++++++-------- src/zencore/include/zencore/sharedbuffer.h | 13 ++++++----- src/zencore/intmath.cpp | 6 +++++ src/zencore/iobuffer.cpp | 20 +++++++++------- src/zencore/trace.cpp | 13 ++++++++--- 6 files changed, 65 insertions(+), 27 deletions(-) diff --git a/src/zencore/include/zencore/hashutils.h b/src/zencore/include/zencore/hashutils.h index 4e877e219..6b9902b3a 100644 --- a/src/zencore/include/zencore/hashutils.h +++ b/src/zencore/include/zencore/hashutils.h @@ -2,6 +2,9 @@ #pragma once +#include +#include + namespace zen { template diff --git a/src/zencore/include/zencore/iobuffer.h b/src/zencore/include/zencore/iobuffer.h index 182768ff6..82c201edd 100644 --- a/src/zencore/include/zencore/iobuffer.h +++ b/src/zencore/include/zencore/iobuffer.h @@ -426,22 +426,39 @@ private: class IoBufferBuilder { public: - static IoBuffer MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset = 0, uint64_t Size = ~0ull); - static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName); - static IoBuffer MakeFromFileHandle(void* FileHandle, uint64_t Offset = 0, uint64_t Size = ~0ull); - /** Make sure buffer data is memory resident, but avoid memory mapping data from files - */ - static IoBuffer ReadFromFileMaybe(const IoBuffer& InBuffer); - inline static IoBuffer MakeFromMemory(MemoryView Memory) { return IoBuffer(IoBuffer::Wrap, Memory.GetData(), Memory.GetSize()); } - inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz) + static IoBuffer MakeFromFile(const std::filesystem::path& FileName, + uint64_t Offset = 0, + uint64_t Size = ~0ull, + ZenContentType ContentType = ZenContentType::kBinary); + static IoBuffer MakeFromTemporaryFile(const std::filesystem::path& FileName, ZenContentType ContentType = ZenContentType::kBinary); + static IoBuffer MakeFromFileHandle(void* FileHandle, + uint64_t Offset = 0, + uint64_t Size = ~0ull, + ZenContentType ContentType = ZenContentType::kBinary); + inline static IoBuffer MakeFromMemory(MemoryView Memory, ZenContentType ContentType = ZenContentType::kBinary) + { + IoBuffer NewBuffer(IoBuffer::Wrap, Memory.GetData(), Memory.GetSize()); + NewBuffer.SetContentType(ContentType); + return NewBuffer; + } + inline static IoBuffer MakeCloneFromMemory(const void* Ptr, size_t Sz, ZenContentType ContentType = ZenContentType::kBinary) { if (Sz) { - return IoBuffer(IoBuffer::Clone, Ptr, Sz); + IoBuffer NewBuffer(IoBuffer::Clone, Ptr, Sz); + NewBuffer.SetContentType(ContentType); + return NewBuffer; } return {}; } - inline static IoBuffer MakeCloneFromMemory(MemoryView Memory) { return MakeCloneFromMemory(Memory.GetData(), Memory.GetSize()); } + inline static IoBuffer MakeCloneFromMemory(MemoryView Memory, ZenContentType ContentType = ZenContentType::kBinary) + { + return MakeCloneFromMemory(Memory.GetData(), Memory.GetSize(), ContentType); + } + + /** Make sure buffer data is memory resident, but avoid memory mapping data from files + */ + static IoBuffer ReadFromFileMaybe(const IoBuffer& InBuffer); }; void iobuffer_forcelink(); diff --git a/src/zencore/include/zencore/sharedbuffer.h b/src/zencore/include/zencore/sharedbuffer.h index c57e9f568..3d4c19282 100644 --- a/src/zencore/include/zencore/sharedbuffer.h +++ b/src/zencore/include/zencore/sharedbuffer.h @@ -116,14 +116,15 @@ public: inline void Reset() { m_Buffer = nullptr; } inline bool GetFileReference(IoBufferFileReference& OutRef) const { - if (const IoBufferExtendedCore* Core = m_Buffer->ExtendedCore()) + if (!IsNull()) { - return Core->GetFileReference(OutRef); - } - else - { - return false; + if (const IoBufferExtendedCore* Core = m_Buffer->ExtendedCore()) + { + return Core->GetFileReference(OutRef); + } } + + return false; } [[nodiscard]] MemoryView GetView() const diff --git a/src/zencore/intmath.cpp b/src/zencore/intmath.cpp index 5a686dc8e..32f82b486 100644 --- a/src/zencore/intmath.cpp +++ b/src/zencore/intmath.cpp @@ -43,6 +43,12 @@ TEST_CASE("intmath") CHECK(FloorLog2_64(0x0000'0001'0000'0000ull) == 32); CHECK(FloorLog2_64(0x8000'0000'0000'0000ull) == 63); + CHECK(CountLeadingZeros(0x8000'0000u) == 0); + CHECK(CountLeadingZeros(0x0000'0000u) == 32); + CHECK(CountLeadingZeros(0x0000'0001u) == 31); + CHECK(CountLeadingZeros(0x0000'8000u) == 16); + CHECK(CountLeadingZeros(0x0001'0000u) == 15); + CHECK(CountLeadingZeros64(0x8000'0000'0000'0000ull) == 0); CHECK(CountLeadingZeros64(0x0000'0000'0000'0000ull) == 64); CHECK(CountLeadingZeros64(0x0000'0000'0000'0001ull) == 63); diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp index be9b39e7a..1c31d6620 100644 --- a/src/zencore/iobuffer.cpp +++ b/src/zencore/iobuffer.cpp @@ -592,15 +592,17 @@ IoBufferBuilder::ReadFromFileMaybe(const IoBuffer& InBuffer) } IoBuffer -IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFileHandle(void* FileHandle, uint64_t Offset, uint64_t Size, ZenContentType ContentType) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromFileHandle"); - return IoBuffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); + IoBuffer Buffer(IoBuffer::BorrowedFile, FileHandle, Offset, Size); + Buffer.SetContentType(ContentType); + return Buffer; } IoBuffer -IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size) +IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Offset, uint64_t Size, ZenContentType ContentType) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromFile"); @@ -632,8 +634,6 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of FileSize = Stat.st_size; #endif // ZEN_PLATFORM_WINDOWS - // TODO: should validate that offset is in range - if (Size == ~0ull) { Size = FileSize - Offset; @@ -652,7 +652,9 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of #if ZEN_PLATFORM_WINDOWS void* Fd = DataFile.Detach(); #endif - return IoBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size, Offset == 0 && Size == FileSize); + IoBuffer NewBuffer(IoBuffer::File, (void*)uintptr_t(Fd), Offset, Size, Offset == 0 && Size == FileSize); + NewBuffer.SetContentType(ContentType); + return NewBuffer; } #if !ZEN_PLATFORM_WINDOWS @@ -664,7 +666,7 @@ IoBufferBuilder::MakeFromFile(const std::filesystem::path& FileName, uint64_t Of } IoBuffer -IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName) +IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName, ZenContentType ContentType) { ZEN_TRACE_CPU("IoBufferBuilder::MakeFromTemporaryFile"); @@ -703,7 +705,9 @@ IoBufferBuilder::MakeFromTemporaryFile(const std::filesystem::path& FileName) Handle = (void*)uintptr_t(Fd); #endif // ZEN_PLATFORM_WINDOWS - return IoBuffer(IoBuffer::File, Handle, 0, FileSize, /*IsWholeFile*/ true); + IoBuffer NewBuffer(IoBuffer::File, Handle, 0, FileSize, /*IsWholeFile*/ true); + NewBuffer.SetContentType(ContentType); + return NewBuffer; } ////////////////////////////////////////////////////////////////////////// diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp index 87035554f..a026974c0 100644 --- a/src/zencore/trace.cpp +++ b/src/zencore/trace.cpp @@ -165,10 +165,17 @@ GetTraceOptionsFromCommandline(TraceOptions& OutOptions) auto MatchesArg = [](std::string_view Option, std::string_view Arg) -> std::optional { if (Arg.starts_with(Option)) { - std::string_view::value_type DelimChar = Arg[Option.length()]; - if (DelimChar == ' ' || DelimChar == '=') + if (Arg.length() > Option.length()) { - return Arg.substr(Option.size() + 1); + std::string_view::value_type DelimChar = Arg[Option.length()]; + if (DelimChar == ' ' || DelimChar == '=') + { + return Arg.substr(Option.size() + 1); + } + } + else + { + return ""sv; } } return {}; -- cgit v1.2.3 From 241e4faf64be83711dc509ad8a25ff4e8ae95c12 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 25 Feb 2026 10:15:41 +0100 Subject: HttpService/Frontend improvements (#782) - zenhttp: added `GetServiceUri()`/`GetExternalHost()` - enables code to quickly generate an externally reachable URI for a given service - frontend: improved Uri handling (better defaults) - added support for 404 page (to make it easier to find a good URL) --- CHANGELOG.md | 1 + src/zenhttp/httpserver.cpp | 24 +++++++++++- src/zenhttp/include/zenhttp/httpserver.h | 16 ++++++++ src/zenhttp/servers/httpasio.cpp | 48 ++++++++++++++++++++---- src/zenhttp/servers/httpmulti.cpp | 10 +++++ src/zenhttp/servers/httpmulti.h | 13 ++++--- src/zenhttp/servers/httpsys.cpp | 49 ++++++++++++++++++++++--- src/zenserver/frontend/frontend.cpp | 57 +++++++++++++++++++++-------- src/zenserver/frontend/html.zip | Bin 183939 -> 279965 bytes src/zenserver/storage/zenstorageserver.cpp | 9 +++++ 10 files changed, 192 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e9d3b79c8..e555dd86f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## - Feature: Add `--allow-partial-block-requests` to `zen oplog-import` - Feature: `zen ui` can be used to open dashboards for local instances +- Feature: Added 404 page to dashboard, to make it easier to find your way back to a valid URL - Improvement: `zen oplog-import` now uses partial block requests to reduce download size - Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests - Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index f2fe4738f..3cefa0ad8 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1014,7 +1015,28 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) int HttpServer::Initialize(int BasePort, std::filesystem::path DataDir) { - return OnInitialize(BasePort, std::move(DataDir)); + m_EffectivePort = OnInitialize(BasePort, std::move(DataDir)); + m_ExternalHost = OnGetExternalHost(); + return m_EffectivePort; +} + +std::string +HttpServer::OnGetExternalHost() const +{ + return GetMachineName(); +} + +std::string +HttpServer::GetServiceUri(const HttpService* Service) const +{ + if (Service) + { + return fmt::format("http://{}:{}{}", m_ExternalHost, m_EffectivePort, Service->BaseUri()); + } + else + { + return fmt::format("http://{}:{}", m_ExternalHost, m_EffectivePort); + } } void diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 350532126..00cbc6c14 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -219,8 +219,21 @@ public: void RequestExit(); void Close(); + /** Returns a canonical http:// URI for the given service, using the external + * IP and the port the server is actually listening on. Only valid + * after Initialize() has returned successfully. + */ + std::string GetServiceUri(const HttpService* Service) const; + + /** Returns the external host string (IP or hostname) determined during Initialize(). + * Only valid after Initialize() has returned successfully. + */ + std::string_view GetExternalHost() const { return m_ExternalHost; } + private: std::vector m_KnownServices; + int m_EffectivePort = 0; + std::string m_ExternalHost; virtual void OnRegisterService(HttpService& Service) = 0; virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) = 0; @@ -228,6 +241,9 @@ private: virtual void OnRun(bool IsInteractiveSession) = 0; virtual void OnRequestExit() = 0; virtual void OnClose() = 0; + +protected: + virtual std::string OnGetExternalHost() const; }; struct HttpServerPluginConfig diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index fbc7fe401..0c0238886 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -506,6 +507,8 @@ public: HttpService* RouteRequest(std::string_view Url); IHttpRequestFilter::Result FilterRequest(HttpServerRequest& Request); + bool IsLoopbackOnly() const; + asio::io_service m_IoService; asio::io_service::work m_Work{m_IoService}; std::unique_ptr m_Acceptor; @@ -1601,7 +1604,8 @@ struct HttpAcceptor void StopAccepting() { m_IsStopped = true; } - int GetAcceptPort() { return m_Acceptor.local_endpoint().port(); } + int GetAcceptPort() const { return m_Acceptor.local_endpoint().port(); } + bool IsLoopbackOnly() const { return m_Acceptor.local_endpoint().address().is_loopback(); } bool IsValid() const { return m_IsValid; } @@ -1975,6 +1979,12 @@ HttpAsioServerImpl::FilterRequest(HttpServerRequest& Request) return RequestFilter->FilterRequest(Request); } +bool +HttpAsioServerImpl::IsLoopbackOnly() const +{ + return m_Acceptor && m_Acceptor->IsLoopbackOnly(); +} + } // namespace zen::asio_http ////////////////////////////////////////////////////////////////////////// @@ -1987,12 +1997,13 @@ public: HttpAsioServer(const AsioConfig& Config); ~HttpAsioServer(); - virtual void OnRegisterService(HttpService& Service) override; - virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; - virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; - virtual void OnRun(bool IsInteractiveSession) override; - virtual void OnRequestExit() override; - virtual void OnClose() override; + virtual void OnRegisterService(HttpService& Service) override; + virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; + virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; + virtual void OnRun(bool IsInteractiveSession) override; + virtual void OnRequestExit() override; + virtual void OnClose() override; + virtual std::string OnGetExternalHost() const override; private: Event m_ShutdownEvent; @@ -2067,6 +2078,29 @@ HttpAsioServer::OnInitialize(int BasePort, std::filesystem::path DataDir) return m_BasePort; } +std::string +HttpAsioServer::OnGetExternalHost() const +{ + if (m_Impl->IsLoopbackOnly()) + { + return "127.0.0.1"; + } + + // Use the UDP connect trick: connecting a UDP socket to an external address + // causes the OS to select the appropriate local interface without sending any data. + try + { + asio::io_service IoService; + asio::ip::udp::socket Sock(IoService, asio::ip::udp::v4()); + Sock.connect(asio::ip::udp::endpoint(asio::ip::address::from_string("8.8.8.8"), 80)); + return Sock.local_endpoint().address().to_string(); + } + catch (const std::exception&) + { + return GetMachineName(); + } +} + void HttpAsioServer::OnRun(bool IsInteractive) { diff --git a/src/zenhttp/servers/httpmulti.cpp b/src/zenhttp/servers/httpmulti.cpp index 310ac9dc0..584e06cbf 100644 --- a/src/zenhttp/servers/httpmulti.cpp +++ b/src/zenhttp/servers/httpmulti.cpp @@ -117,6 +117,16 @@ HttpMultiServer::OnClose() } } +std::string +HttpMultiServer::OnGetExternalHost() const +{ + if (!m_Servers.empty()) + { + return std::string(m_Servers.front()->GetExternalHost()); + } + return HttpServer::OnGetExternalHost(); +} + void HttpMultiServer::AddServer(Ref Server) { diff --git a/src/zenhttp/servers/httpmulti.h b/src/zenhttp/servers/httpmulti.h index 1897587a9..97699828a 100644 --- a/src/zenhttp/servers/httpmulti.h +++ b/src/zenhttp/servers/httpmulti.h @@ -15,12 +15,13 @@ public: HttpMultiServer(); ~HttpMultiServer(); - virtual void OnRegisterService(HttpService& Service) override; - virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; - virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; - virtual void OnRun(bool IsInteractiveSession) override; - virtual void OnRequestExit() override; - virtual void OnClose() override; + virtual void OnRegisterService(HttpService& Service) override; + virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; + virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; + virtual void OnRun(bool IsInteractiveSession) override; + virtual void OnRequestExit() override; + virtual void OnClose() override; + virtual std::string OnGetExternalHost() const override; void AddServer(Ref Server); diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 6995ffca9..e93ae4853 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +28,7 @@ # include # include +# include // for resolving addresses for GetExternalHost namespace zen { @@ -93,12 +95,13 @@ public: // HttpServer interface implementation - virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; - virtual void OnRun(bool TestMode) override; - virtual void OnRequestExit() override; - virtual void OnRegisterService(HttpService& Service) override; - virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; - virtual void OnClose() override; + virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) override; + virtual void OnRun(bool TestMode) override; + virtual void OnRequestExit() override; + virtual void OnRegisterService(HttpService& Service) override; + virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; + virtual void OnClose() override; + virtual std::string OnGetExternalHost() const override; WorkerThreadPool& WorkPool(); @@ -2290,6 +2293,40 @@ HttpSysServer::OnRequestExit() m_ShutdownEvent.Set(); } +std::string +HttpSysServer::OnGetExternalHost() const +{ + // Check whether we registered a public wildcard URL (http://*:port/) or fell back to loopback + bool IsPublic = false; + for (const auto& Uri : m_BaseUris) + { + if (Uri.find(L'*') != std::wstring::npos) + { + IsPublic = true; + break; + } + } + + if (!IsPublic) + { + return "127.0.0.1"; + } + + // Use the UDP connect trick: connecting a UDP socket to an external address + // causes the OS to select the appropriate local interface without sending any data. + try + { + asio::io_service IoService; + asio::ip::udp::socket Sock(IoService, asio::ip::udp::v4()); + Sock.connect(asio::ip::udp::endpoint(asio::ip::address::from_string("8.8.8.8"), 80)); + return Sock.local_endpoint().address().to_string(); + } + catch (const std::exception&) + { + return GetMachineName(); + } +} + void HttpSysServer::OnRegisterService(HttpService& Service) { diff --git a/src/zenserver/frontend/frontend.cpp b/src/zenserver/frontend/frontend.cpp index 1cf451e91..579a65c5a 100644 --- a/src/zenserver/frontend/frontend.cpp +++ b/src/zenserver/frontend/frontend.cpp @@ -114,6 +114,8 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) { using namespace std::literals; + ExtendableStringBuilder<256> UriBuilder; + std::string_view Uri = Request.RelativeUriWithExtension(); for (; Uri.length() > 0 && Uri[0] == '/'; Uri = Uri.substr(1)) ; @@ -121,6 +123,11 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) { Uri = "index.html"sv; } + else if (Uri.back() == '/') + { + UriBuilder << Uri << "index.html"sv; + Uri = UriBuilder; + } // Dismiss if the URI contains .. anywhere to prevent arbitrary file reads if (Uri.find("..") != Uri.npos) @@ -145,27 +152,47 @@ HttpFrontendService::HandleRequest(zen::HttpServerRequest& Request) return Request.WriteResponse(HttpResponseCode::Forbidden); } - // The given content directory overrides any zip-fs discovered in the binary - if (!m_Directory.empty()) - { - auto FullPath = m_Directory / std::filesystem::path(Uri).make_preferred(); - FileContents File = ReadFile(FullPath); - - if (!File.ErrorCode) + auto WriteResponseForUri = [this, + &Request](std::string_view InUri, HttpResponseCode ResponseCode, HttpContentType ContentType) -> bool { + // The given content directory overrides any zip-fs discovered in the binary + if (!m_Directory.empty()) { - return Request.WriteResponse(HttpResponseCode::OK, ContentType, File.Data[0]); + auto FullPath = m_Directory / std::filesystem::path(InUri).make_preferred(); + FileContents File = ReadFile(FullPath); + + if (!File.ErrorCode) + { + Request.WriteResponse(ResponseCode, ContentType, File.Data[0]); + + return true; + } } - } - if (m_ZipFs) - { - if (IoBuffer FileBuffer = m_ZipFs->GetFile(Uri)) + if (m_ZipFs) { - return Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer); + if (IoBuffer FileBuffer = m_ZipFs->GetFile(InUri)) + { + Request.WriteResponse(HttpResponseCode::OK, ContentType, FileBuffer); + + return true; + } } - } - Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + return false; + }; + + if (WriteResponseForUri(Uri, HttpResponseCode::OK, ContentType)) + { + return; + } + else if (WriteResponseForUri("404.html"sv, HttpResponseCode::NotFound, HttpContentType::kHTML)) + { + return; + } + else + { + Request.WriteResponse(HttpResponseCode::NotFound, HttpContentType::kText, "Not found"sv); + } } } // namespace zen diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index d70a5a62b..3d90c18a8 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index ff854b72d..3d81db656 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -700,6 +700,15 @@ ZenStorageServer::Run() ZEN_INFO(ZEN_APP_NAME " now running (pid: {})", GetCurrentProcessId()); + if (m_FrontendService) + { + ZEN_INFO("frontend link: {}", m_Http->GetServiceUri(m_FrontendService.get())); + } + else + { + ZEN_INFO("frontend service disabled"); + } + #if ZEN_PLATFORM_WINDOWS if (zen::windows::IsRunningOnWine()) { -- cgit v1.2.3 From d7354c2ad34858d8ee99fb307685956c24abd897 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 25 Feb 2026 18:49:31 +0100 Subject: work around doctest shutdown issues with static CRT (#784) * tweaked doctest.h to avoid shutdown issues due to thread_local variables running destructors after the main thread has torn down everything including the heap * disabled zenserver exit thread waiting since doctest should hopefully not be causing issues during shutdown anymore after my workaround This should help reduce the duration of tests spawning lots of server instances --- src/zenserver/main.cpp | 9 +++++++++ thirdparty/doctest/doctest/doctest.h | 29 +++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index ee783d2a6..571dd3b4f 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -267,6 +267,14 @@ main(int argc, char* argv[]) using namespace zen; using namespace std::literals; + // note: doctest has locally (in thirdparty) been fixed to not cause shutdown + // crashes due to TLS destructors + // + // mimalloc on the other hand might still be causing issues, in which case + // we should work out either how to eliminate the mimalloc dependency or how + // to configure it in a way that doesn't cause shutdown issues + +#if 0 auto _ = zen::MakeGuard([] { // Allow some time for worker threads to unravel, in an effort // to prevent shutdown races in TLS object destruction, mainly due to @@ -277,6 +285,7 @@ main(int argc, char* argv[]) // shutdown crashes observed in some situations. WaitForThreads(1000); }); +#endif enum { diff --git a/thirdparty/doctest/doctest/doctest.h b/thirdparty/doctest/doctest/doctest.h index f297fa0b2..5299d9c1a 100644 --- a/thirdparty/doctest/doctest/doctest.h +++ b/thirdparty/doctest/doctest/doctest.h @@ -3337,7 +3337,8 @@ namespace { } // namespace namespace detail { - DOCTEST_THREAD_LOCAL class + // Leaked heap allocation - see g_infoContexts comment above. + class ThreadLocalStringStream { std::vector stack; std::stringstream ss; @@ -3358,7 +3359,12 @@ namespace detail { ss.rdbuf()->pubseekpos(pos, std::ios::in | std::ios::out); return String(ss, sz); } - } g_oss; + }; + static ThreadLocalStringStream& g_oss_ref() { + DOCTEST_THREAD_LOCAL auto* p = new ThreadLocalStringStream(); + return *p; + } + #define g_oss g_oss_ref() // NOLINT std::ostream* tlssPush() { return g_oss.push(); @@ -4600,7 +4606,14 @@ namespace detail { getExceptionTranslators().push_back(et); } - DOCTEST_THREAD_LOCAL std::vector g_infoContexts; // for logging with INFO() + // Use a leaked heap allocation for thread-local state to avoid calling destructors + // during shutdown. With static CRT linking, thread-local destructors can run after + // the CRT has already been torn down on secondary threads, causing crashes. + static std::vector& g_infoContexts_ref() { + DOCTEST_THREAD_LOCAL auto* p = new std::vector(); + return *p; + } + #define g_infoContexts g_infoContexts_ref() // NOLINT ContextScopeBase::ContextScopeBase() { g_infoContexts.push_back(this); @@ -6440,13 +6453,18 @@ namespace { #ifdef DOCTEST_PLATFORM_WINDOWS struct DebugOutputWindowReporter : public ConsoleReporter { - DOCTEST_THREAD_LOCAL static std::ostringstream oss; + // Leaked heap allocation - see g_infoContexts comment in detail namespace. + static std::ostringstream& oss_ref() { + DOCTEST_THREAD_LOCAL auto* p = new std::ostringstream(); + return *p; + } DebugOutputWindowReporter(const ContextOptions& co) - : ConsoleReporter(co, oss) {} + : ConsoleReporter(co, oss_ref()) {} #define DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(func, type, arg) \ void func(type arg) override { \ + auto& oss = oss_ref(); \ bool with_col = g_no_colors; \ g_no_colors = false; \ ConsoleReporter::func(arg); \ @@ -6470,7 +6488,6 @@ namespace { DOCTEST_DEBUG_OUTPUT_REPORTER_OVERRIDE(test_case_skipped, const TestCaseData&, in) }; - DOCTEST_THREAD_LOCAL std::ostringstream DebugOutputWindowReporter::oss; #endif // DOCTEST_PLATFORM_WINDOWS // the implementation of parseOption() -- cgit v1.2.3 From c1838da092c31c4ebe1e9c3f3909a1bef37d34a2 Mon Sep 17 00:00:00 2001 From: zousar Date: Thu, 26 Feb 2026 10:58:50 -0700 Subject: updatefrontend --- src/zenserver/frontend/html.zip | Bin 183939 -> 238188 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index d70a5a62b..4767029c0 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ -- cgit v1.2.3 From 7c7e25d55ebb593aaa6a42903e2db4629f3b7051 Mon Sep 17 00:00:00 2001 From: zousar Date: Thu, 26 Feb 2026 11:09:36 -0700 Subject: updatefrontend --- src/zenserver/frontend/html.zip | Bin 279965 -> 238188 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index 3d90c18a8..4767029c0 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ -- cgit v1.2.3 From 91885b9fc6b1954d78d14bdf39e2ba91a5aa9f67 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 26 Feb 2026 20:14:07 +0100 Subject: adding HttpClient tests (#785) Add comprehensive `HttpClient` test suite. Covers: - **HTTP verbs** -- GET, POST, PUT, DELETE, HEAD dispatch correctly - **GET/POST/PUT/Upload/Download** -- payload round-trips (IoBuffer, CbObject, CompositeBuffer), content types, large payloads, file-spill downloads - **Status codes** -- 2xx/4xx/5xx classification, exact code matching - **Response API** -- IsSuccess, AsText, AsObject, ToText, ErrorMessage, ThrowError - **Error handling** -- connection refused, request timeout, nonexistent endpoints - **Session management** -- default ID, SetSessionId, reset to zero - **Authentication** -- token provider, expired tokens, bearer verification - **Content type detection** -- text, JSON, binary, CbObject - **Request metadata** -- elapsed time, upload/download byte counts - **Retry logic** -- retry after transient 503s, no-retry baseline - **Latency measurement** -- MeasureLatency against live and unreachable servers - **KeyValueMap** -- construction from pairs, string_views, initializer lists - **Transport-level faults (GET)** -- connection reset/close before response, partial headers, truncated body, mid-body reset, stalled response timeout, retry after RST - **Transport-level faults (POST)** -- server reset/close before consuming body, mid-body reset, early 503 without consuming upload, stalled upload timeout, retry with large body after transient failures Also adds zenhttp-test to the xmake test runner (xmake test --run=http). --- src/zenhttp/httpclient_test.cpp | 1362 ++++++++++++++++++++++++++++++ src/zenhttp/include/zenhttp/httpclient.h | 3 +- src/zenhttp/zenhttp.cpp | 1 + xmake.lua | 4 +- 4 files changed, 1367 insertions(+), 3 deletions(-) create mode 100644 src/zenhttp/httpclient_test.cpp diff --git a/src/zenhttp/httpclient_test.cpp b/src/zenhttp/httpclient_test.cpp new file mode 100644 index 000000000..509b56371 --- /dev/null +++ b/src/zenhttp/httpclient_test.cpp @@ -0,0 +1,1362 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include + +#if ZEN_WITH_TESTS + +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include "servers/httpasio.h" + +# include +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// +// Test service + +class HttpClientTestService : public HttpService +{ +public: + HttpClientTestService() + { + m_Router.AddMatcher("statuscode", [](std::string_view Str) -> bool { + for (char C : Str) + { + if (C < '0' || C > '9') + { + return false; + } + } + return !Str.empty(); + }); + + m_Router.RegisterRoute( + "hello", + [](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "hello world"); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "echo", + [](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + IoBuffer Body = HttpReq.ReadPayload(); + HttpContentType CT = HttpReq.RequestContentType(); + HttpReq.WriteResponse(HttpResponseCode::OK, CT, Body); + }, + HttpVerb::kPost | HttpVerb::kPut); + + m_Router.RegisterRoute( + "echo/headers", + [](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + std::string_view Auth = HttpReq.GetAuthorizationHeader(); + CbObjectWriter Writer; + if (!Auth.empty()) + { + Writer.AddString("Authorization", Auth); + } + HttpReq.WriteResponse(HttpResponseCode::OK, Writer.Save()); + }, + HttpVerb::kGet | HttpVerb::kPost); + + m_Router.RegisterRoute( + "echo/method", + [](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + std::string_view Method = ToString(HttpReq.RequestVerb()); + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Method); + }, + HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kPut | HttpVerb::kDelete | HttpVerb::kHead); + + m_Router.RegisterRoute( + "json", + [](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.AddBool("ok", true); + Obj.AddString("message", "test"); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "nocontent", + [](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::NoContent); }, + HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kPut | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "created", + [](HttpRouterRequest& Req) { + Req.ServerRequest().WriteResponse(HttpResponseCode::Created, HttpContentType::kText, "resource created"); + }, + HttpVerb::kPost | HttpVerb::kPut); + + m_Router.RegisterRoute( + "content-type/text", + [](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "plain text"); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "content-type/json", + [](HttpRouterRequest& Req) { + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kJSON, "{\"key\":\"value\"}"); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "content-type/binary", + [](HttpRouterRequest& Req) { + uint8_t Data[] = {0xDE, 0xAD, 0xBE, 0xEF}; + IoBuffer Buf(IoBuffer::Clone, Data, sizeof(Data)); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Buf); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "content-type/cbobject", + [](HttpRouterRequest& Req) { + CbObjectWriter Obj; + Obj.AddString("type", "cbobject"); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "auth/bearer", + [](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + std::string_view Auth = HttpReq.GetAuthorizationHeader(); + if (Auth.starts_with("Bearer ") && Auth.size() > 7) + { + HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "authenticated"); + } + else + { + HttpReq.WriteResponse(HttpResponseCode::Unauthorized, HttpContentType::kText, "unauthorized"); + } + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "slow", + [](HttpRouterRequest& Req) { + Req.ServerRequest().WriteResponseAsync([](HttpServerRequest& Request) { + Sleep(2000); + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "slow response"); + }); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "large", + [](HttpRouterRequest& Req) { + constexpr size_t Size = 64 * 1024; + IoBuffer Buf(Size); + uint8_t* Ptr = static_cast(Buf.MutableData()); + for (size_t i = 0; i < Size; ++i) + { + Ptr[i] = static_cast(i & 0xFF); + } + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kBinary, Buf); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "status/{statuscode}", + [](HttpRouterRequest& Req) { + std::string_view CodeStr = Req.GetCapture(1); + int Code = std::stoi(std::string{CodeStr}); + const HttpResponseCode ResponseCode = static_cast(Code); + Req.ServerRequest().WriteResponse(ResponseCode); + }, + HttpVerb::kGet | HttpVerb::kPost | HttpVerb::kPut | HttpVerb::kDelete | HttpVerb::kHead); + + m_Router.RegisterRoute( + "attempt-counter", + [this](HttpRouterRequest& Req) { + uint32_t Count = m_AttemptCounter.fetch_add(1); + if (Count < m_FailCount) + { + Req.ServerRequest().WriteResponse(HttpResponseCode::ServiceUnavailable); + } + else + { + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "success after retries"); + } + }, + HttpVerb::kGet); + } + + virtual const char* BaseUri() const override { return "/api/test/"; } + virtual void HandleRequest(HttpServerRequest& Request) override { m_Router.HandleRequest(Request); } + + void ResetAttemptCounter(uint32_t FailCount) + { + m_AttemptCounter.store(0); + m_FailCount = FailCount; + } + +private: + HttpRequestRouter m_Router; + std::atomic m_AttemptCounter{0}; + uint32_t m_FailCount = 2; +}; + +////////////////////////////////////////////////////////////////////////// +// Test server fixture + +struct TestServerFixture +{ + HttpClientTestService TestService; + ScopedTemporaryDirectory TmpDir; + Ref Server; + std::thread ServerThread; + int Port = -1; + + TestServerFixture() + { + Server = CreateHttpAsioServer(AsioConfig{}); + Port = Server->Initialize(7600, TmpDir.Path()); + ZEN_ASSERT(Port != -1); + Server->RegisterService(TestService); + ServerThread = std::thread([this]() { Server->Run(false); }); + } + + ~TestServerFixture() + { + Server->RequestExit(); + if (ServerThread.joinable()) + { + ServerThread.join(); + } + Server->Close(); + } + + HttpClient MakeClient(HttpClientSettings Settings = {}) + { + return HttpClient(fmt::format("127.0.0.1:{}", Port), Settings, /*CheckIfAbortFunction*/ {}); + } +}; + +////////////////////////////////////////////////////////////////////////// +// Tests + +TEST_CASE("httpclient.verbs") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + SUBCASE("GET returns 200 with expected body") + { + HttpClient::Response Resp = Client.Get("/api/test/echo/method"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "GET"); + } + + SUBCASE("POST dispatches correctly") + { + HttpClient::Response Resp = Client.Post("/api/test/echo/method"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "POST"); + } + + SUBCASE("PUT dispatches correctly") + { + HttpClient::Response Resp = Client.Put("/api/test/echo/method"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "PUT"); + } + + SUBCASE("DELETE dispatches correctly") + { + HttpClient::Response Resp = Client.Delete("/api/test/echo/method"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "DELETE"); + } + + SUBCASE("HEAD returns 200 with empty body") + { + HttpClient::Response Resp = Client.Head("/api/test/echo/method"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), ""sv); + } +} + +TEST_CASE("httpclient.get") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + SUBCASE("simple GET with text response") + { + HttpClient::Response Resp = Client.Get("/api/test/hello"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.StatusCode, HttpResponseCode::OK); + CHECK_EQ(Resp.AsText(), "hello world"); + } + + SUBCASE("GET with auth header via echo") + { + HttpClient::Response Resp = + Client.Get("/api/test/echo/headers", std::pair("Authorization", "Bearer test-token-123")); + CHECK(Resp.IsSuccess()); + CbObject Obj = Resp.AsObject(); + CHECK_EQ(Obj["Authorization"].AsString(), "Bearer test-token-123"); + } + + SUBCASE("GET returning CbObject") + { + HttpClient::Response Resp = Client.Get("/api/test/json"); + CHECK(Resp.IsSuccess()); + CbObject Obj = Resp.AsObject(); + CHECK(Obj["ok"].AsBool() == true); + CHECK_EQ(Obj["message"].AsString(), "test"); + } + + SUBCASE("GET large payload") + { + HttpClient::Response Resp = Client.Get("/api/test/large"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.ResponsePayload.GetSize(), 64u * 1024u); + + const uint8_t* Data = static_cast(Resp.ResponsePayload.GetData()); + bool Valid = true; + for (size_t i = 0; i < 64 * 1024; ++i) + { + if (Data[i] != static_cast(i & 0xFF)) + { + Valid = false; + break; + } + } + CHECK(Valid); + } +} + +TEST_CASE("httpclient.post") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + SUBCASE("POST with IoBuffer payload echo round-trip") + { + const char* Payload = "test payload data"; + IoBuffer Buf(IoBuffer::Clone, Payload, strlen(Payload)); + Buf.SetContentType(ZenContentType::kText); + + HttpClient::Response Resp = Client.Post("/api/test/echo", Buf); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "test payload data"); + } + + SUBCASE("POST with IoBuffer and explicit content type") + { + const char* Payload = "{\"key\":\"value\"}"; + IoBuffer Buf(IoBuffer::Clone, Payload, strlen(Payload)); + + HttpClient::Response Resp = Client.Post("/api/test/echo", Buf, ZenContentType::kJSON); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "{\"key\":\"value\"}"); + } + + SUBCASE("POST with CbObject payload round-trip") + { + CbObjectWriter Writer; + Writer.AddBool("enabled", true); + Writer.AddString("name", "testobj"); + CbObject Obj = Writer.Save(); + + HttpClient::Response Resp = Client.Post("/api/test/echo", Obj); + CHECK(Resp.IsSuccess()); + CbObject RoundTripped = Resp.AsObject(); + CHECK(RoundTripped["enabled"].AsBool() == true); + CHECK_EQ(RoundTripped["name"].AsString(), "testobj"); + } + + SUBCASE("POST with CompositeBuffer payload") + { + const char* Part1 = "hello "; + const char* Part2 = "composite"; + IoBuffer Buf1(IoBuffer::Clone, Part1, strlen(Part1)); + IoBuffer Buf2(IoBuffer::Clone, Part2, strlen(Part2)); + + SharedBuffer Seg1{Buf1}; + SharedBuffer Seg2{Buf2}; + CompositeBuffer Composite{std::move(Seg1), std::move(Seg2)}; + + HttpClient::Response Resp = Client.Post("/api/test/echo", Composite, ZenContentType::kText); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "hello composite"); + } + + SUBCASE("POST with custom headers") + { + HttpClient::Response Resp = Client.Post("/api/test/echo/headers", HttpClient::KeyValueMap{}, HttpClient::KeyValueMap{}); + CHECK(Resp.IsSuccess()); + } + + SUBCASE("POST with empty body to nocontent endpoint") + { + HttpClient::Response Resp = Client.Post("/api/test/nocontent"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.StatusCode, HttpResponseCode::NoContent); + } +} + +TEST_CASE("httpclient.put") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + SUBCASE("PUT with IoBuffer payload echo round-trip") + { + const char* Payload = "put payload data"; + IoBuffer Buf(IoBuffer::Clone, Payload, strlen(Payload)); + Buf.SetContentType(ZenContentType::kText); + + HttpClient::Response Resp = Client.Put("/api/test/echo", Buf); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "put payload data"); + } + + SUBCASE("PUT with parameters only") + { + HttpClient::Response Resp = Client.Put("/api/test/nocontent"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.StatusCode, HttpResponseCode::NoContent); + } + + SUBCASE("PUT to created endpoint") + { + const char* Payload = "new resource"; + IoBuffer Buf(IoBuffer::Clone, Payload, strlen(Payload)); + Buf.SetContentType(ZenContentType::kText); + + HttpClient::Response Resp = Client.Put("/api/test/created", Buf); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.StatusCode, HttpResponseCode::Created); + CHECK_EQ(Resp.AsText(), "resource created"); + } +} + +TEST_CASE("httpclient.upload") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + SUBCASE("Upload IoBuffer") + { + constexpr size_t Size = 128 * 1024; + IoBuffer Blob = CreateSemiRandomBlob(Size); + + HttpClient::Response Resp = Client.Upload("/api/test/echo", Blob); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.ResponsePayload.GetSize(), Size); + } + + SUBCASE("Upload CompositeBuffer") + { + IoBuffer Buf1 = CreateSemiRandomBlob(32 * 1024); + IoBuffer Buf2 = CreateSemiRandomBlob(32 * 1024); + + SharedBuffer Seg1{Buf1}; + SharedBuffer Seg2{Buf2}; + CompositeBuffer Composite{std::move(Seg1), std::move(Seg2)}; + + HttpClient::Response Resp = Client.Upload("/api/test/echo", Composite, ZenContentType::kBinary); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.ResponsePayload.GetSize(), 64u * 1024u); + } +} + +TEST_CASE("httpclient.download") +{ + TestServerFixture Fixture; + ScopedTemporaryDirectory DownloadDir; + + SUBCASE("Download small payload stays in memory") + { + HttpClient Client = Fixture.MakeClient(); + + HttpClient::Response Resp = Client.Download("/api/test/hello", DownloadDir.Path()); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "hello world"); + } + + SUBCASE("Download with reduced MaximumInMemoryDownloadSize forces file spill") + { + HttpClientSettings Settings; + Settings.MaximumInMemoryDownloadSize = 4; + HttpClient Client = Fixture.MakeClient(Settings); + + HttpClient::Response Resp = Client.Download("/api/test/large", DownloadDir.Path()); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.ResponsePayload.GetSize(), 64u * 1024u); + } +} + +TEST_CASE("httpclient.status-codes") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + SUBCASE("2xx are success") + { + CHECK(Client.Get("/api/test/status/200").IsSuccess()); + CHECK(Client.Get("/api/test/status/201").IsSuccess()); + CHECK(Client.Get("/api/test/status/204").IsSuccess()); + } + + SUBCASE("4xx are not success") + { + CHECK(!Client.Get("/api/test/status/400").IsSuccess()); + CHECK(!Client.Get("/api/test/status/401").IsSuccess()); + CHECK(!Client.Get("/api/test/status/403").IsSuccess()); + CHECK(!Client.Get("/api/test/status/404").IsSuccess()); + CHECK(!Client.Get("/api/test/status/409").IsSuccess()); + } + + SUBCASE("5xx are not success") + { + CHECK(!Client.Get("/api/test/status/500").IsSuccess()); + CHECK(!Client.Get("/api/test/status/502").IsSuccess()); + CHECK(!Client.Get("/api/test/status/503").IsSuccess()); + } + + SUBCASE("status code values match") + { + CHECK_EQ(Client.Get("/api/test/status/200").StatusCode, HttpResponseCode::OK); + CHECK_EQ(Client.Get("/api/test/status/201").StatusCode, HttpResponseCode::Created); + CHECK_EQ(Client.Get("/api/test/status/204").StatusCode, HttpResponseCode::NoContent); + CHECK_EQ(Client.Get("/api/test/status/400").StatusCode, HttpResponseCode::BadRequest); + CHECK_EQ(Client.Get("/api/test/status/401").StatusCode, HttpResponseCode::Unauthorized); + CHECK_EQ(Client.Get("/api/test/status/403").StatusCode, HttpResponseCode::Forbidden); + CHECK_EQ(Client.Get("/api/test/status/404").StatusCode, HttpResponseCode::NotFound); + CHECK_EQ(Client.Get("/api/test/status/409").StatusCode, HttpResponseCode::Conflict); + CHECK_EQ(Client.Get("/api/test/status/500").StatusCode, HttpResponseCode::InternalServerError); + CHECK_EQ(Client.Get("/api/test/status/502").StatusCode, HttpResponseCode::BadGateway); + CHECK_EQ(Client.Get("/api/test/status/503").StatusCode, HttpResponseCode::ServiceUnavailable); + } +} + +TEST_CASE("httpclient.response") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + SUBCASE("IsSuccess and operator bool for success") + { + HttpClient::Response Resp = Client.Get("/api/test/hello"); + CHECK(Resp.IsSuccess()); + CHECK(static_cast(Resp)); + } + + SUBCASE("IsSuccess and operator bool for failure") + { + HttpClient::Response Resp = Client.Get("/api/test/status/404"); + CHECK(!Resp.IsSuccess()); + CHECK(!static_cast(Resp)); + } + + SUBCASE("AsText returns body") + { + HttpClient::Response Resp = Client.Get("/api/test/hello"); + CHECK_EQ(Resp.AsText(), "hello world"); + } + + SUBCASE("AsText returns empty for no-content") + { + HttpClient::Response Resp = Client.Get("/api/test/nocontent"); + CHECK(Resp.AsText().empty()); + } + + SUBCASE("AsObject parses CbObject") + { + HttpClient::Response Resp = Client.Get("/api/test/json"); + CbObject Obj = Resp.AsObject(); + CHECK(Obj["ok"].AsBool() == true); + CHECK_EQ(Obj["message"].AsString(), "test"); + } + + SUBCASE("AsObject returns empty for non-CB content") + { + HttpClient::Response Resp = Client.Get("/api/test/hello"); + CbObject Obj = Resp.AsObject(); + CHECK(!Obj); + } + + SUBCASE("ToText for text content") + { + HttpClient::Response Resp = Client.Get("/api/test/content-type/text"); + CHECK_EQ(Resp.ToText(), "plain text"); + } + + SUBCASE("ToText for CbObject content") + { + HttpClient::Response Resp = Client.Get("/api/test/json"); + std::string Text = Resp.ToText(); + CHECK(!Text.empty()); + // ToText for CbObject converts to JSON string representation + CHECK(Text.find("ok") != std::string::npos); + CHECK(Text.find("test") != std::string::npos); + } + + SUBCASE("ErrorMessage includes status code on failure") + { + HttpClient::Response Resp = Client.Get("/api/test/status/404"); + std::string Msg = Resp.ErrorMessage("test-prefix"); + CHECK(Msg.find("test-prefix") != std::string::npos); + CHECK(Msg.find("404") != std::string::npos); + } + + SUBCASE("ThrowError throws on failure") + { + HttpClient::Response Resp = Client.Get("/api/test/status/500"); + CHECK_THROWS_AS(Resp.ThrowError("test"), HttpClientError); + } + + SUBCASE("ThrowError does not throw on success") + { + HttpClient::Response Resp = Client.Get("/api/test/hello"); + CHECK_NOTHROW(Resp.ThrowError("test")); + } + + SUBCASE("HttpClientError carries response code") + { + HttpClient::Response Resp = Client.Get("/api/test/status/403"); + try + { + Resp.ThrowError("test"); + CHECK(false); // should not reach + } + catch (const HttpClientError& Err) + { + CHECK_EQ(Err.GetHttpResponseCode(), HttpResponseCode::Forbidden); + } + } +} + +TEST_CASE("httpclient.error-handling") +{ + SUBCASE("Connection refused") + { + HttpClient Client("127.0.0.1:19999", HttpClientSettings{}, /*CheckIfAbortFunction*/ {}); + HttpClient::Response Resp = Client.Get("/api/test/hello"); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + } + + SUBCASE("Request timeout") + { + TestServerFixture Fixture; + HttpClientSettings Settings; + Settings.Timeout = std::chrono::milliseconds(500); + HttpClient Client = Fixture.MakeClient(Settings); + + HttpClient::Response Resp = Client.Get("/api/test/slow"); + CHECK(!Resp.IsSuccess()); + } + + SUBCASE("Nonexistent endpoint returns failure") + { + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + HttpClient::Response Resp = Client.Get("/api/test/does-not-exist"); + CHECK(!Resp.IsSuccess()); + } +} + +TEST_CASE("httpclient.session") +{ + TestServerFixture Fixture; + + SUBCASE("Default session ID is non-empty") + { + HttpClient Client = Fixture.MakeClient(); + CHECK(!Client.GetSessionId().empty()); + } + + SUBCASE("SetSessionId changes ID") + { + HttpClient Client = Fixture.MakeClient(); + Oid NewId = Oid::NewOid(); + std::string OldId = std::string(Client.GetSessionId()); + Client.SetSessionId(NewId); + CHECK_EQ(Client.GetSessionId(), NewId.ToString()); + CHECK_NE(Client.GetSessionId(), OldId); + } + + SUBCASE("SetSessionId with Zero resets") + { + HttpClient Client = Fixture.MakeClient(); + Oid NewId = Oid::NewOid(); + Client.SetSessionId(NewId); + CHECK_EQ(Client.GetSessionId(), NewId.ToString()); + Client.SetSessionId(Oid::Zero); + // After resetting, should get a session string (not empty, not the custom one) + CHECK(!Client.GetSessionId().empty()); + CHECK_NE(Client.GetSessionId(), NewId.ToString()); + } +} + +TEST_CASE("httpclient.authentication") +{ + TestServerFixture Fixture; + + SUBCASE("Authenticate returns false without provider") + { + HttpClient Client = Fixture.MakeClient(); + CHECK(!Client.Authenticate()); + } + + SUBCASE("Authenticate returns true with valid token") + { + HttpClientSettings Settings; + Settings.AccessTokenProvider = []() -> HttpClientAccessToken { + return HttpClientAccessToken{ + .Value = "valid-token", + .ExpireTime = HttpClientAccessToken::Clock::now() + std::chrono::hours(1), + }; + }; + HttpClient Client = Fixture.MakeClient(Settings); + CHECK(Client.Authenticate()); + } + + SUBCASE("Authenticate returns false with expired token") + { + HttpClientSettings Settings; + Settings.AccessTokenProvider = []() -> HttpClientAccessToken { + return HttpClientAccessToken{ + .Value = "expired-token", + .ExpireTime = HttpClientAccessToken::Clock::now() - std::chrono::hours(1), + }; + }; + HttpClient Client = Fixture.MakeClient(Settings); + CHECK(!Client.Authenticate()); + } + + SUBCASE("Bearer token verified by auth endpoint") + { + HttpClient Client = Fixture.MakeClient(); + + HttpClient::Response AuthResp = + Client.Get("/api/test/auth/bearer", std::pair("Authorization", "Bearer my-secret-token")); + CHECK(AuthResp.IsSuccess()); + CHECK_EQ(AuthResp.AsText(), "authenticated"); + } + + SUBCASE("Request without token to auth endpoint gets 401") + { + HttpClient Client = Fixture.MakeClient(); + + HttpClient::Response Resp = Client.Get("/api/test/auth/bearer"); + CHECK(!Resp.IsSuccess()); + CHECK_EQ(Resp.StatusCode, HttpResponseCode::Unauthorized); + } +} + +TEST_CASE("httpclient.content-types") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + SUBCASE("text content type") + { + HttpClient::Response Resp = Client.Get("/api/test/content-type/text"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.ResponsePayload.GetContentType(), ZenContentType::kText); + } + + SUBCASE("JSON content type") + { + HttpClient::Response Resp = Client.Get("/api/test/content-type/json"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.ResponsePayload.GetContentType(), ZenContentType::kJSON); + } + + SUBCASE("binary content type") + { + HttpClient::Response Resp = Client.Get("/api/test/content-type/binary"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.ResponsePayload.GetContentType(), ZenContentType::kBinary); + } + + SUBCASE("CbObject content type") + { + HttpClient::Response Resp = Client.Get("/api/test/content-type/cbobject"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.ResponsePayload.GetContentType(), ZenContentType::kCbObject); + } +} + +TEST_CASE("httpclient.metadata") +{ + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + SUBCASE("ElapsedSeconds is positive") + { + HttpClient::Response Resp = Client.Get("/api/test/hello"); + CHECK(Resp.IsSuccess()); + CHECK(Resp.ElapsedSeconds > 0.0); + } + + SUBCASE("DownloadedBytes populated for GET") + { + HttpClient::Response Resp = Client.Get("/api/test/hello"); + CHECK(Resp.IsSuccess()); + CHECK(Resp.DownloadedBytes > 0); + } + + SUBCASE("UploadedBytes populated for POST with payload") + { + const char* Payload = "some upload data"; + IoBuffer Buf(IoBuffer::Clone, Payload, strlen(Payload)); + Buf.SetContentType(ZenContentType::kText); + + HttpClient::Response Resp = Client.Post("/api/test/echo", Buf); + CHECK(Resp.IsSuccess()); + CHECK(Resp.UploadedBytes > 0); + } +} + +TEST_CASE("httpclient.retry") +{ + TestServerFixture Fixture; + + SUBCASE("Retry succeeds after transient failures") + { + Fixture.TestService.ResetAttemptCounter(2); + + HttpClientSettings Settings; + Settings.RetryCount = 3; + HttpClient Client = Fixture.MakeClient(Settings); + + HttpClient::Response Resp = Client.Get("/api/test/attempt-counter"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "success after retries"); + } + + SUBCASE("No retry returns 503 immediately") + { + Fixture.TestService.ResetAttemptCounter(2); + + HttpClientSettings Settings; + Settings.RetryCount = 0; + HttpClient Client = Fixture.MakeClient(Settings); + + HttpClient::Response Resp = Client.Get("/api/test/attempt-counter"); + CHECK(!Resp.IsSuccess()); + CHECK_EQ(Resp.StatusCode, HttpResponseCode::ServiceUnavailable); + } +} + +TEST_CASE("httpclient.measurelatency") +{ + SUBCASE("Successful measurement against live server") + { + TestServerFixture Fixture; + HttpClient Client = Fixture.MakeClient(); + + LatencyTestResult Result = MeasureLatency(Client, "/api/test/hello"); + CHECK(Result.Success); + CHECK(Result.LatencySeconds > 0.0); + } + + SUBCASE("Failed measurement against unreachable port") + { + HttpClient Client("127.0.0.1:19999", HttpClientSettings{}, /*CheckIfAbortFunction*/ {}); + LatencyTestResult Result = MeasureLatency(Client, "/api/test/hello"); + CHECK(!Result.Success); + CHECK(!Result.FailureReason.empty()); + } +} + +TEST_CASE("httpclient.keyvaluemap") +{ + SUBCASE("Default construction is empty") + { + HttpClient::KeyValueMap Map; + CHECK(Map->empty()); + } + + SUBCASE("Construction from pair") + { + HttpClient::KeyValueMap Map(std::pair("key", "value")); + CHECK_EQ(Map->size(), 1u); + CHECK_EQ(Map->at("key"), "value"); + } + + SUBCASE("Construction from string_view pair") + { + HttpClient::KeyValueMap Map(std::pair("key"sv, "value"sv)); + CHECK_EQ(Map->size(), 1u); + CHECK_EQ(Map->at("key"), "value"); + } + + SUBCASE("Construction from initializer list") + { + HttpClient::KeyValueMap Map({{"a"sv, "1"sv}, {"b"sv, "2"sv}}); + CHECK_EQ(Map->size(), 2u); + CHECK_EQ(Map->at("a"), "1"); + CHECK_EQ(Map->at("b"), "2"); + } +} + +////////////////////////////////////////////////////////////////////////// +// Transport fault testing + +static std::string +MakeRawHttpResponse(int StatusCode, std::string_view Body) +{ + return fmt::format( + "HTTP/1.1 {} OK\r\n" + "Content-Type: text/plain\r\n" + "Content-Length: {}\r\n" + "\r\n" + "{}", + StatusCode, + Body.size(), + Body); +} + +static std::string +MakeRawHttpHeaders(int StatusCode, size_t ContentLength) +{ + return fmt::format( + "HTTP/1.1 {} OK\r\n" + "Content-Type: application/octet-stream\r\n" + "Content-Length: {}\r\n" + "\r\n", + StatusCode, + ContentLength); +} + +static void +DrainHttpRequest(asio::ip::tcp::socket& Socket) +{ + asio::streambuf Buf; + std::error_code Ec; + asio::read_until(Socket, Buf, "\r\n\r\n", Ec); +} + +static void +DrainFullHttpRequest(asio::ip::tcp::socket& Socket) +{ + // Read until end of headers + asio::streambuf Buf; + std::error_code Ec; + asio::read_until(Socket, Buf, "\r\n\r\n", Ec); + if (Ec) + { + return; + } + + // Extract headers to find Content-Length + std::string Headers(asio::buffers_begin(Buf.data()), asio::buffers_end(Buf.data())); + + size_t ContentLength = 0; + auto Pos = Headers.find("Content-Length: "); + if (Pos == std::string::npos) + { + Pos = Headers.find("content-length: "); + } + if (Pos != std::string::npos) + { + size_t ValStart = Pos + 16; // length of "Content-Length: " + size_t ValEnd = Headers.find("\r\n", ValStart); + if (ValEnd != std::string::npos) + { + ContentLength = std::stoull(Headers.substr(ValStart, ValEnd - ValStart)); + } + } + + // Calculate how many body bytes were already read past the header boundary. + // asio::read_until may read past the delimiter, so Buf.data() contains everything read. + size_t HeaderEnd = Headers.find("\r\n\r\n") + 4; + size_t BodyBytesInBuf = Headers.size() > HeaderEnd ? Headers.size() - HeaderEnd : 0; + size_t Remaining = ContentLength > BodyBytesInBuf ? ContentLength - BodyBytesInBuf : 0; + + if (Remaining > 0) + { + std::vector BodyBuf(Remaining); + asio::read(Socket, asio::buffer(BodyBuf), Ec); + } +} + +static void +DrainPartialBody(asio::ip::tcp::socket& Socket, size_t BytesToRead) +{ + // Read headers first + asio::streambuf Buf; + std::error_code Ec; + asio::read_until(Socket, Buf, "\r\n\r\n", Ec); + if (Ec) + { + return; + } + + // Determine how many body bytes were already buffered past headers + std::string All(asio::buffers_begin(Buf.data()), asio::buffers_end(Buf.data())); + size_t HeaderEnd = All.find("\r\n\r\n") + 4; + size_t BodyBytesInBuf = All.size() > HeaderEnd ? All.size() - HeaderEnd : 0; + + if (BodyBytesInBuf < BytesToRead) + { + size_t Remaining = BytesToRead - BodyBytesInBuf; + std::vector BodyBuf(Remaining); + asio::read(Socket, asio::buffer(BodyBuf), Ec); + } +} + +struct FaultTcpServer +{ + using FaultHandler = std::function; + + asio::io_context m_IoContext; + asio::ip::tcp::acceptor m_Acceptor; + FaultHandler m_Handler; + std::thread m_Thread; + int m_Port; + + explicit FaultTcpServer(FaultHandler Handler) + : m_Acceptor(m_IoContext, asio::ip::tcp::endpoint(asio::ip::address_v4::loopback(), 0)) + , m_Handler(std::move(Handler)) + { + m_Port = m_Acceptor.local_endpoint().port(); + StartAccept(); + m_Thread = std::thread([this]() { m_IoContext.run(); }); + } + + ~FaultTcpServer() + { + std::error_code Ec; + m_Acceptor.close(Ec); + m_IoContext.stop(); + if (m_Thread.joinable()) + { + m_Thread.join(); + } + } + + FaultTcpServer(const FaultTcpServer&) = delete; + FaultTcpServer& operator=(const FaultTcpServer&) = delete; + + void StartAccept() + { + m_Acceptor.async_accept([this](std::error_code Ec, asio::ip::tcp::socket Socket) { + if (!Ec) + { + m_Handler(Socket); + } + if (m_Acceptor.is_open()) + { + StartAccept(); + } + }); + } + + HttpClient MakeClient(HttpClientSettings Settings = {}) + { + return HttpClient(fmt::format("127.0.0.1:{}", m_Port), Settings, /*CheckIfAbortFunction*/ {}); + } +}; + +TEST_CASE("httpclient.transport-faults") +{ + SUBCASE("connection reset before response") + { + FaultTcpServer Server([](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + std::error_code Ec; + Socket.set_option(asio::socket_base::linger(true, 0), Ec); + Socket.close(Ec); + }); + HttpClient Client = Server.MakeClient(); + HttpClient::Response Resp = Client.Get("/test"); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + } + + SUBCASE("connection closed before response") + { + FaultTcpServer Server([](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + std::error_code Ec; + Socket.shutdown(asio::ip::tcp::socket::shutdown_both, Ec); + Socket.close(Ec); + }); + HttpClient Client = Server.MakeClient(); + HttpClient::Response Resp = Client.Get("/test"); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + } + + SUBCASE("partial headers then close") + { + // libcurl parses the status line (200 OK) and accepts the response even though + // headers are truncated mid-field. It reports success with an empty body instead + // of an error. Ideally this should be detected as a transport failure. + FaultTcpServer Server([](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + std::string Partial = "HTTP/1.1 200 OK\r\nContent-"; + std::error_code Ec; + asio::write(Socket, asio::buffer(Partial), Ec); + Socket.close(Ec); + }); + HttpClient Client = Server.MakeClient(); + HttpClient::Response Resp = Client.Get("/test"); + WARN(!Resp.IsSuccess()); + WARN(Resp.Error.has_value()); + } + + SUBCASE("truncated body") + { + FaultTcpServer Server([](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + std::string Headers = MakeRawHttpHeaders(200, 1000); + std::error_code Ec; + asio::write(Socket, asio::buffer(Headers), Ec); + std::string PartialBody(100, 'x'); + asio::write(Socket, asio::buffer(PartialBody), Ec); + Socket.close(Ec); + }); + HttpClient Client = Server.MakeClient(); + HttpClient::Response Resp = Client.Get("/test"); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + } + + SUBCASE("connection reset mid-body") + { + FaultTcpServer Server([](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + std::string Headers = MakeRawHttpHeaders(200, 10000); + std::error_code Ec; + asio::write(Socket, asio::buffer(Headers), Ec); + std::string PartialBody(1000, 'x'); + asio::write(Socket, asio::buffer(PartialBody), Ec); + Socket.set_option(asio::socket_base::linger(true, 0), Ec); + Socket.close(Ec); + }); + HttpClient Client = Server.MakeClient(); + HttpClient::Response Resp = Client.Get("/test"); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + } + + SUBCASE("stalled response triggers timeout") + { + std::atomic StallActive{true}; + FaultTcpServer Server([&StallActive](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + std::string Headers = MakeRawHttpHeaders(200, 1000); + std::error_code Ec; + asio::write(Socket, asio::buffer(Headers), Ec); + while (StallActive.load()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + }); + + HttpClientSettings Settings; + Settings.Timeout = std::chrono::milliseconds(500); + HttpClient Client = Server.MakeClient(Settings); + + HttpClient::Response Resp = Client.Get("/test"); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + StallActive.store(false); + } + + SUBCASE("retry succeeds after transient failures") + { + std::atomic ConnCount{0}; + FaultTcpServer Server([&ConnCount](asio::ip::tcp::socket& Socket) { + int N = ConnCount.fetch_add(1); + DrainHttpRequest(Socket); + if (N < 2) + { + // Connection reset produces NETWORK_SEND_FAILURE which is retryable + std::error_code Ec; + Socket.set_option(asio::socket_base::linger(true, 0), Ec); + Socket.close(Ec); + } + else + { + std::string Response = MakeRawHttpResponse(200, "recovered"); + std::error_code Ec; + asio::write(Socket, asio::buffer(Response), Ec); + } + }); + + HttpClientSettings Settings; + Settings.RetryCount = 3; + HttpClient Client = Server.MakeClient(Settings); + + HttpClient::Response Resp = Client.Get("/test"); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "recovered"); + } +} + +TEST_CASE("httpclient.transport-faults-post") +{ + constexpr size_t kPostBodySize = 256 * 1024; + + auto MakePostBody = []() -> IoBuffer { + IoBuffer Buf(kPostBodySize); + uint8_t* Ptr = static_cast(Buf.MutableData()); + for (size_t i = 0; i < kPostBodySize; ++i) + { + Ptr[i] = static_cast(i & 0xFF); + } + Buf.SetContentType(ZenContentType::kBinary); + return Buf; + }; + + SUBCASE("POST: server resets before consuming body") + { + FaultTcpServer Server([](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + std::error_code Ec; + Socket.set_option(asio::socket_base::linger(true, 0), Ec); + Socket.close(Ec); + }); + HttpClient Client = Server.MakeClient(); + IoBuffer Body = MakePostBody(); + HttpClient::Response Resp = Client.Post("/test", Body); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + } + + SUBCASE("POST: server closes before consuming body") + { + FaultTcpServer Server([](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + std::error_code Ec; + Socket.shutdown(asio::ip::tcp::socket::shutdown_both, Ec); + Socket.close(Ec); + }); + HttpClient Client = Server.MakeClient(); + IoBuffer Body = MakePostBody(); + HttpClient::Response Resp = Client.Post("/test", Body); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + } + + SUBCASE("POST: server resets mid-body") + { + FaultTcpServer Server([](asio::ip::tcp::socket& Socket) { + DrainPartialBody(Socket, 8 * 1024); + std::error_code Ec; + Socket.set_option(asio::socket_base::linger(true, 0), Ec); + Socket.close(Ec); + }); + HttpClient Client = Server.MakeClient(); + IoBuffer Body = MakePostBody(); + HttpClient::Response Resp = Client.Post("/test", Body); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + } + + SUBCASE("POST: early error response before consuming body") + { + FaultTcpServer Server([](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + std::string Response = MakeRawHttpResponse(503, "service busy"); + std::error_code Ec; + asio::write(Socket, asio::buffer(Response), Ec); + Socket.shutdown(asio::ip::tcp::socket::shutdown_both, Ec); + Socket.close(Ec); + }); + HttpClient Client = Server.MakeClient(); + IoBuffer Body = MakePostBody(); + HttpClient::Response Resp = Client.Post("/test", Body); + CHECK(!Resp.IsSuccess()); + // With a large upload body, the server may RST the connection before the client + // reads the 503 response. Either outcome is valid: the client sees the HTTP 503 + // status, or it sees a transport-level error from the RST. + CHECK((Resp.StatusCode == HttpResponseCode::ServiceUnavailable || Resp.Error.has_value())); + } + + SUBCASE("POST: stalled upload triggers timeout") + { + std::atomic StallActive{true}; + FaultTcpServer Server([&StallActive](asio::ip::tcp::socket& Socket) { + DrainHttpRequest(Socket); + // Stop reading body — TCP window will fill and client send will stall + while (StallActive.load()) + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + }); + + HttpClientSettings Settings; + Settings.Timeout = std::chrono::milliseconds(2000); + HttpClient Client = Server.MakeClient(Settings); + + IoBuffer Body = MakePostBody(); + HttpClient::Response Resp = Client.Post("/test", Body); + CHECK(!Resp.IsSuccess()); + CHECK(Resp.Error.has_value()); + StallActive.store(false); + } + + SUBCASE("POST: retry with large body after transient failure") + { + std::atomic ConnCount{0}; + FaultTcpServer Server([&ConnCount](asio::ip::tcp::socket& Socket) { + int N = ConnCount.fetch_add(1); + if (N < 2) + { + DrainHttpRequest(Socket); + std::error_code Ec; + Socket.set_option(asio::socket_base::linger(true, 0), Ec); + Socket.close(Ec); + } + else + { + DrainFullHttpRequest(Socket); + std::string Response = MakeRawHttpResponse(200, "upload-ok"); + std::error_code Ec; + asio::write(Socket, asio::buffer(Response), Ec); + } + }); + + HttpClientSettings Settings; + Settings.RetryCount = 3; + HttpClient Client = Server.MakeClient(Settings); + + IoBuffer Body = MakePostBody(); + HttpClient::Response Resp = Client.Post("/test", Body); + CHECK(Resp.IsSuccess()); + CHECK_EQ(Resp.AsText(), "upload-ok"); + } +} + +void +httpclient_test_forcelink() +{ +} + +} // namespace zen + +#endif diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index 7a129a98c..336a3deee 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -269,6 +269,7 @@ struct LatencyTestResult LatencyTestResult MeasureLatency(HttpClient& Client, std::string_view Url); -void httpclient_forcelink(); // internal +void httpclient_forcelink(); // internal +void httpclient_test_forcelink(); // internal } // namespace zen diff --git a/src/zenhttp/zenhttp.cpp b/src/zenhttp/zenhttp.cpp index 0b5408453..ad14ecb8d 100644 --- a/src/zenhttp/zenhttp.cpp +++ b/src/zenhttp/zenhttp.cpp @@ -16,6 +16,7 @@ zenhttp_forcelinktests() { http_forcelink(); httpclient_forcelink(); + httpclient_test_forcelink(); forcelink_packageformat(); passwordsecurity_forcelink(); } diff --git a/xmake.lua b/xmake.lua index 3537c618d..d49743cb2 100644 --- a/xmake.lua +++ b/xmake.lua @@ -344,10 +344,10 @@ task("sln") task("test") set_menu { - usage = "xmake test --run=[core|store|server|integration|all]", + usage = "xmake test --run=[core|store|http|server|integration|all]", description = "Run Zen tests", options = { - {'r', "run", "kv", "all", "Run test(s)", " - all", " - core ", " - remotestore", " - store", " - server", " - integration"}, + {'r', "run", "kv", "all", "Run test(s)", " - all", " - core ", " - remotestore", " - store", " - http", " - server", " - integration"}, {'j', "junit", "k", nil, "Enable junit report output"} } } -- cgit v1.2.3 From 4d5caf7d011bf73c7b90ff1d8c1cfdad817fa2f5 Mon Sep 17 00:00:00 2001 From: Martin Ridgers Date: Fri, 27 Feb 2026 11:47:00 +0100 Subject: Ported "lane trace" feature from UE (by way of IAX) (#771) * Ported "lane trace" feature from UE (by way of IAX) --- CHANGELOG.md | 1 + src/zencore/include/zencore/trace.h | 1 + thirdparty/trace/lane_trace.h | 146 ++++++++ thirdparty/trace/lane_trace.inl | 646 ++++++++++++++++++++++++++++++++++++ thirdparty/trace/trace.h | 22 +- 5 files changed, 805 insertions(+), 11 deletions(-) create mode 100644 thirdparty/trace/lane_trace.h create mode 100644 thirdparty/trace/lane_trace.inl diff --git a/CHANGELOG.md b/CHANGELOG.md index e555dd86f..873e88761 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Feature: Added 404 page to dashboard, to make it easier to find your way back to a valid URL - Improvement: `zen oplog-import` now uses partial block requests to reduce download size - Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests +- Improvement: IAX's lane tracing - Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly - Bugfix: Time spent indexing local and remote state during `zen builds download` now show the correct time diff --git a/src/zencore/include/zencore/trace.h b/src/zencore/include/zencore/trace.h index 99a565151..d17e018ea 100644 --- a/src/zencore/include/zencore/trace.h +++ b/src/zencore/include/zencore/trace.h @@ -13,6 +13,7 @@ ZEN_THIRD_PARTY_INCLUDES_START # define TRACE_IMPLEMENT 0 #endif #include +#include #undef TRACE_IMPLEMENT ZEN_THIRD_PARTY_INCLUDES_END diff --git a/thirdparty/trace/lane_trace.h b/thirdparty/trace/lane_trace.h new file mode 100644 index 000000000..170526eb2 --- /dev/null +++ b/thirdparty/trace/lane_trace.h @@ -0,0 +1,146 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#if __has_include() +# include +# include +# include +# include +#else +# define UE_LANETRACE_ENABLED 2 +# define IOSTOREHTTPCLIENT_API +#endif + +//////////////////////////////////////////////////////////////////////////////// +#if !defined(UE_LANETRACE_ENABLED) +# define UE_LANETRACE_ENABLED CPUPROFILERTRACE_ENABLED && !UE_BUILD_SHIPPING +#endif + +#if UE_LANETRACE_ENABLED +# define LANETRACE_OFF_IMPL(...) +# define UE_API IOSTOREHTTPCLIENT_API +#else +# define LANETRACE_OFF_IMPL(...) { return __VA_ARGS__ ; } +# define UE_API inline +#endif + + +//////////////////////////////////////////////////////////////////////////////// +#if UE_LANETRACE_ENABLED == 2 +namespace trace { +#endif + +//////////////////////////////////////////////////////////////////////////////// +struct FLaneTraceSpec +{ + FAnsiStringView Name; + FAnsiStringView Group = "Lanes"; + const void* Channel; + int32 Weight = 100; +}; + +class FLaneTrace; +UE_API FLaneTrace* LaneTrace_New(const FLaneTraceSpec& Spec) LANETRACE_OFF_IMPL(nullptr); +UE_API void LaneTrace_Delete(FLaneTrace* Lane) LANETRACE_OFF_IMPL(); +UE_API uint32 LaneTrace_NewScope(const FAnsiStringView& Name) LANETRACE_OFF_IMPL(1); +UE_API void LaneTrace_Enter(FLaneTrace* Lane, uint32 ScopeId) LANETRACE_OFF_IMPL(); +UE_API void LaneTrace_Change(FLaneTrace* Lane, uint32 ScopeId) LANETRACE_OFF_IMPL(); +UE_API void LaneTrace_Leave(FLaneTrace* Lane) LANETRACE_OFF_IMPL(); +UE_API void LaneTrace_LeaveAll(FLaneTrace* Lane) LANETRACE_OFF_IMPL(); + + + +//////////////////////////////////////////////////////////////////////////////// +struct FLanePostcode +{ + FLanePostcode(const void* In) : Value(UPTRINT(In)) {} + FLanePostcode(UPTRINT In) : Value(In) {} + UPTRINT Value; +}; + +class FLaneEstate; +UE_API FLaneEstate* LaneEstate_New(const FLaneTraceSpec& Spec) LANETRACE_OFF_IMPL(nullptr); +UE_API void LaneEstate_Delete(FLaneEstate* Estate) LANETRACE_OFF_IMPL(); +UE_API FLaneTrace* LaneEstate_Build(FLaneEstate* Estate, FLanePostcode Postcode) LANETRACE_OFF_IMPL(nullptr); +UE_API FLaneTrace* LaneEstate_Lookup(FLaneEstate* Estate, FLanePostcode Postcode) LANETRACE_OFF_IMPL(nullptr); +UE_API void LaneEstate_Demolish(FLaneEstate* Estate, FLanePostcode Postcode)LANETRACE_OFF_IMPL(); + +#undef LANETRACE_OFF_IMPL +#undef UE_API + + + +#if UE_LANETRACE_ENABLED + +//////////////////////////////////////////////////////////////////////////////// +class FLaneTraceScope +{ +public: + FLaneTraceScope(FLaneTrace* InLane, uint32 Scope); + FLaneTraceScope() = default; + ~FLaneTraceScope(); + FLaneTraceScope(FLaneTraceScope&& Rhs); + FLaneTraceScope& operator = (FLaneTraceScope&& Rhs); + void Change(uint32 Scope) const; + +private: + FLaneTraceScope(const FLaneTraceScope&) = delete; + FLaneTraceScope& operator = (const FLaneTraceScope&) = delete; + FLaneTrace* Lane = nullptr; +}; + +//////////////////////////////////////////////////////////////////////////////// +inline FLaneTraceScope::FLaneTraceScope(FLaneTrace* InLane, uint32 Scope) +: Lane(InLane) +{ + LaneTrace_Enter(Lane, Scope); +} + +//////////////////////////////////////////////////////////////////////////////// +inline FLaneTraceScope::~FLaneTraceScope() +{ + if (Lane) + { + LaneTrace_Leave(Lane); + } +} + +//////////////////////////////////////////////////////////////////////////////// +inline FLaneTraceScope::FLaneTraceScope(FLaneTraceScope&& Rhs) +{ + Swap(Rhs.Lane, Lane); +} + +//////////////////////////////////////////////////////////////////////////////// +inline FLaneTraceScope& FLaneTraceScope::operator = (FLaneTraceScope&& Rhs) +{ + Swap(Rhs.Lane, Lane); + return *this; +} + +//////////////////////////////////////////////////////////////////////////////// +inline void FLaneTraceScope::Change(uint32 Scope) const +{ + LaneTrace_Change(Lane, Scope); +} + +#else // UE_LANETRACE_ENABLED + +class FLaneTraceScope +{ +public: + FLaneTraceScope(...) {} + void Change(uint32) const {} +}; + +#endif // UE_LANETRACE_ENABLED + +//////////////////////////////////////////////////////////////////////////////// +#if UE_LANETRACE_ENABLED == 2 +} // namespace trace +#endif + +#if defined(TRACE_IMPLEMENT) && TRACE_IMPLEMENT +# include "lane_trace.inl" +#endif diff --git a/thirdparty/trace/lane_trace.inl b/thirdparty/trace/lane_trace.inl new file mode 100644 index 000000000..abfaab027 --- /dev/null +++ b/thirdparty/trace/lane_trace.inl @@ -0,0 +1,646 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#if __has_include() +# include +# if UE_LANETRACE_ENABLED +# include +# include +# include +# include +# endif +#endif + +#if UE_LANETRACE_ENABLED + +//////////////////////////////////////////////////////////////////////////////// +#if UE_LANETRACE_ENABLED == 2 + +#include + +#define check(e) do { if ((e) == false) *(int volatile*)0 = 0; } while (false) + +struct FCriticalSection +{ + void Lock() { Inner.lock(); } + void Unlock() { Inner.unlock(); } + std::mutex Inner; +}; + +struct FScopeLock +{ + FScopeLock(FCriticalSection* Lock) : Inner(Lock->Inner) {} + std::unique_lock Inner; +}; + +namespace trace { +#endif + +// For this to work Insights would need to know it there may be a ThreadId field +#define LANETRACE_USE_V2_EVENTS 0 + +namespace LaneTraceDetail +{ + +//////////////////////////////////////////////////////////////////////////////// +UE_TRACE_EVENT_BEGIN($Trace, ThreadInfo, NoSync|Important) + UE_TRACE_EVENT_FIELD(uint32, ThreadId) + UE_TRACE_EVENT_FIELD(int32, SortHint) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, Name) +UE_TRACE_EVENT_END() + +UE_TRACE_EVENT_BEGIN(CpuProfiler, EventBatch) + UE_TRACE_EVENT_FIELD(uint8[], Data) + UE_TRACE_EVENT_FIELD(uint16, ThreadId) +UE_TRACE_EVENT_END() + +#if LANETRACE_USE_V2_EVENTS +UE_TRACE_EVENT_BEGIN(CpuProfiler, EventBatchV2) + UE_TRACE_EVENT_FIELD(uint8[], Data) + UE_TRACE_EVENT_FIELD(uint16, ThreadId) +UE_TRACE_EVENT_END() +#endif + +//////////////////////////////////////////////////////////////////////////////// +static int32 Encode32_7bit(int32 Value, void* __restrict Out) +{ + // Calculate the number of bytes + int32 Length = 1; + + Length += (Value >= (1 << 7)); + Length += (Value >= (1 << 14)); + Length += (Value >= (1 << 21)); + + // Add a gap every eigth bit for the continuations + int32 Ret = Value; + Ret = (Ret & 0x0000'3fff) | ((Ret & 0x0fff'c000) << 2); + Ret = (Ret & 0x007f'007f) | ((Ret & 0x3f80'3f80) << 1); + + // Set the bits indicating another byte follows + int32 Continuations = 0x0080'8080; + Continuations >>= (sizeof(Value) - Length) * 8; + Ret |= Continuations; + + ::memcpy(Out, &Ret, sizeof(Value)); + + return Length; +} + +//////////////////////////////////////////////////////////////////////////////// +static int32 Encode64_7bit(int64 Value, void* __restrict Out) +{ + // Calculate the output length + uint32 Length = 1; + Length += (Value >= (1ll << 7)); + Length += (Value >= (1ll << 14)); + Length += (Value >= (1ll << 21)); + Length += (Value >= (1ll << 28)); + Length += (Value >= (1ll << 35)); + Length += (Value >= (1ll << 42)); + Length += (Value >= (1ll << 49)); + + // Add a gap every eigth bit for the continuations + int64 Ret = Value; + Ret = (Ret & 0x0000'0000'0fff'ffffull) | ((Ret & 0x00ff'ffff'f000'0000ull) << 4); + Ret = (Ret & 0x0000'3fff'0000'3fffull) | ((Ret & 0x0fff'c000'0fff'c000ull) << 2); + Ret = (Ret & 0x007f'007f'007f'007full) | ((Ret & 0x3f80'3f80'3f80'3f80ull) << 1); + + // Set the bits indicating another byte follows + int64 Continuations = 0x0080'8080'8080'8080ull; + Continuations >>= (sizeof(Value) - Length) * 8; + Ret |= Continuations; + + ::memcpy(Out, &Ret, sizeof(Value)); + + return Length; +} + +//////////////////////////////////////////////////////////////////////////////// +static uint64 TimeGetTimestamp() +{ +#if UE_LANETRACE_ENABLED != 2 + return FPlatformTime::Cycles64(); +#else + return trace::detail::TimeGetTimestamp(); +#endif +} + + + +//////////////////////////////////////////////////////////////////////////////// +class FScopeBuffer +{ +public: + FScopeBuffer(UE::Trace::FChannel& InChannel); + void SetThreadId(uint32 Value); + bool IsInScope() const; + uint32 GetDepth() const; + void Flush(bool Force=false); + void Enter(uint64 Timestamp, uint32 ScopeId); + void Leave(uint64 Timestamp); + +private: + enum + { + BufferSize = 128, + Overflow = 24, + EnterLsb = 1, + LeaveLsb = 0, + TraceEventBatchVer = 1 + LANETRACE_USE_V2_EVENTS, + }; + uint64 LastTimestamp = 0; + uint64 PrevTimestamp = 0; + uint8* Cursor = Buffer; + UE::Trace::FChannel& Channel; + uint32 ThreadIdOverride = 0; + uint16 Depth = 0; + uint8 Buffer[BufferSize]; +}; + +//////////////////////////////////////////////////////////////////////////////// +FScopeBuffer::FScopeBuffer(UE::Trace::FChannel& InChannel) +: Channel(InChannel) +{ +} + +//////////////////////////////////////////////////////////////////////////////// +void FScopeBuffer::SetThreadId(uint32 Value) +{ + ThreadIdOverride = Value; +} + +//////////////////////////////////////////////////////////////////////////////// +bool FScopeBuffer::IsInScope() const +{ + return GetDepth() > 0; +} + +//////////////////////////////////////////////////////////////////////////////// +uint32 FScopeBuffer::GetDepth() const +{ + return Depth; +} + +//////////////////////////////////////////////////////////////////////////////// +void FScopeBuffer::Flush(bool Force) +{ + if (Cursor == Buffer) + { + return; + } + + if (Depth > 0 && !Force && (Cursor <= (Buffer + BufferSize - Overflow))) + { + return; + } + + if (TraceEventBatchVer == 1) + { + UE_TRACE_LOG(CpuProfiler, EventBatch, Channel) + << EventBatch.ThreadId(uint16(ThreadIdOverride)) + << EventBatch.Data(Buffer, uint32(ptrdiff_t(Cursor - Buffer))); + } +#if LANETRACE_USE_V2_EVENTS + else + { + UE_TRACE_LOG(CpuProfiler, EventBatchV2, Channel) + << EventBatchV2.ThreadId(uint16(ThreadIdOverride)) + << EventBatchV2.Data(Buffer, uint32(ptrdiff_t(Cursor - Buffer))); + + // Both protocols should really do this rebase but it make analysis go + // bonkers and I'm not looking into that right now + PrevTimestamp = 0; + } +#endif + + LastTimestamp = 0; + PrevTimestamp = 0; + + Cursor = Buffer; +} + +//////////////////////////////////////////////////////////////////////////////// +void FScopeBuffer::Enter(uint64 Timestamp, uint32 ScopeId) +{ + check(Timestamp >= LastTimestamp); + LastTimestamp = Timestamp; + + PrevTimestamp += (Timestamp -= PrevTimestamp); + enum { Shift = (TraceEventBatchVer == 1) ? 1 : 2 }; + Cursor += Encode64_7bit((Timestamp << Shift) | EnterLsb, Cursor); + Cursor += Encode32_7bit(ScopeId, Cursor); + Depth++; +} + +//////////////////////////////////////////////////////////////////////////////// +void FScopeBuffer::Leave(uint64 Timestamp) +{ + check(Timestamp >= LastTimestamp); + LastTimestamp = Timestamp; + + if (Depth == 0) + { + return; + } + + PrevTimestamp += (Timestamp -= PrevTimestamp); + enum { Shift = (TraceEventBatchVer == 1) ? 1 : 2 }; + Cursor += Encode64_7bit((Timestamp << Shift) | LeaveLsb, Cursor); + Depth--; +} + + + +//////////////////////////////////////////////////////////////////////////////// +struct FLoctight +{ + struct FScope + { + FScope() = default; + ~FScope(); + FScope(FScope&& Rhs); + FScope(const FScope&) = delete; + FScope& operator = (const FScope&) = delete; + FScope& operator = (FScope&&) = delete; + FCriticalSection* Outer = nullptr; + }; + FScope Scope() const; + mutable FCriticalSection Loch; +}; + +//////////////////////////////////////////////////////////////////////////////// +FLoctight::FScope::~FScope() +{ + if (Outer) + { + Outer->Unlock(); + } +} + +//////////////////////////////////////////////////////////////////////////////// +FLoctight::FScope::FScope(FScope&& Rhs) +{ + Swap(Outer, Rhs.Outer); +} + +//////////////////////////////////////////////////////////////////////////////// +FLoctight::FScope FLoctight::Scope() const +{ + Loch.Lock(); + FScope Ret; + Ret.Outer = &Loch; + return Ret; +} + + + +//////////////////////////////////////////////////////////////////////////////// +class FScopeBufferTs + : protected FLoctight + , protected FScopeBuffer +{ +public: + void SetThreadId(uint32 Value); + bool IsInScope() const; + void Flush(bool Force=false); + void Enter(uint32 ScopeId); + void Leave(); +}; + +//////////////////////////////////////////////////////////////////////////////// +void FScopeBufferTs::SetThreadId(uint32 Value) +{ + FScope _ = Scope(); + FScopeBuffer::SetThreadId(Value); +} + +//////////////////////////////////////////////////////////////////////////////// +bool FScopeBufferTs::IsInScope() const +{ + FScope _ = Scope(); + return FScopeBuffer::IsInScope(); +} + +//////////////////////////////////////////////////////////////////////////////// +void FScopeBufferTs::Flush(bool Force) +{ + FScope _ = Scope(); + FScopeBuffer::Flush(Force); +} + +//////////////////////////////////////////////////////////////////////////////// +void FScopeBufferTs::Enter(uint32 ScopeId) +{ + uint64 Timestamp = TimeGetTimestamp(); + FScope _ = Scope(); + FScopeBuffer::Enter(Timestamp, ScopeId); +} + +//////////////////////////////////////////////////////////////////////////////// +void FScopeBufferTs::Leave() +{ + uint64 Timestamp = TimeGetTimestamp(); + FScope _ = Scope(); + FScopeBuffer::Leave(Timestamp); +} + + + +//////////////////////////////////////////////////////////////////////////////// +class FLane +{ +public: + FLane(const FLaneTraceSpec& Spec); + ~FLane(); + static uint32 NewScope(const FAnsiStringView& Name); + void Enter(uint32 ScopeId); + void Change(uint32 ScopeId); + void Leave(); + void LeaveAll(); + +private: + FScopeBuffer Buffer; +}; + +//////////////////////////////////////////////////////////////////////////////// +FLane::FLane(const FLaneTraceSpec& Spec) +: Buffer(*(UE::Trace::FChannel*)(Spec.Channel)) +{ + static uint32 volatile NextId = 0; + uint32 Id = UE::Trace::Private::AtomicAddRelaxed(&NextId, 1u) + 1; + Id += 2 << 10; + + uint32 NameSize = uint32(Spec.Name.Len()); + UE_TRACE_LOG($Trace, ThreadInfo, true, NameSize) + << ThreadInfo.ThreadId(Id) + << ThreadInfo.SortHint(Spec.Weight) + << ThreadInfo.Name(Spec.Name.GetData(), NameSize); + + Buffer.SetThreadId(Id); +} + +//////////////////////////////////////////////////////////////////////////////// +FLane::~FLane() +{ + Buffer.Flush(true); +} + +//////////////////////////////////////////////////////////////////////////////// +uint32 FLane::NewScope(const FAnsiStringView& Name) +{ +#if UE_LANETRACE_ENABLED != 2 + return FCpuProfilerTrace::OutputEventType(Name.GetData(), "", 0u); +#else + return uint32(ScopeNew(Name)); +#endif +} + +//////////////////////////////////////////////////////////////////////////////// +void FLane::Enter(uint32 ScopeId) +{ + uint64 Timestamp = TimeGetTimestamp(); + Buffer.Enter(Timestamp, ScopeId); + Buffer.Flush(false); +} + +//////////////////////////////////////////////////////////////////////////////// +void FLane::Change(uint32 ScopeId) +{ + uint64 Timestamp = TimeGetTimestamp(); + Buffer.Leave(Timestamp); + Buffer.Enter(Timestamp, ScopeId); + Buffer.Flush(false); +} + +//////////////////////////////////////////////////////////////////////////////// +void FLane::Leave() +{ + uint64 Timestamp = TimeGetTimestamp(); + Buffer.Leave(Timestamp); + Buffer.Flush(false); +} + +//////////////////////////////////////////////////////////////////////////////// +void FLane::LeaveAll() +{ + uint64 Timestamp = TimeGetTimestamp(); + while (Buffer.IsInScope()) + { + Buffer.Leave(Timestamp); + } + Buffer.Flush(true); +} + +} // namespace LaneTraceDetail + + + +//////////////////////////////////////////////////////////////////////////////// +class FLaneTrace + : public LaneTraceDetail::FLane +{ + using LaneTraceDetail::FLane::FLane; +}; + +//////////////////////////////////////////////////////////////////////////////// +FLaneTrace* LaneTrace_New(const FLaneTraceSpec& Spec) +{ + return new FLaneTrace(Spec); +} + +//////////////////////////////////////////////////////////////////////////////// +void LaneTrace_Delete(FLaneTrace* Lane) +{ + delete Lane; +} + +//////////////////////////////////////////////////////////////////////////////// +uint32 LaneTrace_NewScope(const FAnsiStringView& Name) +{ + return FLaneTrace::NewScope(Name); +} + +//////////////////////////////////////////////////////////////////////////////// +void LaneTrace_Enter(FLaneTrace* Lane, uint32 ScopeId) +{ + Lane->Enter(ScopeId); +} + +//////////////////////////////////////////////////////////////////////////////// +void LaneTrace_Change(FLaneTrace* Lane, uint32 ScopeId) +{ + Lane->Change(ScopeId); +} + +//////////////////////////////////////////////////////////////////////////////// +void LaneTrace_Leave(FLaneTrace* Lane) +{ + Lane->Leave(); +} + +//////////////////////////////////////////////////////////////////////////////// +void LaneTrace_LeaveAll(FLaneTrace* Lane) +{ + Lane->LeaveAll(); +} + + + +namespace LaneTraceDetail +{ + +//////////////////////////////////////////////////////////////////////////////// +class FEstate +{ +public: + FEstate(const FLaneTraceSpec& Spec); + ~FEstate(); + FLaneTrace* Build(UPTRINT Postcode); + FLaneTrace* Lookup(UPTRINT Postcode); + void Demolish(UPTRINT Postcode); + +private: + struct FEntry + { + UPTRINT Postcode = 0; + FLaneTrace* Lane = nullptr; + }; + + enum { GROWTH_SIZE = 4 }; + + FLaneTraceSpec LaneSpec; + FCriticalSection Lock; + TArray Directory; +}; + +//////////////////////////////////////////////////////////////////////////////// +FEstate::FEstate(const FLaneTraceSpec& InSpec) +: LaneSpec(InSpec) +{ + Directory.SetNum(GROWTH_SIZE); +} + +//////////////////////////////////////////////////////////////////////////////// +FEstate::~FEstate() +{ + for (FEntry& Entry : Directory) + { + if (Entry.Lane != nullptr) + { + LaneTrace_Delete(Entry.Lane); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +FLaneTrace* FEstate::Build(UPTRINT Postcode) +{ + auto UseEstate = [this] (UPTRINT Postcode, FEntry& Entry) + { + Entry.Postcode = Postcode; + if (Entry.Lane == nullptr) + { + Entry.Lane = LaneTrace_New(LaneSpec); + } + + return Entry.Lane; + }; + + FScopeLock _(&Lock); + + for (FEntry& Entry : Directory) + { + if (Entry.Postcode == 0) + { + return UseEstate(Postcode, Entry); + } + } + + int32 NextSize = Directory.Num() + GROWTH_SIZE; + Directory.SetNum(NextSize); + return UseEstate(Postcode, Directory[NextSize - GROWTH_SIZE]); +} + +//////////////////////////////////////////////////////////////////////////////// +FLaneTrace* FEstate::Lookup(UPTRINT Postcode) +{ + FScopeLock _(&Lock); + + for (const FEntry& Entry : Directory) + { + if (Entry.Postcode == Postcode) + { + return Entry.Lane; + } + } + + checkf(false, TEXT("Invalid/unknown postcode given, unable to find estate: %llx"), Postcode); + return nullptr; +} + +//////////////////////////////////////////////////////////////////////////////// +void FEstate::Demolish(UPTRINT Postcode) +{ + FScopeLock _(&Lock); + + for (FEntry& Entry : Directory) + { + if (Entry.Postcode == Postcode) + { + LaneTrace_LeaveAll(Entry.Lane); + Entry.Postcode = 0; + return; + } + } + + checkf(false, TEXT("Invalid/unknown postcode given, unable to demolish estate: %llx"), Postcode); +} + +} // namespace LaneTraceDetail + + + +//////////////////////////////////////////////////////////////////////////////// +class FLaneEstate + : public LaneTraceDetail::FEstate +{ +public: + using LaneTraceDetail::FEstate::FEstate; +}; + +//////////////////////////////////////////////////////////////////////////////// +FLaneEstate*LaneEstate_New(const FLaneTraceSpec& Spec) +{ + return new FLaneEstate(Spec); +} + +//////////////////////////////////////////////////////////////////////////////// +void LaneEstate_Delete(FLaneEstate* Estate) +{ + delete Estate; +} + +//////////////////////////////////////////////////////////////////////////////// +FLaneTrace* LaneEstate_Build(FLaneEstate* Estate, FLanePostcode Postcode) +{ + return Estate->Build(Postcode.Value); +} + +//////////////////////////////////////////////////////////////////////////////// +FLaneTrace* LaneEstate_Lookup(FLaneEstate* Estate, FLanePostcode Postcode) +{ + return Estate->Lookup(Postcode.Value); +} + +//////////////////////////////////////////////////////////////////////////////// +void LaneEstate_Demolish(FLaneEstate* Estate, FLanePostcode Postcode) +{ + return Estate->Demolish(Postcode.Value); +} + +#undef LANETRACE_UNTESTED + +#if UE_LANETRACE_ENABLED == 2 +} // namespace trace +#endif + +#endif // UE_LANETRACE_ENABLED diff --git a/thirdparty/trace/trace.h b/thirdparty/trace/trace.h index a1fce80a6..0835781ba 100644 --- a/thirdparty/trace/trace.h +++ b/thirdparty/trace/trace.h @@ -190,7 +190,7 @@ struct TArray : public std::vector { using Super = std::vector; - using Super::vector; + using Super::Super; using Super::back; using Super::begin; using Super::clear; @@ -260,7 +260,7 @@ struct TStringViewAdapter : public std::basic_string_view { using Super = std::basic_string_view; - using Super::basic_string_view; + using Super::Super; using Super::size; using Super::data; size_t Len() const { return size(); } @@ -596,8 +596,8 @@ namespace UE { namespace Trace { enum AnsiString {}; enum WideString {}; -template -struct TEventRef +template +struct TEventRef { using ReferenceType = IdType; TEventRef(IdType InId, uint32 InTypeId) @@ -632,7 +632,7 @@ inline uint64 TEventRef::GetHash() const return (uint64(RefTypeId) << 32) ^ Id; } typedef TEventRef FEventRef8; -typedef TEventRef FEventRef16; +typedef TEventRef FEventRef16; typedef TEventRef FEventRef32; typedef TEventRef FEventRef64; template @@ -653,7 +653,7 @@ struct FChannelInfo; typedef void* AllocFunc(SIZE_T, uint32); typedef void FreeFunc(void*, SIZE_T); typedef void ChannelIterFunc(const ANSICHAR*, bool, void*); -/* The callback provides information about a channel and a user provided pointer. +/* The callback provides information about a channel and a user provided pointer. Returning false from the callback will stop the enumeration */ typedef bool ChannelIterCallback(const FChannelInfo& Info, void*/*User*/); struct FStatistics @@ -673,7 +673,7 @@ struct FSendFlags }; UE_TRACE_API void SetMemoryHooks(AllocFunc Alloc, FreeFunc Free) UE_TRACE_IMPL(); UE_TRACE_API void Initialize(const FInitializeDesc& Desc) UE_TRACE_IMPL(); -UE_TRACE_API void StartWorkerThread() UE_TRACE_IMPL(); +UE_TRACE_API void StartWorkerThread() UE_TRACE_IMPL(); UE_TRACE_API void Shutdown() UE_TRACE_IMPL(); UE_TRACE_API void Update() UE_TRACE_IMPL(); UE_TRACE_API void GetStatistics(FStatistics& Out) UE_TRACE_IMPL(); @@ -718,7 +718,7 @@ struct FChannelInfo bool bIsEnabled; bool bIsReadOnly; }; -typedef void ChannelIterFunc(const ANSICHAR*, bool, void*); +typedef void ChannelIterFunc(const ANSICHAR*, bool, void*); typedef bool ChannelIterCallback(const FChannelInfo& OutChannelInfo, void*); /* A named channel which can be used to filter trace events. Channels can be @@ -1277,7 +1277,7 @@ struct FNewEventEvent struct { uint16 Offset; - uint16 RefUid; + uint16 RefUid; uint8 TypeInfo; uint8 NameSize; } Reference; @@ -2519,7 +2519,7 @@ FChannel* FChannel::FindChannel(const ANSICHAR* ChannelName) } return nullptr; } -void FChannel::EnumerateChannels(ChannelIterFunc Func, void* User) +void FChannel::EnumerateChannels(ChannelIterFunc Func, void* User) { using namespace Private; FChannel* ChannelLists[] = @@ -4985,7 +4985,7 @@ UPTRINT TcpSocketConnect(const ANSICHAR* Host, uint16 Port) { struct FAddrInfoPtr { - ~FAddrInfoPtr() + ~FAddrInfoPtr() { if (Value != nullptr) { -- cgit v1.2.3 From 9e7019aa16b19cd87aa6af3ef39825edb039c8be Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 27 Feb 2026 13:12:10 +0100 Subject: add support in http client to accept multi-range responses (#788) * add support in http client to accept multi-range responses --- CHANGELOG.md | 1 + src/zenhttp/clients/httpclientcommon.cpp | 315 ++++++++++++ src/zenhttp/clients/httpclientcommon.h | 109 ++++- src/zenhttp/clients/httpclientcpr.cpp | 530 ++++++++++++--------- src/zenhttp/clients/httpclientcpr.h | 16 +- src/zenhttp/httpclient.cpp | 41 ++ src/zenhttp/include/zenhttp/httpclient.h | 14 + .../builds/buildstorageoperations.cpp | 152 +++--- 8 files changed, 859 insertions(+), 319 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 873e88761..dee9a26e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Feature: Add `--allow-partial-block-requests` to `zen oplog-import` - Feature: `zen ui` can be used to open dashboards for local instances - Feature: Added 404 page to dashboard, to make it easier to find your way back to a valid URL +- Feature: HttpClient now properly handles multi-part request/response - Improvement: `zen oplog-import` now uses partial block requests to reduce download size - Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests - Improvement: IAX's lane tracing diff --git a/src/zenhttp/clients/httpclientcommon.cpp b/src/zenhttp/clients/httpclientcommon.cpp index 47425e014..312ca16d2 100644 --- a/src/zenhttp/clients/httpclientcommon.cpp +++ b/src/zenhttp/clients/httpclientcommon.cpp @@ -382,6 +382,178 @@ namespace detail { return Result; } + MultipartBoundaryParser::MultipartBoundaryParser() : BoundaryEndMatcher("--"), HeaderEndMatcher("\r\n\r\n") {} + + bool MultipartBoundaryParser::Init(const std::string_view ContentTypeHeaderValue) + { + std::string LowerCaseValue = ToLower(ContentTypeHeaderValue); + if (LowerCaseValue.starts_with("multipart/byteranges")) + { + size_t BoundaryPos = LowerCaseValue.find("boundary="); + if (BoundaryPos != std::string::npos) + { + // Yes, we do a substring of the non-lowercase value string as we want the exact boundary string + std::string_view BoundaryName = std::string_view(ContentTypeHeaderValue).substr(BoundaryPos + 9); + if (!BoundaryName.empty()) + { + size_t BoundaryEnd = std::string::npos; + while (BoundaryName[0] == ' ') + { + BoundaryName = BoundaryName.substr(1); + } + if (!BoundaryName.empty()) + { + if (BoundaryName.size() > 2 && BoundaryName.front() == '"' && BoundaryName.back() == '"') + { + BoundaryEnd = BoundaryName.find('"', 1); + if (BoundaryEnd != std::string::npos) + { + BoundaryBeginMatcher.Init(fmt::format("\r\n--{}", BoundaryName.substr(1, BoundaryEnd - 1))); + return true; + } + } + else + { + BoundaryEnd = BoundaryName.find_first_of(" \r\n"); + BoundaryBeginMatcher.Init(fmt::format("\r\n--{}", BoundaryName.substr(0, BoundaryEnd))); + return true; + } + } + } + } + } + return false; + } + + void MultipartBoundaryParser::InternalParseInput(std::string_view data) + { + size_t ScanPos = 0; + while (ScanPos < data.length()) + { + const char ScanChar = data[ScanPos]; + if (BoundaryBeginMatcher.MatchState == IncrementalStringMatcher::EMatchState::Complete) + { + if (PayloadOffset + ScanPos < (BoundaryBeginMatcher.GetMatchEndOffset() + BoundaryEndMatcher.GetMatchString().length())) + { + BoundaryEndMatcher.Match(PayloadOffset + ScanPos, ScanChar); + if (BoundaryEndMatcher.MatchState == IncrementalStringMatcher::EMatchState::Complete) + { + BoundaryBeginMatcher.Reset(); + HeaderEndMatcher.Reset(); + BoundaryEndMatcher.Reset(); + BoundaryHeader.Reset(); + break; + } + } + + BoundaryHeader.Append(ScanChar); + + HeaderEndMatcher.Match(PayloadOffset + ScanPos, ScanChar); + + if (HeaderEndMatcher.MatchState == IncrementalStringMatcher::EMatchState::Complete) + { + const uint64_t HeaderStartOffset = BoundaryBeginMatcher.GetMatchEndOffset(); + const uint64_t HeaderEndOffset = HeaderEndMatcher.GetMatchStartOffset(); + const uint64_t HeaderLength = HeaderEndOffset - HeaderStartOffset; + std::string_view HeaderText(BoundaryHeader.ToView().substr(0, HeaderLength)); + + uint64_t OffsetInPayload = PayloadOffset + ScanPos + 1; + + uint64_t RangeOffset = 0; + uint64_t RangeLength = 0; + HttpContentType ContentType = HttpContentType::kBinary; + + ForEachStrTok(HeaderText, "\r\n", [&](std::string_view Line) { + const std::pair KeyAndValue = GetHeaderKeyAndValue(Line); + const std::string_view Key = KeyAndValue.first; + const std::string_view Value = KeyAndValue.second; + if (Key == "Content-Range") + { + std::pair ContentRange = ParseContentRange(Value); + if (ContentRange.second != 0) + { + RangeOffset = ContentRange.first; + RangeLength = ContentRange.second; + } + } + else if (Key == "Content-Type") + { + ContentType = ParseContentType(Value); + } + + return true; + }); + + if (RangeLength > 0) + { + Boundaries.push_back(HttpClient::Response::MultipartBoundary{.OffsetInPayload = OffsetInPayload, + .RangeOffset = RangeOffset, + .RangeLength = RangeLength, + .ContentType = ContentType}); + } + + BoundaryBeginMatcher.Reset(); + HeaderEndMatcher.Reset(); + BoundaryEndMatcher.Reset(); + BoundaryHeader.Reset(); + } + } + else + { + BoundaryBeginMatcher.Match(PayloadOffset + ScanPos, ScanChar); + } + ScanPos++; + } + PayloadOffset += data.length(); + } + + std::pair GetHeaderKeyAndValue(std::string_view HeaderString) + { + size_t DelimiterPos = HeaderString.find(':'); + if (DelimiterPos != std::string::npos) + { + std::string_view Key = HeaderString.substr(0, DelimiterPos); + constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n"); + Key = AsciiSet::TrimSuffixWith(Key, WhitespaceCharacters); + Key = AsciiSet::TrimPrefixWith(Key, WhitespaceCharacters); + + std::string_view Value = HeaderString.substr(DelimiterPos + 1); + Value = AsciiSet::TrimSuffixWith(Value, WhitespaceCharacters); + Value = AsciiSet::TrimPrefixWith(Value, WhitespaceCharacters); + return std::make_pair(Key, Value); + } + return std::make_pair(HeaderString, std::string_view{}); + } + + std::pair ParseContentRange(std::string_view Value) + { + if (Value.starts_with("bytes ")) + { + size_t RangeSplitPos = Value.find('-', 6); + if (RangeSplitPos != std::string::npos) + { + size_t RangeEndLength = Value.find('/', RangeSplitPos + 1); + if (RangeEndLength == std::string::npos) + { + RangeEndLength = Value.length() - (RangeSplitPos + 1); + } + else + { + RangeEndLength = RangeEndLength - (RangeSplitPos + 1); + } + std::optional RequestedRangeStart = ParseInt(Value.substr(6, RangeSplitPos - 6)); + std::optional RequestedRangeEnd = ParseInt(Value.substr(RangeSplitPos + 1, RangeEndLength)); + if (RequestedRangeStart.has_value() && RequestedRangeEnd.has_value()) + { + uint64_t RangeOffset = RequestedRangeStart.value(); + uint64_t RangeLength = RequestedRangeEnd.value() - RangeOffset + 1; + return std::make_pair(RangeOffset, RangeLength); + } + } + } + return {0, 0}; + } + } // namespace detail } // namespace zen @@ -470,5 +642,148 @@ TEST_CASE("CompositeBufferReadStream") CHECK_EQ(IoHash::HashBuffer(Data), testutil::HashComposite(Data)); } +TEST_CASE("MultipartBoundaryParser") +{ + uint64_t Range1Offset = 2638; + uint64_t Range1Length = (5111437 - Range1Offset) + 1; + + uint64_t Range2Offset = 5118199; + uint64_t Range2Length = (9147741 - Range2Offset) + 1; + + std::string_view ContentTypeHeaderValue1 = "multipart/byteranges; boundary=00000000000000019229"; + std::string_view ContentTypeHeaderValue2 = "multipart/byteranges; boundary=\"00000000000000019229\""; + + { + std::string_view Example1 = + "\r\n--00000000000000019229\r\n" + "Content-Type: application/x-ue-comp\r\n" + "Content-Range: bytes 2638-5111437/44369878\r\n" + "\r\n" + "datadatadatadata" + "\r\n--00000000000000019229\r\n" + "Content-Type: application/x-ue-comp\r\n" + "Content-Range: bytes 5118199-9147741/44369878\r\n" + "\r\n" + "ditaditadita" + "\r\n--00000000000000019229--"; + + detail::MultipartBoundaryParser ParserExample1; + ParserExample1.Init(ContentTypeHeaderValue1); + + const size_t InputWindow = 7; + for (size_t Offset = 0; Offset < Example1.length(); Offset += InputWindow) + { + ParserExample1.ParseInput(Example1.substr(Offset, Min(Example1.length() - Offset, InputWindow))); + } + + CHECK(ParserExample1.Boundaries.size() == 2); + + CHECK(ParserExample1.Boundaries[0].RangeOffset == Range1Offset); + CHECK(ParserExample1.Boundaries[0].RangeLength == Range1Length); + CHECK(ParserExample1.Boundaries[1].RangeOffset == Range2Offset); + CHECK(ParserExample1.Boundaries[1].RangeLength == Range2Length); + } + + { + std::string_view Example2 = + "\r\n--00000000000000019229\r\n" + "Content-Type: application/x-ue-comp\r\n" + "Content-Range: bytes 2638-5111437/*\r\n" + "\r\n" + "datadatadatadata" + "\r\n--00000000000000019229\r\n" + "Content-Type: application/x-ue-comp\r\n" + "Content-Range: bytes 5118199-9147741/44369878\r\n" + "\r\n" + "ditaditadita" + "\r\n--00000000000000019229--"; + + detail::MultipartBoundaryParser ParserExample2; + ParserExample2.Init(ContentTypeHeaderValue1); + + const size_t InputWindow = 3; + for (size_t Offset = 0; Offset < Example2.length(); Offset += InputWindow) + { + std::string_view Window = Example2.substr(Offset, Min(Example2.length() - Offset, InputWindow)); + ParserExample2.ParseInput(Window); + } + + CHECK(ParserExample2.Boundaries.size() == 2); + + CHECK(ParserExample2.Boundaries[0].RangeOffset == Range1Offset); + CHECK(ParserExample2.Boundaries[0].RangeLength == Range1Length); + CHECK(ParserExample2.Boundaries[1].RangeOffset == Range2Offset); + CHECK(ParserExample2.Boundaries[1].RangeLength == Range2Length); + } + + { + std::string_view Example3 = + "\r\n--00000000000000019229\r\n" + "Content-Type: application/x-ue-comp\r\n" + "Content-Range: bytes 2638-5111437/*\r\n" + "\r\n" + "datadatadatadata" + "\r\n--00000000000000019229\r\n" + "Content-Type: application/x-ue-comp\r\n" + "Content-Range: bytes 5118199-9147741/44369878\r\n" + "\r\n" + "ditaditadita"; + + detail::MultipartBoundaryParser ParserExample3; + ParserExample3.Init(ContentTypeHeaderValue2); + + const size_t InputWindow = 31; + for (size_t Offset = 0; Offset < Example3.length(); Offset += InputWindow) + { + ParserExample3.ParseInput(Example3.substr(Offset, Min(Example3.length() - Offset, InputWindow))); + } + + CHECK(ParserExample3.Boundaries.size() == 2); + + CHECK(ParserExample3.Boundaries[0].RangeOffset == Range1Offset); + CHECK(ParserExample3.Boundaries[0].RangeLength == Range1Length); + CHECK(ParserExample3.Boundaries[1].RangeOffset == Range2Offset); + CHECK(ParserExample3.Boundaries[1].RangeLength == Range2Length); + } + + { + std::string_view Example4 = + "\r\n--00000000000000019229\r\n" + "Content-Type: application/x-ue-comp\r\n" + "Content-Range: bytes 2638-5111437/*\r\n" + "Not: really\r\n" + "\r\n" + "datadatadatadata" + "\r\n--000000000bait0019229\r\n" + "\r\n--00\r\n--000000000bait001922\r\n" + "\r\n\r\n\r\r\n--00000000000000019229\r\n" + "Content-Type: application/x-ue-comp\r\n" + "Content-Range: bytes 5118199-9147741/44369878\r\n" + "\r\n" + "ditaditadita" + "Content-Type: application/x-ue-comp\r\n" + "ditaditadita" + "Content-Range: bytes 5118199-9147741/44369878\r\n" + "\r\n---\r\n--00000000000000019229--"; + + detail::MultipartBoundaryParser ParserExample4; + ParserExample4.Init(ContentTypeHeaderValue1); + + const size_t InputWindow = 3; + for (size_t Offset = 0; Offset < Example4.length(); Offset += InputWindow) + { + std::string_view Window = Example4.substr(Offset, Min(Example4.length() - Offset, InputWindow)); + ParserExample4.ParseInput(Window); + } + + CHECK(ParserExample4.Boundaries.size() == 2); + + CHECK(ParserExample4.Boundaries[0].RangeOffset == Range1Offset); + CHECK(ParserExample4.Boundaries[0].RangeLength == Range1Length); + CHECK(ParserExample4.Boundaries[1].RangeOffset == Range2Offset); + CHECK(ParserExample4.Boundaries[1].RangeLength == Range2Length); + } +} + } // namespace zen #endif diff --git a/src/zenhttp/clients/httpclientcommon.h b/src/zenhttp/clients/httpclientcommon.h index 1d0b7f9ea..8bb1e9268 100644 --- a/src/zenhttp/clients/httpclientcommon.h +++ b/src/zenhttp/clients/httpclientcommon.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include #include @@ -87,7 +88,7 @@ namespace detail { std::error_code Write(std::string_view DataString); IoBuffer DetachToIoBuffer(); IoBuffer BorrowIoBuffer(); - inline uint64_t GetSize() const { return m_WriteOffset; } + inline uint64_t GetSize() const { return m_WriteOffset + m_CacheBufferOffset; } void ResetWritePos(uint64_t WriteOffset); private: @@ -143,6 +144,112 @@ namespace detail { uint64_t m_BytesLeftInSegment; }; + class IncrementalStringMatcher + { + public: + enum class EMatchState + { + None, + Partial, + Complete + }; + + EMatchState MatchState = EMatchState::None; + + IncrementalStringMatcher() {} + + IncrementalStringMatcher(std::string&& InMatchString) : MatchString(std::move(InMatchString)) {} + + void Init(std::string&& InMatchString) { MatchString = std::move(InMatchString); } + + void Reset() + { + MatchLength = 0; + MatchStartOffset = 0; + MatchState = EMatchState::None; + } + + inline uint64_t GetMatchEndOffset() const + { + if (MatchState == EMatchState::Complete) + { + return MatchStartOffset + MatchString.length(); + } + return 0; + } + + inline uint64_t GetMatchStartOffset() const + { + ZEN_ASSERT(MatchState == EMatchState::Complete); + return MatchStartOffset; + } + + void Match(uint64_t Offset, char C) + { + ZEN_ASSERT_SLOW(!MatchString.empty()); + + if (MatchState == EMatchState::Complete) + { + Reset(); + } + if (C == MatchString[MatchLength]) + { + if (MatchLength == 0) + { + MatchStartOffset = Offset; + } + MatchLength++; + if (MatchLength == MatchString.length()) + { + MatchState = EMatchState::Complete; + } + else + { + MatchState = EMatchState::Partial; + } + } + else if (MatchLength != 0) + { + Reset(); + Match(Offset, C); + } + else + { + Reset(); + } + } + inline const std::string& GetMatchString() const { return MatchString; } + + private: + std::string MatchString; + + uint64_t MatchLength = 0; + uint64_t MatchStartOffset = 0; + }; + + class MultipartBoundaryParser + { + public: + std::vector Boundaries; + + MultipartBoundaryParser(); + bool Init(const std::string_view ContentTypeHeaderValue); + inline void ParseInput(std::string_view data) { InternalParseInput(data); } + + private: + IncrementalStringMatcher BoundaryBeginMatcher; + IncrementalStringMatcher BoundaryEndMatcher; + IncrementalStringMatcher HeaderEndMatcher; + + ExtendableStringBuilder<64> BoundaryHeader; + uint64_t PayloadOffset = 0; + + void InternalParseInput(std::string_view data); + }; + + std::pair GetHeaderKeyAndValue(std::string_view HeaderString); + std::pair ParseContentRange(std::string_view Value); + } // namespace detail } // namespace zen diff --git a/src/zenhttp/clients/httpclientcpr.cpp b/src/zenhttp/clients/httpclientcpr.cpp index 5d92b3b6b..6bc63db09 100644 --- a/src/zenhttp/clients/httpclientcpr.cpp +++ b/src/zenhttp/clients/httpclientcpr.cpp @@ -162,10 +162,11 @@ CprHttpClient::~CprHttpClient() } HttpClient::Response -CprHttpClient::ResponseWithPayload(std::string_view SessionId, - cpr::Response&& HttpResponse, - const HttpResponseCode WorkResponseCode, - IoBuffer&& Payload) +CprHttpClient::ResponseWithPayload(std::string_view SessionId, + cpr::Response&& HttpResponse, + const HttpResponseCode WorkResponseCode, + IoBuffer&& Payload, + std::vector&& BoundaryPositions) { // This ends up doing a memcpy, would be good to get rid of it by streaming results // into buffer directly @@ -174,7 +175,6 @@ CprHttpClient::ResponseWithPayload(std::string_view SessionId, if (auto It = HttpResponse.header.find("Content-Type"); It != HttpResponse.header.end()) { const HttpContentType ContentType = ParseContentType(It->second); - ResponseBuffer.SetContentType(ContentType); } @@ -188,16 +188,26 @@ CprHttpClient::ResponseWithPayload(std::string_view SessionId, } } + std::sort(BoundaryPositions.begin(), + BoundaryPositions.end(), + [](const HttpClient::Response::MultipartBoundary& Lhs, const HttpClient::Response::MultipartBoundary& Rhs) { + return Lhs.RangeOffset < Rhs.RangeOffset; + }); + return HttpClient::Response{.StatusCode = WorkResponseCode, .ResponsePayload = std::move(ResponseBuffer), .Header = HttpClient::KeyValueMap(HttpResponse.header.begin(), HttpResponse.header.end()), .UploadedBytes = gsl::narrow(HttpResponse.uploaded_bytes), .DownloadedBytes = gsl::narrow(HttpResponse.downloaded_bytes), - .ElapsedSeconds = HttpResponse.elapsed}; + .ElapsedSeconds = HttpResponse.elapsed, + .Ranges = std::move(BoundaryPositions)}; } HttpClient::Response -CprHttpClient::CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffer&& Payload) +CprHttpClient::CommonResponse(std::string_view SessionId, + cpr::Response&& HttpResponse, + IoBuffer&& Payload, + std::vector&& BoundaryPositions) { const HttpResponseCode WorkResponseCode = HttpResponseCode(HttpResponse.status_code); if (HttpResponse.error) @@ -235,7 +245,7 @@ CprHttpClient::CommonResponse(std::string_view SessionId, cpr::Response&& HttpRe } else { - return ResponseWithPayload(SessionId, std::move(HttpResponse), WorkResponseCode, std::move(Payload)); + return ResponseWithPayload(SessionId, std::move(HttpResponse), WorkResponseCode, std::move(Payload), std::move(BoundaryPositions)); } } @@ -896,236 +906,280 @@ CprHttpClient::Download(std::string_view Url, const std::filesystem::path& TempF std::string PayloadString; std::unique_ptr PayloadFile; - cpr::Response Response = DoWithRetry( - m_SessionId, - [&]() { - auto GetHeader = [&](std::string header) -> std::pair { - size_t DelimiterPos = header.find(':'); - if (DelimiterPos != std::string::npos) - { - std::string Key = header.substr(0, DelimiterPos); - constexpr AsciiSet WhitespaceCharacters(" \v\f\t\r\n"); - Key = AsciiSet::TrimSuffixWith(Key, WhitespaceCharacters); - Key = AsciiSet::TrimPrefixWith(Key, WhitespaceCharacters); - - std::string Value = header.substr(DelimiterPos + 1); - Value = AsciiSet::TrimSuffixWith(Value, WhitespaceCharacters); - Value = AsciiSet::TrimPrefixWith(Value, WhitespaceCharacters); - - return std::make_pair(Key, Value); - } - return std::make_pair(header, ""); - }; - - auto DownloadCallback = [&](std::string data, intptr_t) { - if (m_CheckIfAbortFunction && m_CheckIfAbortFunction()) - { - return false; - } - if (PayloadFile) - { - ZEN_ASSERT(PayloadString.empty()); - std::error_code Ec = PayloadFile->Write(data); - if (Ec) - { - ZEN_WARN("Failed to write to temp file in '{}' for HttpClient::Download. Reason: {}", - TempFolderPath.string(), - Ec.message()); - return false; - } - } - else - { - PayloadString.append(data); - } - return true; - }; - - uint64_t RequestedContentLength = (uint64_t)-1; - if (auto RangeIt = AdditionalHeader.Entries.find("Range"); RangeIt != AdditionalHeader.Entries.end()) - { - if (RangeIt->second.starts_with("bytes")) - { - size_t RangeStartPos = RangeIt->second.find('=', 5); - if (RangeStartPos != std::string::npos) - { - RangeStartPos++; - size_t RangeSplitPos = RangeIt->second.find('-', RangeStartPos); - if (RangeSplitPos != std::string::npos) - { - std::optional RequestedRangeStart = - ParseInt(RangeIt->second.substr(RangeStartPos, RangeSplitPos - RangeStartPos)); - std::optional RequestedRangeEnd = ParseInt(RangeIt->second.substr(RangeStartPos + 1)); - if (RequestedRangeStart.has_value() && RequestedRangeEnd.has_value()) - { - RequestedContentLength = RequestedRangeEnd.value() - 1; - } - } - } - } - } - - cpr::Response Response; - { - std::vector> ReceivedHeaders; - auto HeaderCallback = [&](std::string header, intptr_t) { - std::pair Header = GetHeader(header); - if (Header.first == "Content-Length"sv) - { - std::optional ContentLength = ParseInt(Header.second); - if (ContentLength.has_value()) - { - if (ContentLength.value() > m_ConnectionSettings.MaximumInMemoryDownloadSize) - { - PayloadFile = std::make_unique(); - std::error_code Ec = PayloadFile->Open(TempFolderPath, ContentLength.value()); - if (Ec) - { - ZEN_WARN("Failed to create temp file in '{}' for HttpClient::Download. Reason: {}", - TempFolderPath.string(), - Ec.message()); - PayloadFile.reset(); - } - } - else - { - PayloadString.reserve(ContentLength.value()); - } - } - } - if (!Header.first.empty()) - { - ReceivedHeaders.emplace_back(std::move(Header)); - } - return 1; - }; - - Session Sess = AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); - Response = Sess.Download(cpr::WriteCallback{DownloadCallback}, cpr::HeaderCallback{HeaderCallback}); - for (const std::pair& H : ReceivedHeaders) - { - Response.header.insert_or_assign(H.first, H.second); - } - } - if (m_ConnectionSettings.AllowResume) - { - auto SupportsRanges = [](const cpr::Response& Response) -> bool { - if (Response.header.find("Content-Range") != Response.header.end()) - { - return true; - } - if (auto It = Response.header.find("Accept-Ranges"); It != Response.header.end()) - { - return It->second == "bytes"sv; - } - return false; - }; - - auto ShouldResume = [&SupportsRanges](const cpr::Response& Response) -> bool { - if (ShouldRetry(Response)) - { - return SupportsRanges(Response); - } - return false; - }; - - if (ShouldResume(Response)) - { - auto It = Response.header.find("Content-Length"); - if (It != Response.header.end()) - { - uint64_t ContentLength = RequestedContentLength; - if (ContentLength == uint64_t(-1)) - { - if (auto ParsedContentLength = ParseInt(It->second); ParsedContentLength.has_value()) - { - ContentLength = ParsedContentLength.value(); - } - } - - std::vector> ReceivedHeaders; - - auto HeaderCallback = [&](std::string header, intptr_t) { - std::pair Header = GetHeader(header); - if (!Header.first.empty()) - { - ReceivedHeaders.emplace_back(std::move(Header)); - } - - if (Header.first == "Content-Range"sv) - { - if (Header.second.starts_with("bytes "sv)) - { - size_t RangeStartEnd = Header.second.find('-', 6); - if (RangeStartEnd != std::string::npos) - { - const auto Start = ParseInt(Header.second.substr(6, RangeStartEnd - 6)); - if (Start) - { - uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); - if (Start.value() == DownloadedSize) - { - return 1; - } - else if (Start.value() > DownloadedSize) - { - return 0; - } - if (PayloadFile) - { - PayloadFile->ResetWritePos(Start.value()); - } - else - { - PayloadString = PayloadString.substr(0, Start.value()); - } - return 1; - } - } - } - return 0; - } - return 1; - }; - - KeyValueMap HeadersWithRange(AdditionalHeader); - do - { - uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); - - std::string Range = fmt::format("bytes={}-{}", DownloadedSize, DownloadedSize + ContentLength - 1); - if (auto RangeIt = HeadersWithRange.Entries.find("Range"); RangeIt != HeadersWithRange.Entries.end()) - { - if (RangeIt->second == Range) - { - // If we didn't make any progress, abort - break; - } - } - HeadersWithRange.Entries.insert_or_assign("Range", Range); - - Session Sess = - AllocSession(m_BaseUri, Url, m_ConnectionSettings, HeadersWithRange, {}, m_SessionId, GetAccessToken()); - Response = Sess.Download(cpr::WriteCallback{DownloadCallback}, cpr::HeaderCallback{HeaderCallback}); - for (const std::pair& H : ReceivedHeaders) - { - Response.header.insert_or_assign(H.first, H.second); - } - ReceivedHeaders.clear(); - } while (ShouldResume(Response)); - } - } - } - - if (!PayloadString.empty()) - { - Response.text = std::move(PayloadString); - } - return Response; - }, - PayloadFile); - - return CommonResponse(m_SessionId, std::move(Response), PayloadFile ? PayloadFile->DetachToIoBuffer() : IoBuffer{}); + + HttpContentType ContentType = HttpContentType::kUnknownContentType; + detail::MultipartBoundaryParser BoundaryParser; + bool IsMultiRangeResponse = false; + + cpr::Response Response = DoWithRetry( + m_SessionId, + [&]() { + auto DownloadCallback = [&](std::string data, intptr_t) { + if (m_CheckIfAbortFunction && m_CheckIfAbortFunction()) + { + return false; + } + + if (IsMultiRangeResponse) + { + BoundaryParser.ParseInput(data); + } + + if (PayloadFile) + { + ZEN_ASSERT(PayloadString.empty()); + std::error_code Ec = PayloadFile->Write(data); + if (Ec) + { + ZEN_WARN("Failed to write to temp file in '{}' for HttpClient::Download. Reason: {}", + TempFolderPath.string(), + Ec.message()); + return false; + } + } + else + { + PayloadString.append(data); + } + return true; + }; + + uint64_t RequestedContentLength = (uint64_t)-1; + if (auto RangeIt = AdditionalHeader.Entries.find("Range"); RangeIt != AdditionalHeader.Entries.end()) + { + if (RangeIt->second.starts_with("bytes")) + { + std::string_view RangeValue(RangeIt->second); + size_t RangeStartPos = RangeValue.find('=', 5); + if (RangeStartPos != std::string::npos) + { + RangeStartPos++; + while (RangeValue[RangeStartPos] == ' ') + { + RangeStartPos++; + } + RequestedContentLength = 0; + + while (RangeStartPos < RangeValue.length()) + { + size_t RangeEnd = RangeValue.find_first_of(", \r\n", RangeStartPos); + if (RangeEnd == std::string::npos) + { + RangeEnd = RangeValue.length(); + } + + std::string_view RangeString = RangeValue.substr(RangeStartPos, RangeEnd - RangeStartPos); + size_t RangeSplitPos = RangeString.find('-'); + if (RangeSplitPos != std::string::npos) + { + std::optional RequestedRangeStart = ParseInt(RangeString.substr(0, RangeSplitPos)); + std::optional RequestedRangeEnd = ParseInt(RangeString.substr(RangeSplitPos + 1)); + if (RequestedRangeStart.has_value() && RequestedRangeEnd.has_value()) + { + RequestedContentLength += RequestedRangeEnd.value() - 1; + } + } + RangeStartPos = RangeEnd; + while (RangeStartPos != RangeValue.length() && + (RangeValue[RangeStartPos] == ',' || RangeValue[RangeStartPos] == ' ')) + { + RangeStartPos++; + } + } + } + } + } + + cpr::Response Response; + { + std::vector> ReceivedHeaders; + auto HeaderCallback = [&](std::string header, intptr_t) { + const std::pair Header = detail::GetHeaderKeyAndValue(header); + if (Header.first == "Content-Length"sv) + { + std::optional ContentLength = ParseInt(Header.second); + if (ContentLength.has_value()) + { + if (ContentLength.value() > m_ConnectionSettings.MaximumInMemoryDownloadSize) + { + PayloadFile = std::make_unique(); + std::error_code Ec = PayloadFile->Open(TempFolderPath, ContentLength.value()); + if (Ec) + { + ZEN_WARN("Failed to create temp file in '{}' for HttpClient::Download. Reason: {}", + TempFolderPath.string(), + Ec.message()); + PayloadFile.reset(); + } + } + else + { + PayloadString.reserve(ContentLength.value()); + } + } + } + else if (Header.first == "Content-Type") + { + IsMultiRangeResponse = BoundaryParser.Init(Header.second); + if (!IsMultiRangeResponse) + { + ContentType = ParseContentType(Header.second); + } + } + else if (Header.first == "Content-Range") + { + if (!IsMultiRangeResponse) + { + std::pair Range = detail::ParseContentRange(Header.second); + if (Range.second != 0) + { + BoundaryParser.Boundaries.push_back(HttpClient::Response::MultipartBoundary{.OffsetInPayload = 0, + .RangeOffset = Range.first, + .RangeLength = Range.second, + .ContentType = ContentType}); + } + } + } + if (!Header.first.empty()) + { + ReceivedHeaders.emplace_back(std::move(Header)); + } + return 1; + }; + + Session Sess = AllocSession(m_BaseUri, Url, m_ConnectionSettings, AdditionalHeader, {}, m_SessionId, GetAccessToken()); + Response = Sess.Download(cpr::WriteCallback{DownloadCallback}, cpr::HeaderCallback{HeaderCallback}); + for (const std::pair& H : ReceivedHeaders) + { + Response.header.insert_or_assign(H.first, H.second); + } + } + if (m_ConnectionSettings.AllowResume) + { + auto SupportsRanges = [](const cpr::Response& Response) -> bool { + if (Response.header.find("Content-Range") != Response.header.end()) + { + return true; + } + if (auto It = Response.header.find("Accept-Ranges"); It != Response.header.end()) + { + return It->second == "bytes"sv; + } + return false; + }; + + auto ShouldResume = [&SupportsRanges, &IsMultiRangeResponse](const cpr::Response& Response) -> bool { + if (IsMultiRangeResponse) + { + return false; + } + if (ShouldRetry(Response)) + { + return SupportsRanges(Response); + } + return false; + }; + + if (ShouldResume(Response)) + { + auto It = Response.header.find("Content-Length"); + if (It != Response.header.end()) + { + uint64_t ContentLength = RequestedContentLength; + if (ContentLength == uint64_t(-1)) + { + if (auto ParsedContentLength = ParseInt(It->second); ParsedContentLength.has_value()) + { + ContentLength = ParsedContentLength.value(); + } + } + + std::vector> ReceivedHeaders; + + auto HeaderCallback = [&](std::string header, intptr_t) { + const std::pair Header = detail::GetHeaderKeyAndValue(header); + if (!Header.first.empty()) + { + ReceivedHeaders.emplace_back(std::move(Header)); + } + + if (Header.first == "Content-Range"sv) + { + if (Header.second.starts_with("bytes "sv)) + { + size_t RangeStartEnd = Header.second.find('-', 6); + if (RangeStartEnd != std::string::npos) + { + const auto Start = ParseInt(Header.second.substr(6, RangeStartEnd - 6)); + if (Start) + { + uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); + if (Start.value() == DownloadedSize) + { + return 1; + } + else if (Start.value() > DownloadedSize) + { + return 0; + } + if (PayloadFile) + { + PayloadFile->ResetWritePos(Start.value()); + } + else + { + PayloadString = PayloadString.substr(0, Start.value()); + } + return 1; + } + } + } + return 0; + } + return 1; + }; + + KeyValueMap HeadersWithRange(AdditionalHeader); + do + { + uint64_t DownloadedSize = PayloadFile ? PayloadFile->GetSize() : PayloadString.length(); + + std::string Range = fmt::format("bytes={}-{}", DownloadedSize, DownloadedSize + ContentLength - 1); + if (auto RangeIt = HeadersWithRange.Entries.find("Range"); RangeIt != HeadersWithRange.Entries.end()) + { + if (RangeIt->second == Range) + { + // If we didn't make any progress, abort + break; + } + } + HeadersWithRange.Entries.insert_or_assign("Range", Range); + + Session Sess = + AllocSession(m_BaseUri, Url, m_ConnectionSettings, HeadersWithRange, {}, m_SessionId, GetAccessToken()); + Response = Sess.Download(cpr::WriteCallback{DownloadCallback}, cpr::HeaderCallback{HeaderCallback}); + for (const std::pair& H : ReceivedHeaders) + { + Response.header.insert_or_assign(H.first, H.second); + } + ReceivedHeaders.clear(); + } while (ShouldResume(Response)); + } + } + } + + if (!PayloadString.empty()) + { + Response.text = std::move(PayloadString); + } + return Response; + }, + PayloadFile); + + return CommonResponse(m_SessionId, + std::move(Response), + PayloadFile ? PayloadFile->DetachToIoBuffer() : IoBuffer{}, + std::move(BoundaryParser.Boundaries)); } } // namespace zen diff --git a/src/zenhttp/clients/httpclientcpr.h b/src/zenhttp/clients/httpclientcpr.h index 40af53b5d..cf2d3bd14 100644 --- a/src/zenhttp/clients/httpclientcpr.h +++ b/src/zenhttp/clients/httpclientcpr.h @@ -157,12 +157,16 @@ private: bool ValidatePayload(cpr::Response& Response, std::unique_ptr& PayloadFile); - HttpClient::Response CommonResponse(std::string_view SessionId, cpr::Response&& HttpResponse, IoBuffer&& Payload); - - HttpClient::Response ResponseWithPayload(std::string_view SessionId, - cpr::Response&& HttpResponse, - const HttpResponseCode WorkResponseCode, - IoBuffer&& Payload); + HttpClient::Response CommonResponse(std::string_view SessionId, + cpr::Response&& HttpResponse, + IoBuffer&& Payload, + std::vector&& BoundaryPositions = {}); + + HttpClient::Response ResponseWithPayload(std::string_view SessionId, + cpr::Response&& HttpResponse, + const HttpResponseCode WorkResponseCode, + IoBuffer&& Payload, + std::vector&& BoundaryPositions); }; } // namespace zen diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 078e27b34..998eb27ea 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -104,6 +104,47 @@ HttpClientBase::GetAccessToken() ////////////////////////////////////////////////////////////////////////// +std::vector> +HttpClient::Response::GetRanges(std::span> OffsetAndLengthPairs) const +{ + std::vector> Result; + Result.reserve(OffsetAndLengthPairs.size()); + if (Ranges.empty()) + { + for (const std::pair& Range : OffsetAndLengthPairs) + { + Result.emplace_back(std::make_pair(Range.first, Range.second)); + } + return Result; + } + + auto BoundaryIt = Ranges.begin(); + auto OffsetAndLengthPairIt = OffsetAndLengthPairs.begin(); + while (OffsetAndLengthPairIt != OffsetAndLengthPairs.end()) + { + uint64_t Offset = OffsetAndLengthPairIt->first; + uint64_t Length = OffsetAndLengthPairIt->second; + while (Offset >= BoundaryIt->RangeOffset + BoundaryIt->RangeLength) + { + BoundaryIt++; + if (BoundaryIt == Ranges.end()) + { + throw std::runtime_error("HttpClient::Response can not fulfill requested range"); + } + } + if (Offset + Length > BoundaryIt->RangeOffset + BoundaryIt->RangeLength || Offset < BoundaryIt->RangeOffset) + { + throw std::runtime_error("HttpClient::Response can not fulfill requested range"); + } + uint64_t OffsetIntoRange = Offset - BoundaryIt->RangeOffset; + uint64_t RangePayloadOffset = BoundaryIt->OffsetInPayload + OffsetIntoRange; + Result.emplace_back(std::make_pair(RangePayloadOffset, Length)); + + OffsetAndLengthPairIt++; + } + return Result; +} + CbObject HttpClient::Response::AsObject() const { diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index 336a3deee..41a7ce621 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -179,6 +179,20 @@ public: // The elapsed time in seconds for the request to execute double ElapsedSeconds; + struct MultipartBoundary + { + uint64_t OffsetInPayload; + uint64_t RangeOffset; + uint64_t RangeLength; + HttpContentType ContentType; + }; + + // Ranges will map out all recevied ranges, both single and multi-range responses + // If no range was requested Ranges will be empty + std::vector Ranges; + + std::vector> GetRanges(std::span> OffsetAndLengthPairs) const; + // This contains any errors from the HTTP stack. It won't contain information on // why the server responded with a non-success HTTP status, that may be gleaned // from the response payload itself depending on what the server provides. diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 5219e86d8..08a896f37 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -7828,105 +7828,109 @@ TEST_CASE("buildstorageoperations.memorychunkingcache") TEST_CASE("buildstorageoperations.upload.multipart") { - using namespace buildstorageoperations_testutils; + // Disabled since it relies on authentication and specific block being present in cloud storage + if (false) + { + using namespace buildstorageoperations_testutils; - FastRandom BaseRandom; + FastRandom BaseRandom; - const size_t FileCount = 11; + const size_t FileCount = 11; - const std::string Paths[FileCount] = {{"file_1"}, - {"file_2.exe"}, - {"file_3.txt"}, - {"dir_1/dir1_file_1.exe"}, - {"dir_1/dir1_file_2.pdb"}, - {"dir_1/dir1_file_3.txt"}, - {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, - {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, - {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, - {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, - {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; - const uint64_t Sizes[FileCount] = - {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; + const std::string Paths[FileCount] = {{"file_1"}, + {"file_2.exe"}, + {"file_3.txt"}, + {"dir_1/dir1_file_1.exe"}, + {"dir_1/dir1_file_2.pdb"}, + {"dir_1/dir1_file_3.txt"}, + {"dir_2/dir2_dir1/dir2_dir1_file_1.exe"}, + {"dir_2/dir2_dir1/dir2_dir1_file_2.pdb"}, + {"dir_2/dir2_dir1/dir2_dir1_file_3.dll"}, + {"dir_2/dir2_dir2/dir2_dir2_file_1.txt"}, + {"dir_2/dir2_dir2/dir2_dir2_file_2.json"}}; + const uint64_t Sizes[FileCount] = + {6u * 1024u, 0, 798, 19u * 1024u, 7u * 1024u, 93, 31u * 1024u, 17u * 1024u, 13u * 1024u, 2u * 1024u, 3u * 1024u}; - ScopedTemporaryDirectory SourceFolder; - TestState State(SourceFolder.Path()); - State.Initialize(); - State.CreateSourceData("source", Paths, Sizes); + ScopedTemporaryDirectory SourceFolder; + TestState State(SourceFolder.Path()); + State.Initialize(); + State.CreateSourceData("source", Paths, Sizes); - std::span ManifestFiles1(Paths); - ManifestFiles1 = ManifestFiles1.subspan(0, FileCount / 2); + std::span ManifestFiles1(Paths); + ManifestFiles1 = ManifestFiles1.subspan(0, FileCount / 2); - std::span ManifestSizes1(Sizes); - ManifestSizes1 = ManifestSizes1.subspan(0, FileCount / 2); + std::span ManifestSizes1(Sizes); + ManifestSizes1 = ManifestSizes1.subspan(0, FileCount / 2); - std::span ManifestFiles2(Paths); - ManifestFiles2 = ManifestFiles2.subspan(FileCount / 2 - 1); + std::span ManifestFiles2(Paths); + ManifestFiles2 = ManifestFiles2.subspan(FileCount / 2 - 1); - std::span ManifestSizes2(Sizes); - ManifestSizes2 = ManifestSizes2.subspan(FileCount / 2 - 1); + std::span ManifestSizes2(Sizes); + ManifestSizes2 = ManifestSizes2.subspan(FileCount / 2 - 1); - const Oid BuildPart1Id = Oid::NewOid(); - const std::string BuildPart1Name = "part1"; - const Oid BuildPart2Id = Oid::NewOid(); - const std::string BuildPart2Name = "part2"; - { - CbObjectWriter Writer; - Writer.BeginObject("parts"sv); + const Oid BuildPart1Id = Oid::NewOid(); + const std::string BuildPart1Name = "part1"; + const Oid BuildPart2Id = Oid::NewOid(); + const std::string BuildPart2Name = "part2"; { - Writer.BeginObject(BuildPart1Name); + CbObjectWriter Writer; + Writer.BeginObject("parts"sv); { - Writer.AddObjectId("partId"sv, BuildPart1Id); - Writer.BeginArray("files"sv); - for (const std::string& ManifestFile : ManifestFiles1) + Writer.BeginObject(BuildPart1Name); { - Writer.AddString(ManifestFile); + Writer.AddObjectId("partId"sv, BuildPart1Id); + Writer.BeginArray("files"sv); + for (const std::string& ManifestFile : ManifestFiles1) + { + Writer.AddString(ManifestFile); + } + Writer.EndArray(); // files } - Writer.EndArray(); // files - } - Writer.EndObject(); // part1 + Writer.EndObject(); // part1 - Writer.BeginObject(BuildPart2Name); - { - Writer.AddObjectId("partId"sv, BuildPart2Id); - Writer.BeginArray("files"sv); - for (const std::string& ManifestFile : ManifestFiles2) + Writer.BeginObject(BuildPart2Name); { - Writer.AddString(ManifestFile); + Writer.AddObjectId("partId"sv, BuildPart2Id); + Writer.BeginArray("files"sv); + for (const std::string& ManifestFile : ManifestFiles2) + { + Writer.AddString(ManifestFile); + } + Writer.EndArray(); // files } - Writer.EndArray(); // files + Writer.EndObject(); // part2 } - Writer.EndObject(); // part2 - } - Writer.EndObject(); // parts + Writer.EndObject(); // parts - ExtendableStringBuilder<1024> Manifest; - CompactBinaryToJson(Writer.Save(), Manifest); - WriteFile(State.RootPath / "manifest.json", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size())); - } + ExtendableStringBuilder<1024> Manifest; + CompactBinaryToJson(Writer.Save(), Manifest); + WriteFile(State.RootPath / "manifest.json", IoBuffer(IoBuffer::Wrap, Manifest.Data(), Manifest.Size())); + } - const Oid BuildId = Oid::NewOid(); + const Oid BuildId = Oid::NewOid(); - auto Result = State.Upload(BuildId, {}, {}, "source", State.RootPath / "manifest.json"); + auto Result = State.Upload(BuildId, {}, {}, "source", State.RootPath / "manifest.json"); - CHECK_EQ(Result.size(), 2u); - CHECK_EQ(Result[0].first, BuildPart1Id); - CHECK_EQ(Result[0].second, BuildPart1Name); - CHECK_EQ(Result[1].first, BuildPart2Id); - CHECK_EQ(Result[1].second, BuildPart2Name); - State.ValidateUpload(BuildId, Result); + CHECK_EQ(Result.size(), 2u); + CHECK_EQ(Result[0].first, BuildPart1Id); + CHECK_EQ(Result[0].second, BuildPart1Name); + CHECK_EQ(Result[1].first, BuildPart2Id); + CHECK_EQ(Result[1].second, BuildPart2Name); + State.ValidateUpload(BuildId, Result); - FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); - State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); + FolderContent DownloadContent = State.Download(BuildId, Oid::Zero, {}, "download", /* Append */ false); + State.ValidateDownload(Paths, Sizes, "source", "download", DownloadContent); - FolderContent Part1DownloadContent = State.Download(BuildId, BuildPart1Id, {}, "download_part1", /* Append */ false); - State.ValidateDownload(ManifestFiles1, ManifestSizes1, "source", "download_part1", Part1DownloadContent); + FolderContent Part1DownloadContent = State.Download(BuildId, BuildPart1Id, {}, "download_part1", /* Append */ false); + State.ValidateDownload(ManifestFiles1, ManifestSizes1, "source", "download_part1", Part1DownloadContent); - FolderContent Part2DownloadContent = State.Download(BuildId, Oid::Zero, BuildPart2Name, "download_part2", /* Append */ false); - State.ValidateDownload(ManifestFiles2, ManifestSizes2, "source", "download_part2", Part2DownloadContent); + FolderContent Part2DownloadContent = State.Download(BuildId, Oid::Zero, BuildPart2Name, "download_part2", /* Append */ false); + State.ValidateDownload(ManifestFiles2, ManifestSizes2, "source", "download_part2", Part2DownloadContent); - (void)State.Download(BuildId, BuildPart1Id, BuildPart1Name, "download_part1+2", /* Append */ false); - FolderContent Part1And2DownloadContent = State.Download(BuildId, BuildPart2Id, {}, "download_part1+2", /* Append */ true); - State.ValidateDownload(Paths, Sizes, "source", "download_part1+2", Part1And2DownloadContent); + (void)State.Download(BuildId, BuildPart1Id, BuildPart1Name, "download_part1+2", /* Append */ false); + FolderContent Part1And2DownloadContent = State.Download(BuildId, BuildPart2Id, {}, "download_part1+2", /* Append */ true); + State.ValidateDownload(Paths, Sizes, "source", "download_part1+2", Part1And2DownloadContent); + } } void -- cgit v1.2.3 From 226ba2cf432ae6c0a787c6156a172f343fc71887 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 27 Feb 2026 13:57:18 +0100 Subject: MeasureLatency now bails out quickly if it experiences a connection error (#789) previously it would stall some 40s in this case --- src/zenhttp/clients/httpclientcpr.cpp | 15 +++++++++++++++ src/zenhttp/httpclient.cpp | 7 +++++++ src/zenhttp/include/zenhttp/httpclient.h | 3 +++ 3 files changed, 25 insertions(+) diff --git a/src/zenhttp/clients/httpclientcpr.cpp b/src/zenhttp/clients/httpclientcpr.cpp index 6bc63db09..90dcfacbb 100644 --- a/src/zenhttp/clients/httpclientcpr.cpp +++ b/src/zenhttp/clients/httpclientcpr.cpp @@ -23,6 +23,21 @@ CreateCprHttpClient(std::string_view BaseUri, const HttpClientSettings& Connecti static std::atomic HttpClientRequestIdCounter{0}; +bool +HttpClient::ErrorContext::IsConnectionError() const +{ + switch (static_cast(ErrorCode)) + { + case cpr::ErrorCode::CONNECTION_FAILURE: + case cpr::ErrorCode::OPERATION_TIMEDOUT: + case cpr::ErrorCode::HOST_RESOLUTION_FAILURE: + case cpr::ErrorCode::PROXY_RESOLUTION_FAILURE: + return true; + default: + return false; + } +} + // If we want to support different HTTP client implementations then we'll need to make this more abstract HttpClientError::ResponseClass diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 998eb27ea..1cfddb366 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -400,6 +400,13 @@ MeasureLatency(HttpClient& Client, std::string_view Url) else { ErrorMessage = MeasureResponse.ErrorMessage(fmt::format("Unable to measure latency using {}", Url)); + + // Connection-level failures (timeout, refused, DNS) mean the endpoint is unreachable. + // Bail out immediately — retrying will just burn the connect timeout each time. + if (MeasureResponse.Error && MeasureResponse.Error->IsConnectionError()) + { + break; + } } } diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index 41a7ce621..f00bbec63 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -125,6 +125,9 @@ public: { int ErrorCode; std::string ErrorMessage; + + /** True when the error is a transport-level connection failure (connect timeout, refused, DNS) */ + bool IsConnectionError() const; }; struct KeyValueMap -- cgit v1.2.3 From 87aff23c1246abd2838d8b0e589fe61015effa9c Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 27 Feb 2026 16:39:04 +0100 Subject: optimize string matching (#791) --- src/zenhttp/clients/httpclientcommon.cpp | 12 +++++++----- src/zenhttp/clients/httpclientcommon.h | 26 ++++++++++++++++---------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/zenhttp/clients/httpclientcommon.cpp b/src/zenhttp/clients/httpclientcommon.cpp index 312ca16d2..c016e1c3c 100644 --- a/src/zenhttp/clients/httpclientcommon.cpp +++ b/src/zenhttp/clients/httpclientcommon.cpp @@ -425,12 +425,14 @@ namespace detail { return false; } - void MultipartBoundaryParser::InternalParseInput(std::string_view data) + void MultipartBoundaryParser::ParseInput(std::string_view data) { - size_t ScanPos = 0; - while (ScanPos < data.length()) + const char* InputPtr = data.data(); + size_t InputLength = data.length(); + size_t ScanPos = 0; + while (ScanPos < InputLength) { - const char ScanChar = data[ScanPos]; + const char ScanChar = InputPtr[ScanPos]; if (BoundaryBeginMatcher.MatchState == IncrementalStringMatcher::EMatchState::Complete) { if (PayloadOffset + ScanPos < (BoundaryBeginMatcher.GetMatchEndOffset() + BoundaryEndMatcher.GetMatchString().length())) @@ -504,7 +506,7 @@ namespace detail { } ScanPos++; } - PayloadOffset += data.length(); + PayloadOffset += InputLength; } std::pair GetHeaderKeyAndValue(std::string_view HeaderString) diff --git a/src/zenhttp/clients/httpclientcommon.h b/src/zenhttp/clients/httpclientcommon.h index 8bb1e9268..5ed946541 100644 --- a/src/zenhttp/clients/httpclientcommon.h +++ b/src/zenhttp/clients/httpclientcommon.h @@ -158,11 +158,18 @@ namespace detail { IncrementalStringMatcher() {} - IncrementalStringMatcher(std::string&& InMatchString) : MatchString(std::move(InMatchString)) {} + IncrementalStringMatcher(std::string&& InMatchString) : MatchString(std::move(InMatchString)) + { + RawMatchString = MatchString.data(); + } - void Init(std::string&& InMatchString) { MatchString = std::move(InMatchString); } + void Init(std::string&& InMatchString) + { + MatchString = std::move(InMatchString); + RawMatchString = MatchString.data(); + } - void Reset() + inline void Reset() { MatchLength = 0; MatchStartOffset = 0; @@ -186,13 +193,13 @@ namespace detail { void Match(uint64_t Offset, char C) { - ZEN_ASSERT_SLOW(!MatchString.empty()); + ZEN_ASSERT_SLOW(RawMatchString != nullptr); if (MatchState == EMatchState::Complete) { Reset(); } - if (C == MatchString[MatchLength]) + if (C == RawMatchString[MatchLength]) { if (MatchLength == 0) { @@ -222,8 +229,9 @@ namespace detail { private: std::string MatchString; + const char* RawMatchString = nullptr; + uint64_t MatchLength = 0; - uint64_t MatchLength = 0; uint64_t MatchStartOffset = 0; }; @@ -233,8 +241,8 @@ namespace detail { std::vector Boundaries; MultipartBoundaryParser(); - bool Init(const std::string_view ContentTypeHeaderValue); - inline void ParseInput(std::string_view data) { InternalParseInput(data); } + bool Init(const std::string_view ContentTypeHeaderValue); + void ParseInput(std::string_view data); private: IncrementalStringMatcher BoundaryBeginMatcher; @@ -243,8 +251,6 @@ namespace detail { ExtendableStringBuilder<64> BoundaryHeader; uint64_t PayloadOffset = 0; - - void InternalParseInput(std::string_view data); }; std::pair GetHeaderKeyAndValue(std::string_view HeaderString); -- cgit v1.2.3 From 65eefdfe5a216b546f0d3d8fdfc5e9e58916e5f8 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 27 Feb 2026 17:12:00 +0100 Subject: add sentry-sdk logger (#793) eliminates spurious sentry log output during startup as the new channel defaults to WARN The level can be overridden via `--log-debug=sentry-sdk` or `--log-info=sentry-sdk` --- CHANGELOG.md | 4 +++ src/zencore/sentryintegration.cpp | 62 ++++++++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dee9a26e4..fe5ad365c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ - Improvement: `zen oplog-import` now uses partial block requests to reduce download size - Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests - Improvement: IAX's lane tracing +- Improvement: Eliminated spurious sentry logging during startup and allocated `sentry-sdk` logging channel + defaulting to WARN. The logging level can be changed via --log-debug=sentry-sdk or --log-info=sentry-sdk etc + Though it should be noted that since Sentry is initialized very early there is a small period where the + default will be WARN no matter what you pass on the command line. - Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly - Bugfix: Time spent indexing local and remote state during `zen builds download` now show the correct time diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 00e67dc85..636e182b4 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -145,6 +145,8 @@ SentryAssertImpl::OnAssert(const char* Filename, namespace zen { # if ZEN_USE_SENTRY +ZEN_DEFINE_LOG_CATEGORY_STATIC(LogSentry, "sentry-sdk"); + static void SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) { @@ -163,26 +165,61 @@ SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[may MessagePtr = LogMessage.c_str(); } + // SentryLogFunction can be called before the logging system is initialized + // (during sentry_init which runs before InitializeLogging). Fall back to + // console logging when the category logger is not yet available. + // + // Since we want to default to WARN level but this runs before logging has + // been configured, we ignore the callbacks for DEBUG/INFO explicitly here + // which means users don't see every possible log message if they're trying + // to configure the levels using --log-debug=sentry-sdk + if (!TheDefaultLogger) + { + switch (Level) + { + case SENTRY_LEVEL_DEBUG: + // ZEN_CONSOLE_DEBUG("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_INFO: + // ZEN_CONSOLE_INFO("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_WARNING: + ZEN_CONSOLE_WARN("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_ERROR: + ZEN_CONSOLE_ERROR("sentry: {}", MessagePtr); + break; + + case SENTRY_LEVEL_FATAL: + ZEN_CONSOLE_CRITICAL("sentry: {}", MessagePtr); + break; + } + return; + } + switch (Level) { case SENTRY_LEVEL_DEBUG: - ZEN_CONSOLE_DEBUG("sentry: {}", MessagePtr); + ZEN_LOG_DEBUG(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_INFO: - ZEN_CONSOLE_INFO("sentry: {}", MessagePtr); + ZEN_LOG_INFO(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_WARNING: - ZEN_CONSOLE_WARN("sentry: {}", MessagePtr); + ZEN_LOG_WARN(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_ERROR: - ZEN_CONSOLE_ERROR("sentry: {}", MessagePtr); + ZEN_LOG_ERROR(LogSentry, "sentry: {}", MessagePtr); break; case SENTRY_LEVEL_FATAL: - ZEN_CONSOLE_CRITICAL("sentry: {}", MessagePtr); + ZEN_LOG_CRITICAL(LogSentry, "sentry: {}", MessagePtr); break; } } @@ -310,22 +347,31 @@ SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine void SentryIntegration::LogStartupInformation() { + // Initialize the sentry-sdk log category at Warn level to reduce startup noise. + // The level can be overridden via --log-debug=sentry-sdk or --log-info=sentry-sdk + LogSentry.Logger().SetLogLevel(logging::level::Warn); + if (m_IsInitialized) { if (m_SentryErrorCode == 0) { if (m_AllowPII) { - ZEN_INFO("sentry initialized, username: '{}', hostname: '{}', id: '{}'", m_SentryUserName, m_SentryHostName, m_SentryId); + ZEN_LOG_INFO(LogSentry, + "sentry initialized, username: '{}', hostname: '{}', id: '{}'", + m_SentryUserName, + m_SentryHostName, + m_SentryId); } else { - ZEN_INFO("sentry initialized with anonymous reports"); + ZEN_LOG_INFO(LogSentry, "sentry initialized with anonymous reports"); } } else { - ZEN_WARN( + ZEN_LOG_WARN( + LogSentry, "sentry_init returned failure! (error code: {}) note that sentry expects crashpad_handler to exist alongside the running " "executable", m_SentryErrorCode); -- cgit v1.2.3 From 0a41fd42aa43080fbc991e7d976dde70aeaec594 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 27 Feb 2026 17:13:40 +0100 Subject: add full WebSocket (RFC 6455) client/server support for zenhttp (#792) * This branch adds full WebSocket (RFC 6455) support to the HTTP server layer, covering both transport backends, a client, and tests. - **`websocket.h`** -- Core interfaces: `WebSocketOpcode`, `WebSocketMessage`, `WebSocketConnection` (ref-counted), and `IWebSocketHandler`. Services opt in to WebSocket support by implementing `IWebSocketHandler` alongside their existing `HttpService`. - **`httpwsclient.h`** -- `HttpWsClient`: an ASIO-backed `ws://` client with both standalone (own thread) and shared `io_context` modes. Supports connect timeout and optional auth token injection via `IWsClientHandler` callbacks. - **`wsasio.cpp/h`** -- `WsAsioConnection`: WebSocket over ASIO TCP. Takes over the socket after the HTTP 101 handshake and runs an async read/write loop with a queued write path (guarded by `RwLock`). - **`wshttpsys.cpp/h`** -- `WsHttpSysConnection`: WebSocket over http.sys opaque-mode connections (Windows only). Uses `HttpReceiveRequestEntityBody` / `HttpSendResponseEntityBody` via IOCP, sharing the same threadpool as normal http.sys traffic. Self-ref lifetime management ensures graceful drain of outstanding async ops. - **`httpsys_iocontext.h`** -- Tagged `OVERLAPPED` wrapper (`HttpSysIoContext`) used to distinguish normal HTTP transactions from WebSocket read/write completions in the single IOCP callback. - **`wsframecodec.cpp/h`** -- `WsFrameCodec`: static helpers for parsing (unmasked and masked) and building (unmasked server frames and masked client frames) RFC 6455 frames across all three payload length encodings (7-bit, 16-bit, 64-bit). Also computes `Sec-WebSocket-Accept` keys. - **`clients/httpwsclient.cpp`** -- `HttpWsClient::Impl`: ASIO-based client that performs the HTTP upgrade handshake, then hands off to the frame codec for the read loop. Manages its own `io_context` thread or plugs into an external one. - **`httpasio.cpp`** -- ASIO server now detects `Upgrade: websocket` requests, checks the matching `HttpService` for `IWebSocketHandler` via `dynamic_cast`, performs the RFC 6455 handshake (101 response), and spins up a `WsAsioConnection`. - **`httpsys.cpp`** -- Same upgrade detection and handshake logic for the http.sys backend, using `WsHttpSysConnection` and `HTTP_SEND_RESPONSE_FLAG_OPAQUE`. - **`httpparser.cpp/h`** -- Extended to surface the `Upgrade` / `Connection` / `Sec-WebSocket-Key` headers needed by the handshake. - **`httpcommon.h`** -- Minor additions (probably new header constants or response codes for the WS upgrade). - **`httpserver.h`** -- Small interface changes to support WebSocket registration. - **`zenhttp.cpp` / `xmake.lua`** -- New source files wired in; build config updated. - **Unit tests** (`websocket.framecodec`): round-trip encode/decode for text, binary, close frames; all three payload sizes; masked and unmasked variants; RFC 6455 `Sec-WebSocket-Accept` test vector. - **Integration tests** (`websocket.integration`): full ASIO server tests covering handshake (101), normal HTTP coexistence, echo, server-push broadcast, client close handshake, ping/pong auto-response, sequential messages, and rejection of upgrades on non-WS services. - **Client tests** (`websocket.client`): `HttpWsClient` connect+echo+close, connection failure (bad port -> close code 1006), and server-initiated close. * changed HttpRequestParser::ParseCurrentHeader to use switch instead of if/else chain * remove spurious printf --------- Co-authored-by: Stefan Boberg --- src/zencore/filesystem.cpp | 1 - src/zenhttp/clients/httpwsclient.cpp | 568 ++++++++++++++++++ src/zenhttp/include/zenhttp/httpcommon.h | 7 + src/zenhttp/include/zenhttp/httpserver.h | 3 +- src/zenhttp/include/zenhttp/httpwsclient.h | 79 +++ src/zenhttp/include/zenhttp/websocket.h | 65 ++ src/zenhttp/servers/httpasio.cpp | 49 ++ src/zenhttp/servers/httpparser.cpp | 148 +++-- src/zenhttp/servers/httpparser.h | 7 + src/zenhttp/servers/httpsys.cpp | 180 ++++-- src/zenhttp/servers/httpsys_iocontext.h | 40 ++ src/zenhttp/servers/wsasio.cpp | 297 ++++++++++ src/zenhttp/servers/wsasio.h | 71 +++ src/zenhttp/servers/wsframecodec.cpp | 229 +++++++ src/zenhttp/servers/wsframecodec.h | 74 +++ src/zenhttp/servers/wshttpsys.cpp | 466 +++++++++++++++ src/zenhttp/servers/wshttpsys.h | 104 ++++ src/zenhttp/servers/wstest.cpp | 922 +++++++++++++++++++++++++++++ src/zenhttp/xmake.lua | 1 + src/zenhttp/zenhttp.cpp | 1 + 20 files changed, 3203 insertions(+), 109 deletions(-) create mode 100644 src/zenhttp/clients/httpwsclient.cpp create mode 100644 src/zenhttp/include/zenhttp/httpwsclient.h create mode 100644 src/zenhttp/include/zenhttp/websocket.h create mode 100644 src/zenhttp/servers/httpsys_iocontext.h create mode 100644 src/zenhttp/servers/wsasio.cpp create mode 100644 src/zenhttp/servers/wsasio.h create mode 100644 src/zenhttp/servers/wsframecodec.cpp create mode 100644 src/zenhttp/servers/wsframecodec.h create mode 100644 src/zenhttp/servers/wshttpsys.cpp create mode 100644 src/zenhttp/servers/wshttpsys.h create mode 100644 src/zenhttp/servers/wstest.cpp diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 553897407..03398860b 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -3533,7 +3533,6 @@ TEST_CASE("PathBuilder") Path.Reset(); Path.Append(fspath(L"/\u0119oo/")); Path /= L"bar"; - printf("%ls\n", Path.ToPath().c_str()); CHECK(Path.ToView() == L"/\u0119oo/bar"); CHECK(Path.ToPath() == L"\\\u0119oo\\bar"); # endif diff --git a/src/zenhttp/clients/httpwsclient.cpp b/src/zenhttp/clients/httpwsclient.cpp new file mode 100644 index 000000000..36a6f081b --- /dev/null +++ b/src/zenhttp/clients/httpwsclient.cpp @@ -0,0 +1,568 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include "../servers/wsframecodec.h" + +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#include +#include +#include + +namespace zen { + +////////////////////////////////////////////////////////////////////////// + +struct HttpWsClient::Impl +{ + Impl(std::string_view Url, IWsClientHandler& Handler, const HttpWsClientSettings& Settings) + : m_Handler(Handler) + , m_Settings(Settings) + , m_Log(logging::Get(Settings.LogCategory)) + , m_OwnedIoContext(std::make_unique()) + , m_IoContext(*m_OwnedIoContext) + { + ParseUrl(Url); + } + + Impl(std::string_view Url, IWsClientHandler& Handler, asio::io_context& IoContext, const HttpWsClientSettings& Settings) + : m_Handler(Handler) + , m_Settings(Settings) + , m_Log(logging::Get(Settings.LogCategory)) + , m_IoContext(IoContext) + { + ParseUrl(Url); + } + + ~Impl() + { + // Release work guard so io_context::run() can return + m_WorkGuard.reset(); + + // Close the socket to cancel pending async ops + if (m_Socket) + { + asio::error_code Ec; + m_Socket->close(Ec); + } + + if (m_IoThread.joinable()) + { + m_IoThread.join(); + } + } + + void ParseUrl(std::string_view Url) + { + // Expected format: ws://host:port/path + if (Url.substr(0, 5) == "ws://") + { + Url.remove_prefix(5); + } + + auto SlashPos = Url.find('/'); + std::string_view HostPort; + if (SlashPos != std::string_view::npos) + { + HostPort = Url.substr(0, SlashPos); + m_Path = std::string(Url.substr(SlashPos)); + } + else + { + HostPort = Url; + m_Path = "/"; + } + + auto ColonPos = HostPort.find(':'); + if (ColonPos != std::string_view::npos) + { + m_Host = std::string(HostPort.substr(0, ColonPos)); + m_Port = std::string(HostPort.substr(ColonPos + 1)); + } + else + { + m_Host = std::string(HostPort); + m_Port = "80"; + } + } + + void Connect() + { + if (m_OwnedIoContext) + { + m_WorkGuard = std::make_unique(m_IoContext); + m_IoThread = std::thread([this] { m_IoContext.run(); }); + } + + asio::post(m_IoContext, [this] { DoResolve(); }); + } + + void DoResolve() + { + m_Resolver = std::make_unique(m_IoContext); + + m_Resolver->async_resolve(m_Host, m_Port, [this](const asio::error_code& Ec, asio::ip::tcp::resolver::results_type Results) { + if (Ec) + { + ZEN_LOG_DEBUG(m_Log, "WebSocket resolve failed for {}:{}: {}", m_Host, m_Port, Ec.message()); + m_Handler.OnWsClose(1006, "resolve failed"); + return; + } + + DoConnect(Results); + }); + } + + void DoConnect(const asio::ip::tcp::resolver::results_type& Endpoints) + { + m_Socket = std::make_unique(m_IoContext); + + // Start connect timeout timer + m_Timer = std::make_unique(m_IoContext, m_Settings.ConnectTimeout); + m_Timer->async_wait([this](const asio::error_code& Ec) { + if (!Ec && !m_IsOpen.load(std::memory_order_relaxed)) + { + ZEN_LOG_DEBUG(m_Log, "WebSocket connect timeout for {}:{}", m_Host, m_Port); + if (m_Socket) + { + asio::error_code CloseEc; + m_Socket->close(CloseEc); + } + } + }); + + asio::async_connect(*m_Socket, Endpoints, [this](const asio::error_code& Ec, const asio::ip::tcp::endpoint&) { + if (Ec) + { + m_Timer->cancel(); + ZEN_LOG_DEBUG(m_Log, "WebSocket connect failed for {}:{}: {}", m_Host, m_Port, Ec.message()); + m_Handler.OnWsClose(1006, "connect failed"); + return; + } + + DoHandshake(); + }); + } + + void DoHandshake() + { + // Generate random Sec-WebSocket-Key (16 random bytes, base64 encoded) + uint8_t KeyBytes[16]; + { + static thread_local std::mt19937 s_Rng(std::random_device{}()); + for (int i = 0; i < 4; ++i) + { + uint32_t Val = s_Rng(); + std::memcpy(KeyBytes + i * 4, &Val, 4); + } + } + + char KeyBase64[Base64::GetEncodedDataSize(16) + 1]; + uint32_t KeyLen = Base64::Encode(KeyBytes, 16, KeyBase64); + KeyBase64[KeyLen] = '\0'; + m_WebSocketKey = std::string(KeyBase64, KeyLen); + + // Build the HTTP upgrade request + ExtendableStringBuilder<512> Request; + Request << "GET " << m_Path << " HTTP/1.1\r\n" + << "Host: " << m_Host << ":" << m_Port << "\r\n" + << "Upgrade: websocket\r\n" + << "Connection: Upgrade\r\n" + << "Sec-WebSocket-Key: " << m_WebSocketKey << "\r\n" + << "Sec-WebSocket-Version: 13\r\n"; + + // Add Authorization header if access token provider is set + if (m_Settings.AccessTokenProvider) + { + HttpClientAccessToken Token = (*m_Settings.AccessTokenProvider)(); + if (Token.IsValid()) + { + Request << "Authorization: Bearer " << Token.Value << "\r\n"; + } + } + + Request << "\r\n"; + + std::string_view ReqStr = Request.ToView(); + + m_HandshakeBuffer = std::make_shared(ReqStr); + + asio::async_write(*m_Socket, + asio::buffer(m_HandshakeBuffer->data(), m_HandshakeBuffer->size()), + [this](const asio::error_code& Ec, std::size_t) { + if (Ec) + { + m_Timer->cancel(); + ZEN_LOG_DEBUG(m_Log, "WebSocket handshake write failed: {}", Ec.message()); + m_Handler.OnWsClose(1006, "handshake write failed"); + return; + } + + DoReadHandshakeResponse(); + }); + } + + void DoReadHandshakeResponse() + { + asio::async_read_until(*m_Socket, m_ReadBuffer, "\r\n\r\n", [this](const asio::error_code& Ec, std::size_t) { + m_Timer->cancel(); + + if (Ec) + { + ZEN_LOG_DEBUG(m_Log, "WebSocket handshake read failed: {}", Ec.message()); + m_Handler.OnWsClose(1006, "handshake read failed"); + return; + } + + // Parse the response + const auto& Data = m_ReadBuffer.data(); + std::string Response(asio::buffers_begin(Data), asio::buffers_end(Data)); + + // Consume the headers from the read buffer (any extra data stays for frame parsing) + auto HeaderEnd = Response.find("\r\n\r\n"); + if (HeaderEnd != std::string::npos) + { + m_ReadBuffer.consume(HeaderEnd + 4); + } + + // Validate 101 response + if (Response.find("101") == std::string::npos) + { + ZEN_LOG_DEBUG(m_Log, "WebSocket handshake rejected (no 101): {}", Response.substr(0, 80)); + m_Handler.OnWsClose(1006, "handshake rejected"); + return; + } + + // Validate Sec-WebSocket-Accept + std::string ExpectedAccept = WsFrameCodec::ComputeAcceptKey(m_WebSocketKey); + if (Response.find(ExpectedAccept) == std::string::npos) + { + ZEN_LOG_DEBUG(m_Log, "WebSocket handshake: invalid Sec-WebSocket-Accept"); + m_Handler.OnWsClose(1006, "invalid accept key"); + return; + } + + m_IsOpen.store(true); + m_Handler.OnWsOpen(); + EnqueueRead(); + }); + } + + ////////////////////////////////////////////////////////////////////////// + // + // Read loop + // + + void EnqueueRead() + { + if (!m_IsOpen.load(std::memory_order_relaxed)) + { + return; + } + + asio::async_read(*m_Socket, m_ReadBuffer, asio::transfer_at_least(1), [this](const asio::error_code& Ec, std::size_t) { + OnDataReceived(Ec); + }); + } + + void OnDataReceived(const asio::error_code& Ec) + { + if (Ec) + { + if (Ec != asio::error::eof && Ec != asio::error::operation_aborted) + { + ZEN_LOG_DEBUG(m_Log, "WebSocket read error: {}", Ec.message()); + } + + if (m_IsOpen.exchange(false)) + { + m_Handler.OnWsClose(1006, "connection lost"); + } + return; + } + + ProcessReceivedData(); + + if (m_IsOpen.load(std::memory_order_relaxed)) + { + EnqueueRead(); + } + } + + void ProcessReceivedData() + { + while (m_ReadBuffer.size() > 0) + { + const auto& InputBuffer = m_ReadBuffer.data(); + const auto* RawData = static_cast(InputBuffer.data()); + const auto Size = InputBuffer.size(); + + WsFrameParseResult Frame = WsFrameCodec::TryParseFrame(RawData, Size); + if (!Frame.IsValid) + { + break; + } + + m_ReadBuffer.consume(Frame.BytesConsumed); + + switch (Frame.Opcode) + { + case WebSocketOpcode::kText: + case WebSocketOpcode::kBinary: + { + WebSocketMessage Msg; + Msg.Opcode = Frame.Opcode; + Msg.Payload = IoBuffer(IoBuffer::Clone, Frame.Payload.data(), Frame.Payload.size()); + m_Handler.OnWsMessage(Msg); + break; + } + + case WebSocketOpcode::kPing: + { + // Auto-respond with masked pong + std::vector PongFrame = WsFrameCodec::BuildMaskedFrame(WebSocketOpcode::kPong, Frame.Payload); + EnqueueWrite(std::move(PongFrame)); + break; + } + + case WebSocketOpcode::kPong: + break; + + case WebSocketOpcode::kClose: + { + uint16_t Code = 1000; + std::string_view Reason; + + if (Frame.Payload.size() >= 2) + { + Code = (uint16_t(Frame.Payload[0]) << 8) | uint16_t(Frame.Payload[1]); + if (Frame.Payload.size() > 2) + { + Reason = + std::string_view(reinterpret_cast(Frame.Payload.data() + 2), Frame.Payload.size() - 2); + } + } + + // Echo masked close frame if we haven't sent one yet + if (!m_CloseSent) + { + m_CloseSent = true; + std::vector CloseFrame = WsFrameCodec::BuildMaskedCloseFrame(Code); + EnqueueWrite(std::move(CloseFrame)); + } + + m_IsOpen.store(false); + m_Handler.OnWsClose(Code, Reason); + return; + } + + default: + ZEN_LOG_WARN(m_Log, "Unknown WebSocket opcode: {:#x}", static_cast(Frame.Opcode)); + break; + } + } + } + + ////////////////////////////////////////////////////////////////////////// + // + // Write queue + // + + void EnqueueWrite(std::vector Frame) + { + bool ShouldFlush = false; + + m_WriteLock.WithExclusiveLock([&] { + m_WriteQueue.push_back(std::move(Frame)); + if (!m_IsWriting) + { + m_IsWriting = true; + ShouldFlush = true; + } + }); + + if (ShouldFlush) + { + FlushWriteQueue(); + } + } + + void FlushWriteQueue() + { + std::vector Frame; + + m_WriteLock.WithExclusiveLock([&] { + if (m_WriteQueue.empty()) + { + m_IsWriting = false; + return; + } + Frame = std::move(m_WriteQueue.front()); + m_WriteQueue.pop_front(); + }); + + if (Frame.empty()) + { + return; + } + + auto OwnedFrame = std::make_shared>(std::move(Frame)); + + asio::async_write(*m_Socket, + asio::buffer(OwnedFrame->data(), OwnedFrame->size()), + [this, OwnedFrame](const asio::error_code& Ec, std::size_t) { OnWriteComplete(Ec); }); + } + + void OnWriteComplete(const asio::error_code& Ec) + { + if (Ec) + { + if (Ec != asio::error::operation_aborted) + { + ZEN_LOG_DEBUG(m_Log, "WebSocket write error: {}", Ec.message()); + } + + m_WriteLock.WithExclusiveLock([&] { + m_IsWriting = false; + m_WriteQueue.clear(); + }); + + if (m_IsOpen.exchange(false)) + { + m_Handler.OnWsClose(1006, "write error"); + } + return; + } + + FlushWriteQueue(); + } + + ////////////////////////////////////////////////////////////////////////// + // + // Public operations + // + + void SendText(std::string_view Text) + { + if (!m_IsOpen.load(std::memory_order_relaxed)) + { + return; + } + + std::span Payload(reinterpret_cast(Text.data()), Text.size()); + std::vector Frame = WsFrameCodec::BuildMaskedFrame(WebSocketOpcode::kText, Payload); + EnqueueWrite(std::move(Frame)); + } + + void SendBinary(std::span Data) + { + if (!m_IsOpen.load(std::memory_order_relaxed)) + { + return; + } + + std::vector Frame = WsFrameCodec::BuildMaskedFrame(WebSocketOpcode::kBinary, Data); + EnqueueWrite(std::move(Frame)); + } + + void DoClose(uint16_t Code, std::string_view Reason) + { + if (!m_IsOpen.exchange(false)) + { + return; + } + + if (!m_CloseSent) + { + m_CloseSent = true; + std::vector CloseFrame = WsFrameCodec::BuildMaskedCloseFrame(Code, Reason); + EnqueueWrite(std::move(CloseFrame)); + } + } + + IWsClientHandler& m_Handler; + HttpWsClientSettings m_Settings; + LoggerRef m_Log; + + std::string m_Host; + std::string m_Port; + std::string m_Path; + + // io_context: owned (standalone) or external (shared) + std::unique_ptr m_OwnedIoContext; + asio::io_context& m_IoContext; + std::unique_ptr m_WorkGuard; + std::thread m_IoThread; + + // Connection state + std::unique_ptr m_Resolver; + std::unique_ptr m_Socket; + std::unique_ptr m_Timer; + asio::streambuf m_ReadBuffer; + std::string m_WebSocketKey; + std::shared_ptr m_HandshakeBuffer; + + // Write queue + RwLock m_WriteLock; + std::deque> m_WriteQueue; + bool m_IsWriting = false; + + std::atomic m_IsOpen{false}; + bool m_CloseSent = false; +}; + +////////////////////////////////////////////////////////////////////////// + +HttpWsClient::HttpWsClient(std::string_view Url, IWsClientHandler& Handler, const HttpWsClientSettings& Settings) +: m_Impl(std::make_unique(Url, Handler, Settings)) +{ +} + +HttpWsClient::HttpWsClient(std::string_view Url, + IWsClientHandler& Handler, + asio::io_context& IoContext, + const HttpWsClientSettings& Settings) +: m_Impl(std::make_unique(Url, Handler, IoContext, Settings)) +{ +} + +HttpWsClient::~HttpWsClient() = default; + +void +HttpWsClient::Connect() +{ + m_Impl->Connect(); +} + +void +HttpWsClient::SendText(std::string_view Text) +{ + m_Impl->SendText(Text); +} + +void +HttpWsClient::SendBinary(std::span Data) +{ + m_Impl->SendBinary(Data); +} + +void +HttpWsClient::Close(uint16_t Code, std::string_view Reason) +{ + m_Impl->DoClose(Code, Reason); +} + +bool +HttpWsClient::IsOpen() const +{ + return m_Impl->m_IsOpen.load(std::memory_order_relaxed); +} + +} // namespace zen diff --git a/src/zenhttp/include/zenhttp/httpcommon.h b/src/zenhttp/include/zenhttp/httpcommon.h index bc18549c9..8fca35ac5 100644 --- a/src/zenhttp/include/zenhttp/httpcommon.h +++ b/src/zenhttp/include/zenhttp/httpcommon.h @@ -184,6 +184,13 @@ IsHttpSuccessCode(HttpResponseCode HttpCode) noexcept return IsHttpSuccessCode(int(HttpCode)); } +[[nodiscard]] inline bool +IsHttpOk(HttpResponseCode HttpCode) noexcept +{ + return HttpCode == HttpResponseCode::OK || HttpCode == HttpResponseCode::Created || HttpCode == HttpResponseCode::Accepted || + HttpCode == HttpResponseCode::NoContent; +} + std::string_view ToString(HttpResponseCode HttpCode); } // namespace zen diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 00cbc6c14..fee932daa 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -462,6 +462,7 @@ struct IHttpStatsService virtual void UnregisterHandler(std::string_view Id, IHttpStatsProvider& Provider) = 0; }; -void http_forcelink(); // internal +void http_forcelink(); // internal +void websocket_forcelink(); // internal } // namespace zen diff --git a/src/zenhttp/include/zenhttp/httpwsclient.h b/src/zenhttp/include/zenhttp/httpwsclient.h new file mode 100644 index 000000000..926ec1e3d --- /dev/null +++ b/src/zenhttp/include/zenhttp/httpwsclient.h @@ -0,0 +1,79 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zenhttp.h" + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zen { + +/** + * Callback interface for WebSocket client events + * + * Separate from the server-side IWebSocketHandler because the caller + * already owns the HttpWsClient — no Ref needed. + */ +class IWsClientHandler +{ +public: + virtual ~IWsClientHandler() = default; + + virtual void OnWsOpen() = 0; + virtual void OnWsMessage(const WebSocketMessage& Msg) = 0; + virtual void OnWsClose(uint16_t Code, std::string_view Reason) = 0; +}; + +struct HttpWsClientSettings +{ + std::string LogCategory = "wsclient"; + std::chrono::milliseconds ConnectTimeout{5000}; + std::optional> AccessTokenProvider; +}; + +/** + * WebSocket client over TCP (ws:// scheme) + * + * Uses ASIO for async I/O. Two construction modes: + * - Internal io_context + background thread (standalone use) + * - External io_context (shared event loop, no internal thread) + * + * Thread-safe for SendText/SendBinary/Close. + */ +class HttpWsClient +{ +public: + HttpWsClient(std::string_view Url, IWsClientHandler& Handler, const HttpWsClientSettings& Settings = {}); + HttpWsClient(std::string_view Url, IWsClientHandler& Handler, asio::io_context& IoContext, const HttpWsClientSettings& Settings = {}); + + ~HttpWsClient(); + + HttpWsClient(const HttpWsClient&) = delete; + HttpWsClient& operator=(const HttpWsClient&) = delete; + + void Connect(); + void SendText(std::string_view Text); + void SendBinary(std::span Data); + void Close(uint16_t Code = 1000, std::string_view Reason = {}); + bool IsOpen() const; + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +} // namespace zen diff --git a/src/zenhttp/include/zenhttp/websocket.h b/src/zenhttp/include/zenhttp/websocket.h new file mode 100644 index 000000000..7a6fb33dd --- /dev/null +++ b/src/zenhttp/include/zenhttp/websocket.h @@ -0,0 +1,65 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include +#include +#include + +namespace zen { + +enum class WebSocketOpcode : uint8_t +{ + kText = 0x1, + kBinary = 0x2, + kClose = 0x8, + kPing = 0x9, + kPong = 0xA +}; + +struct WebSocketMessage +{ + WebSocketOpcode Opcode; + IoBuffer Payload; + uint16_t CloseCode = 0; +}; + +/** + * Represents an active WebSocket connection + * + * Derived classes implement the actual transport (e.g. ASIO sockets). + * Instances are reference-counted so that both the service layer and + * the async read/write loop can share ownership. + */ +class WebSocketConnection : public RefCounted +{ +public: + virtual ~WebSocketConnection() = default; + + virtual void SendText(std::string_view Text) = 0; + virtual void SendBinary(std::span Data) = 0; + virtual void Close(uint16_t Code = 1000, std::string_view Reason = {}) = 0; + virtual bool IsOpen() const = 0; +}; + +/** + * Interface for services that accept WebSocket upgrades + * + * An HttpService may additionally implement this interface to indicate + * it supports WebSocket connections. The HTTP server checks for this + * via dynamic_cast when it sees an Upgrade: websocket request. + */ +class IWebSocketHandler +{ +public: + virtual ~IWebSocketHandler() = default; + + virtual void OnWebSocketOpen(Ref Connection) = 0; + virtual void OnWebSocketMessage(WebSocketConnection& Conn, const WebSocketMessage& Msg) = 0; + virtual void OnWebSocketClose(WebSocketConnection& Conn, uint16_t Code, std::string_view Reason) = 0; +}; + +} // namespace zen diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 0c0238886..8c2dcd116 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -14,6 +14,8 @@ #include #include "httpparser.h" +#include "wsasio.h" +#include "wsframecodec.h" #include @@ -1159,6 +1161,53 @@ HttpServerConnection::HandleRequest() { ZEN_MEMSCOPE(GetHttpasioTag()); + // WebSocket upgrade detection must happen before the keep-alive check below, + // because Upgrade requests have "Connection: Upgrade" which the HTTP parser + // treats as non-keep-alive, causing a premature shutdown of the receive side. + if (m_RequestData.IsWebSocketUpgrade()) + { + if (HttpService* Service = m_Server.RouteRequest(m_RequestData.Url())) + { + IWebSocketHandler* WsHandler = dynamic_cast(Service); + if (WsHandler && !m_RequestData.SecWebSocketKey().empty()) + { + std::string AcceptKey = WsFrameCodec::ComputeAcceptKey(m_RequestData.SecWebSocketKey()); + + auto ResponseStr = std::make_shared(); + ResponseStr->reserve(256); + ResponseStr->append( + "HTTP/1.1 101 Switching Protocols\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + "Sec-WebSocket-Accept: "); + ResponseStr->append(AcceptKey); + ResponseStr->append("\r\n\r\n"); + + // Send the 101 response on the current socket, then hand the socket off + // to a WsAsioConnection for the WebSocket protocol. + asio::async_write(*m_Socket, + asio::buffer(ResponseStr->data(), ResponseStr->size()), + [Conn = AsSharedPtr(), WsHandler, OwnedResponse = ResponseStr](const asio::error_code& Ec, std::size_t) { + if (Ec) + { + ZEN_WARN("WebSocket 101 send failed: {}", Ec.message()); + return; + } + + Ref WsConn(new WsAsioConnection(std::move(Conn->m_Socket), *WsHandler)); + Ref WsConnRef(WsConn.Get()); + + WsHandler->OnWebSocketOpen(std::move(WsConnRef)); + WsConn->Start(); + }); + + m_RequestState = RequestState::kDone; + return; + } + } + // Service doesn't support WebSocket or missing key — fall through to normal handling + } + if (!m_RequestData.IsKeepAlive()) { m_RequestState = RequestState::kWritingFinal; diff --git a/src/zenhttp/servers/httpparser.cpp b/src/zenhttp/servers/httpparser.cpp index f0485aa25..3b1229375 100644 --- a/src/zenhttp/servers/httpparser.cpp +++ b/src/zenhttp/servers/httpparser.cpp @@ -12,14 +12,17 @@ namespace zen { using namespace std::literals; -static constinit uint32_t HashContentLength = HashStringAsLowerDjb2("Content-Length"sv); -static constinit uint32_t HashContentType = HashStringAsLowerDjb2("Content-Type"sv); -static constinit uint32_t HashAccept = HashStringAsLowerDjb2("Accept"sv); -static constinit uint32_t HashExpect = HashStringAsLowerDjb2("Expect"sv); -static constinit uint32_t HashSession = HashStringAsLowerDjb2("UE-Session"sv); -static constinit uint32_t HashRequest = HashStringAsLowerDjb2("UE-Request"sv); -static constinit uint32_t HashRange = HashStringAsLowerDjb2("Range"sv); -static constinit uint32_t HashAuthorization = HashStringAsLowerDjb2("Authorization"sv); +static constexpr uint32_t HashContentLength = HashStringAsLowerDjb2("Content-Length"sv); +static constexpr uint32_t HashContentType = HashStringAsLowerDjb2("Content-Type"sv); +static constexpr uint32_t HashAccept = HashStringAsLowerDjb2("Accept"sv); +static constexpr uint32_t HashExpect = HashStringAsLowerDjb2("Expect"sv); +static constexpr uint32_t HashSession = HashStringAsLowerDjb2("UE-Session"sv); +static constexpr uint32_t HashRequest = HashStringAsLowerDjb2("UE-Request"sv); +static constexpr uint32_t HashRange = HashStringAsLowerDjb2("Range"sv); +static constexpr uint32_t HashAuthorization = HashStringAsLowerDjb2("Authorization"sv); +static constexpr uint32_t HashUpgrade = HashStringAsLowerDjb2("Upgrade"sv); +static constexpr uint32_t HashSecWebSocketKey = HashStringAsLowerDjb2("Sec-WebSocket-Key"sv); +static constexpr uint32_t HashSecWebSocketVersion = HashStringAsLowerDjb2("Sec-WebSocket-Version"sv); ////////////////////////////////////////////////////////////////////////// // @@ -143,45 +146,62 @@ HttpRequestParser::ParseCurrentHeader() const uint32_t HeaderHash = HashStringAsLowerDjb2(HeaderName); const int8_t CurrentHeaderIndex = int8_t(CurrentHeaderCount - 1); - if (HeaderHash == HashContentLength) + switch (HeaderHash) { - m_ContentLengthHeaderIndex = CurrentHeaderIndex; - } - else if (HeaderHash == HashAccept) - { - m_AcceptHeaderIndex = CurrentHeaderIndex; - } - else if (HeaderHash == HashContentType) - { - m_ContentTypeHeaderIndex = CurrentHeaderIndex; - } - else if (HeaderHash == HashAuthorization) - { - m_AuthorizationHeaderIndex = CurrentHeaderIndex; - } - else if (HeaderHash == HashSession) - { - m_SessionId = Oid::TryFromHexString(HeaderValue); - } - else if (HeaderHash == HashRequest) - { - std::from_chars(HeaderValue.data(), HeaderValue.data() + HeaderValue.size(), m_RequestId); - } - else if (HeaderHash == HashExpect) - { - if (HeaderValue == "100-continue"sv) - { - // We don't currently do anything with this - m_Expect100Continue = true; - } - else - { - ZEN_INFO("Unexpected expect - Expect: {}", HeaderValue); - } - } - else if (HeaderHash == HashRange) - { - m_RangeHeaderIndex = CurrentHeaderIndex; + case HashContentLength: + m_ContentLengthHeaderIndex = CurrentHeaderIndex; + break; + + case HashAccept: + m_AcceptHeaderIndex = CurrentHeaderIndex; + break; + + case HashContentType: + m_ContentTypeHeaderIndex = CurrentHeaderIndex; + break; + + case HashAuthorization: + m_AuthorizationHeaderIndex = CurrentHeaderIndex; + break; + + case HashSession: + m_SessionId = Oid::TryFromHexString(HeaderValue); + break; + + case HashRequest: + std::from_chars(HeaderValue.data(), HeaderValue.data() + HeaderValue.size(), m_RequestId); + break; + + case HashExpect: + if (HeaderValue == "100-continue"sv) + { + // We don't currently do anything with this + m_Expect100Continue = true; + } + else + { + ZEN_INFO("Unexpected expect - Expect: {}", HeaderValue); + } + break; + + case HashRange: + m_RangeHeaderIndex = CurrentHeaderIndex; + break; + + case HashUpgrade: + m_UpgradeHeaderIndex = CurrentHeaderIndex; + break; + + case HashSecWebSocketKey: + m_SecWebSocketKeyHeaderIndex = CurrentHeaderIndex; + break; + + case HashSecWebSocketVersion: + m_SecWebSocketVersionHeaderIndex = CurrentHeaderIndex; + break; + + default: + break; } } @@ -361,14 +381,17 @@ HttpRequestParser::ResetState() m_HeaderEntries.clear(); - m_ContentLengthHeaderIndex = -1; - m_AcceptHeaderIndex = -1; - m_ContentTypeHeaderIndex = -1; - m_RangeHeaderIndex = -1; - m_AuthorizationHeaderIndex = -1; - m_Expect100Continue = false; - m_BodyBuffer = {}; - m_BodyPosition = 0; + m_ContentLengthHeaderIndex = -1; + m_AcceptHeaderIndex = -1; + m_ContentTypeHeaderIndex = -1; + m_RangeHeaderIndex = -1; + m_AuthorizationHeaderIndex = -1; + m_UpgradeHeaderIndex = -1; + m_SecWebSocketKeyHeaderIndex = -1; + m_SecWebSocketVersionHeaderIndex = -1; + m_Expect100Continue = false; + m_BodyBuffer = {}; + m_BodyPosition = 0; m_HeaderData.clear(); m_NormalizedUrl.clear(); @@ -425,4 +448,21 @@ HttpRequestParser::OnMessageComplete() } } +bool +HttpRequestParser::IsWebSocketUpgrade() const +{ + std::string_view Upgrade = GetHeaderValue(m_UpgradeHeaderIndex); + if (Upgrade.empty()) + { + return false; + } + + // Case-insensitive check for "websocket" + if (Upgrade.size() != 9) + { + return false; + } + return StrCaseCompare(Upgrade.data(), "websocket", 9) == 0; +} + } // namespace zen diff --git a/src/zenhttp/servers/httpparser.h b/src/zenhttp/servers/httpparser.h index ff56ca970..d40a5aeb0 100644 --- a/src/zenhttp/servers/httpparser.h +++ b/src/zenhttp/servers/httpparser.h @@ -48,6 +48,10 @@ struct HttpRequestParser std::string_view AuthorizationHeader() const { return GetHeaderValue(m_AuthorizationHeaderIndex); } + std::string_view UpgradeHeader() const { return GetHeaderValue(m_UpgradeHeaderIndex); } + std::string_view SecWebSocketKey() const { return GetHeaderValue(m_SecWebSocketKeyHeaderIndex); } + bool IsWebSocketUpgrade() const; + private: struct HeaderRange { @@ -86,6 +90,9 @@ private: int8_t m_ContentTypeHeaderIndex; int8_t m_RangeHeaderIndex; int8_t m_AuthorizationHeaderIndex; + int8_t m_UpgradeHeaderIndex; + int8_t m_SecWebSocketKeyHeaderIndex; + int8_t m_SecWebSocketVersionHeaderIndex; HttpVerb m_RequestVerb; std::atomic_bool m_KeepAlive{false}; bool m_Expect100Continue = false; diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index e93ae4853..23d57af57 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -156,6 +156,10 @@ private: #if ZEN_WITH_HTTPSYS +# include "httpsys_iocontext.h" +# include "wshttpsys.h" +# include "wsframecodec.h" + # include # include # pragma comment(lib, "httpapi.lib") @@ -380,7 +384,7 @@ public: PTP_IO Iocp(); HANDLE RequestQueueHandle(); - inline OVERLAPPED* Overlapped() { return &m_HttpOverlapped; } + inline OVERLAPPED* Overlapped() { return &m_IoContext.Overlapped; } inline HttpSysServer& Server() { return m_HttpServer; } inline HTTP_REQUEST* HttpRequest() { return m_InitialHttpHandler.HttpRequest(); } @@ -397,8 +401,8 @@ public: }; private: - OVERLAPPED m_HttpOverlapped{}; - HttpSysServer& m_HttpServer; + HttpSysIoContext m_IoContext{}; + HttpSysServer& m_HttpServer; // Tracks which handler is due to handle the next I/O completion event HttpSysRequestHandler* m_CompletionHandler = nullptr; @@ -1555,7 +1559,23 @@ HttpSysTransaction::IoCompletionCallback(PTP_CALLBACK_INSTANCE Instance, // than one thread at any given moment. This means we need to be careful about what // happens in here - HttpSysTransaction* Transaction = CONTAINING_RECORD(pOverlapped, HttpSysTransaction, m_HttpOverlapped); + HttpSysIoContext* IoContext = CONTAINING_RECORD(pOverlapped, HttpSysIoContext, Overlapped); + + switch (IoContext->ContextType) + { + case HttpSysIoContext::Type::kWebSocketRead: + static_cast(IoContext->Owner)->OnReadCompletion(IoResult, NumberOfBytesTransferred); + return; + + case HttpSysIoContext::Type::kWebSocketWrite: + static_cast(IoContext->Owner)->OnWriteCompletion(IoResult, NumberOfBytesTransferred); + return; + + case HttpSysIoContext::Type::kTransaction: + break; + } + + HttpSysTransaction* Transaction = CONTAINING_RECORD(IoContext, HttpSysTransaction, m_IoContext); if (Transaction->HandleCompletion(IoResult, NumberOfBytesTransferred) == HttpSysTransaction::Status::kDone) { @@ -2111,64 +2131,118 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT { HTTP_REQUEST* HttpReq = HttpRequest(); -# if 0 - for (int i = 0; i < HttpReq->RequestInfoCount; ++i) + if (HttpService* Service = reinterpret_cast(HttpReq->UrlContext)) { - auto& ReqInfo = HttpReq->pRequestInfo[i]; - - switch (ReqInfo.InfoType) + // WebSocket upgrade detection + if (m_IsInitialRequest) { - case HttpRequestInfoTypeRequestTiming: + const HTTP_KNOWN_HEADER& UpgradeHeader = HttpReq->Headers.KnownHeaders[HttpHeaderUpgrade]; + if (UpgradeHeader.RawValueLength > 0 && + StrCaseCompare(UpgradeHeader.pRawValue, "websocket", UpgradeHeader.RawValueLength) == 0) + { + if (IWebSocketHandler* WsHandler = dynamic_cast(Service)) { - const HTTP_REQUEST_TIMING_INFO* TimingInfo = reinterpret_cast(ReqInfo.pInfo); + // Extract Sec-WebSocket-Key from the unknown headers + // (http.sys has no known-header slot for it) + std::string_view SecWebSocketKey; + for (USHORT i = 0; i < HttpReq->Headers.UnknownHeaderCount; ++i) + { + const HTTP_UNKNOWN_HEADER& Hdr = HttpReq->Headers.pUnknownHeaders[i]; + if (Hdr.NameLength == 17 && _strnicmp(Hdr.pName, "Sec-WebSocket-Key", 17) == 0) + { + SecWebSocketKey = std::string_view(Hdr.pRawValue, Hdr.RawValueLength); + break; + } + } - ZEN_INFO(""); - } - break; - case HttpRequestInfoTypeAuth: - ZEN_INFO(""); - break; - case HttpRequestInfoTypeChannelBind: - ZEN_INFO(""); - break; - case HttpRequestInfoTypeSslProtocol: - ZEN_INFO(""); - break; - case HttpRequestInfoTypeSslTokenBindingDraft: - ZEN_INFO(""); - break; - case HttpRequestInfoTypeSslTokenBinding: - ZEN_INFO(""); - break; - case HttpRequestInfoTypeTcpInfoV0: - { - const TCP_INFO_v0* TcpInfo = reinterpret_cast(ReqInfo.pInfo); + if (SecWebSocketKey.empty()) + { + ZEN_WARN("WebSocket upgrade missing Sec-WebSocket-Key header"); + return nullptr; + } - ZEN_INFO(""); - } - break; - case HttpRequestInfoTypeRequestSizing: - { - const HTTP_REQUEST_SIZING_INFO* SizingInfo = reinterpret_cast(ReqInfo.pInfo); - ZEN_INFO(""); - } - break; - case HttpRequestInfoTypeQuicStats: - ZEN_INFO(""); - break; - case HttpRequestInfoTypeTcpInfoV1: - { - const TCP_INFO_v1* TcpInfo = reinterpret_cast(ReqInfo.pInfo); + const std::string AcceptKey = WsFrameCodec::ComputeAcceptKey(SecWebSocketKey); + + HANDLE RequestQueueHandle = Transaction().RequestQueueHandle(); + HTTP_REQUEST_ID RequestId = HttpReq->RequestId; + + // Build the 101 Switching Protocols response + HTTP_RESPONSE Response = {}; + Response.StatusCode = 101; + Response.pReason = "Switching Protocols"; + Response.ReasonLength = (USHORT)strlen(Response.pReason); + + Response.Headers.KnownHeaders[HttpHeaderUpgrade].pRawValue = "websocket"; + Response.Headers.KnownHeaders[HttpHeaderUpgrade].RawValueLength = 9; + + eastl::fixed_vector UnknownHeaders; + + // IMPORTANT: Due to some quirk in HttpSendHttpResponse, this cannot use KnownHeaders + // despite there being an entry for it there (HttpHeaderConnection). If you try to do + // that you get an ERROR_INVALID_PARAMETERS error from HttpSendHttpResponse below + + UnknownHeaders.push_back({.NameLength = 10, .RawValueLength = 7, .pName = "Connection", .pRawValue = "Upgrade"}); + + UnknownHeaders.push_back({.NameLength = 20, + .RawValueLength = (USHORT)AcceptKey.size(), + .pName = "Sec-WebSocket-Accept", + .pRawValue = AcceptKey.c_str()}); + + Response.Headers.UnknownHeaderCount = (USHORT)UnknownHeaders.size(); + Response.Headers.pUnknownHeaders = UnknownHeaders.data(); + + const ULONG Flags = HTTP_SEND_RESPONSE_FLAG_OPAQUE | HTTP_SEND_RESPONSE_FLAG_MORE_DATA; + + // Use an OVERLAPPED with an event so we can wait synchronously. + // The request queue is IOCP-associated, so passing NULL for pOverlapped + // may return ERROR_IO_PENDING. Setting the low-order bit of hEvent + // prevents IOCP delivery and lets us wait on the event directly. + OVERLAPPED SendOverlapped = {}; + HANDLE SendEvent = CreateEventW(nullptr, TRUE, FALSE, nullptr); + SendOverlapped.hEvent = (HANDLE)((uintptr_t)SendEvent | 1); + + ULONG SendResult = HttpSendHttpResponse(RequestQueueHandle, + RequestId, + Flags, + &Response, + nullptr, // CachePolicy + nullptr, // BytesSent + nullptr, // Reserved1 + 0, // Reserved2 + &SendOverlapped, + nullptr // LogData + ); + + if (SendResult == ERROR_IO_PENDING) + { + WaitForSingleObject(SendEvent, INFINITE); + SendResult = (SendOverlapped.Internal == 0) ? NO_ERROR : ERROR_IO_INCOMPLETE; + } + + CloseHandle(SendEvent); + + if (SendResult == NO_ERROR) + { + Ref WsConn( + new WsHttpSysConnection(RequestQueueHandle, RequestId, *WsHandler, Transaction().Iocp())); + Ref WsConnRef(WsConn.Get()); + + WsHandler->OnWebSocketOpen(std::move(WsConnRef)); + WsConn->Start(); + + return nullptr; + } - ZEN_INFO(""); + ZEN_WARN("WebSocket 101 send failed: {}", SendResult); + + // WebSocket upgrade failed — return nullptr since ServerRequest() + // was never populated (no InvokeRequestHandler call) + return nullptr; } - break; + // Service doesn't support WebSocket or missing key — fall through to normal handling + } } - } -# endif - if (HttpService* Service = reinterpret_cast(HttpReq->UrlContext)) - { if (m_IsInitialRequest) { m_ContentLength = GetContentLength(HttpReq); diff --git a/src/zenhttp/servers/httpsys_iocontext.h b/src/zenhttp/servers/httpsys_iocontext.h new file mode 100644 index 000000000..4f8a97012 --- /dev/null +++ b/src/zenhttp/servers/httpsys_iocontext.h @@ -0,0 +1,40 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#if ZEN_WITH_HTTPSYS +# define _WINSOCKAPI_ +# include + +# include + +namespace zen { + +/** + * Tagged OVERLAPPED wrapper for http.sys IOCP dispatch + * + * Both HttpSysTransaction (for normal HTTP request I/O) and WsHttpSysConnection + * (for WebSocket read/write) embed this struct. The single IoCompletionCallback + * bound to the request queue uses the ContextType tag to dispatch to the correct + * handler. + * + * The Overlapped member must be first so that CONTAINING_RECORD works to recover + * the HttpSysIoContext from the OVERLAPPED pointer provided by the threadpool. + */ +struct HttpSysIoContext +{ + OVERLAPPED Overlapped{}; + + enum class Type : uint8_t + { + kTransaction, + kWebSocketRead, + kWebSocketWrite, + } ContextType = Type::kTransaction; + + void* Owner = nullptr; +}; + +} // namespace zen + +#endif // ZEN_WITH_HTTPSYS diff --git a/src/zenhttp/servers/wsasio.cpp b/src/zenhttp/servers/wsasio.cpp new file mode 100644 index 000000000..dfc1eac38 --- /dev/null +++ b/src/zenhttp/servers/wsasio.cpp @@ -0,0 +1,297 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "wsasio.h" +#include "wsframecodec.h" + +#include + +namespace zen::asio_http { + +static LoggerRef +WsLog() +{ + static LoggerRef g_Logger = logging::Get("ws"); + return g_Logger; +} + +////////////////////////////////////////////////////////////////////////// + +WsAsioConnection::WsAsioConnection(std::unique_ptr Socket, IWebSocketHandler& Handler) +: m_Socket(std::move(Socket)) +, m_Handler(Handler) +{ +} + +WsAsioConnection::~WsAsioConnection() +{ + m_IsOpen.store(false); +} + +void +WsAsioConnection::Start() +{ + EnqueueRead(); +} + +bool +WsAsioConnection::IsOpen() const +{ + return m_IsOpen.load(std::memory_order_relaxed); +} + +////////////////////////////////////////////////////////////////////////// +// +// Read loop +// + +void +WsAsioConnection::EnqueueRead() +{ + if (!m_IsOpen.load(std::memory_order_relaxed)) + { + return; + } + + Ref Self(this); + + asio::async_read(*m_Socket, m_ReadBuffer, asio::transfer_at_least(1), [Self](const asio::error_code& Ec, std::size_t ByteCount) { + Self->OnDataReceived(Ec, ByteCount); + }); +} + +void +WsAsioConnection::OnDataReceived(const asio::error_code& Ec, [[maybe_unused]] std::size_t ByteCount) +{ + if (Ec) + { + if (Ec != asio::error::eof && Ec != asio::error::operation_aborted) + { + ZEN_LOG_DEBUG(WsLog(), "WebSocket read error: {}", Ec.message()); + } + + if (m_IsOpen.exchange(false)) + { + m_Handler.OnWebSocketClose(*this, 1006, "connection lost"); + } + return; + } + + ProcessReceivedData(); + + if (m_IsOpen.load(std::memory_order_relaxed)) + { + EnqueueRead(); + } +} + +void +WsAsioConnection::ProcessReceivedData() +{ + while (m_ReadBuffer.size() > 0) + { + const auto& InputBuffer = m_ReadBuffer.data(); + const auto* Data = static_cast(InputBuffer.data()); + const auto Size = InputBuffer.size(); + + WsFrameParseResult Frame = WsFrameCodec::TryParseFrame(Data, Size); + if (!Frame.IsValid) + { + break; // not enough data yet + } + + m_ReadBuffer.consume(Frame.BytesConsumed); + + switch (Frame.Opcode) + { + case WebSocketOpcode::kText: + case WebSocketOpcode::kBinary: + { + WebSocketMessage Msg; + Msg.Opcode = Frame.Opcode; + Msg.Payload = IoBuffer(IoBuffer::Clone, Frame.Payload.data(), Frame.Payload.size()); + m_Handler.OnWebSocketMessage(*this, Msg); + break; + } + + case WebSocketOpcode::kPing: + { + // Auto-respond with pong carrying the same payload + std::vector PongFrame = WsFrameCodec::BuildFrame(WebSocketOpcode::kPong, Frame.Payload); + EnqueueWrite(std::move(PongFrame)); + break; + } + + case WebSocketOpcode::kPong: + // Unsolicited pong — ignore per RFC 6455 + break; + + case WebSocketOpcode::kClose: + { + uint16_t Code = 1000; + std::string_view Reason; + + if (Frame.Payload.size() >= 2) + { + Code = (uint16_t(Frame.Payload[0]) << 8) | uint16_t(Frame.Payload[1]); + if (Frame.Payload.size() > 2) + { + Reason = std::string_view(reinterpret_cast(Frame.Payload.data() + 2), Frame.Payload.size() - 2); + } + } + + // Echo close frame back if we haven't sent one yet + if (!m_CloseSent) + { + m_CloseSent = true; + std::vector CloseFrame = WsFrameCodec::BuildCloseFrame(Code); + EnqueueWrite(std::move(CloseFrame)); + } + + m_IsOpen.store(false); + m_Handler.OnWebSocketClose(*this, Code, Reason); + + // Shut down the socket + std::error_code ShutdownEc; + m_Socket->shutdown(asio::socket_base::shutdown_both, ShutdownEc); + m_Socket->close(ShutdownEc); + return; + } + + default: + ZEN_LOG_WARN(WsLog(), "Unknown WebSocket opcode: {:#x}", static_cast(Frame.Opcode)); + break; + } + } +} + +////////////////////////////////////////////////////////////////////////// +// +// Write queue +// + +void +WsAsioConnection::SendText(std::string_view Text) +{ + if (!m_IsOpen.load(std::memory_order_relaxed)) + { + return; + } + + std::span Payload(reinterpret_cast(Text.data()), Text.size()); + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kText, Payload); + EnqueueWrite(std::move(Frame)); +} + +void +WsAsioConnection::SendBinary(std::span Data) +{ + if (!m_IsOpen.load(std::memory_order_relaxed)) + { + return; + } + + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kBinary, Data); + EnqueueWrite(std::move(Frame)); +} + +void +WsAsioConnection::Close(uint16_t Code, std::string_view Reason) +{ + DoClose(Code, Reason); +} + +void +WsAsioConnection::DoClose(uint16_t Code, std::string_view Reason) +{ + if (!m_IsOpen.exchange(false)) + { + return; + } + + if (!m_CloseSent) + { + m_CloseSent = true; + std::vector CloseFrame = WsFrameCodec::BuildCloseFrame(Code, Reason); + EnqueueWrite(std::move(CloseFrame)); + } + + m_Handler.OnWebSocketClose(*this, Code, Reason); +} + +void +WsAsioConnection::EnqueueWrite(std::vector Frame) +{ + bool ShouldFlush = false; + + m_WriteLock.WithExclusiveLock([&] { + m_WriteQueue.push_back(std::move(Frame)); + if (!m_IsWriting) + { + m_IsWriting = true; + ShouldFlush = true; + } + }); + + if (ShouldFlush) + { + FlushWriteQueue(); + } +} + +void +WsAsioConnection::FlushWriteQueue() +{ + std::vector Frame; + + m_WriteLock.WithExclusiveLock([&] { + if (m_WriteQueue.empty()) + { + m_IsWriting = false; + return; + } + Frame = std::move(m_WriteQueue.front()); + m_WriteQueue.pop_front(); + }); + + if (Frame.empty()) + { + return; + } + + Ref Self(this); + + // Move Frame into a shared_ptr so we can create the buffer and capture ownership + // in the same async_write call without evaluation order issues. + auto OwnedFrame = std::make_shared>(std::move(Frame)); + + asio::async_write(*m_Socket, + asio::buffer(OwnedFrame->data(), OwnedFrame->size()), + [Self, OwnedFrame](const asio::error_code& Ec, std::size_t ByteCount) { Self->OnWriteComplete(Ec, ByteCount); }); +} + +void +WsAsioConnection::OnWriteComplete(const asio::error_code& Ec, [[maybe_unused]] std::size_t ByteCount) +{ + if (Ec) + { + if (Ec != asio::error::operation_aborted) + { + ZEN_LOG_DEBUG(WsLog(), "WebSocket write error: {}", Ec.message()); + } + + m_WriteLock.WithExclusiveLock([&] { + m_IsWriting = false; + m_WriteQueue.clear(); + }); + + if (m_IsOpen.exchange(false)) + { + m_Handler.OnWebSocketClose(*this, 1006, "write error"); + } + return; + } + + FlushWriteQueue(); +} + +} // namespace zen::asio_http diff --git a/src/zenhttp/servers/wsasio.h b/src/zenhttp/servers/wsasio.h new file mode 100644 index 000000000..a638ea836 --- /dev/null +++ b/src/zenhttp/servers/wsasio.h @@ -0,0 +1,71 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#include +#include +#include + +namespace zen::asio_http { + +/** + * WebSocket connection over an ASIO TCP socket + * + * Owns the TCP socket (moved from HttpServerConnection after the 101 handshake) + * and runs an async read/write loop to exchange WebSocket frames. + * + * Lifetime is managed solely through intrusive reference counting (RefCounted). + * The async read/write callbacks capture Ref to keep the + * connection alive for the duration of the async operation. The service layer + * also holds a Ref. + */ +class WsAsioConnection : public WebSocketConnection +{ +public: + WsAsioConnection(std::unique_ptr Socket, IWebSocketHandler& Handler); + ~WsAsioConnection() override; + + /** + * Start the async read loop. Must be called once after construction + * and the 101 response has been sent. + */ + void Start(); + + // WebSocketConnection interface + void SendText(std::string_view Text) override; + void SendBinary(std::span Data) override; + void Close(uint16_t Code, std::string_view Reason) override; + bool IsOpen() const override; + +private: + void EnqueueRead(); + void OnDataReceived(const asio::error_code& Ec, std::size_t ByteCount); + void ProcessReceivedData(); + + void EnqueueWrite(std::vector Frame); + void FlushWriteQueue(); + void OnWriteComplete(const asio::error_code& Ec, std::size_t ByteCount); + + void DoClose(uint16_t Code, std::string_view Reason); + + std::unique_ptr m_Socket; + IWebSocketHandler& m_Handler; + asio::streambuf m_ReadBuffer; + + RwLock m_WriteLock; + std::deque> m_WriteQueue; + bool m_IsWriting = false; + + std::atomic m_IsOpen{true}; + bool m_CloseSent = false; +}; + +} // namespace zen::asio_http diff --git a/src/zenhttp/servers/wsframecodec.cpp b/src/zenhttp/servers/wsframecodec.cpp new file mode 100644 index 000000000..a4c5e0f16 --- /dev/null +++ b/src/zenhttp/servers/wsframecodec.cpp @@ -0,0 +1,229 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "wsframecodec.h" + +#include +#include + +#include +#include + +namespace zen { + +////////////////////////////////////////////////////////////////////////// +// +// Frame parsing +// + +WsFrameParseResult +WsFrameCodec::TryParseFrame(const uint8_t* Data, size_t Size) +{ + // Minimum frame: 2 bytes header (unmasked server frames) or 6 bytes (masked client frames) + if (Size < 2) + { + return {}; + } + + const bool Fin = (Data[0] & 0x80) != 0; + const uint8_t OpcodeRaw = Data[0] & 0x0F; + const bool Masked = (Data[1] & 0x80) != 0; + uint64_t PayloadLen = Data[1] & 0x7F; + + size_t HeaderSize = 2; + + if (PayloadLen == 126) + { + if (Size < 4) + { + return {}; + } + PayloadLen = (uint64_t(Data[2]) << 8) | uint64_t(Data[3]); + HeaderSize = 4; + } + else if (PayloadLen == 127) + { + if (Size < 10) + { + return {}; + } + PayloadLen = (uint64_t(Data[2]) << 56) | (uint64_t(Data[3]) << 48) | (uint64_t(Data[4]) << 40) | (uint64_t(Data[5]) << 32) | + (uint64_t(Data[6]) << 24) | (uint64_t(Data[7]) << 16) | (uint64_t(Data[8]) << 8) | uint64_t(Data[9]); + HeaderSize = 10; + } + + const size_t MaskSize = Masked ? 4 : 0; + const size_t TotalFrame = HeaderSize + MaskSize + PayloadLen; + + if (Size < TotalFrame) + { + return {}; + } + + const uint8_t* MaskKey = Masked ? (Data + HeaderSize) : nullptr; + const uint8_t* PayloadData = Data + HeaderSize + MaskSize; + + WsFrameParseResult Result; + Result.IsValid = true; + Result.BytesConsumed = TotalFrame; + Result.Opcode = static_cast(OpcodeRaw); + Result.Fin = Fin; + + Result.Payload.resize(static_cast(PayloadLen)); + if (PayloadLen > 0) + { + std::memcpy(Result.Payload.data(), PayloadData, static_cast(PayloadLen)); + + if (Masked) + { + for (size_t i = 0; i < Result.Payload.size(); ++i) + { + Result.Payload[i] ^= MaskKey[i & 3]; + } + } + } + + return Result; +} + +////////////////////////////////////////////////////////////////////////// +// +// Frame building (server-to-client, no masking) +// + +std::vector +WsFrameCodec::BuildFrame(WebSocketOpcode Opcode, std::span Payload) +{ + std::vector Frame; + + const size_t PayloadLen = Payload.size(); + + // FIN + opcode + Frame.push_back(0x80 | static_cast(Opcode)); + + // Payload length (no mask bit for server frames) + if (PayloadLen < 126) + { + Frame.push_back(static_cast(PayloadLen)); + } + else if (PayloadLen <= 0xFFFF) + { + Frame.push_back(126); + Frame.push_back(static_cast((PayloadLen >> 8) & 0xFF)); + Frame.push_back(static_cast(PayloadLen & 0xFF)); + } + else + { + Frame.push_back(127); + for (int i = 7; i >= 0; --i) + { + Frame.push_back(static_cast((PayloadLen >> (i * 8)) & 0xFF)); + } + } + + Frame.insert(Frame.end(), Payload.begin(), Payload.end()); + + return Frame; +} + +std::vector +WsFrameCodec::BuildCloseFrame(uint16_t Code, std::string_view Reason) +{ + std::vector Payload; + Payload.push_back(static_cast((Code >> 8) & 0xFF)); + Payload.push_back(static_cast(Code & 0xFF)); + Payload.insert(Payload.end(), Reason.begin(), Reason.end()); + + return BuildFrame(WebSocketOpcode::kClose, Payload); +} + +////////////////////////////////////////////////////////////////////////// +// +// Frame building (client-to-server, with masking) +// + +std::vector +WsFrameCodec::BuildMaskedFrame(WebSocketOpcode Opcode, std::span Payload) +{ + std::vector Frame; + + const size_t PayloadLen = Payload.size(); + + // FIN + opcode + Frame.push_back(0x80 | static_cast(Opcode)); + + // Payload length with mask bit set + if (PayloadLen < 126) + { + Frame.push_back(0x80 | static_cast(PayloadLen)); + } + else if (PayloadLen <= 0xFFFF) + { + Frame.push_back(0x80 | 126); + Frame.push_back(static_cast((PayloadLen >> 8) & 0xFF)); + Frame.push_back(static_cast(PayloadLen & 0xFF)); + } + else + { + Frame.push_back(0x80 | 127); + for (int i = 7; i >= 0; --i) + { + Frame.push_back(static_cast((PayloadLen >> (i * 8)) & 0xFF)); + } + } + + // Generate random 4-byte mask key + static thread_local std::mt19937 s_Rng(std::random_device{}()); + uint32_t MaskValue = s_Rng(); + uint8_t MaskKey[4]; + std::memcpy(MaskKey, &MaskValue, 4); + + Frame.insert(Frame.end(), MaskKey, MaskKey + 4); + + // Masked payload + for (size_t i = 0; i < PayloadLen; ++i) + { + Frame.push_back(Payload[i] ^ MaskKey[i & 3]); + } + + return Frame; +} + +std::vector +WsFrameCodec::BuildMaskedCloseFrame(uint16_t Code, std::string_view Reason) +{ + std::vector Payload; + Payload.push_back(static_cast((Code >> 8) & 0xFF)); + Payload.push_back(static_cast(Code & 0xFF)); + Payload.insert(Payload.end(), Reason.begin(), Reason.end()); + + return BuildMaskedFrame(WebSocketOpcode::kClose, Payload); +} + +////////////////////////////////////////////////////////////////////////// +// +// Sec-WebSocket-Accept key computation (RFC 6455 section 4.2.2) +// + +static constexpr std::string_view kWebSocketMagicGuid = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + +std::string +WsFrameCodec::ComputeAcceptKey(std::string_view ClientKey) +{ + // Concatenate client key with the magic GUID + std::string Combined; + Combined.reserve(ClientKey.size() + kWebSocketMagicGuid.size()); + Combined.append(ClientKey); + Combined.append(kWebSocketMagicGuid); + + // SHA1 hash + SHA1 Hash = SHA1::HashMemory(Combined.data(), Combined.size()); + + // Base64 encode the 20-byte hash + char Base64Buf[Base64::GetEncodedDataSize(20) + 1]; + uint32_t EncodedLen = Base64::Encode(Hash.Hash, 20, Base64Buf); + Base64Buf[EncodedLen] = '\0'; + + return std::string(Base64Buf, EncodedLen); +} + +} // namespace zen diff --git a/src/zenhttp/servers/wsframecodec.h b/src/zenhttp/servers/wsframecodec.h new file mode 100644 index 000000000..2d90b6fa1 --- /dev/null +++ b/src/zenhttp/servers/wsframecodec.h @@ -0,0 +1,74 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace zen { + +/** + * Result of attempting to parse a single WebSocket frame from a byte buffer + */ +struct WsFrameParseResult +{ + bool IsValid = false; // true if a complete frame was successfully parsed + size_t BytesConsumed = 0; // number of bytes consumed from the input buffer + WebSocketOpcode Opcode = WebSocketOpcode::kText; + bool Fin = false; + std::vector Payload; +}; + +/** + * RFC 6455 WebSocket frame codec + * + * Provides static helpers for parsing client-to-server frames (which are + * always masked) and building server-to-client frames (which are never masked). + */ +struct WsFrameCodec +{ + /** + * Try to parse one complete frame from the front of the buffer. + * + * Returns a result with IsValid == false and BytesConsumed == 0 when + * there is not enough data yet. The caller should accumulate more data + * and retry. + */ + static WsFrameParseResult TryParseFrame(const uint8_t* Data, size_t Size); + + /** + * Build a server-to-client frame (no masking) + */ + static std::vector BuildFrame(WebSocketOpcode Opcode, std::span Payload); + + /** + * Build a close frame with a status code and optional reason string + */ + static std::vector BuildCloseFrame(uint16_t Code, std::string_view Reason = {}); + + /** + * Build a client-to-server frame (with masking per RFC 6455) + */ + static std::vector BuildMaskedFrame(WebSocketOpcode Opcode, std::span Payload); + + /** + * Build a masked close frame with status code and optional reason + */ + static std::vector BuildMaskedCloseFrame(uint16_t Code, std::string_view Reason = {}); + + /** + * Compute the Sec-WebSocket-Accept value per RFC 6455 section 4.2.2 + * + * accept = Base64(SHA1(clientKey + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11")) + */ + static std::string ComputeAcceptKey(std::string_view ClientKey); +}; + +} // namespace zen diff --git a/src/zenhttp/servers/wshttpsys.cpp b/src/zenhttp/servers/wshttpsys.cpp new file mode 100644 index 000000000..3f0f0b447 --- /dev/null +++ b/src/zenhttp/servers/wshttpsys.cpp @@ -0,0 +1,466 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "wshttpsys.h" + +#if ZEN_WITH_HTTPSYS + +# include "wsframecodec.h" + +# include + +namespace zen { + +static LoggerRef +WsHttpSysLog() +{ + static LoggerRef g_Logger = logging::Get("ws_httpsys"); + return g_Logger; +} + +////////////////////////////////////////////////////////////////////////// + +WsHttpSysConnection::WsHttpSysConnection(HANDLE RequestQueueHandle, HTTP_REQUEST_ID RequestId, IWebSocketHandler& Handler, PTP_IO Iocp) +: m_RequestQueueHandle(RequestQueueHandle) +, m_RequestId(RequestId) +, m_Handler(Handler) +, m_Iocp(Iocp) +, m_ReadBuffer(8192) +{ + m_ReadIoContext.ContextType = HttpSysIoContext::Type::kWebSocketRead; + m_ReadIoContext.Owner = this; + m_WriteIoContext.ContextType = HttpSysIoContext::Type::kWebSocketWrite; + m_WriteIoContext.Owner = this; +} + +WsHttpSysConnection::~WsHttpSysConnection() +{ + ZEN_ASSERT(m_OutstandingOps.load() == 0); + + if (m_IsOpen.exchange(false)) + { + Disconnect(); + } +} + +void +WsHttpSysConnection::Start() +{ + m_SelfRef = Ref(this); + IssueAsyncRead(); +} + +void +WsHttpSysConnection::Shutdown() +{ + m_ShutdownRequested.store(true, std::memory_order_relaxed); + + if (!m_IsOpen.exchange(false)) + { + return; + } + + // Cancel pending I/O — completions will fire with ERROR_OPERATION_ABORTED + HttpCancelHttpRequest(m_RequestQueueHandle, m_RequestId, nullptr); +} + +bool +WsHttpSysConnection::IsOpen() const +{ + return m_IsOpen.load(std::memory_order_relaxed); +} + +////////////////////////////////////////////////////////////////////////// +// +// Async read path +// + +void +WsHttpSysConnection::IssueAsyncRead() +{ + if (!m_IsOpen.load(std::memory_order_relaxed) || m_ShutdownRequested.load(std::memory_order_relaxed)) + { + MaybeReleaseSelfRef(); + return; + } + + m_OutstandingOps.fetch_add(1, std::memory_order_relaxed); + + ZeroMemory(&m_ReadIoContext.Overlapped, sizeof(OVERLAPPED)); + + StartThreadpoolIo(m_Iocp); + + ULONG Result = HttpReceiveRequestEntityBody(m_RequestQueueHandle, + m_RequestId, + 0, // Flags + m_ReadBuffer.data(), + (ULONG)m_ReadBuffer.size(), + nullptr, // BytesRead (ignored for async) + &m_ReadIoContext.Overlapped); + + if (Result != NO_ERROR && Result != ERROR_IO_PENDING) + { + CancelThreadpoolIo(m_Iocp); + m_OutstandingOps.fetch_sub(1, std::memory_order_relaxed); + + if (m_IsOpen.exchange(false)) + { + m_Handler.OnWebSocketClose(*this, 1006, "read issue failed"); + } + + MaybeReleaseSelfRef(); + } +} + +void +WsHttpSysConnection::OnReadCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) +{ + // Hold a transient ref to prevent mid-callback destruction after MaybeReleaseSelfRef + Ref Guard(this); + + if (IoResult != NO_ERROR) + { + m_OutstandingOps.fetch_sub(1, std::memory_order_relaxed); + + if (m_IsOpen.exchange(false)) + { + if (IoResult == ERROR_HANDLE_EOF) + { + m_Handler.OnWebSocketClose(*this, 1006, "connection closed"); + } + else if (IoResult != ERROR_OPERATION_ABORTED) + { + m_Handler.OnWebSocketClose(*this, 1006, "connection lost"); + } + } + + MaybeReleaseSelfRef(); + return; + } + + if (NumberOfBytesTransferred > 0) + { + m_Accumulated.insert(m_Accumulated.end(), m_ReadBuffer.begin(), m_ReadBuffer.begin() + NumberOfBytesTransferred); + ProcessReceivedData(); + } + + m_OutstandingOps.fetch_sub(1, std::memory_order_relaxed); + + if (m_IsOpen.load(std::memory_order_relaxed)) + { + IssueAsyncRead(); + } + else + { + MaybeReleaseSelfRef(); + } +} + +////////////////////////////////////////////////////////////////////////// +// +// Frame parsing +// + +void +WsHttpSysConnection::ProcessReceivedData() +{ + while (!m_Accumulated.empty()) + { + WsFrameParseResult Frame = WsFrameCodec::TryParseFrame(m_Accumulated.data(), m_Accumulated.size()); + if (!Frame.IsValid) + { + break; // not enough data yet + } + + // Remove consumed bytes + m_Accumulated.erase(m_Accumulated.begin(), m_Accumulated.begin() + Frame.BytesConsumed); + + switch (Frame.Opcode) + { + case WebSocketOpcode::kText: + case WebSocketOpcode::kBinary: + { + WebSocketMessage Msg; + Msg.Opcode = Frame.Opcode; + Msg.Payload = IoBuffer(IoBuffer::Clone, Frame.Payload.data(), Frame.Payload.size()); + m_Handler.OnWebSocketMessage(*this, Msg); + break; + } + + case WebSocketOpcode::kPing: + { + // Auto-respond with pong carrying the same payload + std::vector PongFrame = WsFrameCodec::BuildFrame(WebSocketOpcode::kPong, Frame.Payload); + EnqueueWrite(std::move(PongFrame)); + break; + } + + case WebSocketOpcode::kPong: + // Unsolicited pong — ignore per RFC 6455 + break; + + case WebSocketOpcode::kClose: + { + uint16_t Code = 1000; + std::string_view Reason; + + if (Frame.Payload.size() >= 2) + { + Code = (uint16_t(Frame.Payload[0]) << 8) | uint16_t(Frame.Payload[1]); + if (Frame.Payload.size() > 2) + { + Reason = std::string_view(reinterpret_cast(Frame.Payload.data() + 2), Frame.Payload.size() - 2); + } + } + + // Echo close frame back if we haven't sent one yet + { + bool ShouldSendClose = false; + { + RwLock::ExclusiveLockScope _(m_WriteLock); + if (!m_CloseSent) + { + m_CloseSent = true; + ShouldSendClose = true; + } + } + if (ShouldSendClose) + { + std::vector CloseFrame = WsFrameCodec::BuildCloseFrame(Code); + EnqueueWrite(std::move(CloseFrame)); + } + } + + m_IsOpen.store(false); + m_Handler.OnWebSocketClose(*this, Code, Reason); + Disconnect(); + return; + } + + default: + ZEN_LOG_WARN(WsHttpSysLog(), "Unknown WebSocket opcode: {:#x}", static_cast(Frame.Opcode)); + break; + } + } +} + +////////////////////////////////////////////////////////////////////////// +// +// Async write path +// + +void +WsHttpSysConnection::EnqueueWrite(std::vector Frame) +{ + bool ShouldFlush = false; + + { + RwLock::ExclusiveLockScope _(m_WriteLock); + m_WriteQueue.push_back(std::move(Frame)); + + if (!m_IsWriting) + { + m_IsWriting = true; + ShouldFlush = true; + } + } + + if (ShouldFlush) + { + FlushWriteQueue(); + } +} + +void +WsHttpSysConnection::FlushWriteQueue() +{ + { + RwLock::ExclusiveLockScope _(m_WriteLock); + + if (m_WriteQueue.empty()) + { + m_IsWriting = false; + return; + } + + m_CurrentWriteBuffer = std::move(m_WriteQueue.front()); + m_WriteQueue.pop_front(); + } + + m_OutstandingOps.fetch_add(1, std::memory_order_relaxed); + + ZeroMemory(&m_WriteChunk, sizeof(m_WriteChunk)); + m_WriteChunk.DataChunkType = HttpDataChunkFromMemory; + m_WriteChunk.FromMemory.pBuffer = m_CurrentWriteBuffer.data(); + m_WriteChunk.FromMemory.BufferLength = (ULONG)m_CurrentWriteBuffer.size(); + + ZeroMemory(&m_WriteIoContext.Overlapped, sizeof(OVERLAPPED)); + + StartThreadpoolIo(m_Iocp); + + ULONG Result = HttpSendResponseEntityBody(m_RequestQueueHandle, + m_RequestId, + HTTP_SEND_RESPONSE_FLAG_MORE_DATA, + 1, + &m_WriteChunk, + nullptr, + nullptr, + 0, + &m_WriteIoContext.Overlapped, + nullptr); + + if (Result != NO_ERROR && Result != ERROR_IO_PENDING) + { + CancelThreadpoolIo(m_Iocp); + m_OutstandingOps.fetch_sub(1, std::memory_order_relaxed); + + ZEN_LOG_DEBUG(WsHttpSysLog(), "WebSocket async write failed: {}", Result); + + { + RwLock::ExclusiveLockScope _(m_WriteLock); + m_WriteQueue.clear(); + m_IsWriting = false; + } + m_CurrentWriteBuffer.clear(); + + if (m_IsOpen.exchange(false)) + { + m_Handler.OnWebSocketClose(*this, 1006, "write error"); + } + + MaybeReleaseSelfRef(); + } +} + +void +WsHttpSysConnection::OnWriteCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) +{ + ZEN_UNUSED(NumberOfBytesTransferred); + + // Hold a transient ref to prevent mid-callback destruction + Ref Guard(this); + + m_OutstandingOps.fetch_sub(1, std::memory_order_relaxed); + m_CurrentWriteBuffer.clear(); + + if (IoResult != NO_ERROR) + { + ZEN_LOG_DEBUG(WsHttpSysLog(), "WebSocket write completion error: {}", IoResult); + + { + RwLock::ExclusiveLockScope _(m_WriteLock); + m_WriteQueue.clear(); + m_IsWriting = false; + } + + if (m_IsOpen.exchange(false)) + { + m_Handler.OnWebSocketClose(*this, 1006, "write error"); + } + + MaybeReleaseSelfRef(); + return; + } + + FlushWriteQueue(); +} + +////////////////////////////////////////////////////////////////////////// +// +// Send interface +// + +void +WsHttpSysConnection::SendText(std::string_view Text) +{ + if (!m_IsOpen.load(std::memory_order_relaxed)) + { + return; + } + + std::span Payload(reinterpret_cast(Text.data()), Text.size()); + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kText, Payload); + EnqueueWrite(std::move(Frame)); +} + +void +WsHttpSysConnection::SendBinary(std::span Data) +{ + if (!m_IsOpen.load(std::memory_order_relaxed)) + { + return; + } + + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kBinary, Data); + EnqueueWrite(std::move(Frame)); +} + +void +WsHttpSysConnection::Close(uint16_t Code, std::string_view Reason) +{ + DoClose(Code, Reason); +} + +void +WsHttpSysConnection::DoClose(uint16_t Code, std::string_view Reason) +{ + if (!m_IsOpen.exchange(false)) + { + return; + } + + { + bool ShouldSendClose = false; + { + RwLock::ExclusiveLockScope _(m_WriteLock); + if (!m_CloseSent) + { + m_CloseSent = true; + ShouldSendClose = true; + } + } + if (ShouldSendClose) + { + std::vector CloseFrame = WsFrameCodec::BuildCloseFrame(Code, Reason); + EnqueueWrite(std::move(CloseFrame)); + } + } + + m_Handler.OnWebSocketClose(*this, Code, Reason); + + // Cancel pending read I/O — completions drain via ERROR_OPERATION_ABORTED + HttpCancelHttpRequest(m_RequestQueueHandle, m_RequestId, nullptr); +} + +////////////////////////////////////////////////////////////////////////// +// +// Lifetime management +// + +void +WsHttpSysConnection::MaybeReleaseSelfRef() +{ + if (m_OutstandingOps.load(std::memory_order_relaxed) == 0 && !m_IsOpen.load(std::memory_order_relaxed)) + { + m_SelfRef = nullptr; + } +} + +void +WsHttpSysConnection::Disconnect() +{ + // Send final empty body with DISCONNECT to tell http.sys the connection is done + HttpSendResponseEntityBody(m_RequestQueueHandle, + m_RequestId, + HTTP_SEND_RESPONSE_FLAG_DISCONNECT, + 0, + nullptr, + nullptr, + nullptr, + 0, + nullptr, + nullptr); +} + +} // namespace zen + +#endif // ZEN_WITH_HTTPSYS diff --git a/src/zenhttp/servers/wshttpsys.h b/src/zenhttp/servers/wshttpsys.h new file mode 100644 index 000000000..ab0ca381a --- /dev/null +++ b/src/zenhttp/servers/wshttpsys.h @@ -0,0 +1,104 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include "httpsys_iocontext.h" + +#include + +#if ZEN_WITH_HTTPSYS +# define _WINSOCKAPI_ +# include +# include + +# include +# include +# include + +namespace zen { + +/** + * WebSocket connection over an http.sys opaque-mode connection + * + * After the 101 Switching Protocols response is sent with + * HTTP_SEND_RESPONSE_FLAG_OPAQUE, http.sys stops parsing HTTP on the + * connection. Raw bytes are exchanged via HttpReceiveRequestEntityBody / + * HttpSendResponseEntityBody using the original RequestId. + * + * All I/O is performed asynchronously via the same IOCP threadpool used + * for normal http.sys traffic, eliminating per-connection threads. + * + * Lifetime is managed through intrusive reference counting (RefCounted). + * A self-reference (m_SelfRef) is held from Start() until all outstanding + * async operations have drained, preventing premature destruction. + */ +class WsHttpSysConnection : public WebSocketConnection +{ +public: + WsHttpSysConnection(HANDLE RequestQueueHandle, HTTP_REQUEST_ID RequestId, IWebSocketHandler& Handler, PTP_IO Iocp); + ~WsHttpSysConnection() override; + + /** + * Start the async read loop. Must be called once after construction + * and after the 101 response has been sent. + */ + void Start(); + + /** + * Shut down the connection. Cancels pending I/O; IOCP completions + * will fire with ERROR_OPERATION_ABORTED and drain naturally. + */ + void Shutdown(); + + // WebSocketConnection interface + void SendText(std::string_view Text) override; + void SendBinary(std::span Data) override; + void Close(uint16_t Code, std::string_view Reason) override; + bool IsOpen() const override; + + // Called from IoCompletionCallback via tagged dispatch + void OnReadCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred); + void OnWriteCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred); + +private: + void IssueAsyncRead(); + void ProcessReceivedData(); + void EnqueueWrite(std::vector Frame); + void FlushWriteQueue(); + void DoClose(uint16_t Code, std::string_view Reason); + void Disconnect(); + void MaybeReleaseSelfRef(); + + HANDLE m_RequestQueueHandle; + HTTP_REQUEST_ID m_RequestId; + IWebSocketHandler& m_Handler; + PTP_IO m_Iocp; + + // Tagged OVERLAPPED contexts for concurrent read and write + HttpSysIoContext m_ReadIoContext{}; + HttpSysIoContext m_WriteIoContext{}; + + // Read state + std::vector m_ReadBuffer; + std::vector m_Accumulated; + + // Write state + RwLock m_WriteLock; + std::deque> m_WriteQueue; + std::vector m_CurrentWriteBuffer; + HTTP_DATA_CHUNK m_WriteChunk{}; + bool m_IsWriting = false; + + // Lifetime management + std::atomic m_OutstandingOps{0}; + Ref m_SelfRef; + std::atomic m_ShutdownRequested{false}; + std::atomic m_IsOpen{true}; + bool m_CloseSent = false; +}; + +} // namespace zen + +#endif // ZEN_WITH_HTTPSYS diff --git a/src/zenhttp/servers/wstest.cpp b/src/zenhttp/servers/wstest.cpp new file mode 100644 index 000000000..95f8587df --- /dev/null +++ b/src/zenhttp/servers/wstest.cpp @@ -0,0 +1,922 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#if ZEN_WITH_TESTS + +# include +# include +# include + +# include +# include +# include + +# include "httpasio.h" +# include "wsframecodec.h" + +ZEN_THIRD_PARTY_INCLUDES_START +# if ZEN_PLATFORM_WINDOWS +# include +# else +# include +# include +# endif +# include +ZEN_THIRD_PARTY_INCLUDES_END + +# include +# include +# include +# include +# include +# include +# include +# include + +namespace zen { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// +// +// Unit tests: WsFrameCodec +// + +TEST_CASE("websocket.framecodec") +{ + SUBCASE("ComputeAcceptKey RFC 6455 test vector") + { + // RFC 6455 section 4.2.2 example + std::string AcceptKey = WsFrameCodec::ComputeAcceptKey("dGhlIHNhbXBsZSBub25jZQ=="); + CHECK_EQ(AcceptKey, "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="); + } + + SUBCASE("BuildFrame and TryParseFrame roundtrip - text") + { + std::string_view Text = "Hello, WebSocket!"; + std::span Payload(reinterpret_cast(Text.data()), Text.size()); + + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kText, Payload); + + // Server frames are unmasked — TryParseFrame should handle them + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.BytesConsumed, Frame.size()); + CHECK(Result.Fin); + CHECK_EQ(Result.Opcode, WebSocketOpcode::kText); + CHECK_EQ(Result.Payload.size(), Text.size()); + CHECK_EQ(std::string_view(reinterpret_cast(Result.Payload.data()), Result.Payload.size()), Text); + } + + SUBCASE("BuildFrame and TryParseFrame roundtrip - binary") + { + std::vector BinaryData = {0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD}; + + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kBinary, BinaryData); + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Opcode, WebSocketOpcode::kBinary); + CHECK_EQ(Result.Payload, BinaryData); + } + + SUBCASE("BuildFrame - medium payload (126-65535 bytes)") + { + std::vector Payload(300, 0x42); + + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kBinary, Payload); + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Payload.size(), 300u); + CHECK_EQ(Result.Payload, Payload); + } + + SUBCASE("BuildFrame - large payload (>65535 bytes)") + { + std::vector Payload(70000, 0xAB); + + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kBinary, Payload); + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Payload.size(), 70000u); + } + + SUBCASE("BuildCloseFrame roundtrip") + { + std::vector Frame = WsFrameCodec::BuildCloseFrame(1000, "normal closure"); + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Opcode, WebSocketOpcode::kClose); + REQUIRE(Result.Payload.size() >= 2); + + uint16_t Code = (uint16_t(Result.Payload[0]) << 8) | uint16_t(Result.Payload[1]); + CHECK_EQ(Code, 1000); + + std::string_view Reason(reinterpret_cast(Result.Payload.data() + 2), Result.Payload.size() - 2); + CHECK_EQ(Reason, "normal closure"); + } + + SUBCASE("TryParseFrame - partial data returns invalid") + { + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kText, std::span{}); + + // Pass only 1 byte — not enough for a frame header + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), 1); + CHECK_FALSE(Result.IsValid); + CHECK_EQ(Result.BytesConsumed, 0u); + } + + SUBCASE("TryParseFrame - empty payload") + { + std::vector Frame = WsFrameCodec::BuildFrame(WebSocketOpcode::kText, std::span{}); + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Opcode, WebSocketOpcode::kText); + CHECK(Result.Payload.empty()); + } + + SUBCASE("TryParseFrame - masked client frame") + { + // Build a masked frame manually as a client would send + // Frame: FIN=1, opcode=text, MASK=1, payload_len=5, mask_key=0x37FA213D, payload="Hello" + uint8_t MaskKey[4] = {0x37, 0xFA, 0x21, 0x3D}; + uint8_t MaskedPayload[5] = {}; + const char* Original = "Hello"; + for (int i = 0; i < 5; ++i) + { + MaskedPayload[i] = static_cast(Original[i]) ^ MaskKey[i % 4]; + } + + std::vector Frame; + Frame.push_back(0x81); // FIN + text + Frame.push_back(0x85); // MASK + len=5 + Frame.insert(Frame.end(), MaskKey, MaskKey + 4); + Frame.insert(Frame.end(), MaskedPayload, MaskedPayload + 5); + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Opcode, WebSocketOpcode::kText); + CHECK_EQ(Result.Payload.size(), 5u); + CHECK_EQ(std::string_view(reinterpret_cast(Result.Payload.data()), 5), "Hello"sv); + } + + SUBCASE("BuildMaskedFrame roundtrip - text") + { + std::string_view Text = "Hello, masked WebSocket!"; + std::span Payload(reinterpret_cast(Text.data()), Text.size()); + + std::vector Frame = WsFrameCodec::BuildMaskedFrame(WebSocketOpcode::kText, Payload); + + // Verify mask bit is set + CHECK((Frame[1] & 0x80) != 0); + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.BytesConsumed, Frame.size()); + CHECK(Result.Fin); + CHECK_EQ(Result.Opcode, WebSocketOpcode::kText); + CHECK_EQ(Result.Payload.size(), Text.size()); + CHECK_EQ(std::string_view(reinterpret_cast(Result.Payload.data()), Result.Payload.size()), Text); + } + + SUBCASE("BuildMaskedFrame roundtrip - binary") + { + std::vector BinaryData = {0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD}; + + std::vector Frame = WsFrameCodec::BuildMaskedFrame(WebSocketOpcode::kBinary, BinaryData); + + CHECK((Frame[1] & 0x80) != 0); + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Opcode, WebSocketOpcode::kBinary); + CHECK_EQ(Result.Payload, BinaryData); + } + + SUBCASE("BuildMaskedFrame - medium payload (126-65535 bytes)") + { + std::vector Payload(300, 0x42); + + std::vector Frame = WsFrameCodec::BuildMaskedFrame(WebSocketOpcode::kBinary, Payload); + + CHECK((Frame[1] & 0x80) != 0); + CHECK_EQ((Frame[1] & 0x7F), 126); // 16-bit extended length + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Payload.size(), 300u); + CHECK_EQ(Result.Payload, Payload); + } + + SUBCASE("BuildMaskedFrame - large payload (>65535 bytes)") + { + std::vector Payload(70000, 0xAB); + + std::vector Frame = WsFrameCodec::BuildMaskedFrame(WebSocketOpcode::kBinary, Payload); + + CHECK((Frame[1] & 0x80) != 0); + CHECK_EQ((Frame[1] & 0x7F), 127); // 64-bit extended length + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Payload.size(), 70000u); + } + + SUBCASE("BuildMaskedCloseFrame roundtrip") + { + std::vector Frame = WsFrameCodec::BuildMaskedCloseFrame(1000, "normal closure"); + + CHECK((Frame[1] & 0x80) != 0); + + WsFrameParseResult Result = WsFrameCodec::TryParseFrame(Frame.data(), Frame.size()); + + CHECK(Result.IsValid); + CHECK_EQ(Result.Opcode, WebSocketOpcode::kClose); + REQUIRE(Result.Payload.size() >= 2); + + uint16_t Code = (uint16_t(Result.Payload[0]) << 8) | uint16_t(Result.Payload[1]); + CHECK_EQ(Code, 1000); + + std::string_view Reason(reinterpret_cast(Result.Payload.data() + 2), Result.Payload.size() - 2); + CHECK_EQ(Reason, "normal closure"); + } +} + +////////////////////////////////////////////////////////////////////////// +// +// Integration tests: WebSocket over ASIO +// + +namespace { + + /** + * Helper: Build a masked client-to-server frame per RFC 6455 + */ + std::vector BuildMaskedFrame(WebSocketOpcode Opcode, std::span Payload) + { + std::vector Frame; + + // FIN + opcode + Frame.push_back(0x80 | static_cast(Opcode)); + + // Payload length with mask bit set + if (Payload.size() < 126) + { + Frame.push_back(0x80 | static_cast(Payload.size())); + } + else if (Payload.size() <= 0xFFFF) + { + Frame.push_back(0x80 | 126); + Frame.push_back(static_cast((Payload.size() >> 8) & 0xFF)); + Frame.push_back(static_cast(Payload.size() & 0xFF)); + } + else + { + Frame.push_back(0x80 | 127); + for (int i = 7; i >= 0; --i) + { + Frame.push_back(static_cast((Payload.size() >> (i * 8)) & 0xFF)); + } + } + + // Mask key (use a fixed key for deterministic tests) + uint8_t MaskKey[4] = {0x12, 0x34, 0x56, 0x78}; + Frame.insert(Frame.end(), MaskKey, MaskKey + 4); + + // Masked payload + for (size_t i = 0; i < Payload.size(); ++i) + { + Frame.push_back(Payload[i] ^ MaskKey[i & 3]); + } + + return Frame; + } + + std::vector BuildMaskedTextFrame(std::string_view Text) + { + std::span Payload(reinterpret_cast(Text.data()), Text.size()); + return BuildMaskedFrame(WebSocketOpcode::kText, Payload); + } + + std::vector BuildMaskedCloseFrame(uint16_t Code) + { + std::vector Payload; + Payload.push_back(static_cast((Code >> 8) & 0xFF)); + Payload.push_back(static_cast(Code & 0xFF)); + return BuildMaskedFrame(WebSocketOpcode::kClose, Payload); + } + + /** + * Test service that implements IWebSocketHandler + */ + struct WsTestService : public HttpService, public IWebSocketHandler + { + const char* BaseUri() const override { return "/wstest/"; } + + void HandleRequest(HttpServerRequest& Request) override + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "hello from wstest"); + } + + // IWebSocketHandler + void OnWebSocketOpen(Ref Connection) override + { + m_OpenCount.fetch_add(1); + + m_ConnectionsLock.WithExclusiveLock([&] { m_Connections.push_back(Connection); }); + } + + void OnWebSocketMessage(WebSocketConnection& Conn, const WebSocketMessage& Msg) override + { + m_MessageCount.fetch_add(1); + + if (Msg.Opcode == WebSocketOpcode::kText) + { + std::string_view Text(static_cast(Msg.Payload.Data()), Msg.Payload.Size()); + m_LastMessage = std::string(Text); + + // Echo the message back + Conn.SendText(Text); + } + } + + void OnWebSocketClose(WebSocketConnection& Conn, uint16_t Code, [[maybe_unused]] std::string_view Reason) override + { + m_CloseCount.fetch_add(1); + m_LastCloseCode = Code; + + m_ConnectionsLock.WithExclusiveLock([&] { + auto It = std::remove_if(m_Connections.begin(), m_Connections.end(), [&Conn](const Ref& C) { + return C.Get() == &Conn; + }); + m_Connections.erase(It, m_Connections.end()); + }); + } + + void SendToAll(std::string_view Text) + { + RwLock::SharedLockScope _(m_ConnectionsLock); + for (auto& Conn : m_Connections) + { + if (Conn->IsOpen()) + { + Conn->SendText(Text); + } + } + } + + std::atomic m_OpenCount{0}; + std::atomic m_MessageCount{0}; + std::atomic m_CloseCount{0}; + std::atomic m_LastCloseCode{0}; + std::string m_LastMessage; + + RwLock m_ConnectionsLock; + std::vector> m_Connections; + }; + + /** + * Helper: Perform the WebSocket upgrade handshake on a raw TCP socket + * + * Returns true on success (101 response), false otherwise. + */ + bool DoWebSocketHandshake(asio::ip::tcp::socket& Sock, std::string_view Path, int Port) + { + // Send HTTP upgrade request + ExtendableStringBuilder<512> Request; + Request << "GET " << Path << " HTTP/1.1\r\n" + << "Host: 127.0.0.1:" << Port << "\r\n" + << "Upgrade: websocket\r\n" + << "Connection: Upgrade\r\n" + << "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + << "Sec-WebSocket-Version: 13\r\n" + << "\r\n"; + + std::string_view ReqStr = Request.ToView(); + + asio::write(Sock, asio::buffer(ReqStr.data(), ReqStr.size())); + + // Read the response (look for "101") + asio::streambuf ResponseBuf; + asio::read_until(Sock, ResponseBuf, "\r\n\r\n"); + + std::string Response(asio::buffers_begin(ResponseBuf.data()), asio::buffers_end(ResponseBuf.data())); + + return Response.find("101") != std::string::npos; + } + + /** + * Helper: Read a single server-to-client frame from a socket + * + * Uses a background thread with a synchronous ASIO read and a timeout. + */ + WsFrameParseResult ReadOneFrame(asio::ip::tcp::socket& Sock, int TimeoutMs = 5000) + { + std::vector Buffer; + WsFrameParseResult Result; + std::atomic Done{false}; + + std::thread Reader([&] { + while (!Done.load()) + { + uint8_t Tmp[4096]; + asio::error_code Ec; + size_t BytesRead = Sock.read_some(asio::buffer(Tmp), Ec); + if (Ec || BytesRead == 0) + { + break; + } + + Buffer.insert(Buffer.end(), Tmp, Tmp + BytesRead); + + WsFrameParseResult Frame = WsFrameCodec::TryParseFrame(Buffer.data(), Buffer.size()); + if (Frame.IsValid) + { + Result = std::move(Frame); + Done.store(true); + return; + } + } + }); + + auto Deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(TimeoutMs); + while (!Done.load() && std::chrono::steady_clock::now() < Deadline) + { + Sleep(10); + } + + if (!Done.load()) + { + // Timeout — cancel the read + asio::error_code Ec; + Sock.cancel(Ec); + } + + if (Reader.joinable()) + { + Reader.join(); + } + + return Result; + } + +} // anonymous namespace + +TEST_CASE("websocket.integration") +{ + WsTestService TestService; + ScopedTemporaryDirectory TmpDir; + + Ref Server = CreateHttpAsioServer(AsioConfig{}); + + int Port = Server->Initialize(7575, TmpDir.Path()); + REQUIRE(Port != 0); + + Server->RegisterService(TestService); + + std::thread ServerThread([&]() { Server->Run(false); }); + + auto ServerGuard = MakeGuard([&]() { + Server->RequestExit(); + if (ServerThread.joinable()) + { + ServerThread.join(); + } + Server->Close(); + }); + + // Give server a moment to start accepting + Sleep(100); + + SUBCASE("handshake succeeds with 101") + { + asio::io_context IoCtx; + asio::ip::tcp::socket Sock(IoCtx); + Sock.connect(asio::ip::tcp::endpoint(asio::ip::address::from_string("127.0.0.1"), static_cast(Port))); + + bool Ok = DoWebSocketHandshake(Sock, "/wstest/ws", Port); + CHECK(Ok); + + Sleep(50); + CHECK_EQ(TestService.m_OpenCount.load(), 1); + + Sock.close(); + } + + SUBCASE("normal HTTP still works alongside WebSocket service") + { + asio::io_context IoCtx; + asio::ip::tcp::socket Sock(IoCtx); + Sock.connect(asio::ip::tcp::endpoint(asio::ip::address::from_string("127.0.0.1"), static_cast(Port))); + + // Send a normal HTTP GET (not upgrade) + std::string HttpReq = fmt::format( + "GET /wstest/hello HTTP/1.1\r\n" + "Host: 127.0.0.1:{}\r\n" + "Connection: close\r\n" + "\r\n", + Port); + + asio::write(Sock, asio::buffer(HttpReq)); + + asio::streambuf ResponseBuf; + asio::error_code Ec; + asio::read(Sock, ResponseBuf, asio::transfer_at_least(1), Ec); + + std::string Response(asio::buffers_begin(ResponseBuf.data()), asio::buffers_end(ResponseBuf.data())); + CHECK(Response.find("200") != std::string::npos); + } + + SUBCASE("echo message roundtrip") + { + asio::io_context IoCtx; + asio::ip::tcp::socket Sock(IoCtx); + Sock.connect(asio::ip::tcp::endpoint(asio::ip::address::from_string("127.0.0.1"), static_cast(Port))); + + bool Ok = DoWebSocketHandshake(Sock, "/wstest/ws", Port); + REQUIRE(Ok); + Sleep(50); + + // Send a text message (masked, as client) + std::vector Frame = BuildMaskedTextFrame("ping test"); + asio::write(Sock, asio::buffer(Frame)); + + // Read the echo reply + WsFrameParseResult Reply = ReadOneFrame(Sock); + REQUIRE(Reply.IsValid); + CHECK_EQ(Reply.Opcode, WebSocketOpcode::kText); + std::string_view ReplyText(reinterpret_cast(Reply.Payload.data()), Reply.Payload.size()); + CHECK_EQ(ReplyText, "ping test"sv); + CHECK_EQ(TestService.m_MessageCount.load(), 1); + CHECK_EQ(TestService.m_LastMessage, "ping test"); + + Sock.close(); + } + + SUBCASE("server push to client") + { + asio::io_context IoCtx; + asio::ip::tcp::socket Sock(IoCtx); + Sock.connect(asio::ip::tcp::endpoint(asio::ip::address::from_string("127.0.0.1"), static_cast(Port))); + + bool Ok = DoWebSocketHandshake(Sock, "/wstest/ws", Port); + REQUIRE(Ok); + Sleep(50); + + // Server pushes a message + TestService.SendToAll("server says hello"); + + WsFrameParseResult Frame = ReadOneFrame(Sock); + REQUIRE(Frame.IsValid); + CHECK_EQ(Frame.Opcode, WebSocketOpcode::kText); + std::string_view Text(reinterpret_cast(Frame.Payload.data()), Frame.Payload.size()); + CHECK_EQ(Text, "server says hello"sv); + + Sock.close(); + } + + SUBCASE("client close handshake") + { + asio::io_context IoCtx; + asio::ip::tcp::socket Sock(IoCtx); + Sock.connect(asio::ip::tcp::endpoint(asio::ip::address::from_string("127.0.0.1"), static_cast(Port))); + + bool Ok = DoWebSocketHandshake(Sock, "/wstest/ws", Port); + REQUIRE(Ok); + Sleep(50); + + // Send close frame + std::vector CloseFrame = BuildMaskedCloseFrame(1000); + asio::write(Sock, asio::buffer(CloseFrame)); + + // Server should echo close back + WsFrameParseResult Reply = ReadOneFrame(Sock); + REQUIRE(Reply.IsValid); + CHECK_EQ(Reply.Opcode, WebSocketOpcode::kClose); + + Sleep(50); + CHECK_EQ(TestService.m_CloseCount.load(), 1); + CHECK_EQ(TestService.m_LastCloseCode.load(), 1000); + + Sock.close(); + } + + SUBCASE("multiple concurrent connections") + { + constexpr int NumClients = 5; + + asio::io_context IoCtx; + std::vector Sockets; + + for (int i = 0; i < NumClients; ++i) + { + Sockets.emplace_back(IoCtx); + Sockets.back().connect(asio::ip::tcp::endpoint(asio::ip::address::from_string("127.0.0.1"), static_cast(Port))); + + bool Ok = DoWebSocketHandshake(Sockets.back(), "/wstest/ws", Port); + REQUIRE(Ok); + } + + Sleep(100); + CHECK_EQ(TestService.m_OpenCount.load(), NumClients); + + // Broadcast from server + TestService.SendToAll("broadcast"); + + // Each client should receive the message + for (int i = 0; i < NumClients; ++i) + { + WsFrameParseResult Frame = ReadOneFrame(Sockets[i]); + REQUIRE(Frame.IsValid); + CHECK_EQ(Frame.Opcode, WebSocketOpcode::kText); + std::string_view Text(reinterpret_cast(Frame.Payload.data()), Frame.Payload.size()); + CHECK_EQ(Text, "broadcast"sv); + } + + // Close all + for (auto& S : Sockets) + { + S.close(); + } + } + + SUBCASE("service without IWebSocketHandler rejects upgrade") + { + // Register a plain HTTP service (no WebSocket) + struct PlainService : public HttpService + { + const char* BaseUri() const override { return "/plain/"; } + void HandleRequest(HttpServerRequest& Request) override { Request.WriteResponse(HttpResponseCode::OK); } + }; + + PlainService Plain; + Server->RegisterService(Plain); + + Sleep(50); + + asio::io_context IoCtx; + asio::ip::tcp::socket Sock(IoCtx); + Sock.connect(asio::ip::tcp::endpoint(asio::ip::address::from_string("127.0.0.1"), static_cast(Port))); + + // Attempt WebSocket upgrade on the plain service + ExtendableStringBuilder<512> Request; + Request << "GET /plain/ws HTTP/1.1\r\n" + << "Host: 127.0.0.1:" << Port << "\r\n" + << "Upgrade: websocket\r\n" + << "Connection: Upgrade\r\n" + << "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + << "Sec-WebSocket-Version: 13\r\n" + << "\r\n"; + + std::string_view ReqStr = Request.ToView(); + asio::write(Sock, asio::buffer(ReqStr.data(), ReqStr.size())); + + asio::streambuf ResponseBuf; + asio::read_until(Sock, ResponseBuf, "\r\n\r\n"); + + std::string Response(asio::buffers_begin(ResponseBuf.data()), asio::buffers_end(ResponseBuf.data())); + + // Should NOT get 101 — should fall through to normal request handling + CHECK(Response.find("101") == std::string::npos); + + Sock.close(); + } + + SUBCASE("ping/pong auto-response") + { + asio::io_context IoCtx; + asio::ip::tcp::socket Sock(IoCtx); + Sock.connect(asio::ip::tcp::endpoint(asio::ip::address::from_string("127.0.0.1"), static_cast(Port))); + + bool Ok = DoWebSocketHandshake(Sock, "/wstest/ws", Port); + REQUIRE(Ok); + Sleep(50); + + // Send a ping frame with payload "test" + std::string_view PingPayload = "test"; + std::span PingData(reinterpret_cast(PingPayload.data()), PingPayload.size()); + std::vector PingFrame = BuildMaskedFrame(WebSocketOpcode::kPing, PingData); + asio::write(Sock, asio::buffer(PingFrame)); + + // Should receive a pong with the same payload + WsFrameParseResult Reply = ReadOneFrame(Sock); + REQUIRE(Reply.IsValid); + CHECK_EQ(Reply.Opcode, WebSocketOpcode::kPong); + CHECK_EQ(Reply.Payload.size(), 4u); + std::string_view PongText(reinterpret_cast(Reply.Payload.data()), Reply.Payload.size()); + CHECK_EQ(PongText, "test"sv); + + Sock.close(); + } + + SUBCASE("multiple messages in sequence") + { + asio::io_context IoCtx; + asio::ip::tcp::socket Sock(IoCtx); + Sock.connect(asio::ip::tcp::endpoint(asio::ip::address::from_string("127.0.0.1"), static_cast(Port))); + + bool Ok = DoWebSocketHandshake(Sock, "/wstest/ws", Port); + REQUIRE(Ok); + Sleep(50); + + for (int i = 0; i < 10; ++i) + { + std::string Msg = fmt::format("message {}", i); + std::vector Frame = BuildMaskedTextFrame(Msg); + asio::write(Sock, asio::buffer(Frame)); + + WsFrameParseResult Reply = ReadOneFrame(Sock); + REQUIRE(Reply.IsValid); + CHECK_EQ(Reply.Opcode, WebSocketOpcode::kText); + std::string_view ReplyText(reinterpret_cast(Reply.Payload.data()), Reply.Payload.size()); + CHECK_EQ(ReplyText, Msg); + } + + CHECK_EQ(TestService.m_MessageCount.load(), 10); + + Sock.close(); + } +} + +////////////////////////////////////////////////////////////////////////// +// +// Integration tests: HttpWsClient +// + +namespace { + + struct TestWsClientHandler : public IWsClientHandler + { + void OnWsOpen() override { m_OpenCount.fetch_add(1); } + + void OnWsMessage(const WebSocketMessage& Msg) override + { + m_MessageCount.fetch_add(1); + + if (Msg.Opcode == WebSocketOpcode::kText) + { + std::string_view Text(static_cast(Msg.Payload.Data()), Msg.Payload.Size()); + m_LastMessage = std::string(Text); + } + } + + void OnWsClose(uint16_t Code, [[maybe_unused]] std::string_view Reason) override + { + m_CloseCount.fetch_add(1); + m_LastCloseCode = Code; + } + + std::atomic m_OpenCount{0}; + std::atomic m_MessageCount{0}; + std::atomic m_CloseCount{0}; + std::atomic m_LastCloseCode{0}; + std::string m_LastMessage; + }; + +} // anonymous namespace + +TEST_CASE("websocket.client") +{ + WsTestService TestService; + ScopedTemporaryDirectory TmpDir; + + Ref Server = CreateHttpAsioServer(AsioConfig{}); + + int Port = Server->Initialize(7576, TmpDir.Path()); + REQUIRE(Port != 0); + + Server->RegisterService(TestService); + + std::thread ServerThread([&]() { Server->Run(false); }); + + auto ServerGuard = MakeGuard([&]() { + Server->RequestExit(); + if (ServerThread.joinable()) + { + ServerThread.join(); + } + Server->Close(); + }); + + Sleep(100); + + SUBCASE("connect, echo, close") + { + TestWsClientHandler Handler; + std::string Url = fmt::format("ws://127.0.0.1:{}/wstest/ws", Port); + + HttpWsClient Client(Url, Handler); + Client.Connect(); + + // Wait for OnWsOpen + auto Deadline = std::chrono::steady_clock::now() + 5s; + while (Handler.m_OpenCount.load() == 0 && std::chrono::steady_clock::now() < Deadline) + { + Sleep(10); + } + REQUIRE_EQ(Handler.m_OpenCount.load(), 1); + CHECK(Client.IsOpen()); + + // Send text, expect echo + Client.SendText("hello from client"); + + Deadline = std::chrono::steady_clock::now() + 5s; + while (Handler.m_MessageCount.load() == 0 && std::chrono::steady_clock::now() < Deadline) + { + Sleep(10); + } + CHECK_EQ(Handler.m_MessageCount.load(), 1); + CHECK_EQ(Handler.m_LastMessage, "hello from client"); + + // Close + Client.Close(1000, "done"); + + Deadline = std::chrono::steady_clock::now() + 5s; + while (Handler.m_CloseCount.load() == 0 && std::chrono::steady_clock::now() < Deadline) + { + Sleep(10); + } + + // The server echoes the close frame, which triggers OnWsClose on the client side + // with the server's close code. Allow the connection to settle. + Sleep(50); + CHECK_FALSE(Client.IsOpen()); + } + + SUBCASE("connect to bad port") + { + TestWsClientHandler Handler; + std::string Url = "ws://127.0.0.1:1/wstest/ws"; + + HttpWsClient Client(Url, Handler, HttpWsClientSettings{.ConnectTimeout = std::chrono::milliseconds(2000)}); + Client.Connect(); + + auto Deadline = std::chrono::steady_clock::now() + 5s; + while (Handler.m_CloseCount.load() == 0 && std::chrono::steady_clock::now() < Deadline) + { + Sleep(10); + } + + CHECK_EQ(Handler.m_CloseCount.load(), 1); + CHECK_EQ(Handler.m_LastCloseCode.load(), 1006); + CHECK_EQ(Handler.m_OpenCount.load(), 0); + } + + SUBCASE("server-initiated close") + { + TestWsClientHandler Handler; + std::string Url = fmt::format("ws://127.0.0.1:{}/wstest/ws", Port); + + HttpWsClient Client(Url, Handler); + Client.Connect(); + + auto Deadline = std::chrono::steady_clock::now() + 5s; + while (Handler.m_OpenCount.load() == 0 && std::chrono::steady_clock::now() < Deadline) + { + Sleep(10); + } + REQUIRE_EQ(Handler.m_OpenCount.load(), 1); + + // Copy connections then close them outside the lock to avoid deadlocking + // with OnWebSocketClose which acquires an exclusive lock + std::vector> Conns; + TestService.m_ConnectionsLock.WithSharedLock([&] { Conns = TestService.m_Connections; }); + for (auto& Conn : Conns) + { + Conn->Close(1001, "going away"); + } + + Deadline = std::chrono::steady_clock::now() + 5s; + while (Handler.m_CloseCount.load() == 0 && std::chrono::steady_clock::now() < Deadline) + { + Sleep(10); + } + + CHECK_EQ(Handler.m_CloseCount.load(), 1); + CHECK_EQ(Handler.m_LastCloseCode.load(), 1001); + CHECK_FALSE(Client.IsOpen()); + } +} + +void +websocket_forcelink() +{ +} + +} // namespace zen + +#endif // ZEN_WITH_TESTS diff --git a/src/zenhttp/xmake.lua b/src/zenhttp/xmake.lua index 78876d21b..e8f87b668 100644 --- a/src/zenhttp/xmake.lua +++ b/src/zenhttp/xmake.lua @@ -6,6 +6,7 @@ target('zenhttp') add_headerfiles("**.h") add_files("**.cpp") add_files("servers/httpsys.cpp", {unity_ignored=true}) + add_files("servers/wshttpsys.cpp", {unity_ignored=true}) add_includedirs("include", {public=true}) add_deps("zencore", "zentelemetry", "transport-sdk", "asio", "cpr") add_packages("http_parser", "json11") diff --git a/src/zenhttp/zenhttp.cpp b/src/zenhttp/zenhttp.cpp index ad14ecb8d..3ac8eea8d 100644 --- a/src/zenhttp/zenhttp.cpp +++ b/src/zenhttp/zenhttp.cpp @@ -19,6 +19,7 @@ zenhttp_forcelinktests() httpclient_test_forcelink(); forcelink_packageformat(); passwordsecurity_forcelink(); + websocket_forcelink(); } } // namespace zen -- cgit v1.2.3 From 1ed3139e577f6c8aa6d07f7e76afa3a80d9d4852 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 27 Feb 2026 19:36:22 +0100 Subject: Add test summary table and failure reporting to xmake test (#794) - Add a summary table printed after all test suites complete, showing per-suite test case counts, assertion counts, timings and pass/fail status. - Add failure reporting: individual failing test cases are listed at the end with their file path and line number for easy navigation. - Made zenserver instances spawned by a hub not create new console windows for a better background testing experience - The TestListener in testing.cpp now writes a machine-readable summary file (via `ZEN_TEST_SUMMARY_FILE` env var) containing aggregate counts and per-test-case failure details. This runs as a doctest listener alongside any active reporter, so it works with both console and JUnit modes. - Tests now run in a deterministic order defined by a single ordered list that also serves as the test name/target mapping, replacing the previous unordered table + separate order list. - The `--run` option now accepts comma-separated values (e.g. `--run=core,http,util`) and validates each name, reporting unknown test names early. - Fix platform detection in `xmake test`: the config command now passes `-p` explicitly, fixing "mingw" misdetection when running from Git Bash on Windows. - Add missing "util" entry to the help text for `--run`. --- src/zencore/testing.cpp | 56 ++++- src/zenutil/include/zenutil/zenserverprocess.h | 1 + src/zenutil/zenserverprocess.cpp | 3 +- xmake.lua | 304 +++++++++++++++++++++---- 4 files changed, 310 insertions(+), 54 deletions(-) diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index 936424e0f..ef8fb0480 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -5,6 +5,12 @@ #if ZEN_WITH_TESTS +# include +# include +# include +# include +# include + # include namespace zen::testing { @@ -21,9 +27,35 @@ struct TestListener : public doctest::IReporter void report_query(const doctest::QueryData& /*in*/) override {} - void test_run_start() override {} + void test_run_start() override { RunStart = std::chrono::steady_clock::now(); } + + void test_run_end(const doctest::TestRunStats& in) override + { + auto elapsed = std::chrono::steady_clock::now() - RunStart; + double elapsedSeconds = std::chrono::duration_cast(elapsed).count() / 1000.0; - void test_run_end(const doctest::TestRunStats& /*in*/) override {} + // Write machine-readable summary to file if requested (used by xmake test summary table) + const char* summaryFile = std::getenv("ZEN_TEST_SUMMARY_FILE"); + if (summaryFile && summaryFile[0] != '\0') + { + if (FILE* f = std::fopen(summaryFile, "w")) + { + std::fprintf(f, + "cases_total=%u\ncases_passed=%u\nassertions_total=%d\nassertions_passed=%d\n" + "elapsed_seconds=%.3f\n", + in.numTestCasesPassingFilters, + in.numTestCasesPassingFilters - in.numTestCasesFailed, + in.numAsserts, + in.numAsserts - in.numAssertsFailed, + elapsedSeconds); + for (const auto& failure : FailedTests) + { + std::fprintf(f, "failed=%s|%s|%u\n", failure.Name.c_str(), failure.File.c_str(), failure.Line); + } + std::fclose(f); + } + } + } void test_case_start(const doctest::TestCaseData& in) override { @@ -37,7 +69,14 @@ struct TestListener : public doctest::IReporter ZEN_CONSOLE("{}-------------------------------------------------------------------------------{}", ColorYellow, ColorNone); } - void test_case_end(const doctest::CurrentTestCaseStats& /*in*/) override { Current = nullptr; } + void test_case_end(const doctest::CurrentTestCaseStats& in) override + { + if (!in.testCaseSuccess && Current) + { + FailedTests.push_back({Current->m_name, Current->m_file.c_str(), Current->m_line}); + } + Current = nullptr; + } void test_case_exception(const doctest::TestCaseException& /*in*/) override {} @@ -57,7 +96,16 @@ struct TestListener : public doctest::IReporter void test_case_skipped(const doctest::TestCaseData& /*in*/) override {} - const doctest::TestCaseData* Current = nullptr; + const doctest::TestCaseData* Current = nullptr; + std::chrono::steady_clock::time_point RunStart = {}; + + struct FailedTestInfo + { + std::string Name; + std::string File; + unsigned Line; + }; + std::vector FailedTests; }; struct TestRunner::Impl diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h index d0402640b..b781a03a9 100644 --- a/src/zenutil/include/zenutil/zenserverprocess.h +++ b/src/zenutil/include/zenutil/zenserverprocess.h @@ -42,6 +42,7 @@ public: std::filesystem::path GetTestRootDir(std::string_view Path); inline bool IsInitialized() const { return m_IsInitialized; } inline bool IsTestEnvironment() const { return m_IsTestInstance; } + inline bool IsHubEnvironment() const { return m_IsHubInstance; } inline std::string_view GetServerClass() const { return m_ServerClass; } inline uint16_t GetNewPortNumber() { return m_NextPortNumber.fetch_add(1); } diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index ef2a4fda5..579ba450a 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -934,7 +934,8 @@ ZenServerInstance::SpawnServer(int BasePort, std::string_view AdditionalServerAr CommandLine << " " << AdditionalServerArgs; } - SpawnServerInternal(ChildId, CommandLine, !IsTest, WaitTimeoutMs); + const bool OpenConsole = !IsTest && !m_Env.IsHubEnvironment(); + SpawnServerInternal(ChildId, CommandLine, OpenConsole, WaitTimeoutMs); } void diff --git a/xmake.lua b/xmake.lua index d49743cb2..d7a905981 100644 --- a/xmake.lua +++ b/xmake.lua @@ -344,10 +344,10 @@ task("sln") task("test") set_menu { - usage = "xmake test --run=[core|store|http|server|integration|all]", + usage = "xmake test --run=[core|store|http|server|integration|util|remotestore|all] (comma-separated)", description = "Run Zen tests", options = { - {'r', "run", "kv", "all", "Run test(s)", " - all", " - core ", " - remotestore", " - store", " - http", " - server", " - integration"}, + {'r', "run", "kv", "all", "Run test(s) - comma-separated", " - all", " - core", " - http", " - util", " - store", " - remotestore", " - server", " - integration"}, {'j', "junit", "k", nil, "Enable junit report output"} } } @@ -359,39 +359,61 @@ task("test") config.load() local testname = option.get("run") + + -- Ordered list of available tests (order defines execution order) local available_tests = { - core = "zencore-test", - http = "zenhttp-test", - util = "zenutil-test", - store = "zenstore-test", - remotestore = "zenremotestore-test", - server = "zenserver", - integration = "zenserver-test" + {"core", "zencore-test"}, + {"http", "zenhttp-test"}, + {"util", "zenutil-test"}, + {"store", "zenstore-test"}, + {"remotestore", "zenremotestore-test"}, + {"server", "zenserver"}, + {"integration", "zenserver-test"}, } - local arch + local plat, arch if is_host("windows") then + plat = "windows" arch = "x64" - elseif is_arch("arm64") then - arch = "arm64" + elseif is_host("macosx") then + plat = "macosx" + arch = is_arch("arm64") and "arm64" or "x86_64" else + plat = "linux" arch = "x86_64" end - print(os.exec("xmake config -c -m debug -a "..arch)) + print(os.exec("xmake config -c -m debug -p "..plat.." -a "..arch)) print(os.exec("xmake")) + -- Parse comma-separated test names into a set + local requested = {} + for token in testname:gmatch("[^,]+") do + requested[token:match("^%s*(.-)%s*$")] = true + end + + -- Filter to requested test(s) local tests = {} - local found_match = false + local matched = {} - for name, test in pairs(available_tests) do - if name == testname or testname == "all" then - tests[name] = test - found_match = true + for _, entry in ipairs(available_tests) do + local name, target = entry[1], entry[2] + if requested["all"] or requested[name] then + table.insert(tests, {name = name, target = target}) + matched[name] = true + end + end + + -- Check for unknown test names + if not requested["all"] then + for name, _ in pairs(requested) do + if not matched[name] then + raise("no tests match specification: '%s'", name) + end end end - if not found_match then + if #tests == 0 then raise("no tests match specification: '%s'", testname) end @@ -404,39 +426,223 @@ task("test") os.mkdir(junit_report_dir) end - try - { - function() - for name, test in pairs(tests) do - printf("=== %s ===\n", test) - local cmd = string.format("xmake run %s", test) - if name == "server" then - cmd = string.format("xmake run %s test", test) - end - cmd = string.format("%s --duration=true", cmd) - if use_junit_reporting then - local target = project.target(test) - local junit_report_file = path.join(junit_report_dir, string.format("junit-%s-%s-%s.xml", config.plat(), arch, test)) - junit_report_files[test] = junit_report_file - cmd = string.format("%s --reporters=junit --out=%s", cmd, junit_report_file) - end + -- Results collection for summary table + local results = {} + local any_failed = false + + -- Format a number with thousands separators (e.g. 31103 -> "31,103") + local function format_number(n) + local s = tostring(n) + local pos = #s % 3 + if pos == 0 then pos = 3 end + local result = s:sub(1, pos) + for i = pos + 1, #s, 3 do + result = result .. "," .. s:sub(i, i + 2) + end + return result + end - os.exec(cmd) + -- Center a string within a given width + local function center_str(s, width) + local pad = width - #s + local lpad = math.floor(pad / 2) + local rpad = pad - lpad + return string.rep(" ", lpad) .. s .. string.rep(" ", rpad) + end + + -- Left-align a string within a given width (with 1-space left margin) + local function left_pad_str(s, width) + return " " .. s .. string.rep(" ", width - #s - 1) + end + + -- Format elapsed seconds as a human-readable string + local function format_time(seconds) + if seconds >= 60 then + local mins = math.floor(seconds / 60) + local secs = seconds - mins * 60 + return string.format("%dm %04.1fs", mins, secs) + else + return string.format("%.1fs", seconds) + end + end + + -- Parse test summary file written by TestListener + local function parse_summary_file(filepath) + if not os.isfile(filepath) then return nil end + local content = io.readfile(filepath) + if not content then return nil end + local ct = content:match("cases_total=(%d+)") + local cp = content:match("cases_passed=(%d+)") + local at = content:match("assertions_total=(%d+)") + local ap = content:match("assertions_passed=(%d+)") + if ct then + local failures = {} + for name, file, line in content:gmatch("failed=([^|\n]+)|([^|\n]+)|(%d+)") do + table.insert(failures, {name = name, file = file, line = tonumber(line)}) end - end, - - finally - { - function (ok, errors) - for test, junit_report_file in pairs(junit_report_files) do - printf("=== report - %s ===\n", test) - local data = io.readfile(junit_report_file) - print(data) + local es = content:match("elapsed_seconds=([%d%.]+)") + return { + cases_total = tonumber(ct), + cases_passed = tonumber(cp) or 0, + asserts_total = tonumber(at) or 0, + asserts_passed = tonumber(ap) or 0, + elapsed_seconds = tonumber(es) or 0, + failures = failures + } + end + return nil + end + + -- Temp directory for summary files + local summary_dir = path.join(os.tmpdir(), "zen-test-summary") + os.mkdir(summary_dir) + + -- Run each test suite and collect results + for _, entry in ipairs(tests) do + local name, target = entry.name, entry.target + printf("=== %s ===\n", target) + + local suite_name = target + if name == "server" then + suite_name = "zenserver (test)" + end + + local cmd = string.format("xmake run %s", target) + if name == "server" then + cmd = string.format("xmake run %s test", target) + end + cmd = string.format("%s --duration=true", cmd) + + if use_junit_reporting then + local junit_report_file = path.join(junit_report_dir, string.format("junit-%s-%s-%s.xml", config.plat(), arch, target)) + junit_report_files[target] = junit_report_file + cmd = string.format("%s --reporters=junit --out=%s", cmd, junit_report_file) + end + + -- Tell TestListener where to write the summary + local summary_file = path.join(summary_dir, target .. ".txt") + os.setenv("ZEN_TEST_SUMMARY_FILE", summary_file) + + -- Run test with real-time streaming output + local test_ok = true + try { + function() + os.exec(cmd) + end, + catch { + function(errors) + test_ok = false end - if (errors) then - raise(errors) + } + } + + -- Read summary written by TestListener + local summary = parse_summary_file(summary_file) + os.tryrm(summary_file) + + if not test_ok then + any_failed = true + end + + table.insert(results, { + suite = suite_name, + cases_passed = summary and summary.cases_passed or 0, + cases_total = summary and summary.cases_total or 0, + asserts_passed = summary and summary.asserts_passed or 0, + asserts_total = summary and summary.asserts_total or 0, + elapsed_seconds = summary and summary.elapsed_seconds or 0, + failures = summary and summary.failures or {}, + passed = test_ok + }) + end + + -- Clean up + os.setenv("ZEN_TEST_SUMMARY_FILE", "") + os.tryrm(summary_dir) + + -- Print JUnit reports if requested + for test, junit_report_file in pairs(junit_report_files) do + printf("=== report - %s ===\n", test) + if os.isfile(junit_report_file) then + local data = io.readfile(junit_report_file) + if data then + print(data) + end + end + end + + -- Print summary table + if #results > 0 then + -- Calculate column widths based on content + local col_suite = #("Suite") + local col_cases = #("Cases") + local col_asserts = #("Assertions") + local col_time = #("Time") + local col_status = #("Status") + + for _, r in ipairs(results) do + col_suite = math.max(col_suite, #r.suite) + local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total) + col_cases = math.max(col_cases, #cases_str) + local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total) + col_asserts = math.max(col_asserts, #asserts_str) + col_time = math.max(col_time, #format_time(r.elapsed_seconds)) + local status_str = r.passed and "SUCCESS" or "FAILED" + col_status = math.max(col_status, #status_str) + end + + -- Add padding (1 space each side) + col_suite = col_suite + 2 + col_cases = col_cases + 2 + col_asserts = col_asserts + 2 + col_time = col_time + 2 + col_status = col_status + 2 + + -- Build horizontal border segments + local h_suite = string.rep("-", col_suite) + local h_cases = string.rep("-", col_cases) + local h_asserts = string.rep("-", col_asserts) + local h_time = string.rep("-", col_time) + local h_status = string.rep("-", col_status) + + local top = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+" + local mid = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+" + local bottom = "+" .. h_suite .. "+" .. h_cases .. "+" .. h_asserts .. "+" .. h_time .. "+" .. h_status .. "+" + local vbar = "|" + + local header_msg = any_failed and "Some tests failed:" or "All tests passed:" + printf("\n* %s\n", header_msg) + printf(" %s\n", top) + printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, center_str("Suite", col_suite), vbar, center_str("Cases", col_cases), vbar, center_str("Assertions", col_asserts), vbar, center_str("Time", col_time), vbar, center_str("Status", col_status), vbar) + + for _, r in ipairs(results) do + printf(" %s\n", mid) + local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total) + local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total) + local time_str = format_time(r.elapsed_seconds) + local status_str = r.passed and "SUCCESS" or "FAILED" + printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_pad_str(r.suite, col_suite), vbar, left_pad_str(cases_str, col_cases), vbar, left_pad_str(asserts_str, col_asserts), vbar, left_pad_str(time_str, col_time), vbar, left_pad_str(status_str, col_status), vbar) + end + printf(" %s\n", bottom) + end + + -- Print list of individual failing tests + if any_failed then + printf("\n Failures:\n") + for _, r in ipairs(results) do + if #r.failures > 0 then + printf(" -- %s --\n", r.suite) + for _, f in ipairs(r.failures) do + printf(" FAILED: %s (%s:%d)\n", f.name, f.file, f.line) end + elseif not r.passed then + printf(" -- %s --\n", r.suite) + printf(" (test binary exited with error, no failure details available)\n") end - } - } + end + end + + if any_failed then + raise("one or more test suites failed") + end end) -- cgit v1.2.3 From c32b6042dee8444f4e214f227005a657ec87531e Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 27 Feb 2026 21:22:00 +0100 Subject: add multirange requests to blob store (#795) * add multirange requests to blob store --- CHANGELOG.md | 1 + src/zenhttp/packageformat.cpp | 2 +- src/zenserver-test/buildstore-tests.cpp | 200 ++++++++++++++++++++- .../storage/buildstore/httpbuildstore.cpp | 114 ++++++++++-- 4 files changed, 296 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe5ad365c..899415d3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ - Feature: `zen ui` can be used to open dashboards for local instances - Feature: Added 404 page to dashboard, to make it easier to find your way back to a valid URL - Feature: HttpClient now properly handles multi-part request/response +- Feature: Added `POST` variant to `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` endpoint that accepts a CbObject payload with 1 or more ranges - Improvement: `zen oplog-import` now uses partial block requests to reduce download size - Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests - Improvement: IAX's lane tracing diff --git a/src/zenhttp/packageformat.cpp b/src/zenhttp/packageformat.cpp index 708238224..9a80d07c8 100644 --- a/src/zenhttp/packageformat.cpp +++ b/src/zenhttp/packageformat.cpp @@ -581,7 +581,7 @@ ParsePackageMessage(IoBuffer Payload, std::function CompressedBlobsHashes; + std::vector CompressedBlobsHashes; + std::vector CompressedBlobsSizes; { ZenServerInstance Instance(TestEnv); @@ -51,6 +52,7 @@ TEST_CASE("buildstore.blobs") IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + CompressedBlobsSizes.push_back(CompressedBlob.GetCompressedSize()); IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); Payload.SetContentType(ZenContentType::kCompressedBinary); @@ -107,6 +109,7 @@ TEST_CASE("buildstore.blobs") IoBuffer Blob = CreateSemiRandomBlob(5713 + I * 7); CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + CompressedBlobsSizes.push_back(CompressedBlob.GetCompressedSize()); IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); Payload.SetContentType(ZenContentType::kCompressedBinary); @@ -141,6 +144,201 @@ TEST_CASE("buildstore.blobs") CHECK(IoHash::HashBuffer(Decompressed) == RawHash); } } + + { + // Single-range Get + + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + { + const IoHash& RawHash = CompressedBlobsHashes.front(); + uint64_t BlobSize = CompressedBlobsSizes.front(); + + std::vector> Ranges = {{BlobSize / 16 * 1, BlobSize / 2}}; + + uint64_t RangeSizeSum = Ranges.front().second; + + HttpClient::KeyValueMap Headers; + + Headers.Entries.insert( + {"Range", fmt::format("bytes={}-{}", Ranges.front().first, Ranges.front().first + Ranges.front().second - 1)}); + + HttpClient::Response Result = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), Headers); + REQUIRE(Result); + IoBuffer Payload = Result.ResponsePayload; + CHECK_EQ(RangeSizeSum, Payload.GetSize()); + + HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + HttpClient::Accept(ZenContentType::kCompressedBinary)); + REQUIRE(FullBlobResult); + MemoryView ActualRange = FullBlobResult.ResponsePayload.GetView().Mid(Ranges.front().first, Ranges.front().second); + MemoryView RangeView = Payload.GetView(); + CHECK(ActualRange.EqualBytes(RangeView)); + } + } + + { + // Single-range Post + + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + { + uint64_t RangeSizeSum = 0; + + const IoHash& RawHash = CompressedBlobsHashes.front(); + uint64_t BlobSize = CompressedBlobsSizes.front(); + + std::vector> Ranges = {{BlobSize / 16 * 1, BlobSize / 2}}; + + CbObjectWriter Writer; + Writer.BeginArray("ranges"sv); + { + for (const std::pair& Range : Ranges) + { + Writer.BeginObject(); + { + Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("length"sv, Range.second); + RangeSizeSum += Range.second; + } + Writer.EndObject(); + } + } + Writer.EndArray(); // ranges + + HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + Writer.Save(), + HttpClient::Accept(ZenContentType::kCbPackage)); + REQUIRE(Result); + IoBuffer Payload = Result.ResponsePayload; + REQUIRE(Payload.GetContentType() == ZenContentType::kCbPackage); + + CbPackage ResponsePackage = ParsePackageMessage(Payload); + CbObjectView ResponseObject = ResponsePackage.GetObject(); + + CbArrayView RangeArray = ResponseObject["ranges"sv].AsArrayView(); + CHECK_EQ(RangeArray.Num(), Ranges.size()); + size_t RangeOffset = 0; + for (CbFieldView View : RangeArray) + { + CbObjectView Range = View.AsObjectView(); + CHECK_EQ(Range["offset"sv].AsUInt64(), Ranges[RangeOffset].first); + CHECK_EQ(Range["length"sv].AsUInt64(), Ranges[RangeOffset].second); + RangeOffset++; + } + + const CbAttachment* DataAttachment = ResponsePackage.FindAttachment(RawHash); + REQUIRE(DataAttachment); + SharedBuffer PayloadRanges = DataAttachment->AsBinary(); + CHECK_EQ(RangeSizeSum, PayloadRanges.GetSize()); + + HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + HttpClient::Accept(ZenContentType::kCompressedBinary)); + REQUIRE(FullBlobResult); + + uint64_t Offset = 0; + for (const std::pair& Range : Ranges) + { + MemoryView ActualRange = FullBlobResult.ResponsePayload.GetView().Mid(Range.first, Range.second); + MemoryView RangeView = PayloadRanges.GetView().Mid(Offset, Range.second); + CHECK(ActualRange.EqualBytes(RangeView)); + Offset += Range.second; + } + } + } + + { + // Multi-range + + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = + Instance.SpawnServerAndWaitUntilReady(fmt::format("--buildstore-enabled --system-dir {}", SystemRootPath)); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/builds/"); + + { + uint64_t RangeSizeSum = 0; + + const IoHash& RawHash = CompressedBlobsHashes.front(); + uint64_t BlobSize = CompressedBlobsSizes.front(); + + std::vector> Ranges = { + {BlobSize / 16 * 1, BlobSize / 20}, + {BlobSize / 16 * 3, BlobSize / 32}, + {BlobSize / 16 * 5, BlobSize / 16}, + {BlobSize - BlobSize / 16, BlobSize / 16 - 1}, + }; + + CbObjectWriter Writer; + Writer.BeginArray("ranges"sv); + { + for (const std::pair& Range : Ranges) + { + Writer.BeginObject(); + { + Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("length"sv, Range.second); + RangeSizeSum += Range.second; + } + Writer.EndObject(); + } + } + Writer.EndArray(); // ranges + + HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + Writer.Save(), + HttpClient::Accept(ZenContentType::kCbPackage)); + REQUIRE(Result); + IoBuffer Payload = Result.ResponsePayload; + REQUIRE(Payload.GetContentType() == ZenContentType::kCbPackage); + + CbPackage ResponsePackage = ParsePackageMessage(Payload); + CbObjectView ResponseObject = ResponsePackage.GetObject(); + + CbArrayView RangeArray = ResponseObject["ranges"sv].AsArrayView(); + CHECK_EQ(RangeArray.Num(), Ranges.size()); + size_t RangeOffset = 0; + for (CbFieldView View : RangeArray) + { + CbObjectView Range = View.AsObjectView(); + CHECK_EQ(Range["offset"sv].AsUInt64(), Ranges[RangeOffset].first); + CHECK_EQ(Range["length"sv].AsUInt64(), Ranges[RangeOffset].second); + RangeOffset++; + } + + const CbAttachment* DataAttachment = ResponsePackage.FindAttachment(RawHash); + REQUIRE(DataAttachment); + SharedBuffer PayloadRanges = DataAttachment->AsBinary(); + CHECK_EQ(RangeSizeSum, PayloadRanges.GetSize()); + + HttpClient::Response FullBlobResult = Client.Get(fmt::format("{}/{}/{}/blobs/{}", Namespace, Bucket, BuildId, RawHash), + HttpClient::Accept(ZenContentType::kCompressedBinary)); + REQUIRE(FullBlobResult); + + uint64_t Offset = 0; + for (const std::pair& Range : Ranges) + { + MemoryView ActualRange = FullBlobResult.ResponsePayload.GetView().Mid(Range.first, Range.second); + MemoryView RangeView = PayloadRanges.GetView().Mid(Offset, Range.second); + CHECK(ActualRange.EqualBytes(RangeView)); + Offset += Range.second; + } + } + } } namespace { diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index bf7afcc02..6ada085a5 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -71,7 +71,7 @@ HttpBuildStoreService::Initialize() m_Router.RegisterRoute( "{namespace}/{bucket}/{buildid}/blobs/{hash}", [this](HttpRouterRequest& Req) { GetBlobRequest(Req); }, - HttpVerb::kGet); + HttpVerb::kGet | HttpVerb::kPost); m_Router.RegisterRoute( "{namespace}/{bucket}/{buildid}/blobs/putBlobMetadata", @@ -161,14 +161,49 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) HttpContentType::kText, fmt::format("Invalid blob hash '{}'", Hash)); } - zen::HttpRanges Ranges; - bool HasRange = ServerRequest.TryGetRanges(Ranges); - if (Ranges.size() > 1) + + std::vector> OffsetAndLengthPairs; + if (ServerRequest.RequestVerb() == HttpVerb::kPost) { - // Only a single range is supported - return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, - HttpContentType::kText, - "Multiple ranges in blob request is not supported"); + CbObject RangePayload = ServerRequest.ReadPayloadObject(); + if (RangePayload) + { + CbArrayView RangesArray = RangePayload["ranges"sv].AsArrayView(); + OffsetAndLengthPairs.reserve(RangesArray.Num()); + for (CbFieldView FieldView : RangesArray) + { + CbObjectView RangeView = FieldView.AsObjectView(); + uint64_t RangeOffset = RangeView["offset"sv].AsUInt64(); + uint64_t RangeLength = RangeView["length"sv].AsUInt64(); + OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength)); + } + } + if (OffsetAndLengthPairs.empty()) + { + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Fetching blob without ranges must be done with the GET verb"); + } + } + else + { + HttpRanges Ranges; + bool HasRange = ServerRequest.TryGetRanges(Ranges); + if (HasRange) + { + if (Ranges.size() > 1) + { + // Only a single http range is supported, we have limited support for http multirange responses + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Multiple ranges in blob request is only supported for {} accept type", + ToString(HttpContentType::kCbPackage))); + } + const HttpRange& FirstRange = Ranges.front(); + OffsetAndLengthPairs.push_back(std::make_pair(FirstRange.Start, FirstRange.End - FirstRange.Start + 1)); + } } m_BuildStoreStats.BlobReadCount++; @@ -179,24 +214,65 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) HttpContentType::kText, fmt::format("Blob with hash '{}' could not be found", Hash)); } - // ZEN_INFO("Fetched blob {}. Size: {}", BlobHash, Blob.GetSize()); m_BuildStoreStats.BlobHitCount++; - if (HasRange) + + if (OffsetAndLengthPairs.empty()) { - const HttpRange& Range = Ranges.front(); - const uint64_t BlobSize = Blob.GetSize(); - const uint64_t MaxBlobSize = Range.Start < BlobSize ? BlobSize - Range.Start : 0; - const uint64_t RangeSize = Min(Range.End - Range.Start + 1, MaxBlobSize); - if (Range.Start + RangeSize > BlobSize) + return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + } + + if (ServerRequest.AcceptContentType() == HttpContentType::kCbPackage) + { + const uint64_t BlobSize = Blob.GetSize(); + + CbPackage ResponsePackage; + std::vector RangeBuffers; + CbObjectWriter Writer; + Writer.BeginArray("ranges"sv); + for (const std::pair& Range : OffsetAndLengthPairs) { - return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0; + const uint64_t RangeSize = Min(Range.second, MaxBlobSize); + if (Range.first + RangeSize <= BlobSize) + { + RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize)); + Writer.BeginObject(); + { + Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("length"sv, RangeSize); + } + Writer.EndObject(); + } } - Blob = IoBuffer(Blob, Range.Start, RangeSize); - return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); + Writer.EndArray(); + + CompositeBuffer Ranges(RangeBuffers); + CbAttachment PayloadAttachment(std::move(Ranges), BlobHash); + Writer.AddAttachment("payload", PayloadAttachment); + + CbObject HeaderObject = Writer.Save(); + + ResponsePackage.AddAttachment(PayloadAttachment); + ResponsePackage.SetObject(HeaderObject); + + CompositeBuffer RpcResponseBuffer = FormatPackageMessageBuffer(ResponsePackage); + uint64_t ResponseSize = RpcResponseBuffer.GetSize(); + ZEN_UNUSED(ResponseSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, HttpContentType::kCbPackage, RpcResponseBuffer); } else { - return ServerRequest.WriteResponse(HttpResponseCode::OK, Blob.GetContentType(), Blob); + ZEN_ASSERT(OffsetAndLengthPairs.size() == 1); + const std::pair& OffsetAndLength = OffsetAndLengthPairs.front(); + const uint64_t BlobSize = Blob.GetSize(); + const uint64_t MaxBlobSize = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0; + const uint64_t RangeSize = Min(OffsetAndLength.second, MaxBlobSize); + if (OffsetAndLength.first + RangeSize > BlobSize) + { + return ServerRequest.WriteResponse(HttpResponseCode::NoContent); + } + Blob = IoBuffer(Blob, OffsetAndLength.first, RangeSize); + return ServerRequest.WriteResponse(HttpResponseCode::OK, ZenContentType::kBinary, Blob); } } -- cgit v1.2.3 From c7e0efb9c12f4607d4bc6a844a3e5bd3272bd839 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 28 Feb 2026 15:36:13 +0100 Subject: test running / reporting improvements (#797) **CI/CD improvements (validate.yml):** - Add test reporter (`ue-foundation/test-reporter@v2`) for all three platforms, rendering JUnit test results directly in PR check runs - Add "Trust workspace" step on Windows to fix git safe.directory ownership issue with self-hosted runners - Clean stale report files before each test run to prevent false failures from leftover XML - Broaden `paths-ignore` to skip builds for non-code changes (`*.md`, `LICENSE`, `.gitignore`, `docs/**`) **Test improvements:** - Convert `CHECK` to `REQUIRE` in several test suites (projectstore, integration, http) for fail-fast behavior - Mark some tests with `doctest::skip()` for selective execution - Skip httpclient transport tests pending investigation - Add `--noskip` option to `xmake test` task - Add `--repeat=` option to `xmake test` task, to run tests repeatedly N times or until there is a failure **xmake test output improvements:** - Add totals row to test summary table - Right-justify numeric columns in summary table --- .github/workflows/create_release.yml | 8 +- .github/workflows/validate.yml | 85 +++++++++++---- src/zenhttp/httpclient_test.cpp | 4 +- src/zenserver-test/buildstore-tests.cpp | 16 +-- src/zenserver-test/cache-tests.cpp | 10 +- src/zenserver-test/hub-tests.cpp | 2 +- src/zenserver-test/projectstore-tests.cpp | 34 +++--- src/zenserver-test/workspace-tests.cpp | 4 +- src/zenstore/cache/structuredcachestore.cpp | 2 +- xmake.lua | 162 +++++++++++++++++++--------- 10 files changed, 211 insertions(+), 116 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index f095f3d21..203588d24 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 25 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup xmake uses: ue-foundation/github-action-setup-xmake@v1.1.1 @@ -46,7 +46,7 @@ jobs: timeout-minutes: 25 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install UE Toolchain run: | @@ -89,7 +89,7 @@ jobs: timeout-minutes: 25 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup xmake uses: ue-foundation/github-action-setup-xmake@v1.1.1 @@ -133,7 +133,7 @@ jobs: needs: [bundle-linux, bundle-macos, bundle-windows] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Read VERSION.txt id: read_version diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 17e031ba0..bf7b1c853 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -4,6 +4,10 @@ env: WINDOWS_SDK_VERSION: 22621 XMAKE_GLOBALDIR: ${{ github.workspace }}/../.xmake_shared +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref_name != 'main' }} + on: pull_request: types: @@ -13,8 +17,11 @@ on: branches: - 'main' paths-ignore: + - '*.md' + - 'LICENSE' + - '.gitignore' + - 'docs/**' - 'VERSION.txt' - - 'CHANGELOG.md' - '.github/workflows/create_release.yml' - '.github/workflows/mirror_releases.yml' - '.github/workflows/mirror.yml' @@ -22,31 +29,22 @@ on: branches: - 'main' paths-ignore: + - '*.md' + - 'LICENSE' + - '.gitignore' + - 'docs/**' - 'VERSION.txt' - - 'CHANGELOG.md' - '.github/workflows/create_release.yml' - '.github/workflows/mirror_releases.yml' - '.github/workflows/mirror.yml' jobs: - cancel-old-build: - name: Cancel previous builds - runs-on: [linux, x64, zen] - - steps: - - name: Cancel Previous Runs - if: ${{ github.ref_name != 'main'}} - uses: ue-foundation/cancel-workflow-action@0.11.0 - with: - access_token: ${{ github.token }} - clang-format: - needs: cancel-old-build name: Check clang-format runs-on: [linux, x64, zen] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: clang-format uses: ue-foundation/clang-format-action@epic-batching-v1 @@ -55,7 +53,6 @@ jobs: check-path: 'src' windows-build: - needs: cancel-old-build name: Build & Test Windows runs-on: [windows, x64, zen] timeout-minutes: 25 @@ -69,7 +66,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 + + - name: Trust workspace + run: git config --global --add safe.directory "${{ github.workspace }}" - name: Setup xmake uses: ue-foundation/github-action-setup-xmake@v1.1.1 @@ -80,6 +80,10 @@ jobs: run: | xmake config -vD -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} --zensentry=yes + - name: Clean reports + if: ${{ matrix.config == 'debug' }} + run: if (Test-Path build/reports) { Remove-Item -Recurse -Force build/reports } + - name: Build & Test if: ${{ matrix.config == 'debug' }} run: | @@ -92,9 +96,18 @@ jobs: name: reports-win64 path: build/reports/*.xml + - name: Test Report + if: ${{ (failure() || success()) && (matrix.config == 'debug') }} + uses: ue-foundation/test-reporter@v2 + with: + name: Test Results (win64) + path: build/reports/*.xml + reporter: java-junit + list-tests: 'failed' + - name: Bundle if: ${{ matrix.config == 'release' }} - run: | + run: | xmake bundle -v -y --codesignidentity="Epic Games" - name: Upload zenserver-win64 @@ -105,7 +118,6 @@ jobs: path: build/zenserver-win64.zip linux-build: - needs: cancel-old-build name: Build & Test Linux runs-on: [linux, x64, zen] timeout-minutes: 25 @@ -119,7 +131,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install UE Toolchain run: | @@ -135,6 +147,11 @@ jobs: run: | ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} --zensentry=yes + - name: Clean reports + if: ${{ matrix.config == 'debug' }} + shell: bash + run: rm -rf build/reports + - name: Build & Test if: ${{ matrix.config == 'debug' }} run: | @@ -147,6 +164,15 @@ jobs: name: reports-linux path: build/reports/*.xml + - name: Test Report + if: ${{ (failure() || success()) && (matrix.config == 'debug') }} + uses: ue-foundation/test-reporter@v2 + with: + name: Test Results (linux) + path: build/reports/*.xml + reporter: java-junit + list-tests: 'failed' + - name: Bundle if: ${{ matrix.config == 'release' }} run: | @@ -160,7 +186,6 @@ jobs: path: build/zenserver-linux.zip macos-build: - needs: cancel-old-build name: Build & Test MacOS runs-on: [macos, x64, zen] timeout-minutes: 25 @@ -174,7 +199,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup xmake uses: ue-foundation/github-action-setup-xmake@v1.1.1 @@ -185,6 +210,11 @@ jobs: run: | xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} --zensentry=yes + - name: Clean reports + if: ${{ matrix.config == 'debug' }} + shell: bash + run: rm -rf build/reports + - name: Build & Test if: ${{ matrix.config == 'debug' }} run: | @@ -197,9 +227,18 @@ jobs: name: reports-macos path: build/reports/*.xml + - name: Test Report + if: ${{ (failure() || success()) && (matrix.config == 'debug') }} + uses: ue-foundation/test-reporter@v2 + with: + name: Test Results (macos) + path: build/reports/*.xml + reporter: java-junit + list-tests: 'failed' + - name: Bundle if: ${{ matrix.config == 'release' }} - run: | + run: | xmake bundle -v -y --codesignidentity="Developer ID Application" - name: Upload zenserver-macos diff --git a/src/zenhttp/httpclient_test.cpp b/src/zenhttp/httpclient_test.cpp index 509b56371..91b1a3414 100644 --- a/src/zenhttp/httpclient_test.cpp +++ b/src/zenhttp/httpclient_test.cpp @@ -1079,7 +1079,7 @@ struct FaultTcpServer } }; -TEST_CASE("httpclient.transport-faults") +TEST_CASE("httpclient.transport-faults" * doctest::skip()) { SUBCASE("connection reset before response") { @@ -1217,7 +1217,7 @@ TEST_CASE("httpclient.transport-faults") } } -TEST_CASE("httpclient.transport-faults-post") +TEST_CASE("httpclient.transport-faults-post" * doctest::skip()) { constexpr size_t kPostBodySize = 256 * 1024; diff --git a/src/zenserver-test/buildstore-tests.cpp b/src/zenserver-test/buildstore-tests.cpp index ef48b2362..7cd31db06 100644 --- a/src/zenserver-test/buildstore-tests.cpp +++ b/src/zenserver-test/buildstore-tests.cpp @@ -389,7 +389,7 @@ TEST_CASE("buildstore.metadata") HttpClient::Response Result = Client.Post(fmt::format("{}/{}/{}/blobs/getBlobMetadata", Namespace, Bucket, BuildId), Payload, HttpClient::Accept(ZenContentType::kCbObject)); - CHECK(Result); + REQUIRE(Result); std::vector ResultMetadatas; @@ -570,7 +570,7 @@ TEST_CASE("buildstore.cache") { std::vector Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount; I++) { CHECK(Exists[I].HasBody); @@ -609,7 +609,7 @@ TEST_CASE("buildstore.cache") { std::vector Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount; I++) { CHECK(Exists[I].HasBody); @@ -617,7 +617,7 @@ TEST_CASE("buildstore.cache") } std::vector FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); - CHECK_EQ(BlobCount, FetchedMetadatas.size()); + REQUIRE_EQ(BlobCount, FetchedMetadatas.size()); for (size_t I = 0; I < BlobCount; I++) { @@ -638,7 +638,7 @@ TEST_CASE("buildstore.cache") { std::vector Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount * 2; I++) { CHECK(Exists[I].HasBody); @@ -649,7 +649,7 @@ TEST_CASE("buildstore.cache") CHECK_EQ(BlobCount, MetaDatas.size()); std::vector FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); - CHECK_EQ(BlobCount, FetchedMetadatas.size()); + REQUIRE_EQ(BlobCount, FetchedMetadatas.size()); for (size_t I = 0; I < BlobCount; I++) { @@ -672,7 +672,7 @@ TEST_CASE("buildstore.cache") CreateZenBuildStorageCache(Client, Stats, Namespace, Bucket, TempDir, GetTinyWorkerPool(EWorkloadType::Background))); std::vector Exists = Cache->BlobsExists(BuildId, BlobHashes); - CHECK(Exists.size() == BlobHashes.size()); + REQUIRE(Exists.size() == BlobHashes.size()); for (size_t I = 0; I < BlobCount * 2; I++) { CHECK(Exists[I].HasBody); @@ -691,7 +691,7 @@ TEST_CASE("buildstore.cache") CHECK_EQ(BlobCount, MetaDatas.size()); std::vector FetchedMetadatas = Cache->GetBlobMetadatas(BuildId, BlobHashes); - CHECK_EQ(BlobCount, FetchedMetadatas.size()); + REQUIRE_EQ(BlobCount, FetchedMetadatas.size()); for (size_t I = 0; I < BlobCount; I++) { diff --git a/src/zenserver-test/cache-tests.cpp b/src/zenserver-test/cache-tests.cpp index 0272d3797..745a89253 100644 --- a/src/zenserver-test/cache-tests.cpp +++ b/src/zenserver-test/cache-tests.cpp @@ -145,7 +145,7 @@ TEST_CASE("zcache.cbpackage") for (const zen::CbAttachment& LhsAttachment : LhsAttachments) { const zen::CbAttachment* RhsAttachment = Rhs.FindAttachment(LhsAttachment.GetHash()); - CHECK(RhsAttachment); + REQUIRE(RhsAttachment); zen::SharedBuffer LhsBuffer = LhsAttachment.AsCompressedBinary().Decompress(); CHECK(!LhsBuffer.IsNull()); @@ -1373,14 +1373,8 @@ TEST_CASE("zcache.rpc") } } -TEST_CASE("zcache.failing.upstream") +TEST_CASE("zcache.failing.upstream" * doctest::skip()) { - // This is an exploratory test that takes a long time to run, so lets skip it by default - if (true) - { - return; - } - using namespace std::literals; using namespace utils; diff --git a/src/zenserver-test/hub-tests.cpp b/src/zenserver-test/hub-tests.cpp index 42a5dcae4..bd85a5020 100644 --- a/src/zenserver-test/hub-tests.cpp +++ b/src/zenserver-test/hub-tests.cpp @@ -232,7 +232,7 @@ TEST_CASE("hub.lifecycle.children") TEST_SUITE_END(); -TEST_CASE("hub.consul.lifecycle") +TEST_CASE("hub.consul.lifecycle" * doctest::skip()) { zen::consul::ConsulProcess ConsulProc; ConsulProc.SpawnConsulAgent(); diff --git a/src/zenserver-test/projectstore-tests.cpp b/src/zenserver-test/projectstore-tests.cpp index 735aef159..487832405 100644 --- a/src/zenserver-test/projectstore-tests.cpp +++ b/src/zenserver-test/projectstore-tests.cpp @@ -71,7 +71,7 @@ TEST_CASE("project.basic") { auto Response = Http.Get("/prj/test"sv); - CHECK(Response.StatusCode == HttpResponseCode::OK); + REQUIRE(Response.StatusCode == HttpResponseCode::OK); CbObject ResponseObject = Response.AsObject(); @@ -92,7 +92,7 @@ TEST_CASE("project.basic") { auto Response = Http.Get(""sv); - CHECK(Response.StatusCode == HttpResponseCode::OK); + REQUIRE(Response.StatusCode == HttpResponseCode::OK); CbObject ResponseObject = Response.AsObject(); @@ -213,7 +213,7 @@ TEST_CASE("project.basic") auto Response = Http.Get(ChunkGetUri); REQUIRE(Response); - CHECK(Response.StatusCode == HttpResponseCode::OK); + REQUIRE(Response.StatusCode == HttpResponseCode::OK); IoBuffer Data = Response.ResponsePayload; IoBuffer ReferenceData = IoBufferBuilder::MakeFromFile(RootPath / BinPath); @@ -235,13 +235,13 @@ TEST_CASE("project.basic") auto Response = Http.Get(ChunkGetUri, {{"Accept-Type", "application/x-ue-comp"}}); REQUIRE(Response); - CHECK(Response.StatusCode == HttpResponseCode::OK); + REQUIRE(Response.StatusCode == HttpResponseCode::OK); IoBuffer Data = Response.ResponsePayload; IoHash RawHash; uint64_t RawSize; CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Data), RawHash, RawSize); - CHECK(Compressed); + REQUIRE(Compressed); IoBuffer DataDecompressed = Compressed.Decompress().AsIoBuffer(); IoBuffer ReferenceData = IoBufferBuilder::MakeFromFile(RootPath / BinPath); CHECK(RawSize == ReferenceData.GetSize()); @@ -436,13 +436,13 @@ TEST_CASE("project.remote") HttpClient Http{UrlBase}; HttpClient::Response Response = Http.Post(fmt::format("/prj/{}", ProjectName), ProjectPayload); - CHECK(Response); + REQUIRE(Response); }; auto MakeOplog = [](std::string_view UrlBase, std::string_view ProjectName, std::string_view OplogName) { HttpClient Http{UrlBase}; HttpClient::Response Response = Http.Post(fmt::format("/prj/{}/oplog/{}", ProjectName, OplogName), IoBuffer{}); - CHECK(Response); + REQUIRE(Response); }; auto MakeOp = [](std::string_view UrlBase, std::string_view ProjectName, std::string_view OplogName, const CbPackage& OpPackage) { @@ -453,7 +453,7 @@ TEST_CASE("project.remote") HttpClient Http{UrlBase}; HttpClient::Response Response = Http.Post(fmt::format("/prj/{}/oplog/{}/new", ProjectName, OplogName), Body); - CHECK(Response); + REQUIRE(Response); }; MakeProject(Servers.GetInstance(0).GetBaseUri(), "proj0"); @@ -504,7 +504,7 @@ TEST_CASE("project.remote") HttpClient::Response Response = Http.Post(fmt::format("/prj/{}/oplog/{}/rpc", Project, Oplog), Payload, {{"Accept", "application/x-ue-cbpkg"}}); - CHECK(Response); + REQUIRE(Response); CbPackage ResponsePackage = ParsePackageMessage(Response.ResponsePayload); CHECK(ResponsePackage.GetAttachments().size() == AttachmentHashes.size()); for (auto A : ResponsePackage.GetAttachments()) @@ -519,7 +519,7 @@ TEST_CASE("project.remote") HttpClient Http{Servers.GetInstance(ServerIndex).GetBaseUri()}; HttpClient::Response Response = Http.Get(fmt::format("/prj/{}/oplog/{}/entries", Project, Oplog)); - CHECK(Response); + REQUIRE(Response); IoBuffer Payload(Response.ResponsePayload); CbObject OplogResonse = LoadCompactBinaryObject(Payload); @@ -541,7 +541,7 @@ TEST_CASE("project.remote") auto HttpWaitForCompletion = [](ZenServerInstance& Server, const HttpClient::Response& Response) { REQUIRE(Response); const uint64_t JobId = ParseInt(Response.AsText()).value_or(0); - CHECK(JobId != 0); + REQUIRE(JobId != 0); HttpClient Http{Server.GetBaseUri()}; @@ -549,10 +549,10 @@ TEST_CASE("project.remote") { HttpClient::Response StatusResponse = Http.Get(fmt::format("/admin/jobs/{}", JobId), {{"Accept", ToString(ZenContentType::kCbObject)}}); - CHECK(StatusResponse); + REQUIRE(StatusResponse); CbObject ResponseObject = StatusResponse.AsObject(); std::string_view Status = ResponseObject["Status"sv].AsString(); - CHECK(Status != "Aborted"sv); + REQUIRE(Status != "Aborted"sv); if (Status == "Complete"sv) { return; @@ -887,16 +887,16 @@ TEST_CASE("project.rpcappendop") Project.AddString("project"sv, ""sv); Project.AddString("projectfile"sv, ""sv); HttpClient::Response Response = Client.Post(fmt::format("/prj/{}", ProjectName), Project.Save()); - CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); }; auto MakeOplog = [](HttpClient& Client, std::string_view ProjectName, std::string_view OplogName) { HttpClient::Response Response = Client.Post(fmt::format("/prj/{}/oplog/{}", ProjectName, OplogName)); - CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); }; auto GetOplog = [](HttpClient& Client, std::string_view ProjectName, std::string_view OplogName) { HttpClient::Response Response = Client.Get(fmt::format("/prj/{}/oplog/{}", ProjectName, OplogName)); - CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); return Response.AsObject(); }; @@ -910,7 +910,7 @@ TEST_CASE("project.rpcappendop") } Request.EndArray(); // "ops" HttpClient::Response Response = Client.Post(fmt::format("/prj/{}/oplog/{}/rpc", ProjectName, OplogName), Request.Save()); - CHECK_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); + REQUIRE_MESSAGE(Response.IsSuccess(), Response.ErrorMessage("")); CbObjectView ResponsePayload = Response.AsPackage().GetObject(); CbArrayView NeedArray = ResponsePayload["need"sv].AsArrayView(); diff --git a/src/zenserver-test/workspace-tests.cpp b/src/zenserver-test/workspace-tests.cpp index 7595d790a..aedadf0c3 100644 --- a/src/zenserver-test/workspace-tests.cpp +++ b/src/zenserver-test/workspace-tests.cpp @@ -514,9 +514,9 @@ TEST_CASE("workspaces.share") } IoBuffer BatchResponse = Client.Post(fmt::format("/ws/{}/{}/batch", WorkspaceId, ShareId), BuildChunkBatchRequest(BatchEntries)).ResponsePayload; - CHECK(BatchResponse); + REQUIRE(BatchResponse); std::vector BatchResult = ParseChunkBatchResponse(BatchResponse); - CHECK(BatchResult.size() == Files.size()); + REQUIRE(BatchResult.size() == Files.size()); for (const RequestChunkEntry& Request : BatchEntries) { IoBuffer Result = BatchResult[Request.CorrelationId]; diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 4e8475293..d8a5755c5 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -1551,7 +1551,7 @@ TEST_CASE("cachestore.size") } } -TEST_CASE("cachestore.threadedinsert") // * doctest::skip(true)) +TEST_CASE("cachestore.threadedinsert" * doctest::skip()) { // for (uint32_t i = 0; i < 100; ++i) { diff --git a/xmake.lua b/xmake.lua index d7a905981..1416fbb6a 100644 --- a/xmake.lua +++ b/xmake.lua @@ -348,7 +348,9 @@ task("test") description = "Run Zen tests", options = { {'r', "run", "kv", "all", "Run test(s) - comma-separated", " - all", " - core", " - http", " - util", " - store", " - remotestore", " - server", " - integration"}, - {'j', "junit", "k", nil, "Enable junit report output"} + {'j', "junit", "k", nil, "Enable junit report output"}, + {'n', "noskip", "k", nil, "Run skipped tests (passes --no-skip to doctest)"}, + {nil, "repeat", "kv", nil, "Repeat tests N times (stops on first failure)"} } } on_run(function() @@ -418,6 +420,8 @@ task("test") end local use_junit_reporting = option.get("junit") + local use_noskip = option.get("noskip") + local repeat_count = tonumber(option.get("repeat")) or 1 local junit_report_files = {} local junit_report_dir @@ -451,10 +455,15 @@ task("test") end -- Left-align a string within a given width (with 1-space left margin) - local function left_pad_str(s, width) + local function left_align_str(s, width) return " " .. s .. string.rep(" ", width - #s - 1) end + -- Right-align a string within a given width (with 1-space right margin) + local function right_align_str(s, width) + return string.rep(" ", width - #s - 1) .. s .. " " + end + -- Format elapsed seconds as a human-readable string local function format_time(seconds) if seconds >= 60 then @@ -498,62 +507,78 @@ task("test") os.mkdir(summary_dir) -- Run each test suite and collect results - for _, entry in ipairs(tests) do - local name, target = entry.name, entry.target - printf("=== %s ===\n", target) - - local suite_name = target - if name == "server" then - suite_name = "zenserver (test)" + for iteration = 1, repeat_count do + if repeat_count > 1 then + printf("\n*** Iteration %d/%d ***\n", iteration, repeat_count) end - local cmd = string.format("xmake run %s", target) - if name == "server" then - cmd = string.format("xmake run %s test", target) - end - cmd = string.format("%s --duration=true", cmd) + for _, entry in ipairs(tests) do + local name, target = entry.name, entry.target + printf("=== %s ===\n", target) - if use_junit_reporting then - local junit_report_file = path.join(junit_report_dir, string.format("junit-%s-%s-%s.xml", config.plat(), arch, target)) - junit_report_files[target] = junit_report_file - cmd = string.format("%s --reporters=junit --out=%s", cmd, junit_report_file) - end + local suite_name = target + if name == "server" then + suite_name = "zenserver (test)" + end - -- Tell TestListener where to write the summary - local summary_file = path.join(summary_dir, target .. ".txt") - os.setenv("ZEN_TEST_SUMMARY_FILE", summary_file) - - -- Run test with real-time streaming output - local test_ok = true - try { - function() - os.exec(cmd) - end, - catch { - function(errors) - test_ok = false - end + local cmd = string.format("xmake run %s", target) + if name == "server" then + cmd = string.format("xmake run %s test", target) + end + cmd = string.format("%s --duration=true", cmd) + + if use_junit_reporting then + local junit_report_file = path.join(junit_report_dir, string.format("junit-%s-%s-%s.xml", config.plat(), arch, target)) + junit_report_files[target] = junit_report_file + cmd = string.format("%s --reporters=junit --out=%s", cmd, junit_report_file) + end + if use_noskip then + cmd = string.format("%s --no-skip", cmd) + end + + -- Tell TestListener where to write the summary + local summary_file = path.join(summary_dir, target .. ".txt") + os.setenv("ZEN_TEST_SUMMARY_FILE", summary_file) + + -- Run test with real-time streaming output + local test_ok = true + try { + function() + os.exec(cmd) + end, + catch { + function(errors) + test_ok = false + end + } } - } - -- Read summary written by TestListener - local summary = parse_summary_file(summary_file) - os.tryrm(summary_file) + -- Read summary written by TestListener + local summary = parse_summary_file(summary_file) + os.tryrm(summary_file) + + if not test_ok then + any_failed = true + end - if not test_ok then - any_failed = true + table.insert(results, { + suite = suite_name, + cases_passed = summary and summary.cases_passed or 0, + cases_total = summary and summary.cases_total or 0, + asserts_passed = summary and summary.asserts_passed or 0, + asserts_total = summary and summary.asserts_total or 0, + elapsed_seconds = summary and summary.elapsed_seconds or 0, + failures = summary and summary.failures or {}, + passed = test_ok + }) end - table.insert(results, { - suite = suite_name, - cases_passed = summary and summary.cases_passed or 0, - cases_total = summary and summary.cases_total or 0, - asserts_passed = summary and summary.asserts_passed or 0, - asserts_total = summary and summary.asserts_total or 0, - elapsed_seconds = summary and summary.elapsed_seconds or 0, - failures = summary and summary.failures or {}, - passed = test_ok - }) + if any_failed then + if repeat_count > 1 then + printf("\n*** Failure detected on iteration %d, stopping ***\n", iteration) + end + break + end end -- Clean up @@ -580,6 +605,13 @@ task("test") local col_time = #("Time") local col_status = #("Status") + -- Compute totals + local total_cases_passed = 0 + local total_cases_total = 0 + local total_asserts_passed = 0 + local total_asserts_total = 0 + local total_elapsed = 0 + for _, r in ipairs(results) do col_suite = math.max(col_suite, #r.suite) local cases_str = format_number(r.cases_passed) .. "/" .. format_number(r.cases_total) @@ -589,8 +621,20 @@ task("test") col_time = math.max(col_time, #format_time(r.elapsed_seconds)) local status_str = r.passed and "SUCCESS" or "FAILED" col_status = math.max(col_status, #status_str) + + total_cases_passed = total_cases_passed + r.cases_passed + total_cases_total = total_cases_total + r.cases_total + total_asserts_passed = total_asserts_passed + r.asserts_passed + total_asserts_total = total_asserts_total + r.asserts_total + total_elapsed = total_elapsed + r.elapsed_seconds end + -- Account for totals row in column widths + col_suite = math.max(col_suite, #("Total")) + col_cases = math.max(col_cases, #(format_number(total_cases_passed) .. "/" .. format_number(total_cases_total))) + col_asserts = math.max(col_asserts, #(format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total))) + col_time = math.max(col_time, #format_time(total_elapsed)) + -- Add padding (1 space each side) col_suite = col_suite + 2 col_cases = col_cases + 2 @@ -621,8 +665,26 @@ task("test") local asserts_str = format_number(r.asserts_passed) .. "/" .. format_number(r.asserts_total) local time_str = format_time(r.elapsed_seconds) local status_str = r.passed and "SUCCESS" or "FAILED" - printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_pad_str(r.suite, col_suite), vbar, left_pad_str(cases_str, col_cases), vbar, left_pad_str(asserts_str, col_asserts), vbar, left_pad_str(time_str, col_time), vbar, left_pad_str(status_str, col_status), vbar) + printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str(r.suite, col_suite), vbar, right_align_str(cases_str, col_cases), vbar, right_align_str(asserts_str, col_asserts), vbar, right_align_str(time_str, col_time), vbar, right_align_str(status_str, col_status), vbar) end + + -- Totals row + if #results > 1 then + local h_suite_eq = string.rep("=", col_suite) + local h_cases_eq = string.rep("=", col_cases) + local h_asserts_eq = string.rep("=", col_asserts) + local h_time_eq = string.rep("=", col_time) + local h_status_eq = string.rep("=", col_status) + local totals_sep = "+" .. h_suite_eq .. "+" .. h_cases_eq .. "+" .. h_asserts_eq .. "+" .. h_time_eq .. "+" .. h_status_eq .. "+" + printf(" %s\n", totals_sep) + + local total_cases_str = format_number(total_cases_passed) .. "/" .. format_number(total_cases_total) + local total_asserts_str = format_number(total_asserts_passed) .. "/" .. format_number(total_asserts_total) + local total_time_str = format_time(total_elapsed) + local total_status_str = any_failed and "FAILED" or "SUCCESS" + printf(" %s%s%s%s%s%s%s%s%s%s%s\n", vbar, left_align_str("Total", col_suite), vbar, right_align_str(total_cases_str, col_cases), vbar, right_align_str(total_asserts_str, col_asserts), vbar, right_align_str(total_time_str, col_time), vbar, right_align_str(total_status_str, col_status), vbar) + end + printf(" %s\n", bottom) end -- cgit v1.2.3 From f796ee9e650d5f73844f862ed51a6de6bb33c219 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sat, 28 Feb 2026 15:36:50 +0100 Subject: subprocess tracking using Jobs on Windows/hub (#796) This change introduces job object support on Windows to be able to more accurately track and limit resource usage on storage instances created by the hub service. It also ensures that all child instances can be torn down reliably on exit. Also made it so hub tests no longer pop up console windows while running. --- src/zencore/include/zencore/process.h | 33 ++++++++++ src/zencore/process.cpp | 89 ++++++++++++++++++++++++++ src/zenserver/hub/hubservice.cpp | 49 +++++++++++++- src/zenserver/hub/hubservice.h | 7 ++ src/zenutil/include/zenutil/zenserverprocess.h | 12 +++- src/zenutil/zenserverprocess.cpp | 17 ++++- 6 files changed, 199 insertions(+), 8 deletions(-) diff --git a/src/zencore/include/zencore/process.h b/src/zencore/include/zencore/process.h index c51163a68..809312c7b 100644 --- a/src/zencore/include/zencore/process.h +++ b/src/zencore/include/zencore/process.h @@ -9,6 +9,10 @@ namespace zen { +#if ZEN_PLATFORM_WINDOWS +class JobObject; +#endif + /** Basic process abstraction */ class ProcessHandle @@ -46,6 +50,7 @@ private: /** Basic process creation */ + struct CreateProcOptions { enum @@ -63,6 +68,9 @@ struct CreateProcOptions const std::filesystem::path* WorkingDirectory = nullptr; uint32_t Flags = 0; std::filesystem::path StdoutFile; +#if ZEN_PLATFORM_WINDOWS + JobObject* AssignToJob = nullptr; // When set, the process is created suspended, assigned to the job, then resumed +#endif }; #if ZEN_PLATFORM_WINDOWS @@ -99,6 +107,31 @@ private: std::vector m_ProcessHandles; }; +#if ZEN_PLATFORM_WINDOWS +/** Windows Job Object wrapper + * + * When configured with JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, the OS will + * terminate all assigned child processes when the job handle is closed + * (including abnormal termination of the owning process). This provides + * an OS-level guarantee against orphaned child processes. + */ +class JobObject +{ +public: + JobObject(); + ~JobObject(); + JobObject(const JobObject&) = delete; + JobObject& operator=(const JobObject&) = delete; + + void Initialize(); + bool AssignProcess(void* ProcessHandle); + [[nodiscard]] bool IsValid() const; + +private: + void* m_JobHandle = nullptr; +}; +#endif // ZEN_PLATFORM_WINDOWS + bool IsProcessRunning(int pid); bool IsProcessRunning(int pid, std::error_code& OutEc); int GetCurrentProcessId(); diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 4a2668912..226a94050 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -490,6 +490,8 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma LPSECURITY_ATTRIBUTES ProcessAttributes = nullptr; LPSECURITY_ATTRIBUTES ThreadAttributes = nullptr; + const bool AssignToJob = Options.AssignToJob && Options.AssignToJob->IsValid(); + DWORD CreationFlags = 0; if (Options.Flags & CreateProcOptions::Flag_NewConsole) { @@ -503,6 +505,10 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma { CreationFlags |= CREATE_NEW_PROCESS_GROUP; } + if (AssignToJob) + { + CreationFlags |= CREATE_SUSPENDED; + } const wchar_t* WorkingDir = nullptr; if (Options.WorkingDirectory != nullptr) @@ -571,6 +577,15 @@ CreateProcNormal(const std::filesystem::path& Executable, std::string_view Comma return nullptr; } + if (AssignToJob) + { + if (!Options.AssignToJob->AssignProcess(ProcessInfo.hProcess)) + { + ZEN_WARN("Failed to assign newly created process to job object"); + } + ResumeThread(ProcessInfo.hThread); + } + CloseHandle(ProcessInfo.hThread); return ProcessInfo.hProcess; } @@ -644,6 +659,8 @@ CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view C }; PROCESS_INFORMATION ProcessInfo = {}; + const bool AssignToJob = Options.AssignToJob && Options.AssignToJob->IsValid(); + if (Options.Flags & CreateProcOptions::Flag_NewConsole) { CreateProcFlags |= CREATE_NEW_CONSOLE; @@ -652,6 +669,10 @@ CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view C { CreateProcFlags |= CREATE_NO_WINDOW; } + if (AssignToJob) + { + CreateProcFlags |= CREATE_SUSPENDED; + } ExtendableWideStringBuilder<256> CommandLineZ; CommandLineZ << CommandLine; @@ -679,6 +700,15 @@ CreateProcUnelevated(const std::filesystem::path& Executable, std::string_view C return nullptr; } + if (AssignToJob) + { + if (!Options.AssignToJob->AssignProcess(ProcessInfo.hProcess)) + { + ZEN_WARN("Failed to assign newly created process to job object"); + } + ResumeThread(ProcessInfo.hThread); + } + CloseHandle(ProcessInfo.hThread); return ProcessInfo.hProcess; } @@ -845,6 +875,65 @@ ProcessMonitor::IsActive() const ////////////////////////////////////////////////////////////////////////// +#if ZEN_PLATFORM_WINDOWS +JobObject::JobObject() = default; + +JobObject::~JobObject() +{ + if (m_JobHandle) + { + CloseHandle(m_JobHandle); + m_JobHandle = nullptr; + } +} + +void +JobObject::Initialize() +{ + ZEN_ASSERT(m_JobHandle == nullptr, "JobObject already initialized"); + + m_JobHandle = CreateJobObjectW(nullptr, nullptr); + if (!m_JobHandle) + { + ZEN_WARN("Failed to create job object: {}", zen::GetLastError()); + return; + } + + JOBOBJECT_EXTENDED_LIMIT_INFORMATION LimitInfo = {}; + LimitInfo.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + + if (!SetInformationJobObject(m_JobHandle, JobObjectExtendedLimitInformation, &LimitInfo, sizeof(LimitInfo))) + { + ZEN_WARN("Failed to set job object limits: {}", zen::GetLastError()); + CloseHandle(m_JobHandle); + m_JobHandle = nullptr; + } +} + +bool +JobObject::AssignProcess(void* ProcessHandle) +{ + ZEN_ASSERT(m_JobHandle != nullptr, "JobObject not initialized"); + ZEN_ASSERT(ProcessHandle != nullptr, "ProcessHandle is null"); + + if (!AssignProcessToJobObject(m_JobHandle, ProcessHandle)) + { + ZEN_WARN("Failed to assign process to job object: {}", zen::GetLastError()); + return false; + } + + return true; +} + +bool +JobObject::IsValid() const +{ + return m_JobHandle != nullptr; +} +#endif // ZEN_PLATFORM_WINDOWS + +////////////////////////////////////////////////////////////////////////// + bool IsProcessRunning(int pid, std::error_code& OutEc) { diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp index a00446a75..bf0e294c5 100644 --- a/src/zenserver/hub/hubservice.cpp +++ b/src/zenserver/hub/hubservice.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -150,6 +151,10 @@ struct StorageServerInstance inline uint16_t GetBasePort() const { return m_ServerInstance.GetBasePort(); } +#if ZEN_PLATFORM_WINDOWS + void SetJobObject(JobObject* InJobObject) { m_JobObject = InJobObject; } +#endif + private: void WakeLocked(); RwLock m_Lock; @@ -161,6 +166,9 @@ private: std::filesystem::path m_TempDir; std::filesystem::path m_HydrationPath; ResourceMetrics m_ResourceMetrics; +#if ZEN_PLATFORM_WINDOWS + JobObject* m_JobObject = nullptr; +#endif void SpawnServerProcess(); @@ -191,6 +199,9 @@ StorageServerInstance::SpawnServerProcess() m_ServerInstance.SetServerExecutablePath(GetRunningExecutablePath()); m_ServerInstance.SetDataDir(m_BaseDir); +#if ZEN_PLATFORM_WINDOWS + m_ServerInstance.SetJobObject(m_JobObject); +#endif const uint16_t BasePort = m_ServerInstance.SpawnServerAndWaitUntilReady(); ZEN_DEBUG("Storage server instance for module '{}' started, listening on port {}", m_ModuleId, BasePort); @@ -380,6 +391,21 @@ struct HttpHubService::Impl // flexibility, and to allow running multiple hubs on the same host if // necessary. m_RunEnvironment.SetNextPortNumber(21000); + +#if ZEN_PLATFORM_WINDOWS + if (m_UseJobObject) + { + m_JobObject.Initialize(); + if (m_JobObject.IsValid()) + { + ZEN_INFO("Job object initialized for hub service child process management"); + } + else + { + ZEN_WARN("Failed to initialize job object; child processes will not be auto-terminated on hub crash"); + } + } +#endif } void Cleanup() @@ -422,6 +448,12 @@ struct HttpHubService::Impl IsNewInstance = true; auto NewInstance = std::make_unique(m_RunEnvironment, ModuleId, m_FileHydrationPath, m_HydrationTempPath); +#if ZEN_PLATFORM_WINDOWS + if (m_JobObject.IsValid()) + { + NewInstance->SetJobObject(&m_JobObject); + } +#endif Instance = NewInstance.get(); m_Instances.emplace(std::string(ModuleId), std::move(NewInstance)); @@ -579,10 +611,15 @@ struct HttpHubService::Impl inline int GetInstanceLimit() { return m_InstanceLimit; } inline int GetMaxInstanceCount() { return m_MaxInstanceCount; } + bool m_UseJobObject = true; + private: - ZenServerEnvironment m_RunEnvironment; - std::filesystem::path m_FileHydrationPath; - std::filesystem::path m_HydrationTempPath; + ZenServerEnvironment m_RunEnvironment; + std::filesystem::path m_FileHydrationPath; + std::filesystem::path m_HydrationTempPath; +#if ZEN_PLATFORM_WINDOWS + JobObject m_JobObject; +#endif RwLock m_Lock; std::unordered_map> m_Instances; std::unordered_set m_DeprovisioningModules; @@ -817,6 +854,12 @@ HttpHubService::~HttpHubService() { } +void +HttpHubService::SetUseJobObject(bool Enable) +{ + m_Impl->m_UseJobObject = Enable; +} + const char* HttpHubService::BaseUri() const { diff --git a/src/zenserver/hub/hubservice.h b/src/zenserver/hub/hubservice.h index 1a5a8c57c..ef24bba69 100644 --- a/src/zenserver/hub/hubservice.h +++ b/src/zenserver/hub/hubservice.h @@ -28,6 +28,13 @@ public: void SetNotificationEndpoint(std::string_view UpstreamNotificationEndpoint, std::string_view InstanceId); + /** Enable or disable the use of a Windows Job Object for child process management. + * When enabled, all spawned child processes are assigned to a job object with + * JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, ensuring children are terminated if the hub + * crashes or is force-killed. Must be called before Initialize(). No-op on non-Windows. + */ + void SetUseJobObject(bool Enable); + private: HttpRequestRouter m_Router; diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h index b781a03a9..954916fe2 100644 --- a/src/zenutil/include/zenutil/zenserverprocess.h +++ b/src/zenutil/include/zenutil/zenserverprocess.h @@ -97,9 +97,12 @@ struct ZenServerInstance inline int GetPid() const { return m_Process.Pid(); } inline void SetOwnerPid(int Pid) { m_OwnerPid = Pid; } void* GetProcessHandle() const { return m_Process.Handle(); } - bool IsRunning(); - bool Terminate(); - std::string GetLogOutput() const; +#if ZEN_PLATFORM_WINDOWS + void SetJobObject(JobObject* Job) { m_JobObject = Job; } +#endif + bool IsRunning(); + bool Terminate(); + std::string GetLogOutput() const; inline ServerMode GetServerMode() const { return m_ServerMode; } @@ -148,6 +151,9 @@ private: std::string m_Name; std::filesystem::path m_OutputCapturePath; std::filesystem::path m_ServerExecutablePath; +#if ZEN_PLATFORM_WINDOWS + JobObject* m_JobObject = nullptr; +#endif void CreateShutdownEvent(int BasePort); void SpawnServer(int BasePort, std::string_view AdditionalServerArgs, int WaitTimeoutMs); diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index 579ba450a..0f8ab223d 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -831,8 +831,15 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, m_ServerExecutablePath.empty() ? (BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL) : m_ServerExecutablePath; const std::filesystem::path OutputPath = OpenConsole ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); - CreateProcOptions CreateOptions = {.WorkingDirectory = &CurrentDirectory, .Flags = CreationFlags, .StdoutFile = OutputPath}; - CreateProcResult ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions); + CreateProcOptions CreateOptions = { + .WorkingDirectory = &CurrentDirectory, + .Flags = CreationFlags, + .StdoutFile = OutputPath, +#if ZEN_PLATFORM_WINDOWS + .AssignToJob = m_JobObject, +#endif + }; + CreateProcResult ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions); #if ZEN_PLATFORM_WINDOWS if (!ChildPid) { @@ -841,6 +848,12 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, { ZEN_DEBUG("Regular spawn failed - spawning elevated server"); CreateOptions.Flags |= CreateProcOptions::Flag_Elevated; + // ShellExecuteEx (used by the elevated path) does not support job object assignment + if (CreateOptions.AssignToJob) + { + ZEN_WARN("Elevated process spawn does not support job object assignment; child will not be auto-terminated on parent exit"); + CreateOptions.AssignToJob = nullptr; + } ChildPid = CreateProc(Executable, CommandLine.ToView(), CreateOptions); } else -- cgit v1.2.3 From 4d01aaee0a45f4c9f96e8a4925eff696be98de8d Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Sun, 1 Mar 2026 12:40:20 +0100 Subject: added `--verbose` option to zenserver-test and `xmake test` (#798) * when `--verbose` is specified to zenserver-test, all child process output (typically, zenserver instances) is piped through to stdout. you can also pass `--verbose` to `xmake test` to accomplish the same thing. * this PR also consolidates all test runner `main` function logic (such as from zencore-test, zenhttp-test etc) into central implementation in zencore for consistency and ease of maintenance * also added extended utf8-tests including a fix to `Utf8ToWide()` --- .github/workflows/create_release.yml | 6 +- .github/workflows/validate.yml | 6 +- src/zen/zen.cpp | 1 - src/zencompute-test/zencompute-test.cpp | 22 +--- src/zencore-test/zencore-test.cpp | 36 +------ src/zencore/include/zencore/testing.h | 2 + src/zencore/include/zencore/testutils.h | 27 +++++ src/zencore/string.cpp | 131 ++++++++++++++++++++++-- src/zencore/testing.cpp | 37 ++++++- src/zenhttp-test/zenhttp-test.cpp | 35 +------ src/zennet-test/zennet-test.cpp | 34 +----- src/zenremotestore-test/zenremotestore-test.cpp | 35 +------ src/zenserver-test/zenserver-test.cpp | 11 +- src/zenserver/main.cpp | 1 - src/zenstore-test/zenstore-test.cpp | 34 +----- src/zentelemetry-test/zentelemetry-test.cpp | 34 +----- src/zentest-appstub/zentest-appstub.cpp | 1 - src/zenutil-test/zenutil-test.cpp | 34 +----- src/zenutil/include/zenutil/zenserverprocess.h | 10 +- src/zenutil/zenserverprocess.cpp | 13 +-- xmake.lua | 7 +- 21 files changed, 240 insertions(+), 277 deletions(-) diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 203588d24..163c0cf85 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup xmake - uses: ue-foundation/github-action-setup-xmake@v1.1.1 + uses: ue-foundation/github-action-setup-xmake@v1.2.2 with: xmake-version: ${{env.XMAKE_VERSION}} @@ -54,7 +54,7 @@ jobs: ./scripts/ue_build_linux/get_ue_toolchain.sh ./.tmp-ue-toolchain - name: Setup xmake - uses: ue-foundation/github-action-setup-xmake@v1.1.1 + uses: ue-foundation/github-action-setup-xmake@v1.2.2 with: xmake-version: ${{env.XMAKE_VERSION}} @@ -92,7 +92,7 @@ jobs: - uses: actions/checkout@v4 - name: Setup xmake - uses: ue-foundation/github-action-setup-xmake@v1.1.1 + uses: ue-foundation/github-action-setup-xmake@v1.2.2 with: xmake-version: ${{env.XMAKE_VERSION}} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index bf7b1c853..32b75fe5e 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -72,7 +72,7 @@ jobs: run: git config --global --add safe.directory "${{ github.workspace }}" - name: Setup xmake - uses: ue-foundation/github-action-setup-xmake@v1.1.1 + uses: ue-foundation/github-action-setup-xmake@v1.2.2 with: xmake-version: ${{env.XMAKE_VERSION}} @@ -139,7 +139,7 @@ jobs: ./scripts/ue_build_linux/get_ue_toolchain.sh ./.tmp-ue-toolchain - name: Setup xmake - uses: ue-foundation/github-action-setup-xmake@v1.1.1 + uses: ue-foundation/github-action-setup-xmake@v1.2.2 with: xmake-version: ${{env.XMAKE_VERSION}} @@ -202,7 +202,7 @@ jobs: uses: actions/checkout@v4 - name: Setup xmake - uses: ue-foundation/github-action-setup-xmake@v1.1.1 + uses: ue-foundation/github-action-setup-xmake@v1.2.2 with: xmake-version: ${{env.XMAKE_VERSION}} diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index dc37cb56b..ba8a76bc3 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -56,7 +56,6 @@ #include "progressbar.h" #if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 # include #endif diff --git a/src/zencompute-test/zencompute-test.cpp b/src/zencompute-test/zencompute-test.cpp index 237812e12..60aaeab1d 100644 --- a/src/zencompute-test/zencompute-test.cpp +++ b/src/zencompute-test/zencompute-test.cpp @@ -1,31 +1,15 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include -#include -#include -#include +#include -#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC -# include -# include -# include -#endif - -#if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 -# include -#endif +#include int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { #if ZEN_WITH_TESTS - zen::zencompute_forcelinktests(); - - zen::logging::InitializeLogging(); - zen::MaximizeOpenFileCount(); - - return ZEN_RUN_TESTS(argc, argv); + return zen::testing::RunTestMain(argc, argv, "zencompute-test", zen::zencompute_forcelinktests); #else return 0; #endif diff --git a/src/zencore-test/zencore-test.cpp b/src/zencore-test/zencore-test.cpp index 68fc940ee..3d9a79283 100644 --- a/src/zencore-test/zencore-test.cpp +++ b/src/zencore-test/zencore-test.cpp @@ -1,47 +1,15 @@ // Copyright Epic Games, Inc. All Rights Reserved. -// zencore-test.cpp : Defines the entry point for the console application. -// - -#include -#include -#include +#include #include #include -#if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 -# include -# include -#endif - int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { -#if ZEN_PLATFORM_WINDOWS - setlocale(LC_ALL, "en_us.UTF8"); -#endif // ZEN_PLATFORM_WINDOWS - #if ZEN_WITH_TESTS - zen::zencore_forcelinktests(); - -# if ZEN_PLATFORM_LINUX - zen::IgnoreChildSignals(); -# endif - -# if ZEN_WITH_TRACE - zen::TraceInit("zencore-test"); - zen::TraceOptions TraceCommandlineOptions; - if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) - { - TraceConfigure(TraceCommandlineOptions); - } -# endif // ZEN_WITH_TRACE - zen::logging::InitializeLogging(); - zen::MaximizeOpenFileCount(); - - return ZEN_RUN_TESTS(argc, argv); + return zen::testing::RunTestMain(argc, argv, "zencore-test", zen::zencore_forcelinktests); #else return 0; #endif diff --git a/src/zencore/include/zencore/testing.h b/src/zencore/include/zencore/testing.h index a00ee3166..43bdbbffe 100644 --- a/src/zencore/include/zencore/testing.h +++ b/src/zencore/include/zencore/testing.h @@ -59,6 +59,8 @@ private: return Runner.Run(); \ }() +int RunTestMain(int argc, char* argv[], const char* traceName, void (*forceLink)()); + } // namespace zen::testing #endif diff --git a/src/zencore/include/zencore/testutils.h b/src/zencore/include/zencore/testutils.h index e2a4f8346..2a789d18f 100644 --- a/src/zencore/include/zencore/testutils.h +++ b/src/zencore/include/zencore/testutils.h @@ -59,6 +59,33 @@ struct TrueType static const bool Enabled = true; }; +namespace utf8test { + + // 2-byte UTF-8 (Latin extended) + static constexpr const char kLatin[] = u8"café_résumé"; + static constexpr const wchar_t kLatinW[] = L"café_résumé"; + + // 2-byte UTF-8 (Cyrillic) + static constexpr const char kCyrillic[] = u8"данные"; + static constexpr const wchar_t kCyrillicW[] = L"данные"; + + // 3-byte UTF-8 (CJK) + static constexpr const char kCJK[] = u8"日本語"; + static constexpr const wchar_t kCJKW[] = L"日本語"; + + // Mixed scripts + static constexpr const char kMixed[] = u8"zen_éд日"; + static constexpr const wchar_t kMixedW[] = L"zen_éд日"; + + // 4-byte UTF-8 (supplementary plane) — string tests only, NOT filesystem + static constexpr const char kEmoji[] = u8"📦"; + static constexpr const wchar_t kEmojiW[] = L"📦"; + + // BMP-only test strings suitable for filesystem use + static constexpr const char* kFilenameSafe[] = {kLatin, kCyrillic, kCJK, kMixed}; + +} // namespace utf8test + } // namespace zen #endif // ZEN_WITH_TESTS diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index a9aed6309..ab1c7de58 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -184,7 +185,21 @@ Utf8ToWide(const std::u8string_view& Str8, WideStringBuilderBase& OutString) if (!ByteCount) { +#if ZEN_SIZEOF_WCHAR_T == 2 + if (CurrentOutChar > 0xFFFF) + { + // Supplementary plane: emit a UTF-16 surrogate pair + uint32_t Adjusted = uint32_t(CurrentOutChar - 0x10000); + OutString.Append(wchar_t(0xD800 + (Adjusted >> 10))); + OutString.Append(wchar_t(0xDC00 + (Adjusted & 0x3FF))); + } + else + { + OutString.Append(wchar_t(CurrentOutChar)); + } +#else OutString.Append(wchar_t(CurrentOutChar)); +#endif CurrentOutChar = 0; } } @@ -967,33 +982,131 @@ TEST_CASE("ExtendableWideStringBuilder") TEST_CASE("utf8") { + using namespace utf8test; + SUBCASE("utf8towide") { - // TODO: add more extensive testing here - this covers a very small space - WideStringBuilder<32> wout; Utf8ToWide(u8"abcdefghi", wout); CHECK(StringEquals(L"abcdefghi", wout.c_str())); wout.Reset(); + Utf8ToWide(u8"abc\xC3\xA4\xC3\xB6\xC3\xBC", wout); + CHECK(StringEquals(L"abc\u00E4\u00F6\u00FC", wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kLatin), wout); + CHECK(StringEquals(kLatinW, wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kCyrillic), wout); + CHECK(StringEquals(kCyrillicW, wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kCJK), wout); + CHECK(StringEquals(kCJKW, wout.c_str())); + + wout.Reset(); + Utf8ToWide(std::string_view(kMixed), wout); + CHECK(StringEquals(kMixedW, wout.c_str())); - Utf8ToWide(u8"abc���", wout); - CHECK(StringEquals(L"abc���", wout.c_str())); + wout.Reset(); + Utf8ToWide(std::string_view(kEmoji), wout); + CHECK(StringEquals(kEmojiW, wout.c_str())); } SUBCASE("widetoutf8") { - // TODO: add more extensive testing here - this covers a very small space - - StringBuilder<32> out; + StringBuilder<64> out; WideToUtf8(L"abcdefghi", out); CHECK(StringEquals("abcdefghi", out.c_str())); out.Reset(); + WideToUtf8(kLatinW, out); + CHECK(StringEquals(kLatin, out.c_str())); - WideToUtf8(L"abc���", out); - CHECK(StringEquals(u8"abc���", out.c_str())); + out.Reset(); + WideToUtf8(kCyrillicW, out); + CHECK(StringEquals(kCyrillic, out.c_str())); + + out.Reset(); + WideToUtf8(kCJKW, out); + CHECK(StringEquals(kCJK, out.c_str())); + + out.Reset(); + WideToUtf8(kMixedW, out); + CHECK(StringEquals(kMixed, out.c_str())); + + out.Reset(); + WideToUtf8(kEmojiW, out); + CHECK(StringEquals(kEmoji, out.c_str())); + } + + SUBCASE("roundtrip") + { + // UTF-8 -> Wide -> UTF-8 identity + const char* Utf8Strings[] = {kLatin, kCyrillic, kCJK, kMixed, kEmoji}; + for (const char* Utf8Str : Utf8Strings) + { + ExtendableWideStringBuilder<64> Wide; + Utf8ToWide(std::string_view(Utf8Str), Wide); + + ExtendableStringBuilder<64> Back; + WideToUtf8(std::wstring_view(Wide.c_str()), Back); + CHECK(StringEquals(Utf8Str, Back.c_str())); + } + + // Wide -> UTF-8 -> Wide identity + const wchar_t* WideStrings[] = {kLatinW, kCyrillicW, kCJKW, kMixedW, kEmojiW}; + for (const wchar_t* WideStr : WideStrings) + { + ExtendableStringBuilder<64> Utf8; + WideToUtf8(std::wstring_view(WideStr), Utf8); + + ExtendableWideStringBuilder<64> Back; + Utf8ToWide(std::string_view(Utf8.c_str()), Back); + CHECK(StringEquals(WideStr, Back.c_str())); + } + + // Empty string round-trip + { + ExtendableWideStringBuilder<8> Wide; + Utf8ToWide(std::string_view(""), Wide); + CHECK(Wide.Size() == 0); + + ExtendableStringBuilder<8> Narrow; + WideToUtf8(std::wstring_view(L""), Narrow); + CHECK(Narrow.Size() == 0); + } + } + + SUBCASE("IsValidUtf8") + { + // Valid inputs + CHECK(IsValidUtf8("")); + CHECK(IsValidUtf8("hello world")); + CHECK(IsValidUtf8(kLatin)); + CHECK(IsValidUtf8(kCyrillic)); + CHECK(IsValidUtf8(kCJK)); + CHECK(IsValidUtf8(kMixed)); + CHECK(IsValidUtf8(kEmoji)); + + // Invalid: truncated 2-byte sequence + CHECK(!IsValidUtf8(std::string_view("\xC3", 1))); + + // Invalid: truncated 3-byte sequence + CHECK(!IsValidUtf8(std::string_view("\xE6\x97", 2))); + + // Invalid: truncated 4-byte sequence + CHECK(!IsValidUtf8(std::string_view("\xF0\x9F\x93", 3))); + + // Invalid: bad start byte + CHECK(!IsValidUtf8(std::string_view("\xFF", 1))); + CHECK(!IsValidUtf8(std::string_view("\xFE", 1))); + + // Invalid: overlong encoding of '/' (U+002F) + CHECK(!IsValidUtf8(std::string_view("\xC0\xAF", 2))); } } diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index ef8fb0480..6000bd95c 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -1,18 +1,23 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#define ZEN_TEST_WITH_RUNNER 1 + #include "zencore/testing.h" + +#include "zencore/filesystem.h" #include "zencore/logging.h" +#include "zencore/process.h" +#include "zencore/trace.h" #if ZEN_WITH_TESTS # include +# include # include # include # include # include -# include - namespace zen::testing { using namespace std::literals; @@ -149,6 +154,34 @@ TestRunner::Run() return m_Impl->Session.run(); } +int +RunTestMain(int argc, char* argv[], [[maybe_unused]] const char* traceName, void (*forceLink)()) +{ +# if ZEN_PLATFORM_WINDOWS + setlocale(LC_ALL, "en_us.UTF8"); +# endif + + forceLink(); + +# if ZEN_PLATFORM_LINUX + zen::IgnoreChildSignals(); +# endif + +# if ZEN_WITH_TRACE + zen::TraceInit(traceName); + zen::TraceOptions TraceCommandlineOptions; + if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) + { + TraceConfigure(TraceCommandlineOptions); + } +# endif + + zen::logging::InitializeLogging(); + zen::MaximizeOpenFileCount(); + + return ZEN_RUN_TESTS(argc, argv); +} + } // namespace zen::testing #endif // ZEN_WITH_TESTS diff --git a/src/zenhttp-test/zenhttp-test.cpp b/src/zenhttp-test/zenhttp-test.cpp index c18759beb..b4b406ac8 100644 --- a/src/zenhttp-test/zenhttp-test.cpp +++ b/src/zenhttp-test/zenhttp-test.cpp @@ -1,44 +1,15 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include -#include -#include -#include +#include #include -#if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 -# include -# include -#endif +#include int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { -#if ZEN_PLATFORM_WINDOWS - setlocale(LC_ALL, "en_us.UTF8"); -#endif // ZEN_PLATFORM_WINDOWS - #if ZEN_WITH_TESTS - zen::zenhttp_forcelinktests(); - -# if ZEN_PLATFORM_LINUX - zen::IgnoreChildSignals(); -# endif - -# if ZEN_WITH_TRACE - zen::TraceInit("zenhttp-test"); - zen::TraceOptions TraceCommandlineOptions; - if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) - { - TraceConfigure(TraceCommandlineOptions); - } -# endif // ZEN_WITH_TRACE - - zen::logging::InitializeLogging(); - zen::MaximizeOpenFileCount(); - - return ZEN_RUN_TESTS(argc, argv); + return zen::testing::RunTestMain(argc, argv, "zenhttp-test", zen::zenhttp_forcelinktests); #else return 0; #endif diff --git a/src/zennet-test/zennet-test.cpp b/src/zennet-test/zennet-test.cpp index bc3b8e8e9..1283eb820 100644 --- a/src/zennet-test/zennet-test.cpp +++ b/src/zennet-test/zennet-test.cpp @@ -1,45 +1,15 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include -#include -#include +#include #include #include -#if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 -# include -# include -#endif - int main([[maybe_unused]] int argc, [[maybe_unused]] char** argv) { -#if ZEN_PLATFORM_WINDOWS - setlocale(LC_ALL, "en_us.UTF8"); -#endif // ZEN_PLATFORM_WINDOWS - #if ZEN_WITH_TESTS - zen::zennet_forcelinktests(); - -# if ZEN_PLATFORM_LINUX - zen::IgnoreChildSignals(); -# endif - -# if ZEN_WITH_TRACE - zen::TraceInit("zennet-test"); - zen::TraceOptions TraceCommandlineOptions; - if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) - { - TraceConfigure(TraceCommandlineOptions); - } -# endif // ZEN_WITH_TRACE - - zen::logging::InitializeLogging(); - zen::MaximizeOpenFileCount(); - - return ZEN_RUN_TESTS(argc, argv); + return zen::testing::RunTestMain(argc, argv, "zennet-test", zen::zennet_forcelinktests); #else return 0; #endif diff --git a/src/zenremotestore-test/zenremotestore-test.cpp b/src/zenremotestore-test/zenremotestore-test.cpp index 5db185041..dc47c5aed 100644 --- a/src/zenremotestore-test/zenremotestore-test.cpp +++ b/src/zenremotestore-test/zenremotestore-test.cpp @@ -1,46 +1,15 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include -#include -#include -#include +#include #include #include -#if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 -# include -# include -#endif - int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { -#if ZEN_PLATFORM_WINDOWS - setlocale(LC_ALL, "en_us.UTF8"); -#endif // ZEN_PLATFORM_WINDOWS - #if ZEN_WITH_TESTS - zen::zenremotestore_forcelinktests(); - -# if ZEN_PLATFORM_LINUX - zen::IgnoreChildSignals(); -# endif - -# if ZEN_WITH_TRACE - zen::TraceInit("zenstore-test"); - zen::TraceOptions TraceCommandlineOptions; - if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) - { - TraceConfigure(TraceCommandlineOptions); - } -# endif // ZEN_WITH_TRACE - - zen::logging::InitializeLogging(); - zen::MaximizeOpenFileCount(); - - return ZEN_RUN_TESTS(argc, argv); + return zen::testing::RunTestMain(argc, argv, "zenremotestore-test", zen::zenremotestore_forcelinktests); #else return 0; #endif diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index 4120dec1a..c7ce633d3 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -4,7 +4,6 @@ #if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 # include "zenserver-test.h" # include @@ -97,6 +96,7 @@ main(int argc, char** argv) // somehow in the future std::string ServerClass; + bool Verbose = false; for (int i = 1; i < argc; ++i) { @@ -107,10 +107,19 @@ main(int argc, char** argv) ServerClass = argv[++i]; } } + else if (argv[i] == "--verbose"sv) + { + Verbose = true; + } } zen::tests::TestEnv.InitializeForTest(ProgramBaseDir, TestBaseDir, ServerClass); + if (Verbose) + { + zen::tests::TestEnv.SetPassthroughOutput(true); + } + ZEN_INFO("Running tests...(base dir: '{}')", TestBaseDir); zen::testing::TestRunner Runner; diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index 571dd3b4f..c764cbde6 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -41,7 +41,6 @@ // in some shared code into the executable #if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 # include #endif diff --git a/src/zenstore-test/zenstore-test.cpp b/src/zenstore-test/zenstore-test.cpp index c055dbb64..875373a9d 100644 --- a/src/zenstore-test/zenstore-test.cpp +++ b/src/zenstore-test/zenstore-test.cpp @@ -1,45 +1,15 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include -#include -#include +#include #include #include -#if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 -# include -# include -#endif - int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { -#if ZEN_PLATFORM_WINDOWS - setlocale(LC_ALL, "en_us.UTF8"); -#endif // ZEN_PLATFORM_WINDOWS - #if ZEN_WITH_TESTS - zen::zenstore_forcelinktests(); - -# if ZEN_PLATFORM_LINUX - zen::IgnoreChildSignals(); -# endif - -# if ZEN_WITH_TRACE - zen::TraceInit("zenstore-test"); - zen::TraceOptions TraceCommandlineOptions; - if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) - { - TraceConfigure(TraceCommandlineOptions); - } -# endif // ZEN_WITH_TRACE - - zen::logging::InitializeLogging(); - zen::MaximizeOpenFileCount(); - - return ZEN_RUN_TESTS(argc, argv); + return zen::testing::RunTestMain(argc, argv, "zenstore-test", zen::zenstore_forcelinktests); #else return 0; #endif diff --git a/src/zentelemetry-test/zentelemetry-test.cpp b/src/zentelemetry-test/zentelemetry-test.cpp index 83fd549db..5a2ac74de 100644 --- a/src/zentelemetry-test/zentelemetry-test.cpp +++ b/src/zentelemetry-test/zentelemetry-test.cpp @@ -1,45 +1,15 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include -#include -#include +#include #include #include -#if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 -# include -# include -#endif - int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { -#if ZEN_PLATFORM_WINDOWS - setlocale(LC_ALL, "en_us.UTF8"); -#endif // ZEN_PLATFORM_WINDOWS - #if ZEN_WITH_TESTS - zen::zentelemetry_forcelinktests(); - -# if ZEN_PLATFORM_LINUX - zen::IgnoreChildSignals(); -# endif - -# if ZEN_WITH_TRACE - zen::TraceInit("zenstore-test"); - zen::TraceOptions TraceCommandlineOptions; - if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) - { - TraceConfigure(TraceCommandlineOptions); - } -# endif // ZEN_WITH_TRACE - - zen::logging::InitializeLogging(); - zen::MaximizeOpenFileCount(); - - return ZEN_RUN_TESTS(argc, argv); + return zen::testing::RunTestMain(argc, argv, "zentelemetry-test", zen::zentelemetry_forcelinktests); #else return 0; #endif diff --git a/src/zentest-appstub/zentest-appstub.cpp b/src/zentest-appstub/zentest-appstub.cpp index 926580d96..67fbef532 100644 --- a/src/zentest-appstub/zentest-appstub.cpp +++ b/src/zentest-appstub/zentest-appstub.cpp @@ -9,7 +9,6 @@ #include #if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 # include #endif diff --git a/src/zenutil-test/zenutil-test.cpp b/src/zenutil-test/zenutil-test.cpp index f5cfd5a72..e2b6ac9bd 100644 --- a/src/zenutil-test/zenutil-test.cpp +++ b/src/zenutil-test/zenutil-test.cpp @@ -1,45 +1,15 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include -#include -#include +#include #include #include -#if ZEN_WITH_TESTS -# define ZEN_TEST_WITH_RUNNER 1 -# include -# include -#endif - int main([[maybe_unused]] int argc, [[maybe_unused]] char* argv[]) { -#if ZEN_PLATFORM_WINDOWS - setlocale(LC_ALL, "en_us.UTF8"); -#endif // ZEN_PLATFORM_WINDOWS - #if ZEN_WITH_TESTS - zen::zenutil_forcelinktests(); - -# if ZEN_PLATFORM_LINUX - zen::IgnoreChildSignals(); -# endif - -# if ZEN_WITH_TRACE - zen::TraceInit("zenutil-test"); - zen::TraceOptions TraceCommandlineOptions; - if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) - { - TraceConfigure(TraceCommandlineOptions); - } -# endif // ZEN_WITH_TRACE - - zen::logging::InitializeLogging(); - zen::MaximizeOpenFileCount(); - - return ZEN_RUN_TESTS(argc, argv); + return zen::testing::RunTestMain(argc, argv, "zenutil-test", zen::zenutil_forcelinktests); #else return 0; #endif diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h index 954916fe2..e81b154e8 100644 --- a/src/zenutil/include/zenutil/zenserverprocess.h +++ b/src/zenutil/include/zenutil/zenserverprocess.h @@ -46,6 +46,9 @@ public: inline std::string_view GetServerClass() const { return m_ServerClass; } inline uint16_t GetNewPortNumber() { return m_NextPortNumber.fetch_add(1); } + void SetPassthroughOutput(bool Enable) { m_PassthroughOutput = Enable; } + bool IsPassthroughOutput() const { return m_PassthroughOutput; } + // The defaults will work for a single root process only. For hierarchical // setups (e.g., hub managing storage servers), we need to be able to // allocate distinct child IDs and ports to avoid overlap/conflicts. @@ -55,9 +58,10 @@ public: private: std::filesystem::path m_ProgramBaseDir; std::filesystem::path m_ChildProcessBaseDir; - bool m_IsInitialized = false; - bool m_IsTestInstance = false; - bool m_IsHubInstance = false; + bool m_IsInitialized = false; + bool m_IsTestInstance = false; + bool m_IsHubInstance = false; + bool m_PassthroughOutput = false; std::string m_ServerClass; std::atomic_uint16_t m_NextPortNumber{20000}; }; diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index 0f8ab223d..e127a92d7 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -829,12 +829,13 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, const std::filesystem::path BaseDir = m_Env.ProgramBaseDir(); const std::filesystem::path Executable = m_ServerExecutablePath.empty() ? (BaseDir / "zenserver" ZEN_EXE_SUFFIX_LITERAL) : m_ServerExecutablePath; - const std::filesystem::path OutputPath = - OpenConsole ? std::filesystem::path{} : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); - CreateProcOptions CreateOptions = { - .WorkingDirectory = &CurrentDirectory, - .Flags = CreationFlags, - .StdoutFile = OutputPath, + const std::filesystem::path OutputPath = (OpenConsole || m_Env.IsPassthroughOutput()) + ? std::filesystem::path{} + : std::filesystem::temp_directory_path() / ("zenserver_" + m_Name + ".log"); + CreateProcOptions CreateOptions = { + .WorkingDirectory = &CurrentDirectory, + .Flags = CreationFlags, + .StdoutFile = OutputPath, #if ZEN_PLATFORM_WINDOWS .AssignToJob = m_JobObject, #endif diff --git a/xmake.lua b/xmake.lua index 1416fbb6a..10f5c9c3a 100644 --- a/xmake.lua +++ b/xmake.lua @@ -350,7 +350,8 @@ task("test") {'r', "run", "kv", "all", "Run test(s) - comma-separated", " - all", " - core", " - http", " - util", " - store", " - remotestore", " - server", " - integration"}, {'j', "junit", "k", nil, "Enable junit report output"}, {'n', "noskip", "k", nil, "Run skipped tests (passes --no-skip to doctest)"}, - {nil, "repeat", "kv", nil, "Repeat tests N times (stops on first failure)"} + {nil, "repeat", "kv", nil, "Repeat tests N times (stops on first failure)"}, + {'v', "verbose", "k", nil, "Route child process output to stdout (zenserver-test)"} } } on_run(function() @@ -421,6 +422,7 @@ task("test") local use_junit_reporting = option.get("junit") local use_noskip = option.get("noskip") + local use_verbose = option.get("verbose") local repeat_count = tonumber(option.get("repeat")) or 1 local junit_report_files = {} @@ -535,6 +537,9 @@ task("test") if use_noskip then cmd = string.format("%s --no-skip", cmd) end + if use_verbose and name == "integration" then + cmd = string.format("%s --verbose", cmd) + end -- Tell TestListener where to write the summary local summary_file = path.join(summary_dir, target .. ".txt") -- cgit v1.2.3 From d604351cb5dc3032a7cb8c84d6ad5f1480325e5c Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 2 Mar 2026 09:37:14 +0100 Subject: Add test suites (#799) Makes all test cases part of a test suite. Test suites are named after the module and the name of the file containing the implementation of the test. * This allows for better and more predictable filtering of which test cases to run which should also be able to reduce the time CI spends in tests since it can filter on the tests for that particular module. Also improves `xmake test` behaviour: * instead of an explicit list of projects just enumerate the test projects which are available based on build system state * also introduces logic to avoid running `xmake config` unnecessarily which would invalidate the existing build and do lots of unnecessary work since dependencies were invalidated by the updated config * also invokes build only for the chosen test targets As a bonus, also adds `xmake sln --open` which allows opening IDE after generation of solution/xmake project is done. --- CLAUDE.md | 26 +++++ src/zencore/base64.cpp | 4 + src/zencore/basicfile.cpp | 4 + src/zencore/blake3.cpp | 4 + src/zencore/callstack.cpp | 4 + src/zencore/compactbinary.cpp | 8 +- src/zencore/compactbinarybuilder.cpp | 4 + src/zencore/compactbinaryjson.cpp | 4 + src/zencore/compactbinarypackage.cpp | 4 + src/zencore/compactbinaryvalidation.cpp | 4 + src/zencore/compactbinaryyaml.cpp | 4 + src/zencore/compositebuffer.cpp | 5 + src/zencore/compress.cpp | 4 + src/zencore/crypto.cpp | 4 + src/zencore/filesystem.cpp | 4 + src/zencore/include/zencore/testing.h | 7 +- src/zencore/intmath.cpp | 4 + src/zencore/iobuffer.cpp | 4 + src/zencore/jobqueue.cpp | 4 + src/zencore/logging.cpp | 4 + src/zencore/md5.cpp | 4 + src/zencore/memoryview.cpp | 4 + src/zencore/mpscqueue.cpp | 2 + src/zencore/parallelwork.cpp | 4 + src/zencore/refcount.cpp | 4 + src/zencore/sha1.cpp | 4 + src/zencore/sharedbuffer.cpp | 4 + src/zencore/stream.cpp | 4 + src/zencore/string.cpp | 4 + src/zencore/testing.cpp | 47 +++++++-- src/zencore/uid.cpp | 4 + src/zencore/workthreadpool.cpp | 4 + src/zencore/zencore.cpp | 2 +- src/zenhttp/clients/httpclientcommon.cpp | 4 + src/zenhttp/httpclient.cpp | 4 + src/zenhttp/httpclient_test.cpp | 4 + src/zenhttp/httpserver.cpp | 4 + src/zenhttp/packageformat.cpp | 4 + src/zenhttp/security/passwordsecurity.cpp | 5 + src/zenhttp/servers/wstest.cpp | 4 + src/zennet/statsdclient.cpp | 4 + src/zenremotestore/builds/buildmanifest.cpp | 4 + src/zenremotestore/builds/buildsavedstate.cpp | 4 + .../builds/buildstorageoperations.cpp | 4 + src/zenremotestore/chunking/chunkblock.cpp | 4 + src/zenremotestore/chunking/chunkedcontent.cpp | 4 + src/zenremotestore/chunking/chunkedfile.cpp | 4 + src/zenremotestore/chunking/chunkingcache.cpp | 4 + src/zenremotestore/filesystemutils.cpp | 4 + src/zenserver-test/buildstore-tests.cpp | 4 + src/zenserver-test/cache-tests.cpp | 4 + src/zenserver-test/cacherequests.cpp | 4 + src/zenserver-test/function-tests.cpp | 4 + src/zenserver-test/hub-tests.cpp | 6 +- src/zenserver-test/projectstore-tests.cpp | 4 + src/zenserver-test/workspace-tests.cpp | 4 + src/zenserver-test/zenserver-test.cpp | 5 + src/zenstore/blockstore.cpp | 4 + src/zenstore/buildstore/buildstore.cpp | 4 + src/zenstore/cache/cachepolicy.cpp | 5 + src/zenstore/cache/structuredcachestore.cpp | 4 + src/zenstore/cas.cpp | 4 + src/zenstore/compactcas.cpp | 4 + src/zenstore/filecas.cpp | 4 + src/zenstore/gc.cpp | 4 + src/zenstore/projectstore.cpp | 4 + src/zenstore/workspaces.cpp | 4 + src/zentelemetry/otlptrace.cpp | 4 + src/zentelemetry/stats.cpp | 4 + src/zenutil/config/commandlineoptions.cpp | 4 + src/zenutil/rpcrecording.cpp | 2 +- src/zenutil/wildcard.cpp | 4 + xmake.lua | 113 +++++++++++++++------ 73 files changed, 421 insertions(+), 52 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c859d7ca2..405494acb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -70,6 +70,31 @@ xmake test --run=all --junit Tests use the **doctest** framework. To run server tests: `xmake run zenserver test` +### Test Suite Naming Convention + +All `TEST_CASE` blocks must be wrapped in a `TEST_SUITE_BEGIN`/`TEST_SUITE_END` pair (not the `TEST_SUITE()` wrapper macro, to avoid reformatting). Suite names follow the pattern `{module}.{filename}` where the `zen` prefix is stripped from the module name: + +| Module | File | Suite name | +|---|---|---| +| `zencore` | `filesystem.cpp` | `"core.filesystem"` | +| `zenstore` | `blockstore.cpp` | `"store.blockstore"` | +| `zenhttp` | `httpclient.cpp` | `"http.httpclient"` | +| `zenutil` | `wildcard.cpp` | `"util.wildcard"` | +| `zennet` | `statsdclient.cpp` | `"net.statsdclient"` | +| `zentelemetry` | `stats.cpp` | `"telemetry.stats"` | +| `zenremotestore` | `chunkblock.cpp` | `"remotestore.chunkblock"` | +| `zenserver-test` | `cache-tests.cpp` | `"server.cache"` | + +Each test executable defaults to running only the suites for its own module (e.g. `zencore-test` defaults to `core.*`). This default is derived automatically from the executable name by `RunTestMain` and can be overridden on the command line: + +```bash +# Run only filesystem tests within zencore-test +zencore-test --test-suite=core.filesystem + +# Run all core tests (default behaviour, no flag needed) +zencore-test +``` + ### Code Formatting ```bash @@ -168,6 +193,7 @@ The codebase is organized into layered modules with clear dependencies: **Naming Conventions (UE-inspired with modifications):** - Classes/Structs: `PascalCase` - Functions/Methods: `PascalCase()` +- Function parameters/local variables: `PascalCase` - Member variables: `m_PascalCase` - Global variables: `g_PascalCase` - Static variables: `s_PascalCase` diff --git a/src/zencore/base64.cpp b/src/zencore/base64.cpp index fdf5f2d66..96e121799 100644 --- a/src/zencore/base64.cpp +++ b/src/zencore/base64.cpp @@ -180,6 +180,8 @@ template bool Base64::Decode(const wchar_t* Source, uint32_t Length, ui using namespace std::string_literals; +TEST_SUITE_BEGIN("core.base64"); + TEST_CASE("Base64") { auto EncodeString = [](std::string_view Input) -> std::string { @@ -290,6 +292,8 @@ TEST_CASE("Base64") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/basicfile.cpp b/src/zencore/basicfile.cpp index bd4d119fb..9dcf7663a 100644 --- a/src/zencore/basicfile.cpp +++ b/src/zencore/basicfile.cpp @@ -888,6 +888,8 @@ WriteToTempFile(CompositeBuffer&& Buffer, const std::filesystem::path& Path) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.basicfile"); + TEST_CASE("BasicFile") { ScopedCurrentDirectoryChange _; @@ -1081,6 +1083,8 @@ TEST_CASE("BasicFileBuffer") } } +TEST_SUITE_END(); + void basicfile_forcelink() { diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp index 054f0d3a0..123918de5 100644 --- a/src/zencore/blake3.cpp +++ b/src/zencore/blake3.cpp @@ -200,6 +200,8 @@ BLAKE3Stream::GetHash() // return text; // } +TEST_SUITE_BEGIN("core.blake3"); + TEST_CASE("BLAKE3") { SUBCASE("Basics") @@ -237,6 +239,8 @@ TEST_CASE("BLAKE3") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/callstack.cpp b/src/zencore/callstack.cpp index 8aa1111bf..ee0b0625a 100644 --- a/src/zencore/callstack.cpp +++ b/src/zencore/callstack.cpp @@ -260,6 +260,8 @@ GetCallstackRaw(void* CaptureBuffer, int FramesToSkip, int FramesToCapture) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.callstack"); + TEST_CASE("Callstack.Basic") { void* Addresses[4]; @@ -272,6 +274,8 @@ TEST_CASE("Callstack.Basic") } } +TEST_SUITE_END(); + void callstack_forcelink() { diff --git a/src/zencore/compactbinary.cpp b/src/zencore/compactbinary.cpp index b43cc18f1..9c81305d0 100644 --- a/src/zencore/compactbinary.cpp +++ b/src/zencore/compactbinary.cpp @@ -1512,6 +1512,8 @@ uson_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinary"); + TEST_CASE("guid") { using namespace std::literals; @@ -1704,8 +1706,6 @@ TEST_CASE("uson.datetime") ////////////////////////////////////////////////////////////////////////// -TEST_SUITE_BEGIN("core.datetime"); - TEST_CASE("core.datetime.compare") { DateTime T1(2000, 12, 13); @@ -1732,10 +1732,6 @@ TEST_CASE("core.datetime.add") CHECK(dT + T1 - T2 == dT1); } -TEST_SUITE_END(); - -TEST_SUITE_BEGIN("core.timespan"); - TEST_CASE("core.timespan.compare") { TimeSpan T1(1000); diff --git a/src/zencore/compactbinarybuilder.cpp b/src/zencore/compactbinarybuilder.cpp index 63c0b9c5c..a9ba30750 100644 --- a/src/zencore/compactbinarybuilder.cpp +++ b/src/zencore/compactbinarybuilder.cpp @@ -710,6 +710,8 @@ usonbuilder_forcelink() // return ""; // } +TEST_SUITE_BEGIN("core.compactbinarybuilder"); + TEST_CASE("usonbuilder.object") { using namespace std::literals; @@ -1530,6 +1532,8 @@ TEST_CASE("usonbuilder.stream") CHECK(ValidateCompactBinary(Object.GetBuffer(), CbValidateMode::All) == CbValidateError::None); } } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/compactbinaryjson.cpp b/src/zencore/compactbinaryjson.cpp index abbec360a..da560a449 100644 --- a/src/zencore/compactbinaryjson.cpp +++ b/src/zencore/compactbinaryjson.cpp @@ -654,6 +654,8 @@ cbjson_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinaryjson"); + TEST_CASE("uson.json") { using namespace std::literals; @@ -872,6 +874,8 @@ TEST_CASE("json.uson") } } +TEST_SUITE_END(); + #endif // ZEN_WITH_TESTS } // namespace zen diff --git a/src/zencore/compactbinarypackage.cpp b/src/zencore/compactbinarypackage.cpp index ffe64f2e9..56a292ca6 100644 --- a/src/zencore/compactbinarypackage.cpp +++ b/src/zencore/compactbinarypackage.cpp @@ -805,6 +805,8 @@ usonpackage_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinarypackage"); + TEST_CASE("usonpackage") { using namespace std::literals; @@ -1343,6 +1345,8 @@ TEST_CASE("usonpackage.invalidpackage") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/compactbinaryvalidation.cpp b/src/zencore/compactbinaryvalidation.cpp index d7292f405..3e78f8ef1 100644 --- a/src/zencore/compactbinaryvalidation.cpp +++ b/src/zencore/compactbinaryvalidation.cpp @@ -753,10 +753,14 @@ usonvalidation_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinaryvalidation"); + TEST_CASE("usonvalidation") { SUBCASE("Basic") {} } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/compactbinaryyaml.cpp b/src/zencore/compactbinaryyaml.cpp index b308af418..b7f2c55df 100644 --- a/src/zencore/compactbinaryyaml.cpp +++ b/src/zencore/compactbinaryyaml.cpp @@ -412,6 +412,8 @@ cbyaml_forcelink() { } +TEST_SUITE_BEGIN("core.compactbinaryyaml"); + TEST_CASE("uson.yaml") { using namespace std::literals; @@ -524,6 +526,8 @@ mixed_seq: )"sv); } } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/compositebuffer.cpp b/src/zencore/compositebuffer.cpp index 252ac9045..ed2b16384 100644 --- a/src/zencore/compositebuffer.cpp +++ b/src/zencore/compositebuffer.cpp @@ -297,6 +297,9 @@ CompositeBuffer::IterateRange(uint64_t Offset, } #if ZEN_WITH_TESTS + +TEST_SUITE_BEGIN("core.compositebuffer"); + TEST_CASE("CompositeBuffer Null") { CompositeBuffer Buffer; @@ -462,6 +465,8 @@ TEST_CASE("CompositeBuffer Composite") TestIterateRange(8, 0, MakeMemoryView(FlatArray).Mid(8, 0), FlatView2); } +TEST_SUITE_END(); + void compositebuffer_forcelink() { diff --git a/src/zencore/compress.cpp b/src/zencore/compress.cpp index 25ed0fc46..6aa0adce0 100644 --- a/src/zencore/compress.cpp +++ b/src/zencore/compress.cpp @@ -2420,6 +2420,8 @@ private: #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.compress"); + TEST_CASE("CompressedBuffer") { uint8_t Zeroes[1024]{}; @@ -2967,6 +2969,8 @@ TEST_CASE("CompressedBufferReader") } } +TEST_SUITE_END(); + void compress_forcelink() { diff --git a/src/zencore/crypto.cpp b/src/zencore/crypto.cpp index 09eebb6ae..049854b42 100644 --- a/src/zencore/crypto.cpp +++ b/src/zencore/crypto.cpp @@ -449,6 +449,8 @@ crypto_forcelink() { } +TEST_SUITE_BEGIN("core.crypto"); + TEST_CASE("crypto.bits") { using CryptoBits256Bit = CryptoBits<256>; @@ -500,6 +502,8 @@ TEST_CASE("crypto.aes") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 03398860b..9885b2ada 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -3309,6 +3309,8 @@ filesystem_forcelink() { } +TEST_SUITE_BEGIN("core.filesystem"); + TEST_CASE("filesystem") { using namespace std::filesystem; @@ -3603,6 +3605,8 @@ TEST_CASE("SharedMemory") CHECK(!OpenSharedMemory("SharedMemoryTest0", 482, false)); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/include/zencore/testing.h b/src/zencore/include/zencore/testing.h index 43bdbbffe..8410216c4 100644 --- a/src/zencore/include/zencore/testing.h +++ b/src/zencore/include/zencore/testing.h @@ -43,8 +43,9 @@ public: TestRunner(); ~TestRunner(); - int ApplyCommandLine(int argc, char const* const* argv); - int Run(); + void SetDefaultSuiteFilter(const char* Pattern); + int ApplyCommandLine(int Argc, char const* const* Argv); + int Run(); private: struct Impl; @@ -59,7 +60,7 @@ private: return Runner.Run(); \ }() -int RunTestMain(int argc, char* argv[], const char* traceName, void (*forceLink)()); +int RunTestMain(int Argc, char* Argv[], const char* ExecutableName, void (*ForceLink)()); } // namespace zen::testing #endif diff --git a/src/zencore/intmath.cpp b/src/zencore/intmath.cpp index 32f82b486..fedf76edc 100644 --- a/src/zencore/intmath.cpp +++ b/src/zencore/intmath.cpp @@ -19,6 +19,8 @@ intmath_forcelink() { } +TEST_SUITE_BEGIN("core.intmath"); + TEST_CASE("intmath") { CHECK(FloorLog2(0x00) == 0); @@ -66,6 +68,8 @@ TEST_CASE("intmath") CHECK(ByteSwap(uint64_t(0x214d'6172'7469'6e21ull)) == 0x216e'6974'7261'4d21ull); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/iobuffer.cpp b/src/zencore/iobuffer.cpp index 1c31d6620..c47c54981 100644 --- a/src/zencore/iobuffer.cpp +++ b/src/zencore/iobuffer.cpp @@ -719,6 +719,8 @@ iobuffer_forcelink() { } +TEST_SUITE_BEGIN("core.iobuffer"); + TEST_CASE("IoBuffer") { zen::IoBuffer buffer1; @@ -756,6 +758,8 @@ TEST_CASE("IoBuffer.mmap") # endif } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp index 75c1be42b..35724b07a 100644 --- a/src/zencore/jobqueue.cpp +++ b/src/zencore/jobqueue.cpp @@ -460,6 +460,8 @@ jobqueue_forcelink() { } +TEST_SUITE_BEGIN("core.jobqueue"); + TEST_CASE("JobQueue") { std::unique_ptr Queue(MakeJobQueue(2, "queue")); @@ -580,6 +582,8 @@ TEST_CASE("JobQueue") } JobsLatch.Wait(); } + +TEST_SUITE_END(); #endif } // namespace zen diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index e79c4b41c..e960a2729 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -540,6 +540,8 @@ logging_forcelink() using namespace std::literals; +TEST_SUITE_BEGIN("core.logging"); + TEST_CASE("simple.bread") { ExtendableStringBuilder<256> Crumbs; @@ -588,6 +590,8 @@ TEST_CASE("simple.bread") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/md5.cpp b/src/zencore/md5.cpp index 4ec145697..3baee91c2 100644 --- a/src/zencore/md5.cpp +++ b/src/zencore/md5.cpp @@ -437,6 +437,8 @@ md5_forcelink() // return md5text; // } +TEST_SUITE_BEGIN("core.md5"); + TEST_CASE("MD5") { using namespace std::literals; @@ -458,6 +460,8 @@ TEST_CASE("MD5") CHECK(Output.compare(Buffer)); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/memoryview.cpp b/src/zencore/memoryview.cpp index 1f6a6996c..1654b1766 100644 --- a/src/zencore/memoryview.cpp +++ b/src/zencore/memoryview.cpp @@ -18,6 +18,8 @@ namespace zen { #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.memoryview"); + TEST_CASE("MemoryView") { { @@ -35,6 +37,8 @@ TEST_CASE("MemoryView") CHECK(MakeMemoryView({1.0f, 1.2f}).GetSize() == 8); } +TEST_SUITE_END(); + void memory_forcelink() { diff --git a/src/zencore/mpscqueue.cpp b/src/zencore/mpscqueue.cpp index 29c76c3ca..f749f1c90 100644 --- a/src/zencore/mpscqueue.cpp +++ b/src/zencore/mpscqueue.cpp @@ -8,6 +8,7 @@ namespace zen { #if ZEN_WITH_TESTS && 0 +TEST_SUITE_BEGIN("core.mpscqueue"); TEST_CASE("mpsc") { MpscQueue Queue; @@ -15,6 +16,7 @@ TEST_CASE("mpsc") std::optional Value = Queue.Dequeue(); CHECK_EQ(Value, "hello"); } +TEST_SUITE_END(); #endif void diff --git a/src/zencore/parallelwork.cpp b/src/zencore/parallelwork.cpp index d86d5815f..94696f479 100644 --- a/src/zencore/parallelwork.cpp +++ b/src/zencore/parallelwork.cpp @@ -157,6 +157,8 @@ ParallelWork::RethrowErrors() #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.parallelwork"); + TEST_CASE("parallellwork.nowork") { std::atomic AbortFlag; @@ -255,6 +257,8 @@ TEST_CASE("parallellwork.limitqueue") Work.Wait(); } +TEST_SUITE_END(); + void parallellwork_forcelink() { diff --git a/src/zencore/refcount.cpp b/src/zencore/refcount.cpp index a6a86ee12..f19afe715 100644 --- a/src/zencore/refcount.cpp +++ b/src/zencore/refcount.cpp @@ -33,6 +33,8 @@ refcount_forcelink() { } +TEST_SUITE_BEGIN("core.refcount"); + TEST_CASE("RefPtr") { RefPtr Ref; @@ -60,6 +62,8 @@ TEST_CASE("RefPtr") CHECK(IsDestroyed == true); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/sha1.cpp b/src/zencore/sha1.cpp index 3ee74d7d8..807ae4c30 100644 --- a/src/zencore/sha1.cpp +++ b/src/zencore/sha1.cpp @@ -373,6 +373,8 @@ sha1_forcelink() // return sha1text; // } +TEST_SUITE_BEGIN("core.sha1"); + TEST_CASE("SHA1") { uint8_t sha1_empty[20] = {0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, 0x55, @@ -438,6 +440,8 @@ TEST_CASE("SHA1") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/sharedbuffer.cpp b/src/zencore/sharedbuffer.cpp index 78efb9d42..8dc6d49d8 100644 --- a/src/zencore/sharedbuffer.cpp +++ b/src/zencore/sharedbuffer.cpp @@ -152,10 +152,14 @@ sharedbuffer_forcelink() { } +TEST_SUITE_BEGIN("core.sharedbuffer"); + TEST_CASE("SharedBuffer") { } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/stream.cpp b/src/zencore/stream.cpp index a800ce121..de67303a4 100644 --- a/src/zencore/stream.cpp +++ b/src/zencore/stream.cpp @@ -79,6 +79,8 @@ BufferReader::Serialize(void* V, int64_t Length) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.stream"); + TEST_CASE("binary.writer.span") { BinaryWriter Writer; @@ -91,6 +93,8 @@ TEST_CASE("binary.writer.span") CHECK(memcmp(Result.GetData(), "apa banan", 9) == 0); } +TEST_SUITE_END(); + void stream_forcelink() { diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index ab1c7de58..27635a86c 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -546,6 +546,8 @@ UrlDecode(std::string_view InUrl) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.string"); + TEST_CASE("url") { using namespace std::literals; @@ -1222,6 +1224,8 @@ TEST_CASE("string") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index 6000bd95c..0bae139bd 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -128,18 +128,24 @@ TestRunner::~TestRunner() { } +void +TestRunner::SetDefaultSuiteFilter(const char* Pattern) +{ + m_Impl->Session.setOption("test-suite", Pattern); +} + int -TestRunner::ApplyCommandLine(int argc, char const* const* argv) +TestRunner::ApplyCommandLine(int Argc, char const* const* Argv) { - m_Impl->Session.applyCommandLine(argc, argv); + m_Impl->Session.applyCommandLine(Argc, Argv); - for (int i = 1; i < argc; ++i) + for (int i = 1; i < Argc; ++i) { - if (argv[i] == "--debug"sv) + if (Argv[i] == "--debug"sv) { zen::logging::SetLogLevel(zen::logging::level::Debug); } - else if (argv[i] == "--verbose"sv) + else if (Argv[i] == "--verbose"sv) { zen::logging::SetLogLevel(zen::logging::level::Trace); } @@ -155,20 +161,20 @@ TestRunner::Run() } int -RunTestMain(int argc, char* argv[], [[maybe_unused]] const char* traceName, void (*forceLink)()) +RunTestMain(int Argc, char* Argv[], const char* ExecutableName, void (*ForceLink)()) { # if ZEN_PLATFORM_WINDOWS setlocale(LC_ALL, "en_us.UTF8"); # endif - forceLink(); + ForceLink(); # if ZEN_PLATFORM_LINUX zen::IgnoreChildSignals(); # endif # if ZEN_WITH_TRACE - zen::TraceInit(traceName); + zen::TraceInit(ExecutableName); zen::TraceOptions TraceCommandlineOptions; if (GetTraceOptionsFromCommandline(TraceCommandlineOptions)) { @@ -179,7 +185,30 @@ RunTestMain(int argc, char* argv[], [[maybe_unused]] const char* traceName, void zen::logging::InitializeLogging(); zen::MaximizeOpenFileCount(); - return ZEN_RUN_TESTS(argc, argv); + TestRunner Runner; + + // Derive default suite filter from ExecutableName: "zencore-test" -> "core.*" + if (ExecutableName) + { + std::string_view Name = ExecutableName; + if (Name.starts_with("zen")) + { + Name.remove_prefix(3); + } + if (Name.ends_with("-test")) + { + Name.remove_suffix(5); + } + if (!Name.empty()) + { + std::string Filter(Name); + Filter += ".*"; + Runner.SetDefaultSuiteFilter(Filter.c_str()); + } + } + + Runner.ApplyCommandLine(Argc, Argv); + return Runner.Run(); } } // namespace zen::testing diff --git a/src/zencore/uid.cpp b/src/zencore/uid.cpp index d7636f2ad..971683721 100644 --- a/src/zencore/uid.cpp +++ b/src/zencore/uid.cpp @@ -156,6 +156,8 @@ Oid::FromMemory(const void* Ptr) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("core.uid"); + TEST_CASE("Oid") { SUBCASE("Basic") @@ -185,6 +187,8 @@ TEST_CASE("Oid") } } +TEST_SUITE_END(); + void uid_forcelink() { diff --git a/src/zencore/workthreadpool.cpp b/src/zencore/workthreadpool.cpp index cb84bbe06..1cb338c66 100644 --- a/src/zencore/workthreadpool.cpp +++ b/src/zencore/workthreadpool.cpp @@ -354,6 +354,8 @@ workthreadpool_forcelink() using namespace std::literals; +TEST_SUITE_BEGIN("core.workthreadpool"); + TEST_CASE("threadpool.basic") { WorkerThreadPool Threadpool{1}; @@ -368,6 +370,8 @@ TEST_CASE("threadpool.basic") CHECK_THROWS(FutureThrow.get()); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp index 4ff79edc7..d82474705 100644 --- a/src/zencore/zencore.cpp +++ b/src/zencore/zencore.cpp @@ -285,7 +285,7 @@ zencore_forcelinktests() namespace zen { -TEST_SUITE_BEGIN("core.assert"); +TEST_SUITE_BEGIN("core.zencore"); TEST_CASE("Assert.Default") { diff --git a/src/zenhttp/clients/httpclientcommon.cpp b/src/zenhttp/clients/httpclientcommon.cpp index c016e1c3c..248ae9d70 100644 --- a/src/zenhttp/clients/httpclientcommon.cpp +++ b/src/zenhttp/clients/httpclientcommon.cpp @@ -597,6 +597,8 @@ namespace testutil { } // namespace testutil +TEST_SUITE_BEGIN("http.httpclientcommon"); + TEST_CASE("BufferedReadFileStream") { ScopedTemporaryDirectory TmpDir; @@ -787,5 +789,7 @@ TEST_CASE("MultipartBoundaryParser") } } +TEST_SUITE_END(); + } // namespace zen #endif diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index 1cfddb366..f94c58581 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -430,6 +430,8 @@ MeasureLatency(HttpClient& Client, std::string_view Url) #if ZEN_WITH_TESTS +TEST_SUITE_BEGIN("http.httpclient"); + TEST_CASE("responseformat") { using namespace std::literals; @@ -839,6 +841,8 @@ TEST_CASE("httpclient.password") AsioServer->RequestExit(); } } +TEST_SUITE_END(); + void httpclient_forcelink() { diff --git a/src/zenhttp/httpclient_test.cpp b/src/zenhttp/httpclient_test.cpp index 91b1a3414..52bf149a7 100644 --- a/src/zenhttp/httpclient_test.cpp +++ b/src/zenhttp/httpclient_test.cpp @@ -257,6 +257,8 @@ struct TestServerFixture ////////////////////////////////////////////////////////////////////////// // Tests +TEST_SUITE_BEGIN("http.httpclient"); + TEST_CASE("httpclient.verbs") { TestServerFixture Fixture; @@ -1352,6 +1354,8 @@ TEST_CASE("httpclient.transport-faults-post" * doctest::skip()) } } +TEST_SUITE_END(); + void httpclient_test_forcelink() { diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 3cefa0ad8..2facd8401 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -1322,6 +1322,8 @@ HandlePackageOffers(HttpService& Service, HttpServerRequest& Request, Ref v1 = ParseCommandLine("c:\\my\\exe.exe \"quoted arg\" \"one\",two,\"three\\\""); @@ -235,5 +237,7 @@ TEST_CASE("CommandLine") CHECK_EQ(v3Stripped[5], std::string("--build-part-name=win64")); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zenutil/rpcrecording.cpp b/src/zenutil/rpcrecording.cpp index 54f27dee7..28a0091cb 100644 --- a/src/zenutil/rpcrecording.cpp +++ b/src/zenutil/rpcrecording.cpp @@ -1119,7 +1119,7 @@ rpcrecord_forcelink() { } -TEST_SUITE_BEGIN("rpc.recording"); +TEST_SUITE_BEGIN("util.rpcrecording"); TEST_CASE("rpc.record") { diff --git a/src/zenutil/wildcard.cpp b/src/zenutil/wildcard.cpp index 7a44c0498..7f2f77780 100644 --- a/src/zenutil/wildcard.cpp +++ b/src/zenutil/wildcard.cpp @@ -118,6 +118,8 @@ wildcard_forcelink() { } +TEST_SUITE_BEGIN("util.wildcard"); + TEST_CASE("Wildcard") { CHECK(MatchWildcard("*.*", "normal.txt", true)); @@ -151,5 +153,7 @@ TEST_CASE("Wildcard") CHECK(MatchWildcard("*.d", "dir/path.d", true)); } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/xmake.lua b/xmake.lua index 10f5c9c3a..0d8f53a16 100644 --- a/xmake.lua +++ b/xmake.lua @@ -329,29 +329,46 @@ task("precommit") task("sln") set_menu { - usage = "xmake sln", + usage = "xmake sln [--open]", description = "Generate IDE project files", + options = { + {'o', "open", "k", nil, "Open the generated project in the IDE after generation"}, + } } if is_os("windows") then on_run(function () - print(os.exec("xmake project --yes --kind=vsxmake2022 -m release,debug -a x64")) + import("core.base.option") + os.exec("xmake project --yes --kind=vsxmake2022 -m release,debug -a x64") + if option.get("open") then + local sln = path.join(os.projectdir(), "vsxmake2022", path.filename(os.projectdir()) .. ".sln") + printf("opening %s\n", sln) + try { function() os.execv("explorer", {sln}) end, catch { function() end } } + end end) elseif is_os("macosx") then on_run(function () - print(os.exec("xmake project --yes --kind=xcode -m release,debug -a x64,arm64")) + import("core.base.option") + os.exec("xmake project --yes --kind=xcode -m release,debug -a x64,arm64") + if option.get("open") then + local xcproj = path.join(os.projectdir(), path.filename(os.projectdir()) .. ".xcodeproj") + printf("opening %s\n", xcproj) + os.exec("open \"%s\"", xcproj) + end end) end task("test") set_menu { - usage = "xmake test --run=[core|store|http|server|integration|util|remotestore|all] (comma-separated)", + usage = "xmake test --run=[name|all] [-- extra-args...] (use --list to see available tests)", description = "Run Zen tests", options = { - {'r', "run", "kv", "all", "Run test(s) - comma-separated", " - all", " - core", " - http", " - util", " - store", " - remotestore", " - server", " - integration"}, + {'r', "run", "kv", "all", "Run test(s) - comma-separated"}, + {'l', "list", "k", nil, "List available test names"}, {'j', "junit", "k", nil, "Enable junit report output"}, {'n', "noskip", "k", nil, "Run skipped tests (passes --no-skip to doctest)"}, {nil, "repeat", "kv", nil, "Repeat tests N times (stops on first failure)"}, - {'v', "verbose", "k", nil, "Route child process output to stdout (zenserver-test)"} + {'v', "verbose", "k", nil, "Route child process output to stdout (zenserver-test)"}, + {nil, "arguments", "vs", nil, "Extra arguments passed to test runners (after --)"} } } on_run(function() @@ -361,33 +378,41 @@ task("test") config.load() - local testname = option.get("run") - - -- Ordered list of available tests (order defines execution order) - local available_tests = { - {"core", "zencore-test"}, - {"http", "zenhttp-test"}, - {"util", "zenutil-test"}, - {"store", "zenstore-test"}, - {"remotestore", "zenremotestore-test"}, - {"server", "zenserver"}, - {"integration", "zenserver-test"}, + -- Override table: target name -> short name (for targets that don't follow convention) + local short_name_overrides = { + ["zenserver-test"] = "integration", } - local plat, arch - if is_host("windows") then - plat = "windows" - arch = "x64" - elseif is_host("macosx") then - plat = "macosx" - arch = is_arch("arm64") and "arm64" or "x86_64" - else - plat = "linux" - arch = "x86_64" + -- Build test list from targets in the "tests" group + local available_tests = {} + for name, target in pairs(project.targets()) do + if target:get("group") == "tests" and name:endswith("-test") then + local short = short_name_overrides[name] + if not short then + -- Derive short name: "zencore-test" -> "core" + short = name + if short:startswith("zen") then short = short:sub(4) end + if short:endswith("-test") then short = short:sub(1, -6) end + end + table.insert(available_tests, {short, name}) + end + end + + -- Add non-test-group entries that have a test subcommand + table.insert(available_tests, {"server", "zenserver"}) + + table.sort(available_tests, function(a, b) return a[1] < b[1] end) + + -- Handle --list: print discovered test names and exit + if option.get("list") then + printf("Available tests:\n") + for _, entry in ipairs(available_tests) do + printf(" %-16s -> %s\n", entry[1], entry[2]) + end + return end - print(os.exec("xmake config -c -m debug -p "..plat.." -a "..arch)) - print(os.exec("xmake")) + local testname = option.get("run") -- Parse comma-separated test names into a set local requested = {} @@ -420,10 +445,37 @@ task("test") raise("no tests match specification: '%s'", testname) end + local plat, arch + if is_host("windows") then + plat = "windows" + arch = "x64" + elseif is_host("macosx") then + plat = "macosx" + arch = is_arch("arm64") and "arm64" or "x86_64" + else + plat = "linux" + arch = "x86_64" + end + + -- Only reconfigure if current config doesn't already match + if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch then + os.exec("xmake config -c -m debug -p %s -a %s", plat, arch) + end + + -- Build targets we're going to run + if requested["all"] then + os.exec("xmake build -y") + else + for _, entry in ipairs(tests) do + os.exec("xmake build -y %s", entry.target) + end + end + local use_junit_reporting = option.get("junit") local use_noskip = option.get("noskip") local use_verbose = option.get("verbose") local repeat_count = tonumber(option.get("repeat")) or 1 + local extra_args = option.get("arguments") or {} local junit_report_files = {} local junit_report_dir @@ -540,6 +592,9 @@ task("test") if use_verbose and name == "integration" then cmd = string.format("%s --verbose", cmd) end + for _, arg in ipairs(extra_args) do + cmd = string.format("%s %s", cmd, arg) + end -- Tell TestListener where to write the summary local summary_file = path.join(summary_dir, target .. ".txt") -- cgit v1.2.3 From 627c3357e2b414468772fd4b2caed00e07912df6 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 2 Mar 2026 10:53:56 +0100 Subject: Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 0bc708ff6..36e02d213 100644 --- a/README.md +++ b/README.md @@ -108,8 +108,7 @@ export CXX=g++-11 Install [xmake](https://xmake.io/#/getting_started). ``` -curl -fsSL https://xmake.io/shget.text | bash -xmake --version +curl -fsSL https://xmake.io/shget.text | bash -s v2.9.9 ``` Clone the Zen project and tell `xmake` to use the correct GCC version. -- cgit v1.2.3 From 1558b202663d9d18f87b384110891b190ad24ea2 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 3 Mar 2026 13:17:38 +0100 Subject: fix objectstore uri path parsing (#801) * add objectstore tests * in http router, for last matcher, test if it matches the remaining part of the uri --- CHANGELOG.md | 1 + src/zenhttp/httpserver.cpp | 87 +++++++++++++++++++++++++++----- src/zenserver-test/objectstore-tests.cpp | 74 +++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 14 deletions(-) create mode 100644 src/zenserver-test/objectstore-tests.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 899415d3a..777696097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ default will be WARN no matter what you pass on the command line. - Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly - Bugfix: Time spent indexing local and remote state during `zen builds download` now show the correct time +- Bugfix: ObjectStore failed to correctly parse urls with sub folder paths causing 404 to be returned ## 5.7.21 - Feature: Added `--security-config-path` option to zenserver to configure security settings diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index 2facd8401..d798c46d9 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -746,6 +746,10 @@ HttpRequestRouter::RegisterRoute(const char* UriPattern, HttpRequestRouter::Hand { if (UriPattern[i] == '}') { + if (i == PatternStart) + { + throw std::runtime_error(fmt::format("matcher pattern is empty in URI pattern '{}'", UriPattern)); + } std::string_view Pattern(&UriPattern[PatternStart], i - PatternStart); if (auto it = m_MatcherNameMap.find(std::string(Pattern)); it != m_MatcherNameMap.end()) { @@ -911,8 +915,9 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) CapturedSegments.emplace_back(Uri); - for (int MatcherIndex : Matchers) + for (size_t MatcherOffset = 0; MatcherOffset < Matchers.size(); MatcherOffset++) { + int MatcherIndex = Matchers[MatcherOffset]; if (UriPos >= UriLen) { IsMatch = false; @@ -922,9 +927,9 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) if (MatcherIndex < 0) { // Literal match - int LitIndex = -MatcherIndex - 1; - const std::string& LitStr = m_Literals[LitIndex]; - size_t LitLen = LitStr.length(); + int LitIndex = -MatcherIndex - 1; + std::string_view LitStr = m_Literals[LitIndex]; + size_t LitLen = LitStr.length(); if (Uri.substr(UriPos, LitLen) == LitStr) { @@ -940,9 +945,18 @@ HttpRequestRouter::HandleRequest(zen::HttpServerRequest& Request) { // Matcher function size_t SegmentStart = UriPos; - while (UriPos < UriLen && Uri[UriPos] != '/') + + if (MatcherOffset == (Matchers.size() - 1)) + { + // Last matcher, use the remaining part of the uri + UriPos = UriLen; + } + else { - ++UriPos; + while (UriPos < UriLen && Uri[UriPos] != '/') + { + ++UriPos; + } } std::string_view Segment = Uri.substr(SegmentStart, UriPos - SegmentStart); @@ -1429,20 +1443,33 @@ TEST_CASE("http.common") SUBCASE("router-matcher") { - bool HandledA = false; - bool HandledAA = false; - bool HandledAB = false; - bool HandledAandB = false; + bool HandledA = false; + bool HandledAA = false; + bool HandledAB = false; + bool HandledAandB = false; + bool HandledAandPath = false; std::vector Captures; auto Reset = [&] { - HandledA = HandledAA = HandledAB = HandledAandB = false; + HandledA = HandledAA = HandledAB = HandledAandB = HandledAandPath = false; Captures.clear(); }; TestHttpService Service; HttpRequestRouter r; - r.AddMatcher("a", [](std::string_view In) -> bool { return In.length() % 2 == 0; }); - r.AddMatcher("b", [](std::string_view In) -> bool { return In.length() % 3 == 0; }); + + r.AddMatcher("a", [](std::string_view In) -> bool { return In.length() % 2 == 0 && In.find('/') == std::string_view::npos; }); + r.AddMatcher("b", [](std::string_view In) -> bool { return In.length() % 3 == 0 && In.find('/') == std::string_view::npos; }); + static constexpr AsciiSet ValidPathCharactersSet{"abcdefghijklmnopqrstuvwxyz0123456789/_.,;$~{}+-[]%()]ABCDEFGHIJKLMNOPQRSTUVWXYZ"}; + r.AddMatcher("path", [](std::string_view Str) -> bool { return !Str.empty() && AsciiSet::HasOnly(Str, ValidPathCharactersSet); }); + + r.RegisterRoute( + "path/{a}/{path}", + [&](auto& Req) { + HandledAandPath = true; + Captures = {std::string(Req.GetCapture(1)), std::string(Req.GetCapture(2))}; + }, + HttpVerb::kGet); + r.RegisterRoute( "{a}", [&](auto& Req) { @@ -1471,7 +1498,6 @@ TEST_CASE("http.common") Captures = {std::string(Req.GetCapture(1)), std::string(Req.GetCapture(2))}; }, HttpVerb::kGet); - { Reset(); TestHttpServerRequest req{Service, "ab"sv}; @@ -1479,6 +1505,7 @@ TEST_CASE("http.common") CHECK(HandledA); CHECK(!HandledAA); CHECK(!HandledAB); + CHECK(!HandledAandPath); REQUIRE_EQ(Captures.size(), 1); CHECK_EQ(Captures[0], "ab"sv); @@ -1491,6 +1518,7 @@ TEST_CASE("http.common") CHECK(!HandledA); CHECK(!HandledAA); CHECK(HandledAB); + CHECK(!HandledAandPath); REQUIRE_EQ(Captures.size(), 2); CHECK_EQ(Captures[0], "ab"sv); CHECK_EQ(Captures[1], "def"sv); @@ -1504,6 +1532,7 @@ TEST_CASE("http.common") CHECK(!HandledAA); CHECK(!HandledAB); CHECK(HandledAandB); + CHECK(!HandledAandPath); REQUIRE_EQ(Captures.size(), 2); CHECK_EQ(Captures[0], "ab"sv); CHECK_EQ(Captures[1], "def"sv); @@ -1516,6 +1545,7 @@ TEST_CASE("http.common") CHECK(!HandledA); CHECK(!HandledAA); CHECK(!HandledAB); + CHECK(!HandledAandPath); } { @@ -1525,6 +1555,35 @@ TEST_CASE("http.common") CHECK(HandledA); CHECK(!HandledAA); CHECK(!HandledAB); + CHECK(!HandledAandPath); + REQUIRE_EQ(Captures.size(), 1); + CHECK_EQ(Captures[0], "a123"sv); + } + + { + Reset(); + TestHttpServerRequest req{Service, "path/ab/simple_path.txt"sv}; + r.HandleRequest(req); + CHECK(!HandledA); + CHECK(!HandledAA); + CHECK(!HandledAB); + CHECK(HandledAandPath); + REQUIRE_EQ(Captures.size(), 2); + CHECK_EQ(Captures[0], "ab"sv); + CHECK_EQ(Captures[1], "simple_path.txt"sv); + } + + { + Reset(); + TestHttpServerRequest req{Service, "path/ab/directory/and/path.txt"sv}; + r.HandleRequest(req); + CHECK(!HandledA); + CHECK(!HandledAA); + CHECK(!HandledAB); + CHECK(HandledAandPath); + REQUIRE_EQ(Captures.size(), 2); + CHECK_EQ(Captures[0], "ab"sv); + CHECK_EQ(Captures[1], "directory/and/path.txt"sv); } } diff --git a/src/zenserver-test/objectstore-tests.cpp b/src/zenserver-test/objectstore-tests.cpp new file mode 100644 index 000000000..f3db5fdf6 --- /dev/null +++ b/src/zenserver-test/objectstore-tests.cpp @@ -0,0 +1,74 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#if ZEN_WITH_TESTS +# include "zenserver-test.h" +# include +# include +# include +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::tests { + +using namespace std::literals; + +TEST_SUITE_BEGIN("server.objectstore"); + +TEST_CASE("objectstore.blobs") +{ + std::string_view Bucket = "bkt"sv; + + std::vector CompressedBlobsHashes; + std::vector BlobsSizes; + std::vector CompressedBlobsSizes; + { + ZenServerInstance Instance(TestEnv); + + const uint16_t PortNumber = Instance.SpawnServerAndWaitUntilReady(fmt::format("--objectstore-enabled")); + CHECK(PortNumber != 0); + + HttpClient Client(Instance.GetBaseUri() + "/obj/"); + + for (size_t I = 0; I < 5; I++) + { + IoBuffer Blob = CreateSemiRandomBlob(4711 + I * 7); + BlobsSizes.push_back(Blob.GetSize()); + CompressedBuffer CompressedBlob = CompressedBuffer::Compress(SharedBuffer(std::move(Blob))); + CompressedBlobsHashes.push_back(CompressedBlob.DecodeRawHash()); + CompressedBlobsSizes.push_back(CompressedBlob.GetCompressedSize()); + IoBuffer Payload = std::move(CompressedBlob).GetCompressed().Flatten().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCompressedBinary); + + std::string ObjectPath = fmt::format("{}/{}.utoc", + CompressedBlobsHashes.back().ToHexString().substr(0, 2), + CompressedBlobsHashes.back().ToHexString()); + + HttpClient::Response Result = Client.Put(fmt::format("bucket/{}/{}.utoc", Bucket, ObjectPath), Payload); + CHECK(Result); + } + + for (size_t I = 0; I < 5; I++) + { + std::string ObjectPath = + fmt::format("{}/{}.utoc", CompressedBlobsHashes[I].ToHexString().substr(0, 2), CompressedBlobsHashes[I].ToHexString()); + HttpClient::Response Result = Client.Get(fmt::format("bucket/{}/{}.utoc", Bucket, ObjectPath)); + CHECK(Result); + CHECK_EQ(Result.ResponsePayload.GetSize(), CompressedBlobsSizes[I]); + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer(std::move(Result.ResponsePayload)), RawHash, RawSize); + CHECK(Compressed); + CHECK_EQ(RawHash, CompressedBlobsHashes[I]); + CHECK_EQ(RawSize, BlobsSizes[I]); + } + } +} + +TEST_SUITE_END(); + +} // namespace zen::tests +#endif -- cgit v1.2.3 From 463a0fde16b827c0ec44c9e88fe3c8c8098aa5ea Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Tue, 3 Mar 2026 20:49:01 +0100 Subject: use multi range requests (#800) - Improvement: `zen builds download` now uses multi-range requests for blocks to reduce download size - Improvement: `zen oplog-import` now uses partial block with multi-range requests for blocks to reduce download size - Improvement: Improved feedback in log/console during `zen oplog-import` - Improvement: `--allow-partial-block-requests` now defaults to `true` for `zen builds download` and `zen oplog-import` (was `mixed`) - Improvement: Improved range merging analysis when downloading partial blocks --- CHANGELOG.md | 7 +- src/zen/cmds/builds_cmd.h | 2 +- src/zen/cmds/projectstore_cmd.h | 2 +- src/zen/cmds/workspaces_cmd.cpp | 2 +- src/zen/progressbar.cpp | 5 +- src/zenhttp/clients/httpclientcommon.cpp | 33 +- src/zenhttp/httpclient.cpp | 11 +- src/zenhttp/include/zenhttp/httpclient.h | 4 +- src/zenremotestore/builds/buildstoragecache.cpp | 72 +- .../builds/buildstorageoperations.cpp | 581 ++++++++++---- src/zenremotestore/builds/buildstorageutil.cpp | 17 - src/zenremotestore/builds/filebuildstorage.cpp | 39 + src/zenremotestore/builds/jupiterbuildstorage.cpp | 22 +- src/zenremotestore/chunking/chunkblock.cpp | 63 +- .../include/zenremotestore/builds/buildstorage.h | 21 +- .../zenremotestore/builds/buildstoragecache.h | 8 + .../zenremotestore/builds/buildstorageoperations.h | 11 +- .../zenremotestore/builds/buildstorageutil.h | 1 - .../include/zenremotestore/chunking/chunkblock.h | 31 +- .../zenremotestore/jupiter/jupitersession.h | 12 + .../projectstore/remoteprojectstore.h | 26 +- src/zenremotestore/jupiter/jupitersession.cpp | 65 ++ .../projectstore/buildsremoteprojectstore.cpp | 100 ++- .../projectstore/fileremoteprojectstore.cpp | 68 +- .../projectstore/jupiterremoteprojectstore.cpp | 60 +- .../projectstore/remoteprojectstore.cpp | 879 ++++++++++++--------- .../projectstore/zenremoteprojectstore.cpp | 145 ++-- .../storage/buildstore/httpbuildstore.cpp | 24 +- 28 files changed, 1587 insertions(+), 724 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 777696097..3fc60ee14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,11 @@ - Feature: Added 404 page to dashboard, to make it easier to find your way back to a valid URL - Feature: HttpClient now properly handles multi-part request/response - Feature: Added `POST` variant to `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` endpoint that accepts a CbObject payload with 1 or more ranges -- Improvement: `zen oplog-import` now uses partial block requests to reduce download size -- Improvement: Use latency to Cloud Storage host and Zen Cache host when calculating partial block requests +- Improvement: `zen builds download` now uses multi-range requests for blocks to reduce download size +- Improvement: `zen oplog-import` now uses partial block with multi-range requests for blocks to reduce download size +- Improvement: Improved feedback in log/console during `zen oplog-import` +- Improvement: `--allow-partial-block-requests` now defaults to `true` for `zen builds download` and `zen oplog-import` (was `mixed`) +- Improvement: Improved range merging analysis when downloading partial blocks - Improvement: IAX's lane tracing - Improvement: Eliminated spurious sentry logging during startup and allocated `sentry-sdk` logging channel defaulting to WARN. The logging level can be changed via --log-debug=sentry-sdk or --log-info=sentry-sdk etc diff --git a/src/zen/cmds/builds_cmd.h b/src/zen/cmds/builds_cmd.h index f5c44ab55..5c80beed5 100644 --- a/src/zen/cmds/builds_cmd.h +++ b/src/zen/cmds/builds_cmd.h @@ -71,7 +71,7 @@ private: bool m_AppendNewContent = false; uint8_t m_BlockReuseMinPercentLimit = 85; bool m_AllowMultiparts = true; - std::string m_AllowPartialBlockRequests = "mixed"; + std::string m_AllowPartialBlockRequests = "true"; AuthCommandLineOptions m_AuthOptions; diff --git a/src/zen/cmds/projectstore_cmd.h b/src/zen/cmds/projectstore_cmd.h index 17fd76e9f..1ba98b39e 100644 --- a/src/zen/cmds/projectstore_cmd.h +++ b/src/zen/cmds/projectstore_cmd.h @@ -210,7 +210,7 @@ private: bool m_BoostWorkerMemory = false; bool m_BoostWorkers = false; - std::string m_AllowPartialBlockRequests = "mixed"; + std::string m_AllowPartialBlockRequests = "true"; }; class SnapshotOplogCommand : public ProjectStoreCommand diff --git a/src/zen/cmds/workspaces_cmd.cpp b/src/zen/cmds/workspaces_cmd.cpp index 2661ac9da..af265d898 100644 --- a/src/zen/cmds/workspaces_cmd.cpp +++ b/src/zen/cmds/workspaces_cmd.cpp @@ -815,7 +815,7 @@ WorkspaceShareCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** if (Results.size() != m_ChunkIds.size()) { throw std::runtime_error( - fmt::format("failed to get workspace share batch - invalid result count recevied (expected: {}, received: {}", + fmt::format("failed to get workspace share batch - invalid result count received (expected: {}, received: {}", m_ChunkIds.size(), Results.size())); } diff --git a/src/zen/progressbar.cpp b/src/zen/progressbar.cpp index 732f16e81..9467ed60d 100644 --- a/src/zen/progressbar.cpp +++ b/src/zen/progressbar.cpp @@ -207,8 +207,9 @@ ProgressBar::UpdateState(const State& NewState, bool DoLinebreak) size_t ProgressBarCount = (ProgressBarSize * PercentDone) / 100; uint64_t Completed = NewState.TotalCount - NewState.RemainingCount; uint64_t ETAElapsedMS = ElapsedTimeMS -= m_PausedMS; - uint64_t ETAMS = - (NewState.Status == State::EStatus::Running) && (PercentDone > 5) ? (ETAElapsedMS * NewState.RemainingCount) / Completed : 0; + uint64_t ETAMS = ((m_State.TotalCount == NewState.TotalCount) && (NewState.Status == State::EStatus::Running)) && (PercentDone > 5) + ? (ETAElapsedMS * NewState.RemainingCount) / Completed + : 0; uint32_t ConsoleColumns = TuiConsoleColumns(1024); diff --git a/src/zenhttp/clients/httpclientcommon.cpp b/src/zenhttp/clients/httpclientcommon.cpp index 248ae9d70..9ded23375 100644 --- a/src/zenhttp/clients/httpclientcommon.cpp +++ b/src/zenhttp/clients/httpclientcommon.cpp @@ -394,31 +394,28 @@ namespace detail { { // Yes, we do a substring of the non-lowercase value string as we want the exact boundary string std::string_view BoundaryName = std::string_view(ContentTypeHeaderValue).substr(BoundaryPos + 9); + size_t BoundaryEnd = std::string::npos; + while (!BoundaryName.empty() && BoundaryName[0] == ' ') + { + BoundaryName = BoundaryName.substr(1); + } if (!BoundaryName.empty()) { - size_t BoundaryEnd = std::string::npos; - while (BoundaryName[0] == ' ') - { - BoundaryName = BoundaryName.substr(1); - } - if (!BoundaryName.empty()) + if (BoundaryName.size() > 2 && BoundaryName.front() == '"' && BoundaryName.back() == '"') { - if (BoundaryName.size() > 2 && BoundaryName.front() == '"' && BoundaryName.back() == '"') + BoundaryEnd = BoundaryName.find('"', 1); + if (BoundaryEnd != std::string::npos) { - BoundaryEnd = BoundaryName.find('"', 1); - if (BoundaryEnd != std::string::npos) - { - BoundaryBeginMatcher.Init(fmt::format("\r\n--{}", BoundaryName.substr(1, BoundaryEnd - 1))); - return true; - } - } - else - { - BoundaryEnd = BoundaryName.find_first_of(" \r\n"); - BoundaryBeginMatcher.Init(fmt::format("\r\n--{}", BoundaryName.substr(0, BoundaryEnd))); + BoundaryBeginMatcher.Init(fmt::format("\r\n--{}", BoundaryName.substr(1, BoundaryEnd - 1))); return true; } } + else + { + BoundaryEnd = BoundaryName.find_first_of(" \r\n"); + BoundaryBeginMatcher.Init(fmt::format("\r\n--{}", BoundaryName.substr(0, BoundaryEnd))); + return true; + } } } } diff --git a/src/zenhttp/httpclient.cpp b/src/zenhttp/httpclient.cpp index f94c58581..281d512cf 100644 --- a/src/zenhttp/httpclient.cpp +++ b/src/zenhttp/httpclient.cpp @@ -107,17 +107,14 @@ HttpClientBase::GetAccessToken() std::vector> HttpClient::Response::GetRanges(std::span> OffsetAndLengthPairs) const { - std::vector> Result; - Result.reserve(OffsetAndLengthPairs.size()); if (Ranges.empty()) { - for (const std::pair& Range : OffsetAndLengthPairs) - { - Result.emplace_back(std::make_pair(Range.first, Range.second)); - } - return Result; + return {}; } + std::vector> Result; + Result.reserve(OffsetAndLengthPairs.size()); + auto BoundaryIt = Ranges.begin(); auto OffsetAndLengthPairIt = OffsetAndLengthPairs.begin(); while (OffsetAndLengthPairIt != OffsetAndLengthPairs.end()) diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index f00bbec63..53be36b9a 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -190,10 +190,12 @@ public: HttpContentType ContentType; }; - // Ranges will map out all recevied ranges, both single and multi-range responses + // Ranges will map out all received ranges, both single and multi-range responses // If no range was requested Ranges will be empty std::vector Ranges; + // Map the absolute OffsetAndLengthPairs into ResponsePayload from the ranges received (Ranges). + // If the response was not a partial response, an empty vector will be returned std::vector> GetRanges(std::span> OffsetAndLengthPairs) const; // This contains any errors from the HTTP stack. It won't contain information on diff --git a/src/zenremotestore/builds/buildstoragecache.cpp b/src/zenremotestore/builds/buildstoragecache.cpp index faa85f81b..53d33bd7e 100644 --- a/src/zenremotestore/builds/buildstoragecache.cpp +++ b/src/zenremotestore/builds/buildstoragecache.cpp @@ -151,7 +151,7 @@ public: auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); HttpClient::Response CacheResponse = - m_HttpClient.Upload(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()), + m_HttpClient.Upload(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash), Payload, ContentType); @@ -180,7 +180,7 @@ public: } CreateDirectories(m_TempFolderPath); HttpClient::Response CacheResponse = - m_HttpClient.Download(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash.ToHexString()), + m_HttpClient.Download(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash), m_TempFolderPath, Headers); AddStatistic(CacheResponse); @@ -191,6 +191,74 @@ public: return {}; } + virtual BuildBlobRanges GetBuildBlobRanges(const Oid& BuildId, + const IoHash& RawHash, + std::span> Ranges) override + { + ZEN_TRACE_CPU("ZenBuildStorageCache::GetBuildBlobRanges"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + + CbObjectWriter Writer; + Writer.BeginArray("ranges"sv); + { + for (const std::pair& Range : Ranges) + { + Writer.BeginObject(); + { + Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("length"sv, Range.second); + } + Writer.EndObject(); + } + } + Writer.EndArray(); // ranges + + CreateDirectories(m_TempFolderPath); + HttpClient::Response CacheResponse = + m_HttpClient.Post(fmt::format("/builds/{}/{}/{}/blobs/{}", m_Namespace, m_Bucket, BuildId, RawHash), + Writer.Save(), + HttpClient::Accept(ZenContentType::kCbPackage)); + AddStatistic(CacheResponse); + if (CacheResponse.IsSuccess()) + { + CbPackage ResponsePackage = ParsePackageMessage(CacheResponse.ResponsePayload); + CbObjectView ResponseObject = ResponsePackage.GetObject(); + + CbArrayView RangeArray = ResponseObject["ranges"sv].AsArrayView(); + + std::vector> ReceivedRanges; + ReceivedRanges.reserve(RangeArray.Num()); + + uint64_t OffsetInPayloadRanges = 0; + + for (CbFieldView View : RangeArray) + { + CbObjectView RangeView = View.AsObjectView(); + uint64_t Offset = RangeView["offset"sv].AsUInt64(); + uint64_t Length = RangeView["length"sv].AsUInt64(); + + const std::pair& Range = Ranges[ReceivedRanges.size()]; + + if (Offset != Range.first || Length != Range.second) + { + return {}; + } + ReceivedRanges.push_back(std::make_pair(OffsetInPayloadRanges, Length)); + OffsetInPayloadRanges += Length; + } + + const CbAttachment* DataAttachment = ResponsePackage.FindAttachment(RawHash); + if (DataAttachment) + { + SharedBuffer PayloadRanges = DataAttachment->AsBinary(); + return BuildBlobRanges{.PayloadBuffer = PayloadRanges.AsIoBuffer(), .Ranges = std::move(ReceivedRanges)}; + } + } + return {}; + } + virtual void PutBlobMetadatas(const Oid& BuildId, std::span BlobHashes, std::span MetaDatas) override { ZEN_ASSERT(!IsFlushed); diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 5deb00707..f4b4d592b 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -38,6 +38,7 @@ ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_WITH_TESTS # include # include +# include # include #endif // ZEN_WITH_TESTS @@ -883,12 +884,14 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) BlobsExistsResult ExistsResult; { - ChunkBlockAnalyser BlockAnalyser(m_LogOutput, - m_BlockDescriptions, - ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet, - .IsVerbose = m_Options.IsVerbose, - .HostLatencySec = m_Storage.BuildStorageLatencySec, - .HostHighSpeedLatencySec = m_Storage.CacheLatencySec}); + ChunkBlockAnalyser BlockAnalyser( + m_LogOutput, + m_BlockDescriptions, + ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet, + .IsVerbose = m_Options.IsVerbose, + .HostLatencySec = m_Storage.BuildStorageLatencySec, + .HostHighSpeedLatencySec = m_Storage.CacheLatencySec, + .HostMaxRangeCountPerRequest = BuildStorageBase::MaxRangeCountPerRequest}); std::vector NeededBlocks = BlockAnalyser.GetNeeded( m_RemoteLookup.ChunkHashToChunkIndex, @@ -1027,15 +1030,13 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) const bool BlockExistInCache = ExistsResult.ExistingBlobs.contains(m_BlockDescriptions[BlockIndex].BlockHash); if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::All) { - BlockPartialDownloadModes.push_back(BlockExistInCache - ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed - : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange); + BlockPartialDownloadModes.push_back(BlockExistInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::Exact + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange); } else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::ZenCacheOnly) { - BlockPartialDownloadModes.push_back(BlockExistInCache - ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed - : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); + BlockPartialDownloadModes.push_back(BlockExistInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::Exact + : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); } else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Mixed) { @@ -1045,6 +1046,7 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) } } } + ZEN_ASSERT(BlockPartialDownloadModes.size() == m_BlockDescriptions.size()); ChunkBlockAnalyser::BlockResult PartialBlocks = @@ -1356,90 +1358,105 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) &Work, &PartialBlocks, BlockRangeStartIndex = BlockRangeIndex, - RangeCount](std::atomic&) { + RangeCount = RangeCount](std::atomic&) { if (!m_AbortFlag) { ZEN_TRACE_CPU("Async_GetPartialBlockRanges"); FilteredDownloadedBytesPerSecond.Start(); - for (size_t BlockRangeIndex = BlockRangeStartIndex; BlockRangeIndex < BlockRangeStartIndex + RangeCount; - BlockRangeIndex++) - { - ZEN_TRACE_CPU("GetPartialBlock"); - - const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = PartialBlocks.BlockRanges[BlockRangeIndex]; - - DownloadPartialBlock( - BlockRange, - ExistsResult, - [this, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - &WritePartsComplete, - &WriteCache, - &Work, - TotalRequestCount, - TotalPartWriteCount, - &FilteredDownloadedBytesPerSecond, - &FilteredWrittenBytesPerSecond, - &BlockRange](IoBuffer&& InMemoryBuffer, const std::filesystem::path& OnDiskPath) { - if (m_DownloadStats.RequestsCompleteCount == TotalRequestCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } + DownloadPartialBlock( + PartialBlocks.BlockRanges, + BlockRangeStartIndex, + RangeCount, + ExistsResult, + [this, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + &WritePartsComplete, + &WriteCache, + &Work, + TotalRequestCount, + TotalPartWriteCount, + &FilteredDownloadedBytesPerSecond, + &FilteredWrittenBytesPerSecond, + &PartialBlocks](IoBuffer&& InMemoryBuffer, + const std::filesystem::path& OnDiskPath, + size_t BlockRangeStartIndex, + std::span> OffsetAndLengths) { + if (m_DownloadStats.RequestsCompleteCount == TotalRequestCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } - if (!m_AbortFlag) - { - Work.ScheduleWork( - m_IOWorkerPool, - [this, - &RemoteChunkIndexNeedsCopyFromSourceFlags, - &SequenceIndexChunksLeftToWriteCounters, - &WritePartsComplete, - &WriteCache, - &Work, - TotalPartWriteCount, - &FilteredWrittenBytesPerSecond, - &BlockRange, - BlockChunkPath = std::filesystem::path(OnDiskPath), - BlockPartialBuffer = std::move(InMemoryBuffer)](std::atomic&) mutable { - if (!m_AbortFlag) - { - ZEN_TRACE_CPU("Async_WritePartialBlock"); + if (!m_AbortFlag) + { + Work.ScheduleWork( + m_IOWorkerPool, + [this, + &RemoteChunkIndexNeedsCopyFromSourceFlags, + &SequenceIndexChunksLeftToWriteCounters, + &WritePartsComplete, + &WriteCache, + &Work, + TotalPartWriteCount, + &FilteredWrittenBytesPerSecond, + &PartialBlocks, + BlockRangeStartIndex, + BlockChunkPath = std::filesystem::path(OnDiskPath), + BlockPartialBuffer = std::move(InMemoryBuffer), + OffsetAndLengths = std::vector>(OffsetAndLengths.begin(), + OffsetAndLengths.end())]( + std::atomic&) mutable { + if (!m_AbortFlag) + { + ZEN_TRACE_CPU("Async_WritePartialBlock"); - const uint32_t BlockIndex = BlockRange.BlockIndex; + const uint32_t BlockIndex = PartialBlocks.BlockRanges[BlockRangeStartIndex].BlockIndex; - const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; + const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - if (BlockChunkPath.empty()) - { - ZEN_ASSERT(BlockPartialBuffer); - } - else + if (BlockChunkPath.empty()) + { + ZEN_ASSERT(BlockPartialBuffer); + } + else + { + ZEN_ASSERT(!BlockPartialBuffer); + BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); + if (!BlockPartialBuffer) { - ZEN_ASSERT(!BlockPartialBuffer); - BlockPartialBuffer = IoBufferBuilder::MakeFromFile(BlockChunkPath); - if (!BlockPartialBuffer) - { - throw std::runtime_error( - fmt::format("Could not open downloaded block {} from {}", - BlockDescription.BlockHash, - BlockChunkPath)); - } + throw std::runtime_error( + fmt::format("Could not open downloaded block {} from {}", + BlockDescription.BlockHash, + BlockChunkPath)); } + } + + FilteredWrittenBytesPerSecond.Start(); - FilteredWrittenBytesPerSecond.Start(); - - if (!WritePartialBlockChunksToCache( - BlockDescription, - SequenceIndexChunksLeftToWriteCounters, - Work, - CompositeBuffer(std::move(BlockPartialBuffer)), - BlockRange.ChunkBlockIndexStart, - BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount - 1, - RemoteChunkIndexNeedsCopyFromSourceFlags, - WriteCache)) + size_t RangeCount = OffsetAndLengths.size(); + + for (size_t PartialRangeIndex = 0; PartialRangeIndex < RangeCount; PartialRangeIndex++) + { + const std::pair& OffsetAndLength = + OffsetAndLengths[PartialRangeIndex]; + IoBuffer BlockRangeBuffer(BlockPartialBuffer, + OffsetAndLength.first, + OffsetAndLength.second); + + const ChunkBlockAnalyser::BlockRangeDescriptor& RangeDescriptor = + PartialBlocks.BlockRanges[BlockRangeStartIndex + PartialRangeIndex]; + + if (!WritePartialBlockChunksToCache(BlockDescription, + SequenceIndexChunksLeftToWriteCounters, + Work, + CompositeBuffer(std::move(BlockRangeBuffer)), + RangeDescriptor.ChunkBlockIndexStart, + RangeDescriptor.ChunkBlockIndexStart + + RangeDescriptor.ChunkBlockIndexCount - 1, + RemoteChunkIndexNeedsCopyFromSourceFlags, + WriteCache)) { std::error_code DummyEc; RemoveFile(BlockChunkPath, DummyEc); @@ -1447,28 +1464,27 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) fmt::format("Partial block {} is malformed", BlockDescription.BlockHash)); } - std::error_code Ec = TryRemoveFile(BlockChunkPath); - if (Ec) - { - ZEN_OPERATION_LOG_DEBUG(m_LogOutput, - "Failed removing file '{}', reason: ({}) {}", - BlockChunkPath, - Ec.value(), - Ec.message()); - } - WritePartsComplete++; if (WritePartsComplete == TotalPartWriteCount) { FilteredWrittenBytesPerSecond.Stop(); } } - }, - OnDiskPath.empty() ? WorkerThreadPool::EMode::DisableBacklog - : WorkerThreadPool::EMode::EnableBacklog); - } - }); - } + std::error_code Ec = TryRemoveFile(BlockChunkPath); + if (Ec) + { + ZEN_OPERATION_LOG_DEBUG(m_LogOutput, + "Failed removing file '{}', reason: ({}) {}", + BlockChunkPath, + Ec.value(), + Ec.message()); + } + } + }, + OnDiskPath.empty() ? WorkerThreadPool::EMode::DisableBacklog + : WorkerThreadPool::EMode::EnableBacklog); + } + }); } }); BlockRangeIndex += RangeCount; @@ -3161,45 +3177,40 @@ BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t RemoteChunkInde void BuildsOperationUpdateFolder::DownloadPartialBlock( - const ChunkBlockAnalyser::BlockRangeDescriptor BlockRange, - const BlobsExistsResult& ExistsResult, - std::function&& OnDownloaded) + std::span BlockRanges, + size_t BlockRangeStartIndex, + size_t BlockRangeCount, + const BlobsExistsResult& ExistsResult, + std::function> OffsetAndLengths)>&& OnDownloaded) { - const uint32_t BlockIndex = BlockRange.BlockIndex; + const uint32_t BlockIndex = BlockRanges[BlockRangeStartIndex].BlockIndex; const ChunkBlockDescription& BlockDescription = m_BlockDescriptions[BlockIndex]; - IoBuffer BlockBuffer; - if (m_Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) - { - BlockBuffer = - m_Storage.BuildCacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - } - if (!BlockBuffer) - { - BlockBuffer = - m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - } - if (!BlockBuffer) - { - throw std::runtime_error(fmt::format("Block {} is missing when fetching range {} -> {}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeStart + BlockRange.RangeLength)); - } - if (!m_AbortFlag) - { - uint64_t BlockSize = BlockBuffer.GetSize(); + auto ProcessDownload = [this]( + const ChunkBlockDescription& BlockDescription, + IoBuffer&& BlockRangeBuffer, + size_t BlockRangeStartIndex, + std::span> BlockOffsetAndLengths, + const std::function> OffsetAndLengths)>& OnDownloaded) { + uint64_t BlockRangeBufferSize = BlockRangeBuffer.GetSize(); m_DownloadStats.DownloadedBlockCount++; - m_DownloadStats.DownloadedBlockByteCount += BlockSize; - m_DownloadStats.RequestsCompleteCount++; + m_DownloadStats.DownloadedBlockByteCount += BlockRangeBufferSize; + m_DownloadStats.RequestsCompleteCount += BlockOffsetAndLengths.size(); std::filesystem::path BlockChunkPath; // Check if the dowloaded block is file based and we can move it directly without rewriting it { IoBufferFileReference FileRef; - if (BlockBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && (FileRef.FileChunkSize == BlockSize)) + if (BlockRangeBuffer.GetFileReference(FileRef) && (FileRef.FileChunkOffset == 0) && + (FileRef.FileChunkSize == BlockRangeBufferSize)) { ZEN_TRACE_CPU("MoveTempPartialBlock"); @@ -3207,10 +3218,17 @@ BuildsOperationUpdateFolder::DownloadPartialBlock( std::filesystem::path TempBlobPath = PathFromHandle(FileRef.FileHandle, Ec); if (!Ec) { - BlockBuffer.SetDeleteOnClose(false); - BlockBuffer = {}; - BlockChunkPath = m_TempBlockFolderPath / - fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); + BlockRangeBuffer.SetDeleteOnClose(false); + BlockRangeBuffer = {}; + + IoHashStream RangeId; + for (const std::pair& Range : BlockOffsetAndLengths) + { + RangeId.Append(&Range.first, sizeof(uint64_t)); + RangeId.Append(&Range.second, sizeof(uint64_t)); + } + + BlockChunkPath = m_TempBlockFolderPath / fmt::format("{}_{}", BlockDescription.BlockHash, RangeId.GetHash()); RenameFile(TempBlobPath, BlockChunkPath, Ec); if (Ec) { @@ -3218,27 +3236,137 @@ BuildsOperationUpdateFolder::DownloadPartialBlock( // Re-open the temp file again BasicFile OpenTemp(TempBlobPath, BasicFile::Mode::kDelete); - BlockBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockSize, true); - BlockBuffer.SetDeleteOnClose(true); + BlockRangeBuffer = IoBuffer(IoBuffer::File, OpenTemp.Detach(), 0, BlockRangeBufferSize, true); + BlockRangeBuffer.SetDeleteOnClose(true); } } } } - if (BlockChunkPath.empty() && (BlockSize > m_Options.MaximumInMemoryPayloadSize)) + if (BlockChunkPath.empty() && (BlockRangeBufferSize > m_Options.MaximumInMemoryPayloadSize)) { ZEN_TRACE_CPU("WriteTempPartialBlock"); + + IoHashStream RangeId; + for (const std::pair& Range : BlockOffsetAndLengths) + { + RangeId.Append(&Range.first, sizeof(uint64_t)); + RangeId.Append(&Range.second, sizeof(uint64_t)); + } + // Could not be moved and rather large, lets store it on disk - BlockChunkPath = m_TempBlockFolderPath / - fmt::format("{}_{:x}_{:x}", BlockDescription.BlockHash, BlockRange.RangeStart, BlockRange.RangeLength); - TemporaryFile::SafeWriteFile(BlockChunkPath, BlockBuffer); - BlockBuffer = {}; + BlockChunkPath = m_TempBlockFolderPath / fmt::format("{}_{}", BlockDescription.BlockHash, RangeId.GetHash()); + TemporaryFile::SafeWriteFile(BlockChunkPath, BlockRangeBuffer); + BlockRangeBuffer = {}; } if (!m_AbortFlag) { - OnDownloaded(std::move(BlockBuffer), std::move(BlockChunkPath)); + OnDownloaded(std::move(BlockRangeBuffer), std::move(BlockChunkPath), BlockRangeStartIndex, BlockOffsetAndLengths); + } + }; + + std::vector> Ranges; + Ranges.reserve(BlockRangeCount); + for (size_t BlockRangeIndex = BlockRangeStartIndex; BlockRangeIndex < BlockRangeStartIndex + BlockRangeCount; BlockRangeIndex++) + { + const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = BlockRanges[BlockRangeIndex]; + Ranges.push_back(std::make_pair(BlockRange.RangeStart, BlockRange.RangeLength)); + } + + if (m_Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) + { + BuildStorageCache::BuildBlobRanges RangeBuffers = + m_Storage.BuildCacheStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, Ranges); + if (RangeBuffers.PayloadBuffer) + { + if (!m_AbortFlag) + { + if (RangeBuffers.Ranges.size() != Ranges.size()) + { + throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges", + Ranges.size(), + BlockDescription.BlockHash, + RangeBuffers.Ranges.size())); + } + + std::vector> BlockOffsetAndLengths = std::move(RangeBuffers.Ranges); + ProcessDownload(BlockDescription, + std::move(RangeBuffers.PayloadBuffer), + BlockRangeStartIndex, + BlockOffsetAndLengths, + OnDownloaded); + } + return; } } + + const size_t MaxRangesPerRequestToJupiter = BuildStorageBase::MaxRangeCountPerRequest; + + size_t SubBlockRangeCount = BlockRangeCount; + size_t SubRangeCountComplete = 0; + std::span> RangesSpan(Ranges); + while (SubRangeCountComplete < SubBlockRangeCount) + { + if (m_AbortFlag) + { + break; + } + size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, MaxRangesPerRequestToJupiter); + size_t SubRangeStartIndex = BlockRangeStartIndex + SubRangeCountComplete; + + auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); + + BuildStorageBase::BuildBlobRanges RangeBuffers = + m_Storage.BuildStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, SubRanges); + if (RangeBuffers.PayloadBuffer) + { + if (m_AbortFlag) + { + break; + } + if (RangeBuffers.Ranges.empty()) + { + // Jupiter will ignore the ranges and send the whole payload if it fetches the payload from S3 + // Upload to cache (if enabled) and use the whole payload for the remaining ranges + + if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + { + m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, + BlockDescription.BlockHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(std::vector{RangeBuffers.PayloadBuffer})); + } + + SubRangeCount = Ranges.size() - SubRangeCountComplete; + ProcessDownload(BlockDescription, + std::move(RangeBuffers.PayloadBuffer), + SubRangeStartIndex, + RangesSpan.subspan(SubRangeCountComplete, SubRangeCount), + OnDownloaded); + } + else + { + if (RangeBuffers.Ranges.size() != SubRanges.size()) + { + throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges", + SubRanges.size(), + BlockDescription.BlockHash, + RangeBuffers.Ranges.size())); + } + ProcessDownload(BlockDescription, + std::move(RangeBuffers.PayloadBuffer), + SubRangeStartIndex, + RangeBuffers.Ranges, + OnDownloaded); + } + } + else + { + throw std::runtime_error(fmt::format("Block {} is missing when fetching {} ranges", BlockDescription.BlockHash, SubRangeCount)); + } + + SubRangeCountComplete += SubRangeCount; + } } std::vector @@ -7083,16 +7211,31 @@ GetRemoteContent(OperationLogOutput& Output, // TODO: GetBlockDescriptions for all BlockRawHashes in one go - check for local block descriptions when we cache them { + if (!IsQuiet) + { + ZEN_OPERATION_LOG_INFO(Output, "Fetching metadata for {} blocks", BlockRawHashes.size()); + } + + Stopwatch GetBlockMetadataTimer; + bool AttemptFallback = false; OutBlockDescriptions = GetBlockDescriptions(Output, *Storage.BuildStorage, Storage.BuildCacheStorage.get(), BuildId, - BuildPartId, BlockRawHashes, AttemptFallback, IsQuiet, IsVerbose); + + if (!IsQuiet) + { + ZEN_OPERATION_LOG_INFO(Output, + "GetBlockMetadata for {} took {}. Found {} blocks", + BuildPartId, + NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), + OutBlockDescriptions.size()); + } } CalculateLocalChunkOrders(AbsoluteChunkOrders, @@ -7935,6 +8078,164 @@ TEST_CASE("buildstorageoperations.upload.multipart") } } +TEST_CASE("buildstorageoperations.partial.block.download" * doctest::skip(true)) +{ + const std::string OidcExecutableName = "OidcToken" ZEN_EXE_SUFFIX_LITERAL; + std::filesystem::path OidcTokenExePath = (GetRunningExecutablePath().parent_path() / OidcExecutableName).make_preferred(); + + HttpClientSettings ClientSettings{ + .LogCategory = "httpbuildsclient", + .AccessTokenProvider = + httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, "https://jupiter.devtools.epicgames.com", true, false, false), + .AssumeHttp2 = false, + .AllowResume = true, + .RetryCount = 0, + .Verbose = false}; + + HttpClient HttpClient("https://euc.jupiter.devtools.epicgames.com", ClientSettings); + + const std::string_view Namespace = "fortnite.oplog"; + const std::string_view Bucket = "fortnitegame.staged-build.fortnite-main.ps4-client"; + const Oid BuildId = Oid::FromHexString("09a76ea92ad301d4724fafad"); + + { + HttpClient::Response Response = HttpClient.Get(fmt::format("/api/v2/builds/{}/{}/{}", Namespace, Bucket, BuildId), + HttpClient::Accept(ZenContentType::kCbObject)); + CbValidateError ValidateResult = CbValidateError::None; + CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(Response.ResponsePayload), ValidateResult); + REQUIRE(ValidateResult == CbValidateError::None); + } + + std::vector BlockDescriptions; + { + CbObjectWriter Request; + + Request.BeginArray("blocks"sv); + { + Request.AddHash(IoHash::FromHexString("7c353ed782675a5e8f968e61e51fc797ecdc2882")); + } + Request.EndArray(); + + IoBuffer Payload = Request.Save().GetBuffer().AsIoBuffer(); + Payload.SetContentType(ZenContentType::kCbObject); + + HttpClient::Response BlockDescriptionsResponse = + HttpClient.Post(fmt::format("/api/v2/builds/{}/{}/{}/blocks/getBlockMetadata", Namespace, Bucket, BuildId), + Payload, + HttpClient::Accept(ZenContentType::kCbObject)); + REQUIRE(BlockDescriptionsResponse.IsSuccess()); + + CbValidateError ValidateResult = CbValidateError::None; + CbObject Object = ValidateAndReadCompactBinaryObject(IoBuffer(BlockDescriptionsResponse.ResponsePayload), ValidateResult); + REQUIRE(ValidateResult == CbValidateError::None); + + { + CbArrayView BlocksArray = Object["blocks"sv].AsArrayView(); + for (CbFieldView Block : BlocksArray) + { + ChunkBlockDescription Description = ParseChunkBlockDescription(Block.AsObjectView()); + BlockDescriptions.emplace_back(std::move(Description)); + } + } + } + + REQUIRE(!BlockDescriptions.empty()); + + const IoHash BlockHash = BlockDescriptions.back().BlockHash; + + const ChunkBlockDescription& BlockDescription = BlockDescriptions.front(); + REQUIRE(!BlockDescription.ChunkRawHashes.empty()); + REQUIRE(!BlockDescription.ChunkCompressedLengths.empty()); + + std::vector> ChunkOffsetAndSizes; + uint64_t Offset = gsl::narrow(CompressedBuffer::GetHeaderSizeForNoneEncoder() + BlockDescription.HeaderSize); + + for (uint32_t ChunkCompressedSize : BlockDescription.ChunkCompressedLengths) + { + ChunkOffsetAndSizes.push_back(std::make_pair(Offset, ChunkCompressedSize)); + Offset += ChunkCompressedSize; + } + + ScopedTemporaryDirectory SourceFolder; + + auto Validate = [&](std::span ChunkIndexesToFetch) { + std::vector> Ranges; + for (uint32_t ChunkIndex : ChunkIndexesToFetch) + { + Ranges.push_back(ChunkOffsetAndSizes[ChunkIndex]); + } + + HttpClient::KeyValueMap Headers; + if (!Ranges.empty()) + { + ExtendableStringBuilder<512> SB; + for (const std::pair& R : Ranges) + { + if (SB.Size() > 0) + { + SB << ", "; + } + SB << R.first << "-" << R.first + R.second - 1; + } + Headers.Entries.insert({"Range", fmt::format("bytes={}", SB.ToView())}); + } + + HttpClient::Response GetBlobRangesResponse = HttpClient.Download( + fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect=false", Namespace, Bucket, BuildId, BlockHash), + SourceFolder.Path(), + Headers); + + REQUIRE(GetBlobRangesResponse.IsSuccess()); + MemoryView RangesMemoryView = GetBlobRangesResponse.ResponsePayload.GetView(); + + std::vector> PayloadRanges = GetBlobRangesResponse.GetRanges(Ranges); + if (PayloadRanges.empty()) + { + // We got the whole blob, use the ranges as is + PayloadRanges = Ranges; + } + + REQUIRE(PayloadRanges.size() == Ranges.size()); + + for (uint32_t RangeIndex = 0; RangeIndex < PayloadRanges.size(); RangeIndex++) + { + const std::pair& PayloadRange = PayloadRanges[RangeIndex]; + + CHECK_EQ(PayloadRange.second, Ranges[RangeIndex].second); + + IoBuffer ChunkPayload(GetBlobRangesResponse.ResponsePayload, PayloadRange.first, PayloadRange.second); + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed(SharedBuffer(ChunkPayload), RawHash, RawSize); + CHECK(CompressedChunk); + CHECK_EQ(RawHash, BlockDescription.ChunkRawHashes[ChunkIndexesToFetch[RangeIndex]]); + CHECK_EQ(RawSize, BlockDescription.ChunkRawLengths[ChunkIndexesToFetch[RangeIndex]]); + } + }; + + { + // Single + std::vector ChunkIndexesToFetch{uint32_t(BlockDescription.ChunkCompressedLengths.size() / 2)}; + Validate(ChunkIndexesToFetch); + } + { + // Many + std::vector ChunkIndexesToFetch; + for (uint32_t Index = 0; Index < BlockDescription.ChunkCompressedLengths.size() / 16; Index++) + { + ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7)); + ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7 + 1)); + ChunkIndexesToFetch.push_back(uint32_t(BlockDescription.ChunkCompressedLengths.size() / 6 + Index * 7 + 3)); + } + Validate(ChunkIndexesToFetch); + } + + { + // First and last + std::vector ChunkIndexesToFetch{0, uint32_t(BlockDescription.ChunkCompressedLengths.size() - 1)}; + Validate(ChunkIndexesToFetch); + } +} TEST_SUITE_END(); void diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp index b249d7d52..d65f18b9a 100644 --- a/src/zenremotestore/builds/buildstorageutil.cpp +++ b/src/zenremotestore/builds/buildstorageutil.cpp @@ -251,7 +251,6 @@ GetBlockDescriptions(OperationLogOutput& Output, BuildStorageBase& Storage, BuildStorageCache* OptionalCacheStorage, const Oid& BuildId, - const Oid& BuildPartId, std::span BlockRawHashes, bool AttemptFallback, bool IsQuiet, @@ -259,13 +258,6 @@ GetBlockDescriptions(OperationLogOutput& Output, { using namespace std::literals; - if (!IsQuiet) - { - ZEN_OPERATION_LOG_INFO(Output, "Fetching metadata for {} blocks", BlockRawHashes.size()); - } - - Stopwatch GetBlockMetadataTimer; - std::vector UnorderedList; tsl::robin_map BlockDescriptionLookup; if (OptionalCacheStorage && !BlockRawHashes.empty()) @@ -355,15 +347,6 @@ GetBlockDescriptions(OperationLogOutput& Output, } } - if (!IsQuiet) - { - ZEN_OPERATION_LOG_INFO(Output, - "GetBlockMetadata for {} took {}. Found {} blocks", - BuildPartId, - NiceTimeSpanMs(GetBlockMetadataTimer.GetElapsedTimeMs()), - Result.size()); - } - if (Result.size() != BlockRawHashes.size()) { std::string ErrorDescription = diff --git a/src/zenremotestore/builds/filebuildstorage.cpp b/src/zenremotestore/builds/filebuildstorage.cpp index 55e69de61..2f4904449 100644 --- a/src/zenremotestore/builds/filebuildstorage.cpp +++ b/src/zenremotestore/builds/filebuildstorage.cpp @@ -432,6 +432,45 @@ public: return IoBuffer{}; } + virtual BuildBlobRanges GetBuildBlobRanges(const Oid& BuildId, + const IoHash& RawHash, + std::span> Ranges) override + { + ZEN_TRACE_CPU("FileBuildStorage::GetBuildBlobRanges"); + ZEN_UNUSED(BuildId); + ZEN_ASSERT(!Ranges.empty()); + + uint64_t ReceivedBytes = 0; + uint64_t SentBytes = Ranges.size() * 2 * 8; + + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + + Stopwatch ExecutionTimer; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer, SentBytes, ReceivedBytes); }); + + BuildBlobRanges Result; + + const std::filesystem::path BlockPath = GetBlobPayloadPath(RawHash); + if (IsFile(BlockPath)) + { + BasicFile File(BlockPath, BasicFile::Mode::kRead); + + uint64_t RangeOffset = Ranges.front().first; + uint64_t RangeBytes = Ranges.back().first + Ranges.back().second - RangeOffset; + Result.PayloadBuffer = IoBufferBuilder::MakeFromFileHandle(File.Detach(), RangeOffset, RangeBytes); + + Result.Ranges.reserve(Ranges.size()); + + for (const std::pair& Range : Ranges) + { + Result.Ranges.push_back(std::make_pair(Range.first - RangeOffset, Range.second)); + } + ReceivedBytes = Result.PayloadBuffer.GetSize(); + } + return Result; + } + virtual std::vector> GetLargeBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t ChunkSize, diff --git a/src/zenremotestore/builds/jupiterbuildstorage.cpp b/src/zenremotestore/builds/jupiterbuildstorage.cpp index 23d0ddd4c..8e16da1a9 100644 --- a/src/zenremotestore/builds/jupiterbuildstorage.cpp +++ b/src/zenremotestore/builds/jupiterbuildstorage.cpp @@ -21,7 +21,7 @@ namespace zen { using namespace std::literals; namespace { - void ThrowFromJupiterResult(const JupiterResult& Result, std::string_view Prefix) + [[noreturn]] void ThrowFromJupiterResult(const JupiterResult& Result, std::string_view Prefix) { int Error = Result.ErrorCode < (int)HttpResponseCode::Continue ? Result.ErrorCode : 0; HttpResponseCode Status = @@ -295,6 +295,26 @@ public: return std::move(GetBuildBlobResult.Response); } + virtual BuildBlobRanges GetBuildBlobRanges(const Oid& BuildId, + const IoHash& RawHash, + std::span> Ranges) override + { + ZEN_TRACE_CPU("Jupiter::GetBuildBlob"); + + Stopwatch ExecutionTimer; + auto _ = MakeGuard([&]() { m_Stats.TotalExecutionTimeUs += ExecutionTimer.GetElapsedTimeUs(); }); + CreateDirectories(m_TempFolderPath); + + BuildBlobRangesResult GetBuildBlobResult = + m_Session.GetBuildBlob(m_Namespace, m_Bucket, BuildId, RawHash, m_TempFolderPath, Ranges); + AddStatistic(GetBuildBlobResult); + if (!GetBuildBlobResult.Success) + { + ThrowFromJupiterResult(GetBuildBlobResult, "Failed fetching build blob ranges"sv); + } + return BuildBlobRanges{.PayloadBuffer = std::move(GetBuildBlobResult.Response), .Ranges = std::move(GetBuildBlobResult.Ranges)}; + } + virtual std::vector> GetLargeBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t ChunkSize, diff --git a/src/zenremotestore/chunking/chunkblock.cpp b/src/zenremotestore/chunking/chunkblock.cpp index 3a4e6011d..9c3fe8a0b 100644 --- a/src/zenremotestore/chunking/chunkblock.cpp +++ b/src/zenremotestore/chunking/chunkblock.cpp @@ -608,40 +608,49 @@ ChunkBlockAnalyser::CalculatePartialBlockDownloads(std::span if (PartialBlockDownloadMode != EPartialBlockDownloadMode::Exact && BlockRanges.size() > 1) { - // TODO: Once we have support in our http client to request multiple ranges in one request this - // logic would need to change as the per-request overhead would go away + const uint64_t MaxRangeCountPerRequest = PartialBlockDownloadMode == EPartialBlockDownloadMode::MultiRangeHighSpeed + ? m_Options.HostHighSpeedMaxRangeCountPerRequest + : m_Options.HostMaxRangeCountPerRequest; - const double LatencySec = PartialBlockDownloadMode == EPartialBlockDownloadMode::MultiRangeHighSpeed - ? m_Options.HostHighSpeedLatencySec - : m_Options.HostLatencySec; - if (LatencySec > 0) + ZEN_ASSERT(MaxRangeCountPerRequest != 0); + + if (MaxRangeCountPerRequest != (uint64_t)-1) { - const uint64_t BytesPerSec = PartialBlockDownloadMode == EPartialBlockDownloadMode::MultiRangeHighSpeed - ? m_Options.HostHighSpeedBytesPerSec - : m_Options.HostSpeedBytesPerSec; + const uint64_t ExtraRequestCount = BlockRanges.size() / MaxRangeCountPerRequest; - const double ExtraRequestTimeSec = (BlockRanges.size() - 1) * LatencySec; - const uint64_t ExtraRequestTimeBytes = uint64_t(ExtraRequestTimeSec * BytesPerSec); + const double LatencySec = PartialBlockDownloadMode == EPartialBlockDownloadMode::MultiRangeHighSpeed + ? m_Options.HostHighSpeedLatencySec + : m_Options.HostLatencySec; + if (LatencySec > 0) + { + const uint64_t BytesPerSec = PartialBlockDownloadMode == EPartialBlockDownloadMode::MultiRangeHighSpeed + ? m_Options.HostHighSpeedBytesPerSec + : m_Options.HostSpeedBytesPerSec; - const uint64_t FullRangeSize = - BlockRanges.back().RangeStart + BlockRanges.back().RangeLength - BlockRanges.front().RangeStart; + const double ExtraRequestTimeSec = ExtraRequestCount * LatencySec; + const uint64_t ExtraRequestTimeBytes = uint64_t(ExtraRequestTimeSec * BytesPerSec); - if (ExtraRequestTimeBytes + RequestedSize >= FullRangeSize) - { - BlockRanges = std::vector{MergeBlockRanges(BlockRanges)}; + const uint64_t FullRangeSize = + BlockRanges.back().RangeStart + BlockRanges.back().RangeLength - BlockRanges.front().RangeStart; - if (m_Options.IsVerbose) + if (ExtraRequestTimeBytes + RequestedSize >= FullRangeSize) { - ZEN_OPERATION_LOG_INFO(m_LogOutput, - "Merging {} chunks ({}) from block {} ({}) to single request (extra bytes {})", - NeededBlock.ChunkIndexes.size(), - NiceBytes(RequestedSize), - BlockDescription.BlockHash, - NiceBytes(TotalBlockSize), - NiceBytes(BlockRanges.front().RangeLength - RequestedSize)); + BlockRanges = std::vector{MergeBlockRanges(BlockRanges)}; + + if (m_Options.IsVerbose) + { + ZEN_OPERATION_LOG_INFO( + m_LogOutput, + "Merging {} chunks ({}) from block {} ({}) to single request (extra bytes {})", + NeededBlock.ChunkIndexes.size(), + NiceBytes(RequestedSize), + BlockDescription.BlockHash, + NiceBytes(TotalBlockSize), + NiceBytes(BlockRanges.front().RangeLength - RequestedSize)); + } + + RequestedSize = BlockRanges.front().RangeLength; } - - RequestedSize = BlockRanges.front().RangeLength; } } } @@ -730,7 +739,7 @@ ChunkBlockAnalyser::CalculatePartialBlockDownloads(std::span ZEN_OPERATION_LOG_INFO(m_LogOutput, "Analysis of partial block requests saves download of {} out of {}, {:.1f}% of possible {} using {} extra " - "requests. Completed in {}", + "ranges. Completed in {}", NiceBytes(ActualSkippedSize), NiceBytes(AllBlocksTotalBlocksSize), PercentOfIdealPartialSkippedSize, diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h index 85dabc59f..ce3da41c1 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h @@ -53,15 +53,26 @@ public: std::function&& Transmitter, std::function&& OnSentBytes) = 0; - virtual IoBuffer GetBuildBlob(const Oid& BuildId, - const IoHash& RawHash, - uint64_t RangeOffset = 0, - uint64_t RangeBytes = (uint64_t)-1) = 0; + virtual IoBuffer GetBuildBlob(const Oid& BuildId, + const IoHash& RawHash, + uint64_t RangeOffset = 0, + uint64_t RangeBytes = (uint64_t)-1) = 0; + + static constexpr size_t MaxRangeCountPerRequest = 128u; + + struct BuildBlobRanges + { + IoBuffer PayloadBuffer; + std::vector> Ranges; + }; + virtual BuildBlobRanges GetBuildBlobRanges(const Oid& BuildId, + const IoHash& RawHash, + std::span> Ranges) = 0; virtual std::vector> GetLargeBuildBlob(const Oid& BuildId, const IoHash& RawHash, uint64_t ChunkSize, std::function&& OnReceive, - std::function&& OnComplete) = 0; + std::function&& OnComplete) = 0; [[nodiscard]] virtual bool PutBlockMetadata(const Oid& BuildId, const IoHash& BlockRawHash, const CbObject& MetaData) = 0; virtual CbObject FindBlocks(const Oid& BuildId, uint64_t MaxBlockCount) = 0; diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h index f25ce5b5e..67c93480b 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h @@ -37,6 +37,14 @@ public: const IoHash& RawHash, uint64_t RangeOffset = 0, uint64_t RangeBytes = (uint64_t)-1) = 0; + struct BuildBlobRanges + { + IoBuffer PayloadBuffer; + std::vector> Ranges; + }; + virtual BuildBlobRanges GetBuildBlobRanges(const Oid& BuildId, + const IoHash& RawHash, + std::span> Ranges) = 0; virtual void PutBlobMetadatas(const Oid& BuildId, std::span BlobHashes, std::span MetaDatas) = 0; virtual std::vector GetBlobMetadatas(const Oid& BuildId, std::span BlobHashes) = 0; diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h index 31733569e..875b8593b 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h @@ -263,9 +263,14 @@ private: ParallelWork& Work, std::function&& OnDownloaded); - void DownloadPartialBlock(const ChunkBlockAnalyser::BlockRangeDescriptor BlockRange, - const BlobsExistsResult& ExistsResult, - std::function&& OnDownloaded); + void DownloadPartialBlock(std::span BlockRanges, + size_t BlockRangeIndex, + size_t BlockRangeCount, + const BlobsExistsResult& ExistsResult, + std::function> OffsetAndLengths)>&& OnDownloaded); std::vector WriteLocalChunkToCache(CloneQueryInterface* CloneQuery, const CopyChunkData& CopyData, diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h index 4b85d8f1e..764a24e61 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h @@ -45,7 +45,6 @@ std::vector GetBlockDescriptions(OperationLogOutput& Out BuildStorageBase& Storage, BuildStorageCache* OptionalCacheStorage, const Oid& BuildId, - const Oid& BuildPartId, std::span BlockRawHashes, bool AttemptFallback, bool IsQuiet, diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h index 5a17ef79c..7aae1442e 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h @@ -82,12 +82,14 @@ class ChunkBlockAnalyser public: struct Options { - bool IsQuiet = false; - bool IsVerbose = false; - double HostLatencySec = -1.0; - double HostHighSpeedLatencySec = -1.0; - uint64_t HostSpeedBytesPerSec = (1u * 1024u * 1024u * 1024u) / 8u; // 1GBit - uint64_t HostHighSpeedBytesPerSec = (2u * 1024u * 1024u * 1024u) / 8u; // 2GBit + bool IsQuiet = false; + bool IsVerbose = false; + double HostLatencySec = -1.0; + double HostHighSpeedLatencySec = -1.0; + uint64_t HostSpeedBytesPerSec = (1u * 1024u * 1024u * 1024u) / 8u; // 1GBit + uint64_t HostHighSpeedBytesPerSec = (2u * 1024u * 1024u * 1024u) / 8u; // 2GBit + uint64_t HostMaxRangeCountPerRequest = (uint64_t)-1; + uint64_t HostHighSpeedMaxRangeCountPerRequest = (uint64_t)-1; // No limit }; ChunkBlockAnalyser(OperationLogOutput& LogOutput, std::span BlockDescriptions, const Options& Options); @@ -137,14 +139,15 @@ private: static constexpr uint16_t FullBlockRangePercentLimit = 98; - static constexpr BlockRangeLimit ForceMergeLimits[] = {{.SizePercent = FullBlockRangePercentLimit, .MaxRangeCount = 1}, - {.SizePercent = 90, .MaxRangeCount = 4}, - {.SizePercent = 85, .MaxRangeCount = 16}, - {.SizePercent = 80, .MaxRangeCount = 32}, - {.SizePercent = 75, .MaxRangeCount = 48}, - {.SizePercent = 70, .MaxRangeCount = 64}, - {.SizePercent = 4, .MaxRangeCount = 82}, - {.SizePercent = 0, .MaxRangeCount = 96}}; + static constexpr BlockRangeLimit ForceMergeLimits[] = {{.SizePercent = FullBlockRangePercentLimit, .MaxRangeCount = 8}, + {.SizePercent = 90, .MaxRangeCount = 16}, + {.SizePercent = 85, .MaxRangeCount = 32}, + {.SizePercent = 80, .MaxRangeCount = 48}, + {.SizePercent = 75, .MaxRangeCount = 64}, + {.SizePercent = 70, .MaxRangeCount = 92}, + {.SizePercent = 50, .MaxRangeCount = 128}, + {.SizePercent = 4, .MaxRangeCount = 192}, + {.SizePercent = 0, .MaxRangeCount = 256}}; BlockRangeDescriptor MergeBlockRanges(std::span Ranges); std::optional> MakeOptionalBlockRangeVector(uint64_t TotalBlockSize, diff --git a/src/zenremotestore/include/zenremotestore/jupiter/jupitersession.h b/src/zenremotestore/include/zenremotestore/jupiter/jupitersession.h index eaf6962fd..8721bc37f 100644 --- a/src/zenremotestore/include/zenremotestore/jupiter/jupitersession.h +++ b/src/zenremotestore/include/zenremotestore/jupiter/jupitersession.h @@ -56,6 +56,11 @@ struct FinalizeBuildPartResult : JupiterResult std::vector Needs; }; +struct BuildBlobRangesResult : JupiterResult +{ + std::vector> Ranges; +}; + /** * Context for performing Jupiter operations * @@ -135,6 +140,13 @@ public: uint64_t Offset = 0, uint64_t Size = (uint64_t)-1); + BuildBlobRangesResult GetBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const IoHash& Hash, + std::filesystem::path TempFolderPath, + std::span> Ranges); + JupiterResult PutMultipartBuildBlob(std::string_view Namespace, std::string_view BucketId, const Oid& BuildId, diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h index 152c02ee2..2cf10c664 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h @@ -84,6 +84,12 @@ public: std::vector HasBody; }; + struct LoadAttachmentRangesResult : public Result + { + IoBuffer Bytes; + std::vector> Ranges; + }; + struct RemoteStoreInfo { bool CreateBlocks; @@ -127,15 +133,21 @@ public: virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) = 0; virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) = 0; - struct AttachmentRange + enum ESourceMode { - uint64_t Offset = 0; - uint64_t Bytes = (uint64_t)-1; - - inline operator bool() const { return Offset != 0 || Bytes != (uint64_t)-1; } + kAny, + kCacheOnly, + kHostOnly }; - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) = 0; - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) = 0; + + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode = ESourceMode::kAny) = 0; + + static constexpr size_t MaxRangeCountPerRequest = 128u; + + virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, + std::span> Ranges, + ESourceMode SourceMode = ESourceMode::kAny) = 0; + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode = ESourceMode::kAny) = 0; virtual void Flush() = 0; }; diff --git a/src/zenremotestore/jupiter/jupitersession.cpp b/src/zenremotestore/jupiter/jupitersession.cpp index 1bc6564ce..52f9eb678 100644 --- a/src/zenremotestore/jupiter/jupitersession.cpp +++ b/src/zenremotestore/jupiter/jupitersession.cpp @@ -852,6 +852,71 @@ JupiterSession::GetBuildBlob(std::string_view Namespace, return detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv); } +BuildBlobRangesResult +JupiterSession::GetBuildBlob(std::string_view Namespace, + std::string_view BucketId, + const Oid& BuildId, + const IoHash& Hash, + std::filesystem::path TempFolderPath, + std::span> Ranges) +{ + HttpClient::KeyValueMap Headers; + if (!Ranges.empty()) + { + ExtendableStringBuilder<512> SB; + for (const std::pair& R : Ranges) + { + if (SB.Size() > 0) + { + SB << ", "; + } + SB << R.first << "-" << R.first + R.second - 1; + } + Headers.Entries.insert({"Range", fmt::format("bytes={}", SB.ToView())}); + } + std::string Url = fmt::format("/api/v2/builds/{}/{}/{}/blobs/{}?supportsRedirect={}", + Namespace, + BucketId, + BuildId, + Hash.ToHexString(), + m_AllowRedirect ? "true"sv : "false"sv); + + HttpClient::Response Response = m_HttpClient.Download(Url, TempFolderPath, Headers); + if (Response.StatusCode == HttpResponseCode::RangeNotSatisfiable && Ranges.size() > 1) + { + // Requests to Jupiter that is not served via nginx (content not stored locally in the file system) can not serve multi-range + // requests (asp.net limitation) This rejection is not implemented as of 2026-03-02, it is in the backlog (@joakim.lindqvist) + // If we encounter this error we fall back to a single range which covers all the requested ranges + uint64_t RangeStart = Ranges.front().first; + uint64_t RangeEnd = Ranges.back().first + Ranges.back().second - 1; + Headers.Entries.insert_or_assign("Range", fmt::format("bytes={}-{}", RangeStart, RangeEnd)); + Response = m_HttpClient.Download(Url, TempFolderPath, Headers); + } + if (Response.IsSuccess()) + { + // If we get a redirect to S3 or a non-Jupiter endpoint the content type will not be correct, validate it and set it + if (m_AllowRedirect && (Response.ResponsePayload.GetContentType() == HttpContentType::kBinary)) + { + IoHash ValidateRawHash; + uint64_t ValidateRawSize = 0; + if (!Headers.Entries.contains("Range")) + { + ZEN_ASSERT_SLOW(CompressedBuffer::ValidateCompressedHeader(Response.ResponsePayload, + ValidateRawHash, + ValidateRawSize, + /*OutOptionalTotalCompressedSize*/ nullptr)); + ZEN_ASSERT_SLOW(ValidateRawHash == Hash); + ZEN_ASSERT_SLOW(ValidateRawSize > 0); + ZEN_UNUSED(ValidateRawHash, ValidateRawSize); + Response.ResponsePayload.SetContentType(ZenContentType::kCompressedBinary); + } + } + } + BuildBlobRangesResult Result = {detail::ConvertResponse(Response, "JupiterSession::GetBuildBlob"sv)}; + Result.Ranges = Response.GetRanges(Ranges); + return Result; +} + JupiterResult JupiterSession::PutBlockMetadata(std::string_view Namespace, std::string_view BucketId, diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp index c42373e4d..3400cdbf5 100644 --- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp @@ -478,7 +478,6 @@ public: *m_BuildStorage, m_BuildCacheStorage.get(), m_BuildId, - m_OplogBuildPartId, BlockHashes, /*AttemptFallback*/ false, /*IsQuiet*/ false, @@ -549,7 +548,7 @@ public: return Result; } - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) override + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); @@ -559,25 +558,90 @@ public: try { - if (m_BuildCacheStorage) + if (m_BuildCacheStorage && SourceMode != ESourceMode::kHostOnly) { - IoBuffer CachedBlob = m_BuildCacheStorage->GetBuildBlob(m_BuildId, RawHash, Range.Offset, Range.Bytes); + IoBuffer CachedBlob = m_BuildCacheStorage->GetBuildBlob(m_BuildId, RawHash); if (CachedBlob) { Result.Bytes = std::move(CachedBlob); } } - if (!Result.Bytes) + if (!Result.Bytes && SourceMode != ESourceMode::kCacheOnly) { - Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash, Range.Offset, Range.Bytes); + Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash); if (m_BuildCacheStorage && Result.Bytes && m_PopulateCache) { - if (!Range) + m_BuildCacheStorage->PutBuildBlob(m_BuildId, + RawHash, + Result.Bytes.GetContentType(), + CompositeBuffer(SharedBuffer(Result.Bytes))); + } + } + } + catch (const HttpClientError& Ex) + { + Result.ErrorCode = MakeErrorCode(Ex); + Result.Reason = fmt::format("Failed getting blob {}/{}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + RawHash, + Ex.what()); + } + catch (const std::exception& Ex) + { + Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("Failed getting blob {}/{}/{}/{}/{}. Reason: '{}'", + m_BuildStorageHttp.GetBaseUri(), + m_Namespace, + m_Bucket, + m_BuildId, + RawHash, + Ex.what()); + } + + return Result; + } + + virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, + std::span> Ranges, + ESourceMode SourceMode) override + { + LoadAttachmentRangesResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; }); + + try + { + if (m_BuildCacheStorage && SourceMode != ESourceMode::kHostOnly) + { + BuildStorageCache::BuildBlobRanges BlobRanges = m_BuildCacheStorage->GetBuildBlobRanges(m_BuildId, RawHash, Ranges); + if (BlobRanges.PayloadBuffer) + { + Result.Bytes = std::move(BlobRanges.PayloadBuffer); + Result.Ranges = std::move(BlobRanges.Ranges); + } + } + if (!Result.Bytes && SourceMode != ESourceMode::kCacheOnly) + { + BuildStorageBase::BuildBlobRanges BlobRanges = m_BuildStorage->GetBuildBlobRanges(m_BuildId, RawHash, Ranges); + if (BlobRanges.PayloadBuffer) + { + Result.Bytes = std::move(BlobRanges.PayloadBuffer); + Result.Ranges = std::move(BlobRanges.Ranges); + + if (Result.Ranges.empty()) { - m_BuildCacheStorage->PutBuildBlob(m_BuildId, - RawHash, - Result.Bytes.GetContentType(), - CompositeBuffer(SharedBuffer(Result.Bytes))); + // Jupiter will ignore the ranges and send the whole payload if it fetches the payload from S3/Replicated + // Upload to cache (if enabled) + if (m_BuildCacheStorage && Result.Bytes && m_PopulateCache) + { + m_BuildCacheStorage->PutBuildBlob(m_BuildId, + RawHash, + Result.Bytes.GetContentType(), + CompositeBuffer(SharedBuffer(Result.Bytes))); + } } } } @@ -585,28 +649,32 @@ public: catch (const HttpClientError& Ex) { Result.ErrorCode = MakeErrorCode(Ex); - Result.Reason = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'", + Result.Reason = fmt::format("Failed getting {} ranges for blob {}/{}/{}/{}/{}. Reason: '{}'", + Ranges.size(), m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, + RawHash, Ex.what()); } catch (const std::exception& Ex) { Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("Failed listing known blocks for {}/{}/{}/{}. Reason: '{}'", + Result.Reason = fmt::format("Failed getting {} ranges for blob {}/{}/{}/{}/{}. Reason: '{}'", + Ranges.size(), m_BuildStorageHttp.GetBaseUri(), m_Namespace, m_Bucket, m_BuildId, + RawHash, Ex.what()); } return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) override + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode) override { LoadAttachmentsResult Result; Stopwatch Timer; @@ -614,7 +682,7 @@ public: std::vector AttachmentsLeftToFind = RawHashes; - if (m_BuildCacheStorage) + if (m_BuildCacheStorage && SourceMode != ESourceMode::kHostOnly) { std::vector ExistCheck = m_BuildCacheStorage->BlobsExists(m_BuildId, RawHashes); if (ExistCheck.size() == RawHashes.size()) @@ -648,7 +716,7 @@ public: for (const IoHash& Hash : AttachmentsLeftToFind) { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash, {}); + LoadAttachmentResult ChunkResult = LoadAttachment(Hash, SourceMode); if (ChunkResult.ErrorCode) { return LoadAttachmentsResult{ChunkResult}; diff --git a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp index ec7fb7bbc..f950fd46c 100644 --- a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp @@ -228,28 +228,62 @@ public: return AttachmentExistsInCacheResult{Result{.ErrorCode = 0}, std::vector(RawHashes.size(), false)}; } - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) override + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode) override { - Stopwatch Timer; - LoadAttachmentResult Result; - std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!IsFile(ChunkPath)) + Stopwatch Timer; + LoadAttachmentResult Result; + if (SourceMode != ESourceMode::kCacheOnly) { - Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); - Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; - return Result; + std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); + if (!IsFile(ChunkPath)) + { + Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); + Result.Reason = + fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); + Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; + return Result; + } + { + BasicFile ChunkFile; + ChunkFile.Open(ChunkPath, BasicFile::Mode::kRead); + Result.Bytes = ChunkFile.ReadAll(); + } } + AddStats(0, Result.Bytes.GetSize(), Timer.GetElapsedTimeUs() * 1000); + Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; + return Result; + } + + virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, + std::span> Ranges, + ESourceMode SourceMode) override + { + Stopwatch Timer; + LoadAttachmentRangesResult Result; + if (SourceMode != ESourceMode::kCacheOnly) { - BasicFile ChunkFile; - ChunkFile.Open(ChunkPath, BasicFile::Mode::kRead); - if (Range) + std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); + if (!IsFile(ChunkPath)) { - Result.Bytes = ChunkFile.ReadRange(Range.Offset, Range.Bytes); + Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); + Result.Reason = + fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); + Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; + return Result; } - else { - Result.Bytes = ChunkFile.ReadAll(); + BasicFile ChunkFile; + ChunkFile.Open(ChunkPath, BasicFile::Mode::kRead); + + uint64_t Start = Ranges.front().first; + uint64_t Length = Ranges.back().first + Ranges.back().second - Ranges.front().first; + + Result.Bytes = ChunkFile.ReadRange(Start, Length); + Result.Ranges.reserve(Ranges.size()); + for (const std::pair& Range : Ranges) + { + Result.Ranges.push_back(std::make_pair(Range.first - Start, Range.second)); + } } } AddStats(0, Result.Bytes.GetSize(), Timer.GetElapsedTimeUs() * 1000); @@ -257,13 +291,13 @@ public: return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) override + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode) override { Stopwatch Timer; LoadAttachmentsResult Result; for (const IoHash& Hash : RawHashes) { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash, {}); + LoadAttachmentResult ChunkResult = LoadAttachment(Hash, SourceMode); if (ChunkResult.ErrorCode) { ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; diff --git a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp index f8179831c..514484f30 100644 --- a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp @@ -223,34 +223,62 @@ public: return AttachmentExistsInCacheResult{Result{.ErrorCode = 0}, std::vector(RawHashes.size(), false)}; } - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) override + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode) override { - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); - AddStats(GetResult); - - LoadAttachmentResult Result{ConvertResult(GetResult), std::move(GetResult.Response)}; - if (GetResult.ErrorCode) + LoadAttachmentResult Result; + if (SourceMode != ESourceMode::kCacheOnly) { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - RawHash, - Result.Reason); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); + JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); + AddStats(GetResult); + + Result = {ConvertResult(GetResult), std::move(GetResult.Response)}; + if (GetResult.ErrorCode) + { + Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'", + m_JupiterClient->ServiceUrl(), + m_Namespace, + RawHash, + Result.Reason); + } } - if (!Result.ErrorCode && Range) + return Result; + } + + virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, + std::span> Ranges, + ESourceMode SourceMode) override + { + LoadAttachmentRangesResult Result; + if (SourceMode != ESourceMode::kCacheOnly) { - Result.Bytes = IoBuffer(Result.Bytes, Range.Offset, Range.Bytes); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); + JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); + AddStats(GetResult); + + Result = LoadAttachmentRangesResult{ConvertResult(GetResult), std::move(GetResult.Response)}; + if (GetResult.ErrorCode) + { + Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'", + m_JupiterClient->ServiceUrl(), + m_Namespace, + RawHash, + Result.Reason); + } + else + { + Result.Ranges = std::vector>(Ranges.begin(), Ranges.end()); + } } return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) override + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode) override { LoadAttachmentsResult Result; for (const IoHash& Hash : RawHashes) { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash, {}); + LoadAttachmentResult ChunkResult = LoadAttachment(Hash, SourceMode); if (ChunkResult.ErrorCode) { return LoadAttachmentsResult{ChunkResult}; diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index 2a9da6f58..1882f599a 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -339,9 +339,10 @@ namespace remotestore_impl { uint64_t ChunkSize = It.second.GetCompressedSize(); Info.AttachmentBytesDownloaded.fetch_add(ChunkSize); } - ZEN_INFO("Loaded {} bulk attachments in {}", - Chunks.size(), - NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000))); + remotestore_impl::ReportMessage(OptionalContext, + fmt::format("Loaded {} bulk attachments in {}", + Chunks.size(), + NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000)))); if (RemoteResult.IsError()) { return; @@ -446,7 +447,7 @@ namespace remotestore_impl { { uint64_t Unset = (std::uint64_t)-1; DownloadStartMS.compare_exchange_strong(Unset, LoadAttachmentsTimer.GetElapsedTimeMs()); - RemoteProjectStore::LoadAttachmentResult BlockResult = RemoteStore.LoadAttachment(BlockHash, {}); + RemoteProjectStore::LoadAttachmentResult BlockResult = RemoteStore.LoadAttachment(BlockHash); if (BlockResult.ErrorCode) { ReportMessage(OptionalContext, @@ -506,50 +507,100 @@ namespace remotestore_impl { IoHash RawHash; uint64_t RawSize; CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(Bytes), RawHash, RawSize); + + std::string ErrorString; + if (!Compressed) { - if (RetriesLeft > 0) + ErrorString = + fmt::format("Block attachment {} is malformed, can't parse as compressed binary", BlockHash); + } + else if (RawHash != BlockHash) + { + ErrorString = fmt::format("Block attachment {} has mismatching raw hash ({})", BlockHash, RawHash); + } + else if (CompositeBuffer BlockPayload = Compressed.DecompressToComposite(); !BlockPayload) + { + ErrorString = fmt::format("Block attachment {} is malformed, can't decompress payload", BlockHash); + } + else + { + uint64_t PotentialSize = 0; + uint64_t UsedSize = 0; + uint64_t BlockSize = BlockPayload.GetSize(); + + uint64_t BlockHeaderSize = 0; + + bool StoreChunksOK = IterateChunkBlock( + BlockPayload.Flatten(), + [&AllNeededPartialChunkHashesLookup, + &ChunkDownloadedFlags, + &WriteAttachmentBuffers, + &WriteRawHashes, + &Info, + &PotentialSize](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) { + auto ChunkIndexIt = AllNeededPartialChunkHashesLookup.find(AttachmentRawHash); + if (ChunkIndexIt != AllNeededPartialChunkHashesLookup.end()) + { + bool Expected = false; + if (ChunkDownloadedFlags[ChunkIndexIt->second].compare_exchange_strong(Expected, true)) + { + WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); + IoHash RawHash; + uint64_t RawSize; + ZEN_ASSERT(CompressedBuffer::ValidateCompressedHeader( + WriteAttachmentBuffers.back(), + RawHash, + RawSize, + /*OutOptionalTotalCompressedSize*/ nullptr)); + ZEN_ASSERT(RawHash == AttachmentRawHash); + WriteRawHashes.emplace_back(AttachmentRawHash); + PotentialSize += WriteAttachmentBuffers.back().GetSize(); + } + } + }, + BlockHeaderSize); + + if (!StoreChunksOK) { - ReportMessage( - OptionalContext, - fmt::format( - "Block attachment {} is malformed, can't parse as compressed binary, retrying download", - BlockHash)); - return DownloadAndSaveBlock(ChunkStore, - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, - NetworkWorkerPool, - WorkerPool, - AttachmentsDownloadLatch, - AttachmentsWriteLatch, - RemoteResult, - Info, - LoadAttachmentsTimer, - DownloadStartMS, - BlockHash, - AllNeededPartialChunkHashesLookup, - ChunkDownloadedFlags, - RetriesLeft - 1); + ErrorString = fmt::format("Invalid format for block {}", BlockHash); + } + else + { + if (!WriteAttachmentBuffers.empty()) + { + std::vector Results = + ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (size_t Index = 0; Index < Results.size(); Index++) + { + const CidStore::InsertResult& Result = Results[Index]; + if (Result.New) + { + Info.AttachmentBytesStored.fetch_add(WriteAttachmentBuffers[Index].GetSize()); + Info.AttachmentsStored.fetch_add(1); + UsedSize += WriteAttachmentBuffers[Index].GetSize(); + } + } + if (UsedSize < BlockSize) + { + ZEN_DEBUG("Used {} (skipping {}) out of {} for block {} ({} %) (use of matching {}%)", + NiceBytes(UsedSize), + NiceBytes(BlockSize - UsedSize), + NiceBytes(BlockSize), + BlockHash, + (100 * UsedSize) / BlockSize, + PotentialSize > 0 ? (UsedSize * 100) / PotentialSize : 0); + } + } } - ReportMessage( - OptionalContext, - fmt::format("Block attachment {} is malformed, can't parse as compressed binary", BlockHash)); - RemoteResult.SetError( - gsl::narrow(HttpResponseCode::InternalServerError), - fmt::format("Block attachment {} is malformed, can't parse as compressed binary", BlockHash), - {}); - return; } - CompositeBuffer BlockPayload = Compressed.DecompressToComposite(); - if (!BlockPayload) + + if (!ErrorString.empty()) { if (RetriesLeft > 0) { - ReportMessage( - OptionalContext, - fmt::format("Block attachment {} is malformed, can't decompress payload, retrying download", - BlockHash)); + ReportMessage(OptionalContext, fmt::format("{}, retrying download", ErrorString)); + return DownloadAndSaveBlock(ChunkStore, RemoteStore, IgnoreMissingAttachments, @@ -567,94 +618,12 @@ namespace remotestore_impl { ChunkDownloadedFlags, RetriesLeft - 1); } - ReportMessage(OptionalContext, - fmt::format("Block attachment {} is malformed, can't decompress payload", BlockHash)); - RemoteResult.SetError( - gsl::narrow(HttpResponseCode::InternalServerError), - fmt::format("Block attachment {} is malformed, can't decompress payload", BlockHash), - {}); - return; - } - if (RawHash != BlockHash) - { - ReportMessage(OptionalContext, - fmt::format("Block attachment {} has mismatching raw hash ({})", BlockHash, RawHash)); - RemoteResult.SetError( - gsl::narrow(HttpResponseCode::InternalServerError), - fmt::format("Block attachment {} has mismatching raw hash ({})", BlockHash, RawHash), - {}); - return; - } - - uint64_t PotentialSize = 0; - uint64_t UsedSize = 0; - uint64_t BlockSize = BlockPayload.GetSize(); - - uint64_t BlockHeaderSize = 0; - - bool StoreChunksOK = IterateChunkBlock( - BlockPayload.Flatten(), - [&AllNeededPartialChunkHashesLookup, - &ChunkDownloadedFlags, - &WriteAttachmentBuffers, - &WriteRawHashes, - &Info, - &PotentialSize](CompressedBuffer&& Chunk, const IoHash& AttachmentRawHash) { - auto ChunkIndexIt = AllNeededPartialChunkHashesLookup.find(AttachmentRawHash); - if (ChunkIndexIt != AllNeededPartialChunkHashesLookup.end()) - { - bool Expected = false; - if (ChunkDownloadedFlags[ChunkIndexIt->second].compare_exchange_strong(Expected, true)) - { - WriteAttachmentBuffers.emplace_back(Chunk.GetCompressed().Flatten().AsIoBuffer()); - IoHash RawHash; - uint64_t RawSize; - ZEN_ASSERT( - CompressedBuffer::ValidateCompressedHeader(WriteAttachmentBuffers.back(), - RawHash, - RawSize, - /*OutOptionalTotalCompressedSize*/ nullptr)); - ZEN_ASSERT(RawHash == AttachmentRawHash); - WriteRawHashes.emplace_back(AttachmentRawHash); - PotentialSize += WriteAttachmentBuffers.back().GetSize(); - } - } - }, - BlockHeaderSize); - - if (!StoreChunksOK) - { - ReportMessage(OptionalContext, - fmt::format("Block attachment {} has invalid format ({}): {}", - BlockHash, - RemoteResult.GetError(), - RemoteResult.GetErrorReason())); - RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), - fmt::format("Invalid format for block {}", BlockHash), - {}); - return; - } - - if (!WriteAttachmentBuffers.empty()) - { - auto Results = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); - for (size_t Index = 0; Index < Results.size(); Index++) + else { - const auto& Result = Results[Index]; - if (Result.New) - { - Info.AttachmentBytesStored.fetch_add(WriteAttachmentBuffers[Index].GetSize()); - Info.AttachmentsStored.fetch_add(1); - UsedSize += WriteAttachmentBuffers[Index].GetSize(); - } + ReportMessage(OptionalContext, ErrorString); + RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), ErrorString, {}); + return; } - ZEN_DEBUG("Used {} (matching {}) out of {} for block {} ({} %) (use of matching {}%)", - NiceBytes(UsedSize), - NiceBytes(PotentialSize), - NiceBytes(BlockSize), - BlockHash, - (100 * UsedSize) / BlockSize, - PotentialSize > 0 ? (UsedSize * 100) / PotentialSize : 0); } } catch (const std::exception& Ex) @@ -676,6 +645,119 @@ namespace remotestore_impl { WorkerThreadPool::EMode::EnableBacklog); }; + bool DownloadPartialBlock(RemoteProjectStore& RemoteStore, + bool IgnoreMissingAttachments, + JobContext* OptionalContext, + AsyncRemoteResult& RemoteResult, + DownloadInfo& Info, + double& DownloadTimeSeconds, + const ChunkBlockDescription& BlockDescription, + bool BlockExistsInCache, + std::span BlockRangeDescriptors, + size_t BlockRangeIndexStart, + size_t BlockRangeCount, + std::function> OffsetAndLengths)>&& OnDownloaded) + { + std::vector> Ranges; + Ranges.reserve(BlockRangeDescriptors.size()); + for (size_t BlockRangeIndex = BlockRangeIndexStart; BlockRangeIndex < BlockRangeIndexStart + BlockRangeCount; BlockRangeIndex++) + { + const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = BlockRangeDescriptors[BlockRangeIndex]; + Ranges.push_back(std::make_pair(BlockRange.RangeStart, BlockRange.RangeLength)); + } + + if (BlockExistsInCache) + { + RemoteProjectStore::LoadAttachmentRangesResult BlockResult = + RemoteStore.LoadAttachmentRanges(BlockDescription.BlockHash, Ranges, RemoteProjectStore::ESourceMode::kCacheOnly); + DownloadTimeSeconds += BlockResult.ElapsedSeconds; + if (RemoteResult.IsError()) + { + return false; + } + if (!BlockResult.ErrorCode && BlockResult.Bytes) + { + if (BlockResult.Ranges.size() != Ranges.size()) + { + throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges", + Ranges.size(), + BlockDescription.BlockHash, + BlockResult.Ranges.size())); + } + OnDownloaded(std::move(BlockResult.Bytes), BlockRangeIndexStart, BlockResult.Ranges); + return true; + } + } + + const size_t MaxRangesPerRequestToJupiter = RemoteProjectStore::MaxRangeCountPerRequest; + + size_t SubBlockRangeCount = BlockRangeCount; + size_t SubRangeCountComplete = 0; + std::span> RangesSpan(Ranges); + while (SubRangeCountComplete < SubBlockRangeCount) + { + if (RemoteResult.IsError()) + { + break; + } + size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, MaxRangesPerRequestToJupiter); + size_t SubRangeStartIndex = BlockRangeIndexStart + SubRangeCountComplete; + + auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); + + RemoteProjectStore::LoadAttachmentRangesResult BlockResult = + RemoteStore.LoadAttachmentRanges(BlockDescription.BlockHash, SubRanges, RemoteProjectStore::ESourceMode::kHostOnly); + DownloadTimeSeconds += BlockResult.ElapsedSeconds; + if (RemoteResult.IsError()) + { + return false; + } + if (BlockResult.ErrorCode || !BlockResult.Bytes) + { + ReportMessage(OptionalContext, + fmt::format("Failed to download {} ranges from block attachment '{}' ({}): {}", + SubRanges.size(), + BlockDescription.BlockHash, + BlockResult.ErrorCode, + BlockResult.Reason)); + Info.MissingAttachmentCount.fetch_add(1); + if (!IgnoreMissingAttachments) + { + RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text); + return false; + } + } + else + { + if (BlockResult.Ranges.empty()) + { + // Jupiter will ignore the ranges and send the whole payload if it fetches the payload from S3 + // Use the whole payload for the remaining ranges + SubRangeCount = Ranges.size() - SubRangeCountComplete; + OnDownloaded(std::move(BlockResult.Bytes), + SubRangeStartIndex, + RangesSpan.subspan(SubRangeCountComplete, SubRangeCount)); + } + else + { + if (BlockResult.Ranges.size() != SubRanges.size()) + { + throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges", + SubRanges.size(), + BlockDescription.BlockHash, + BlockResult.Ranges.size())); + } + OnDownloaded(std::move(BlockResult.Bytes), SubRangeStartIndex, BlockResult.Ranges); + } + } + + SubRangeCountComplete += SubRangeCount; + } + return true; + } + void DownloadAndSavePartialBlock(CidStore& ChunkStore, RemoteProjectStore& RemoteStore, bool IgnoreMissingAttachments, @@ -689,6 +771,7 @@ namespace remotestore_impl { Stopwatch& LoadAttachmentsTimer, std::atomic_uint64_t& DownloadStartMS, const ChunkBlockDescription& BlockDescription, + bool BlockExistsInCache, std::span BlockRangeDescriptors, size_t BlockRangeIndexStart, size_t BlockRangeCount, @@ -710,13 +793,14 @@ namespace remotestore_impl { &DownloadStartMS, IgnoreMissingAttachments, OptionalContext, - RetriesLeft, BlockDescription, + BlockExistsInCache, BlockRangeDescriptors, BlockRangeIndexStart, BlockRangeCount, &AllNeededPartialChunkHashesLookup, - ChunkDownloadedFlags]() { + ChunkDownloadedFlags, + RetriesLeft]() { ZEN_TRACE_CPU("DownloadBlockRanges"); auto _ = MakeGuard([&AttachmentsDownloadLatch] { AttachmentsDownloadLatch.CountDown(); }); @@ -728,230 +812,240 @@ namespace remotestore_impl { double DownloadElapsedSeconds = 0; uint64_t DownloadedBytes = 0; - for (size_t BlockRangeIndex = BlockRangeIndexStart; BlockRangeIndex < BlockRangeIndexStart + BlockRangeCount; - BlockRangeIndex++) - { - if (RemoteResult.IsError()) - { - return; - } - - const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = BlockRangeDescriptors[BlockRangeIndex]; + bool Success = DownloadPartialBlock( + RemoteStore, + IgnoreMissingAttachments, + OptionalContext, + RemoteResult, + Info, + DownloadElapsedSeconds, + BlockDescription, + BlockExistsInCache, + BlockRangeDescriptors, + BlockRangeIndexStart, + BlockRangeCount, + [&](IoBuffer&& Buffer, + size_t BlockRangeStartIndex, + std::span> OffsetAndLengths) { + uint64_t BlockPartSize = Buffer.GetSize(); + DownloadedBytes += BlockPartSize; + + Info.AttachmentBlockRangeBytesDownloaded.fetch_add(BlockPartSize); + Info.AttachmentBlocksRangesDownloaded++; + + AttachmentsWriteLatch.AddCount(1); + WorkerPool.ScheduleWork( + [&AttachmentsWriteLatch, + &ChunkStore, + &RemoteStore, + &NetworkWorkerPool, + &WorkerPool, + &AttachmentsDownloadLatch, + &RemoteResult, + &Info, + &LoadAttachmentsTimer, + &DownloadStartMS, + IgnoreMissingAttachments, + OptionalContext, + BlockDescription, + BlockExistsInCache, + BlockRangeDescriptors, + BlockRangeStartIndex, + &AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags, + RetriesLeft, + BlockPayload = std::move(Buffer), + OffsetAndLengths = + std::vector>(OffsetAndLengths.begin(), OffsetAndLengths.end())]() { + auto _ = MakeGuard([&AttachmentsWriteLatch] { AttachmentsWriteLatch.CountDown(); }); + try + { + ZEN_ASSERT(BlockPayload.Size() > 0); - RemoteProjectStore::LoadAttachmentResult BlockResult = - RemoteStore.LoadAttachment(BlockDescription.BlockHash, - {.Offset = BlockRange.RangeStart, .Bytes = BlockRange.RangeLength}); - if (BlockResult.ErrorCode) - { - ReportMessage(OptionalContext, - fmt::format("Failed to download block attachment '{}' range {},{} ({}): {}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength, - BlockResult.ErrorCode, - BlockResult.Reason)); - Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) - { - RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text); - } - return; - } - if (RemoteResult.IsError()) - { - return; - } - uint64_t BlockPartSize = BlockResult.Bytes.GetSize(); - if (BlockPartSize != BlockRange.RangeLength) - { - std::string ErrorString = - fmt::format("Failed to download block attachment '{}' range {},{}, got {} bytes ({}): {}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength, - BlockPartSize, - RemoteResult.GetError(), - RemoteResult.GetErrorReason()); - - ReportMessage(OptionalContext, ErrorString); - Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) - { - RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), - "Mismatching block part range received", - ErrorString); - } - return; - } - Info.AttachmentBlocksRangesDownloaded.fetch_add(1); + size_t RangeCount = OffsetAndLengths.size(); + for (size_t RangeOffset = 0; RangeOffset < RangeCount; RangeOffset++) + { + if (RemoteResult.IsError()) + { + return; + } - DownloadElapsedSeconds += BlockResult.ElapsedSeconds; - DownloadedBytes += BlockPartSize; + const ChunkBlockAnalyser::BlockRangeDescriptor& BlockRange = + BlockRangeDescriptors[BlockRangeStartIndex + RangeOffset]; + const std::pair& OffsetAndLength = OffsetAndLengths[RangeOffset]; + IoBuffer BlockRangeBuffer(BlockPayload, OffsetAndLength.first, OffsetAndLength.second); - Info.AttachmentBlockRangeBytesDownloaded.fetch_add(BlockPartSize); + std::vector WriteAttachmentBuffers; + std::vector WriteRawHashes; - AttachmentsWriteLatch.AddCount(1); - WorkerPool.ScheduleWork( - [&AttachmentsDownloadLatch, - &AttachmentsWriteLatch, - &ChunkStore, - &RemoteStore, - &NetworkWorkerPool, - &WorkerPool, - &RemoteResult, - &Info, - &LoadAttachmentsTimer, - &DownloadStartMS, - IgnoreMissingAttachments, - OptionalContext, - RetriesLeft, - BlockDescription, - BlockRange, - &AllNeededPartialChunkHashesLookup, - ChunkDownloadedFlags, - BlockPayload = std::move(BlockResult.Bytes)]() { - auto _ = MakeGuard([&AttachmentsWriteLatch] { AttachmentsWriteLatch.CountDown(); }); - if (RemoteResult.IsError()) - { - return; - } - try - { - ZEN_ASSERT(BlockPayload.Size() > 0); - std::vector WriteAttachmentBuffers; - std::vector WriteRawHashes; + uint64_t PotentialSize = 0; + uint64_t UsedSize = 0; + uint64_t BlockPartSize = BlockRangeBuffer.GetSize(); - uint64_t PotentialSize = 0; - uint64_t UsedSize = 0; - uint64_t BlockPartSize = BlockPayload.GetSize(); + uint32_t OffsetInBlock = 0; + for (uint32_t ChunkBlockIndex = BlockRange.ChunkBlockIndexStart; + ChunkBlockIndex < BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount; + ChunkBlockIndex++) + { + if (RemoteResult.IsError()) + { + break; + } - uint32_t OffsetInBlock = 0; - for (uint32_t ChunkBlockIndex = BlockRange.ChunkBlockIndexStart; - ChunkBlockIndex < BlockRange.ChunkBlockIndexStart + BlockRange.ChunkBlockIndexCount; - ChunkBlockIndex++) - { - const uint32_t ChunkCompressedSize = BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; - const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; + const uint32_t ChunkCompressedSize = + BlockDescription.ChunkCompressedLengths[ChunkBlockIndex]; + const IoHash& ChunkHash = BlockDescription.ChunkRawHashes[ChunkBlockIndex]; - if (auto ChunkIndexIt = AllNeededPartialChunkHashesLookup.find(ChunkHash); - ChunkIndexIt != AllNeededPartialChunkHashesLookup.end()) - { - bool Expected = false; - if (ChunkDownloadedFlags[ChunkIndexIt->second].compare_exchange_strong(Expected, true)) - { - IoHash VerifyChunkHash; - uint64_t VerifyChunkSize; - CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed( - SharedBuffer(IoBuffer(BlockPayload, OffsetInBlock, ChunkCompressedSize)), - VerifyChunkHash, - VerifyChunkSize); - if (!CompressedChunk) + if (auto ChunkIndexIt = AllNeededPartialChunkHashesLookup.find(ChunkHash); + ChunkIndexIt != AllNeededPartialChunkHashesLookup.end()) { - std::string ErrorString = fmt::format( - "Chunk at {},{} in block attachment '{}' is not a valid compressed buffer", - OffsetInBlock, - ChunkCompressedSize, - BlockDescription.BlockHash); - ReportMessage(OptionalContext, ErrorString); - Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) + if (!ChunkDownloadedFlags[ChunkIndexIt->second]) { - RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), - "Malformed chunk block", - ErrorString); + IoHash VerifyChunkHash; + uint64_t VerifyChunkSize; + CompressedBuffer CompressedChunk = CompressedBuffer::FromCompressed( + SharedBuffer(IoBuffer(BlockRangeBuffer, OffsetInBlock, ChunkCompressedSize)), + VerifyChunkHash, + VerifyChunkSize); + + std::string ErrorString; + + if (!CompressedChunk) + { + ErrorString = fmt::format( + "Chunk at {},{} in block attachment '{}' is not a valid compressed buffer", + OffsetInBlock, + ChunkCompressedSize, + BlockDescription.BlockHash); + } + else if (VerifyChunkHash != ChunkHash) + { + ErrorString = fmt::format( + "Chunk at {},{} in block attachment '{}' has mismatching hash, expected " + "{}, got {}", + OffsetInBlock, + ChunkCompressedSize, + BlockDescription.BlockHash, + ChunkHash, + VerifyChunkHash); + } + else if (VerifyChunkSize != BlockDescription.ChunkRawLengths[ChunkBlockIndex]) + { + ErrorString = fmt::format( + "Chunk at {},{} in block attachment '{}' has mismatching raw size, " + "expected {}, " + "got {}", + OffsetInBlock, + ChunkCompressedSize, + BlockDescription.BlockHash, + BlockDescription.ChunkRawLengths[ChunkBlockIndex], + VerifyChunkSize); + } + + if (!ErrorString.empty()) + { + if (RetriesLeft > 0) + { + ReportMessage(OptionalContext, + fmt::format("{}, retrying download", ErrorString)); + return DownloadAndSavePartialBlock(ChunkStore, + RemoteStore, + IgnoreMissingAttachments, + OptionalContext, + NetworkWorkerPool, + WorkerPool, + AttachmentsDownloadLatch, + AttachmentsWriteLatch, + RemoteResult, + Info, + LoadAttachmentsTimer, + DownloadStartMS, + BlockDescription, + BlockExistsInCache, + BlockRangeDescriptors, + BlockRangeStartIndex, + RangeCount, + AllNeededPartialChunkHashesLookup, + ChunkDownloadedFlags, + RetriesLeft - 1); + } + + ReportMessage(OptionalContext, ErrorString); + Info.MissingAttachmentCount.fetch_add(1); + if (!IgnoreMissingAttachments) + { + RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), + "Malformed chunk block", + ErrorString); + } + } + else + { + bool Expected = false; + if (ChunkDownloadedFlags[ChunkIndexIt->second].compare_exchange_strong(Expected, + true)) + { + WriteAttachmentBuffers.emplace_back( + CompressedChunk.GetCompressed().Flatten().AsIoBuffer()); + WriteRawHashes.emplace_back(ChunkHash); + PotentialSize += WriteAttachmentBuffers.back().GetSize(); + } + } } - continue; } - if (VerifyChunkHash != ChunkHash) + OffsetInBlock += ChunkCompressedSize; + } + + if (!WriteAttachmentBuffers.empty()) + { + std::vector Results = + ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + for (size_t Index = 0; Index < Results.size(); Index++) { - std::string ErrorString = fmt::format( - "Chunk at {},{} in block attachment '{}' has mismatching hash, expected {}, got {}", - OffsetInBlock, - ChunkCompressedSize, - BlockDescription.BlockHash, - ChunkHash, - VerifyChunkHash); - ReportMessage(OptionalContext, ErrorString); - Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) + const CidStore::InsertResult& Result = Results[Index]; + if (Result.New) { - RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), - "Malformed chunk block", - ErrorString); + Info.AttachmentBytesStored.fetch_add(WriteAttachmentBuffers[Index].GetSize()); + Info.AttachmentsStored.fetch_add(1); + UsedSize += WriteAttachmentBuffers[Index].GetSize(); } - continue; } - if (VerifyChunkSize != BlockDescription.ChunkRawLengths[ChunkBlockIndex]) + if (UsedSize < BlockPartSize) { - std::string ErrorString = fmt::format( - "Chunk at {},{} in block attachment '{}' has mismatching raw size, expected {}, " - "got {}", - OffsetInBlock, - ChunkCompressedSize, + ZEN_DEBUG( + "Used {} (skipping {}) out of {} for block {} range {}, {} ({} %) (use of matching " + "{}%)", + NiceBytes(UsedSize), + NiceBytes(BlockPartSize - UsedSize), + NiceBytes(BlockPartSize), BlockDescription.BlockHash, - BlockDescription.ChunkRawLengths[ChunkBlockIndex], - VerifyChunkSize); - ReportMessage(OptionalContext, ErrorString); - Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) - { - RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), - "Malformed chunk block", - ErrorString); - } - continue; + BlockRange.RangeStart, + BlockRange.RangeLength, + (100 * UsedSize) / BlockPartSize, + PotentialSize > 0 ? (UsedSize * 100) / PotentialSize : 0); } - - WriteAttachmentBuffers.emplace_back(CompressedChunk.GetCompressed().Flatten().AsIoBuffer()); - WriteRawHashes.emplace_back(ChunkHash); - PotentialSize += WriteAttachmentBuffers.back().GetSize(); } } - OffsetInBlock += ChunkCompressedSize; } - - if (!WriteAttachmentBuffers.empty()) + catch (const std::exception& Ex) { - auto Results = ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); - for (size_t Index = 0; Index < Results.size(); Index++) - { - const auto& Result = Results[Index]; - if (Result.New) - { - Info.AttachmentBytesStored.fetch_add(WriteAttachmentBuffers[Index].GetSize()); - Info.AttachmentsStored.fetch_add(1); - UsedSize += WriteAttachmentBuffers[Index].GetSize(); - } - } - ZEN_DEBUG("Used {} (matching {}) out of {} for block {} range {}, {} ({} %) (use of matching {}%)", - NiceBytes(UsedSize), - NiceBytes(PotentialSize), - NiceBytes(BlockPartSize), - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength, - (100 * UsedSize) / BlockPartSize, - PotentialSize > 0 ? (UsedSize * 100) / PotentialSize : 0); + RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), + fmt::format("Failed saving {} ranges from block attachment {}", + OffsetAndLengths.size(), + BlockDescription.BlockHash), + Ex.what()); } - } - catch (const std::exception& Ex) - { - RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), - fmt::format("Failed save block attachment {} range {}, {}", - BlockDescription.BlockHash, - BlockRange.RangeStart, - BlockRange.RangeLength), - Ex.what()); - } - }, - WorkerThreadPool::EMode::EnableBacklog); + }, + WorkerThreadPool::EMode::EnableBacklog); + }); + if (Success) + { + ZEN_DEBUG("Loaded {} ranges from block attachment '{}' in {} ({})", + BlockRangeCount, + BlockDescription.BlockHash, + NiceTimeSpanMs(static_cast(DownloadElapsedSeconds * 1000)), + NiceBytes(DownloadedBytes)); } - - ZEN_DEBUG("Loaded {} ranges from block attachment '{}' in {} ({})", - BlockRangeCount, - BlockDescription.BlockHash, - NiceTimeSpanMs(static_cast(DownloadElapsedSeconds * 1000)), - NiceBytes(DownloadedBytes)); } catch (const std::exception& Ex) { @@ -1002,7 +1096,7 @@ namespace remotestore_impl { { uint64_t Unset = (std::uint64_t)-1; DownloadStartMS.compare_exchange_strong(Unset, LoadAttachmentsTimer.GetElapsedTimeMs()); - RemoteProjectStore::LoadAttachmentResult AttachmentResult = RemoteStore.LoadAttachment(RawHash, {}); + RemoteProjectStore::LoadAttachmentResult AttachmentResult = RemoteStore.LoadAttachment(RawHash); if (AttachmentResult.ErrorCode) { ReportMessage(OptionalContext, @@ -3115,6 +3209,12 @@ ParseOplogContainer( std::unordered_set NeededAttachments; { CbArrayView OpsArray = OutOplogSection["ops"sv].AsArrayView(); + + size_t OpCount = OpsArray.Num(); + size_t OpsCompleteCount = 0; + + remotestore_impl::ReportMessage(OptionalContext, fmt::format("Scanning {} ops for attachments", OpCount)); + for (CbFieldView OpEntry : OpsArray) { OpEntry.IterateAttachments([&](CbFieldView FieldView) { NeededAttachments.insert(FieldView.AsAttachment()); }); @@ -3124,6 +3224,16 @@ ParseOplogContainer( .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0, .Reason = "Operation cancelled"}; } + OpsCompleteCount++; + if ((OpsCompleteCount & 4095) == 0) + { + remotestore_impl::ReportProgress( + OptionalContext, + "Scanning oplog"sv, + fmt::format("{} attachments found, {} ops remaining...", NeededAttachments.size(), OpCount - OpsCompleteCount), + OpCount, + OpCount - OpsCompleteCount); + } } } { @@ -3151,13 +3261,27 @@ ParseOplogContainer( { ChunkedInfo Chunked = ReadChunkedInfo(ChunkedFileView); + size_t NeededChunkAttachmentCount = 0; + OnReferencedAttachments(Chunked.ChunkHashes); - NeededAttachments.insert(Chunked.ChunkHashes.begin(), Chunked.ChunkHashes.end()); + for (const IoHash& ChunkHash : Chunked.ChunkHashes) + { + if (!HasAttachment(ChunkHash)) + { + if (NeededAttachments.insert(ChunkHash).second) + { + NeededChunkAttachmentCount++; + } + } + } OnChunkedAttachment(Chunked); - ZEN_INFO("Requesting chunked attachment '{}' ({}) built from {} chunks", - Chunked.RawHash, - NiceBytes(Chunked.RawSize), - Chunked.ChunkHashes.size()); + + remotestore_impl::ReportMessage(OptionalContext, + fmt::format("Requesting chunked attachment '{}' ({}) built from {} chunks, need {} chunks", + Chunked.RawHash, + NiceBytes(Chunked.RawSize), + Chunked.ChunkHashes.size(), + NeededChunkAttachmentCount)); } } if (remotestore_impl::IsCancelled(OptionalContext)) @@ -3490,8 +3614,16 @@ LoadOplog(CidStore& ChunkStore, std::vector DownloadedViaLegacyChunkFlag(AllNeededChunkHashes.size(), false); ChunkBlockAnalyser::BlockResult PartialBlocksResult; + remotestore_impl::ReportMessage(OptionalContext, fmt::format("Fetching descriptions for {} blocks", BlockHashes.size())); + RemoteProjectStore::GetBlockDescriptionsResult BlockDescriptions = RemoteStore.GetBlockDescriptions(BlockHashes); - std::vector BlocksWithDescription; + + remotestore_impl::ReportMessage(OptionalContext, + fmt::format("GetBlockDescriptions took {}. Found {} blocks", + NiceTimeSpanMs(uint64_t(BlockDescriptions.ElapsedSeconds * 1000)), + BlockDescriptions.Blocks.size())); + + std::vector BlocksWithDescription; BlocksWithDescription.reserve(BlockDescriptions.Blocks.size()); for (const ChunkBlockDescription& BlockDescription : BlockDescriptions.Blocks) { @@ -3547,6 +3679,7 @@ LoadOplog(CidStore& ChunkStore, if (!AllNeededChunkHashes.empty()) { std::vector PartialBlockDownloadModes; + std::vector BlockExistsInCache; if (PartialBlockRequestMode == EPartialBlockRequestMode::Off) { @@ -3558,21 +3691,25 @@ LoadOplog(CidStore& ChunkStore, RemoteStore.AttachmentExistsInCache(BlocksWithDescription); if (CacheExistsResult.ErrorCode != 0 || CacheExistsResult.HasBody.size() != BlocksWithDescription.size()) { - CacheExistsResult.HasBody.resize(BlocksWithDescription.size(), false); + BlockExistsInCache.resize(BlocksWithDescription.size(), false); + } + else + { + BlockExistsInCache = std::move(CacheExistsResult.HasBody); } PartialBlockDownloadModes.reserve(BlocksWithDescription.size()); - for (bool ExistsInCache : CacheExistsResult.HasBody) + for (bool ExistsInCache : BlockExistsInCache) { if (PartialBlockRequestMode == EPartialBlockRequestMode::All) { - PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::Exact : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange); } else if (PartialBlockRequestMode == EPartialBlockRequestMode::ZenCacheOnly) { - PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::Exact : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); } else if (PartialBlockRequestMode == EPartialBlockRequestMode::Mixed) @@ -3584,13 +3721,14 @@ LoadOplog(CidStore& ChunkStore, } ZEN_ASSERT(PartialBlockDownloadModes.size() == BlocksWithDescription.size()); - - ChunkBlockAnalyser PartialAnalyser(*LogOutput, - BlockDescriptions.Blocks, - ChunkBlockAnalyser::Options{.IsQuiet = false, - .IsVerbose = false, - .HostLatencySec = HostLatencySec, - .HostHighSpeedLatencySec = CacheLatencySec}); + ChunkBlockAnalyser PartialAnalyser( + *LogOutput, + BlockDescriptions.Blocks, + ChunkBlockAnalyser::Options{.IsQuiet = false, + .IsVerbose = false, + .HostLatencySec = HostLatencySec, + .HostHighSpeedLatencySec = CacheLatencySec, + .HostMaxRangeCountPerRequest = RemoteProjectStore::MaxRangeCountPerRequest}); std::vector NeededBlocks = PartialAnalyser.GetNeeded(AllNeededPartialChunkHashesLookup, @@ -3641,12 +3779,13 @@ LoadOplog(CidStore& ChunkStore, LoadAttachmentsTimer, DownloadStartMS, BlockDescriptions.Blocks[CurrentBlockRange.BlockIndex], + BlockExistsInCache[CurrentBlockRange.BlockIndex], PartialBlocksResult.BlockRanges, BlockRangeIndex, RangeCount, AllNeededPartialChunkHashesLookup, ChunkDownloadedFlags, - 3); + /* RetriesLeft*/ 3); BlockRangeIndex += RangeCount; } @@ -3668,12 +3807,23 @@ LoadOplog(CidStore& ChunkStore, { PartialTransferWallTimeMS += LoadAttachmentsTimer.GetElapsedTimeMs() - DownloadStartMS.load(); } - remotestore_impl::ReportProgress( - OptionalContext, - "Loading attachments"sv, - fmt::format("{} remaining. {}", Remaining, remotestore_impl::GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)), - AttachmentCount.load(), - Remaining); + + uint64_t AttachmentsDownloaded = + Info.AttachmentBlocksDownloaded.load() + Info.AttachmentBlocksRangesDownloaded.load() + Info.AttachmentsDownloaded.load(); + uint64_t AttachmentBytesDownloaded = Info.AttachmentBlockBytesDownloaded.load() + Info.AttachmentBlockRangeBytesDownloaded.load() + + Info.AttachmentBytesDownloaded.load(); + + remotestore_impl::ReportProgress(OptionalContext, + "Loading attachments"sv, + fmt::format("{} ({}) downloaded, {} ({}) stored, {} remaining. {}", + AttachmentsDownloaded, + NiceBytes(AttachmentBytesDownloaded), + Info.AttachmentsStored.load(), + NiceBytes(Info.AttachmentBytesStored.load()), + Remaining, + remotestore_impl::GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)), + AttachmentCount.load(), + Remaining); } if (DownloadStartMS != (uint64_t)-1) { @@ -3700,11 +3850,12 @@ LoadOplog(CidStore& ChunkStore, RemoteResult.SetError(gsl::narrow(HttpResponseCode::OK), "Operation cancelled", ""); } } - remotestore_impl::ReportProgress(OptionalContext, - "Writing attachments"sv, - fmt::format("{} remaining.", Remaining), - AttachmentCount.load(), - Remaining); + remotestore_impl::ReportProgress( + OptionalContext, + "Writing attachments"sv, + fmt::format("{} ({}), {} remaining.", Info.AttachmentsStored.load(), NiceBytes(Info.AttachmentBytesStored.load()), Remaining), + AttachmentCount.load(), + Remaining); } if (AttachmentCount.load() > 0) @@ -3867,18 +4018,20 @@ LoadOplog(CidStore& ChunkStore, TmpFile.Close(); TmpBuffer = IoBufferBuilder::MakeFromTemporaryFile(TempFileName); } + uint64_t TmpBufferSize = TmpBuffer.GetSize(); CidStore::InsertResult InsertResult = ChunkStore.AddChunk(TmpBuffer, Chunked.RawHash, CidStore::InsertMode::kMayBeMovedInPlace); if (InsertResult.New) { - Info.AttachmentBytesStored.fetch_add(TmpBuffer.GetSize()); + Info.AttachmentBytesStored.fetch_add(TmpBufferSize); Info.AttachmentsStored.fetch_add(1); } - ZEN_INFO("Dechunked attachment {} ({}) in {}", - Chunked.RawHash, - NiceBytes(Chunked.RawSize), - NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + remotestore_impl::ReportMessage(OptionalContext, + fmt::format("Dechunked attachment {} ({}) in {}", + Chunked.RawHash, + NiceBytes(Chunked.RawSize), + NiceTimeSpanMs(Timer.GetElapsedTimeMs()))); } catch (const std::exception& Ex) { diff --git a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp index b4c1156ac..ef82c45e0 100644 --- a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp @@ -157,55 +157,59 @@ public: return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) override + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode) override { - std::string LoadRequest = fmt::format("/{}/oplog/{}/rpc"sv, m_Project, m_Oplog); - - CbObject Request; + LoadAttachmentsResult Result; + if (SourceMode != ESourceMode::kCacheOnly) { - CbObjectWriter RequestWriter; - RequestWriter.AddString("method"sv, "getchunks"sv); - RequestWriter.BeginObject("Request"sv); + std::string LoadRequest = fmt::format("/{}/oplog/{}/rpc"sv, m_Project, m_Oplog); + + CbObject Request; { - RequestWriter.BeginArray("Chunks"sv); + CbObjectWriter RequestWriter; + RequestWriter.AddString("method"sv, "getchunks"sv); + RequestWriter.BeginObject("Request"sv); { - for (const IoHash& RawHash : RawHashes) + RequestWriter.BeginArray("Chunks"sv); { - RequestWriter.BeginObject(); + for (const IoHash& RawHash : RawHashes) { - RequestWriter.AddHash("RawHash", RawHash); + RequestWriter.BeginObject(); + { + RequestWriter.AddHash("RawHash", RawHash); + } + RequestWriter.EndObject(); } - RequestWriter.EndObject(); } + RequestWriter.EndArray(); // "chunks" } - RequestWriter.EndArray(); // "chunks" + RequestWriter.EndObject(); + Request = RequestWriter.Save(); } - RequestWriter.EndObject(); - Request = RequestWriter.Save(); - } - HttpClient::Response Response = m_Client.Post(LoadRequest, Request, HttpClient::Accept(ZenContentType::kCbPackage)); - AddStats(Response); + HttpClient::Response Response = m_Client.Post(LoadRequest, Request, HttpClient::Accept(ZenContentType::kCbPackage)); + AddStats(Response); - LoadAttachmentsResult Result = LoadAttachmentsResult{ConvertResult(Response)}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching {} oplog attachments from {}/{}/{}. Reason: '{}'", - RawHashes.size(), - m_ProjectStoreUrl, - m_Project, - m_Oplog, - Result.Reason); - } - else - { - CbPackage Package = Response.AsPackage(); - std::span Attachments = Package.GetAttachments(); - Result.Chunks.reserve(Attachments.size()); - for (const CbAttachment& Attachment : Attachments) + Result = LoadAttachmentsResult{ConvertResult(Response)}; + if (Result.ErrorCode) { - Result.Chunks.emplace_back( - std::pair{Attachment.GetHash(), Attachment.AsCompressedBinary().MakeOwned()}); + Result.Reason = fmt::format("Failed fetching {} oplog attachments from {}/{}/{}. Reason: '{}'", + RawHashes.size(), + m_ProjectStoreUrl, + m_Project, + m_Oplog, + Result.Reason); + } + else + { + CbPackage Package = Response.AsPackage(); + std::span Attachments = Package.GetAttachments(); + Result.Chunks.reserve(Attachments.size()); + for (const CbAttachment& Attachment : Attachments) + { + Result.Chunks.emplace_back( + std::pair{Attachment.GetHash(), Attachment.AsCompressedBinary().MakeOwned()}); + } } } return Result; @@ -260,32 +264,59 @@ public: return AttachmentExistsInCacheResult{Result{.ErrorCode = 0}, std::vector(RawHashes.size(), false)}; } - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, const AttachmentRange& Range) override + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode) override { - std::string LoadRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash); - HttpClient::Response Response = - m_Client.Download(LoadRequest, m_TempFilePath, HttpClient::Accept(ZenContentType::kCompressedBinary)); - AddStats(Response); - - LoadAttachmentResult Result = LoadAttachmentResult{ConvertResult(Response)}; - if (Result.ErrorCode) + LoadAttachmentResult Result; + if (SourceMode != ESourceMode::kCacheOnly) { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'", - m_ProjectStoreUrl, - m_Project, - m_Oplog, - RawHash, - Result.Reason); - } - if (!Result.ErrorCode && Range) - { - Result.Bytes = IoBuffer(Response.ResponsePayload, Range.Offset, Range.Bytes); + std::string LoadRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash); + HttpClient::Response Response = + m_Client.Download(LoadRequest, m_TempFilePath, HttpClient::Accept(ZenContentType::kCompressedBinary)); + AddStats(Response); + + Result = LoadAttachmentResult{ConvertResult(Response)}; + if (Result.ErrorCode) + { + Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'", + m_ProjectStoreUrl, + m_Project, + m_Oplog, + RawHash, + Result.Reason); + } + Result.Bytes = Response.ResponsePayload; + Result.Bytes.MakeOwned(); } - else + return Result; + } + + virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, + std::span> Ranges, + ESourceMode SourceMode) override + { + LoadAttachmentRangesResult Result; + if (SourceMode != ESourceMode::kCacheOnly) { - Result.Bytes = Response.ResponsePayload; + std::string LoadRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash); + HttpClient::Response Response = + m_Client.Download(LoadRequest, m_TempFilePath, HttpClient::Accept(ZenContentType::kCompressedBinary)); + AddStats(Response); + + Result = LoadAttachmentRangesResult{ConvertResult(Response)}; + if (Result.ErrorCode) + { + Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'", + m_ProjectStoreUrl, + m_Project, + m_Oplog, + RawHash, + Result.Reason); + } + else + { + Result.Ranges = std::vector>(Ranges.begin(), Ranges.end()); + } } - Result.Bytes.MakeOwned(); return Result; } diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index 6ada085a5..459e044eb 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -233,16 +233,21 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) { const uint64_t MaxBlobSize = Range.first < BlobSize ? BlobSize - Range.first : 0; const uint64_t RangeSize = Min(Range.second, MaxBlobSize); - if (Range.first + RangeSize <= BlobSize) + Writer.BeginObject(); { - RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize)); - Writer.BeginObject(); + if (Range.first + RangeSize <= BlobSize) { + RangeBuffers.push_back(IoBuffer(Blob, Range.first, RangeSize)); Writer.AddInteger("offset"sv, Range.first); Writer.AddInteger("length"sv, RangeSize); } - Writer.EndObject(); + else + { + Writer.AddInteger("offset"sv, Range.first); + Writer.AddInteger("length"sv, 0); + } } + Writer.EndObject(); } Writer.EndArray(); @@ -262,7 +267,16 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) } else { - ZEN_ASSERT(OffsetAndLengthPairs.size() == 1); + if (OffsetAndLengthPairs.size() != 1) + { + // Only a single http range is supported, we have limited support for http multirange responses + m_BuildStoreStats.BadRequestCount++; + return ServerRequest.WriteResponse( + HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Multiple ranges in blob request is only supported for {} accept type", ToString(HttpContentType::kCbPackage))); + } + const std::pair& OffsetAndLength = OffsetAndLengthPairs.front(); const uint64_t BlobSize = Blob.GetSize(); const uint64_t MaxBlobSize = OffsetAndLength.first < BlobSize ? BlobSize - OffsetAndLength.first : 0; -- cgit v1.2.3 From b67dac7c093cc82b7e8f12f9eb57bfa34dfe26d8 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 08:35:32 +0100 Subject: unity build fixes (#802) Various fixes to make cpp files build in unity build mode as an aside using Unity build doesn't really seem to work on Linux, unsure why but it leads to link-time issues --- src/zen/cmds/builds_cmd.cpp | 9 +++++---- src/zen/cmds/projectstore_cmd.cpp | 18 ++++++++++++++---- src/zen/cmds/wipe_cmd.cpp | 6 ++++-- src/zencore/include/zencore/compactbinaryfile.h | 1 + src/zencore/include/zencore/meta.h | 1 + src/zencore/include/zencore/varint.h | 1 + src/zencore/md5.cpp | 19 ++++++++++++++++++- src/zencore/xmake.lua | 1 + src/zenhttp/include/zenhttp/httpapiservice.h | 1 + src/zenremotestore/chunking/chunkblock.cpp | 8 ++++---- .../projectstore/remoteprojectstore.cpp | 6 +++--- src/zenserver/storage/storageconfig.h | 1 + src/zenstore/include/zenstore/buildstore/buildstore.h | 2 +- 13 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 5254ef3cf..ffdc5fe48 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -67,13 +67,11 @@ ZEN_THIRD_PARTY_INCLUDES_END static const bool DoExtraContentVerify = false; -#define ZEN_CLOUD_STORAGE "Cloud Storage" - namespace zen { using namespace std::literals; -namespace { +namespace builds_impl { static std::atomic AbortFlag = false; static std::atomic PauseFlag = false; @@ -270,6 +268,7 @@ namespace { static bool IsQuiet = false; static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; +#undef ZEN_CONSOLE_VERBOSE #define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ if (IsVerbose) \ { \ @@ -2009,12 +2008,13 @@ namespace { ProgressBar::SetLogOperationProgress(ProgressMode, TaskSteps::Cleanup, TaskSteps::StepCount); } -} // namespace +} // namespace builds_impl ////////////////////////////////////////////////////////////////////////////////////////////////////// BuildsCommand::BuildsCommand() { + using namespace builds_impl; m_Options.add_options()("h,help", "Print help"); auto AddSystemOptions = [this](cxxopts::Options& Ops) { @@ -2655,6 +2655,7 @@ BuildsCommand::~BuildsCommand() = default; void BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace builds_impl; ZEN_UNUSED(GlobalOptions); signal(SIGINT, SignalCallbackHandler); diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index bedab3cfd..dfc6c1650 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -41,12 +41,10 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { -namespace { +namespace projectstore_impl { using namespace std::literals; -#define ZEN_CLOUD_STORAGE "Cloud Storage" - void WriteAuthOptions(CbObjectWriter& Writer, std::string_view JupiterOpenIdProvider, std::string_view JupiterAccessToken, @@ -500,7 +498,7 @@ namespace { return {}; } -} // namespace +} // namespace projectstore_impl /////////////////////////////////////// @@ -522,6 +520,7 @@ DropProjectCommand::~DropProjectCommand() void DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -611,6 +610,7 @@ ProjectInfoCommand::~ProjectInfoCommand() void ProjectInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -697,6 +697,7 @@ CreateProjectCommand::~CreateProjectCommand() = default; void CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); using namespace std::literals; @@ -766,6 +767,7 @@ CreateOplogCommand::~CreateOplogCommand() = default; void CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); using namespace std::literals; @@ -989,6 +991,7 @@ ExportOplogCommand::~ExportOplogCommand() void ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; using namespace std::literals; ZEN_UNUSED(GlobalOptions); @@ -1495,6 +1498,7 @@ ImportOplogCommand::~ImportOplogCommand() void ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; using namespace std::literals; ZEN_UNUSED(GlobalOptions); @@ -1788,6 +1792,7 @@ SnapshotOplogCommand::~SnapshotOplogCommand() void SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; using namespace std::literals; ZEN_UNUSED(GlobalOptions); @@ -1852,6 +1857,7 @@ ProjectStatsCommand::~ProjectStatsCommand() void ProjectStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -1904,6 +1910,7 @@ ProjectOpDetailsCommand::~ProjectOpDetailsCommand() void ProjectOpDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -2019,6 +2026,7 @@ OplogMirrorCommand::~OplogMirrorCommand() void OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -2286,6 +2294,7 @@ OplogValidateCommand::~OplogValidateCommand() void OplogValidateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) @@ -2437,6 +2446,7 @@ OplogDownloadCommand::~OplogDownloadCommand() void OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace projectstore_impl; ZEN_UNUSED(GlobalOptions); if (!ParseOptions(argc, argv)) diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index a5029e1c5..fd9e28a80 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -33,7 +33,7 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { -namespace { +namespace wipe_impl { static std::atomic AbortFlag = false; static std::atomic PauseFlag = false; static bool IsVerbose = false; @@ -49,6 +49,7 @@ namespace { : GetMediumWorkerPool(EWorkloadType::Burst); } +#undef ZEN_CONSOLE_VERBOSE #define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ if (IsVerbose) \ { \ @@ -505,7 +506,7 @@ namespace { } return CleanWipe; } -} // namespace +} // namespace wipe_impl WipeCommand::WipeCommand() { @@ -532,6 +533,7 @@ WipeCommand::~WipeCommand() = default; void WipeCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { + using namespace wipe_impl; ZEN_UNUSED(GlobalOptions); signal(SIGINT, SignalCallbackHandler); diff --git a/src/zencore/include/zencore/compactbinaryfile.h b/src/zencore/include/zencore/compactbinaryfile.h index 00c37e941..33f3e7bea 100644 --- a/src/zencore/include/zencore/compactbinaryfile.h +++ b/src/zencore/include/zencore/compactbinaryfile.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include #include diff --git a/src/zencore/include/zencore/meta.h b/src/zencore/include/zencore/meta.h index 82eb5cc30..20ec4ac6f 100644 --- a/src/zencore/include/zencore/meta.h +++ b/src/zencore/include/zencore/meta.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once /* This file contains utility functions for meta programming * diff --git a/src/zencore/include/zencore/varint.h b/src/zencore/include/zencore/varint.h index 9fe905f25..43ca14d38 100644 --- a/src/zencore/include/zencore/varint.h +++ b/src/zencore/include/zencore/varint.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include "intmath.h" diff --git a/src/zencore/md5.cpp b/src/zencore/md5.cpp index 3baee91c2..83ed53fc8 100644 --- a/src/zencore/md5.cpp +++ b/src/zencore/md5.cpp @@ -342,6 +342,23 @@ Transform(uint32_t* buf, uint32_t* in) #undef G #undef H #undef I +#undef ROTATE_LEFT +#undef S11 +#undef S12 +#undef S13 +#undef S14 +#undef S21 +#undef S22 +#undef S23 +#undef S24 +#undef S31 +#undef S32 +#undef S33 +#undef S34 +#undef S41 +#undef S42 +#undef S43 +#undef S44 namespace zen { @@ -391,7 +408,7 @@ MD5::FromHexString(const char* string) { MD5 md5; - ParseHexBytes(string, 40, md5.Hash); + ParseHexBytes(string, 2 * sizeof md5.Hash, md5.Hash); return md5; } diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index 9a67175a0..2f81b7ec8 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -15,6 +15,7 @@ target('zencore') set_configdir("include/zencore") add_files("**.cpp") add_files("trace.cpp", {unity_ignored = true }) + add_files("testing.cpp", {unity_ignored = true }) if has_config("zenrpmalloc") then add_deps("rpmalloc") diff --git a/src/zenhttp/include/zenhttp/httpapiservice.h b/src/zenhttp/include/zenhttp/httpapiservice.h index 0270973bf..2d384d1d8 100644 --- a/src/zenhttp/include/zenhttp/httpapiservice.h +++ b/src/zenhttp/include/zenhttp/httpapiservice.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include diff --git a/src/zenremotestore/chunking/chunkblock.cpp b/src/zenremotestore/chunking/chunkblock.cpp index 9c3fe8a0b..f80bfc2ba 100644 --- a/src/zenremotestore/chunking/chunkblock.cpp +++ b/src/zenremotestore/chunking/chunkblock.cpp @@ -1037,7 +1037,7 @@ ChunkBlockAnalyser::CalculateBlockRanges(uint32_t BlockIndex, #if ZEN_WITH_TESTS -namespace testutils { +namespace chunkblock_testutils { static std::vector> CreateAttachments( const std::span& Sizes, OodleCompressionLevel CompressionLevel = OodleCompressionLevel::VeryFast, @@ -1054,14 +1054,14 @@ namespace testutils { return Result; } -} // namespace testutils +} // namespace chunkblock_testutils TEST_SUITE_BEGIN("remotestore.chunkblock"); TEST_CASE("chunkblock.block") { using namespace std::literals; - using namespace testutils; + using namespace chunkblock_testutils; std::vector AttachmentSizes({7633, 6825, 5738, 8031, 7225, 566, 3656, 6006, 24, 3466, 1093, 4269, 2257, 3685, 3489, 7194, 6151, 5482, 6217, 3511, 6738, 5061, 7537, 2759, 1916, 8210, 2235, 4024, 1582, 5251, @@ -1089,7 +1089,7 @@ TEST_CASE("chunkblock.block") TEST_CASE("chunkblock.reuseblocks") { using namespace std::literals; - using namespace testutils; + using namespace chunkblock_testutils; std::vector> BlockAttachmentSizes( {std::vector{7633, 6825, 5738, 8031, 7225, 566, 3656, 6006, 24, 3466, 1093, 4269, 2257, 3685, 3489, diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index 1882f599a..570025b6d 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -4186,7 +4186,7 @@ RemoteProjectStore::~RemoteProjectStore() #if ZEN_WITH_TESTS -namespace testutils { +namespace projectstore_testutils { using namespace std::literals; static std::string OidAsString(const Oid& Id) @@ -4238,7 +4238,7 @@ namespace testutils { return Result; } -} // namespace testutils +} // namespace projectstore_testutils struct ExportForceDisableBlocksTrue_ForceTempBlocksFalse { @@ -4265,7 +4265,7 @@ TEST_CASE_TEMPLATE("project.store.export", ExportForceDisableBlocksFalse_ForceTempBlocksTrue) { using namespace std::literals; - using namespace testutils; + using namespace projectstore_testutils; ScopedTemporaryDirectory TempDir; ScopedTemporaryDirectory ExportDir; diff --git a/src/zenserver/storage/storageconfig.h b/src/zenserver/storage/storageconfig.h index b408b0c26..6124cae14 100644 --- a/src/zenserver/storage/storageconfig.h +++ b/src/zenserver/storage/storageconfig.h @@ -1,4 +1,5 @@ // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include "config/config.h" diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h index 76cba05b9..bfc83ba0d 100644 --- a/src/zenstore/include/zenstore/buildstore/buildstore.h +++ b/src/zenstore/include/zenstore/buildstore/buildstore.h @@ -1,5 +1,5 @@ - // Copyright Epic Games, Inc. All Rights Reserved. +#pragma once #include -- cgit v1.2.3 From eafd4d78378c1a642445ed127fdbe51ac559d4e3 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 09:40:49 +0100 Subject: HTTP improvements (#803) - Add GetTotalBytesReceived/GetTotalBytesSent to HttpServer with implementations in ASIO and http.sys backends - Add ExpectedErrorCodes to HttpClientSettings to suppress warn/info logs for anticipated HTTP error codes - Also fixes minor issues in `CprHttpClient::Download` --- src/zencore/process.cpp | 14 ++++++++++++ src/zencore/windows.cpp | 12 +++++----- src/zenhttp/clients/httpclientcpr.cpp | 38 +++++++++++++++++++++++--------- src/zenhttp/clients/httpclientcpr.h | 1 + src/zenhttp/include/zenhttp/httpclient.h | 5 +++++ src/zenhttp/include/zenhttp/httpserver.h | 4 ++++ src/zenhttp/servers/httpasio.cpp | 21 ++++++++++++++++++ src/zenhttp/servers/httpsys.cpp | 23 ++++++++++++++++++- 8 files changed, 99 insertions(+), 19 deletions(-) diff --git a/src/zencore/process.cpp b/src/zencore/process.cpp index 226a94050..f657869dc 100644 --- a/src/zencore/process.cpp +++ b/src/zencore/process.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include @@ -745,6 +746,8 @@ CreateProcElevated(const std::filesystem::path& Executable, std::string_view Com CreateProcResult CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine, const CreateProcOptions& Options) { + ZEN_TRACE_CPU("CreateProc"); + #if ZEN_PLATFORM_WINDOWS if (Options.Flags & CreateProcOptions::Flag_Unelevated) { @@ -776,6 +779,17 @@ CreateProc(const std::filesystem::path& Executable, std::string_view CommandLine ZEN_UNUSED(Result); } + if (!Options.StdoutFile.empty()) + { + int Fd = open(Options.StdoutFile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (Fd >= 0) + { + dup2(Fd, STDOUT_FILENO); + dup2(Fd, STDERR_FILENO); + close(Fd); + } + } + if (execv(Executable.c_str(), ArgV.data()) < 0) { ThrowLastError("Failed to exec() a new process image"); diff --git a/src/zencore/windows.cpp b/src/zencore/windows.cpp index d02fcd35e..87f854b90 100644 --- a/src/zencore/windows.cpp +++ b/src/zencore/windows.cpp @@ -12,14 +12,12 @@ namespace zen::windows { bool IsRunningOnWine() { - HMODULE NtDll = GetModuleHandleA("ntdll.dll"); + static bool s_Result = [] { + HMODULE NtDll = GetModuleHandleA("ntdll.dll"); + return NtDll && !!GetProcAddress(NtDll, "wine_get_version"); + }(); - if (NtDll) - { - return !!GetProcAddress(NtDll, "wine_get_version"); - } - - return false; + return s_Result; } FileMapping::FileMapping(_In_ FileMapping& orig) diff --git a/src/zenhttp/clients/httpclientcpr.cpp b/src/zenhttp/clients/httpclientcpr.cpp index 90dcfacbb..14e40b02a 100644 --- a/src/zenhttp/clients/httpclientcpr.cpp +++ b/src/zenhttp/clients/httpclientcpr.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace zen { @@ -164,6 +165,18 @@ CprHttpClient::CprHttpClient(std::string_view BaseUri, { } +bool +CprHttpClient::ShouldLogErrorCode(HttpResponseCode ResponseCode) const +{ + if (m_CheckIfAbortFunction && m_CheckIfAbortFunction()) + { + // Quiet + return false; + } + const auto& Expected = m_ConnectionSettings.ExpectedErrorCodes; + return std::find(Expected.begin(), Expected.end(), ResponseCode) == Expected.end(); +} + CprHttpClient::~CprHttpClient() { ZEN_TRACE_CPU("CprHttpClient::~CprHttpClient"); @@ -193,11 +206,9 @@ CprHttpClient::ResponseWithPayload(std::string_view SessionId, ResponseBuffer.SetContentType(ContentType); } - const bool Quiet = m_CheckIfAbortFunction && m_CheckIfAbortFunction(); - - if (!Quiet) + if (!IsHttpSuccessCode(WorkResponseCode) && WorkResponseCode != HttpResponseCode::NotFound) { - if (!IsHttpSuccessCode(WorkResponseCode) && WorkResponseCode != HttpResponseCode::NotFound) + if (ShouldLogErrorCode(WorkResponseCode)) { ZEN_WARN("HttpClient request failed (session: {}): {}", SessionId, HttpResponse); } @@ -371,8 +382,7 @@ CprHttpClient::DoWithRetry(std::string_view SessionId, } Sleep(100 * (Attempt + 1)); Attempt++; - const bool Quiet = m_CheckIfAbortFunction && m_CheckIfAbortFunction(); - if (!Quiet) + if (ShouldLogErrorCode(HttpResponseCode(Result.status_code))) { ZEN_INFO("{} Attempt {}/{}", CommonResponse(SessionId, std::move(Result), {}).ErrorMessage("Retry"), @@ -410,8 +420,7 @@ CprHttpClient::DoWithRetry(std::string_view SessionId, } Sleep(100 * (Attempt + 1)); Attempt++; - const bool Quiet = m_CheckIfAbortFunction && m_CheckIfAbortFunction(); - if (!Quiet) + if (ShouldLogErrorCode(HttpResponseCode(Result.status_code))) { ZEN_INFO("{} Attempt {}/{}", CommonResponse(SessionId, std::move(Result), {}).ErrorMessage("Retry"), @@ -646,7 +655,7 @@ CprHttpClient::TransactPackage(std::string_view Url, CbPackage Package, const Ke ResponseBuffer.SetContentType(ContentType); } - return {.StatusCode = HttpResponseCode(FilterResponse.status_code), .ResponsePayload = ResponseBuffer}; + return {.StatusCode = HttpResponseCode(FilterResponse.status_code), .ResponsePayload = std::move(ResponseBuffer)}; } ////////////////////////////////////////////////////////////////////////// @@ -929,6 +938,13 @@ CprHttpClient::Download(std::string_view Url, const std::filesystem::path& TempF cpr::Response Response = DoWithRetry( m_SessionId, [&]() { + // Reset state from any previous attempt + PayloadString.clear(); + PayloadFile.reset(); + BoundaryParser.Boundaries.clear(); + ContentType = HttpContentType::kUnknownContentType; + IsMultiRangeResponse = false; + auto DownloadCallback = [&](std::string data, intptr_t) { if (m_CheckIfAbortFunction && m_CheckIfAbortFunction()) { @@ -969,7 +985,7 @@ CprHttpClient::Download(std::string_view Url, const std::filesystem::path& TempF if (RangeStartPos != std::string::npos) { RangeStartPos++; - while (RangeValue[RangeStartPos] == ' ') + while (RangeStartPos < RangeValue.length() && RangeValue[RangeStartPos] == ' ') { RangeStartPos++; } @@ -991,7 +1007,7 @@ CprHttpClient::Download(std::string_view Url, const std::filesystem::path& TempF std::optional RequestedRangeEnd = ParseInt(RangeString.substr(RangeSplitPos + 1)); if (RequestedRangeStart.has_value() && RequestedRangeEnd.has_value()) { - RequestedContentLength += RequestedRangeEnd.value() - 1; + RequestedContentLength += RequestedRangeEnd.value() - RequestedRangeStart.value() + 1; } } RangeStartPos = RangeEnd; diff --git a/src/zenhttp/clients/httpclientcpr.h b/src/zenhttp/clients/httpclientcpr.h index cf2d3bd14..752d91add 100644 --- a/src/zenhttp/clients/httpclientcpr.h +++ b/src/zenhttp/clients/httpclientcpr.h @@ -155,6 +155,7 @@ private: std::function&& Func, std::function&& Validate = [](cpr::Response&) { return true; }); + bool ShouldLogErrorCode(HttpResponseCode ResponseCode) const; bool ValidatePayload(cpr::Response& Response, std::unique_ptr& PayloadFile); HttpClient::Response CommonResponse(std::string_view SessionId, diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index 53be36b9a..d87082d10 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -13,6 +13,7 @@ #include #include #include +#include namespace zen { @@ -58,6 +59,10 @@ struct HttpClientSettings Oid SessionId = Oid::Zero; bool Verbose = false; uint64_t MaximumInMemoryDownloadSize = 1024u * 1024u; + + /// HTTP status codes that are expected and should not be logged as warnings. + /// 404 is always treated as expected regardless of this list. + std::vector ExpectedErrorCodes; }; class HttpClientError : public std::runtime_error diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index fee932daa..62c080a7b 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -230,6 +230,10 @@ public: */ std::string_view GetExternalHost() const { return m_ExternalHost; } + /** Returns total bytes received and sent across all connections since server start. */ + virtual uint64_t GetTotalBytesReceived() const { return 0; } + virtual uint64_t GetTotalBytesSent() const { return 0; } + private: std::vector m_KnownServices; int m_EffectivePort = 0; diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 8c2dcd116..c4d9ee777 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -528,6 +528,9 @@ public: RwLock m_Lock; std::vector m_UriHandlers; + + std::atomic m_TotalBytesReceived{0}; + std::atomic m_TotalBytesSent{0}; }; /** @@ -1043,6 +1046,8 @@ HttpServerConnection::OnDataReceived(const asio::error_code& Ec, [[maybe_unused] } } + m_Server.m_TotalBytesReceived.fetch_add(ByteCount, std::memory_order_relaxed); + ZEN_TRACE_VERBOSE("on data received, connection: {}, request: {}, thread: {}, bytes: {}", m_ConnectionId, m_RequestCounter.load(std::memory_order_relaxed), @@ -1096,6 +1101,8 @@ HttpServerConnection::OnResponseDataSent(const asio::error_code& Ec, return; } + m_Server.m_TotalBytesSent.fetch_add(ByteCount, std::memory_order_relaxed); + ZEN_TRACE_VERBOSE("on data sent, connection: {}, request: {}, thread: {}, bytes: {}", m_ConnectionId, RequestNumber, @@ -2053,6 +2060,8 @@ public: virtual void OnRequestExit() override; virtual void OnClose() override; virtual std::string OnGetExternalHost() const override; + virtual uint64_t GetTotalBytesReceived() const override; + virtual uint64_t GetTotalBytesSent() const override; private: Event m_ShutdownEvent; @@ -2150,6 +2159,18 @@ HttpAsioServer::OnGetExternalHost() const } } +uint64_t +HttpAsioServer::GetTotalBytesReceived() const +{ + return m_Impl->m_TotalBytesReceived.load(std::memory_order_relaxed); +} + +uint64_t +HttpAsioServer::GetTotalBytesSent() const +{ + return m_Impl->m_TotalBytesSent.load(std::memory_order_relaxed); +} + void HttpAsioServer::OnRun(bool IsInteractive) { diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index 23d57af57..a48f1d316 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -88,6 +88,8 @@ class HttpSysServerRequest; class HttpSysServer : public HttpServer { friend class HttpSysTransaction; + friend class HttpMessageResponseRequest; + friend struct InitialRequestHandler; public: explicit HttpSysServer(const HttpSysConfig& Config); @@ -102,6 +104,8 @@ public: virtual void OnSetHttpRequestFilter(IHttpRequestFilter* RequestFilter) override; virtual void OnClose() override; virtual std::string OnGetExternalHost() const override; + virtual uint64_t GetTotalBytesReceived() const override; + virtual uint64_t GetTotalBytesSent() const override; WorkerThreadPool& WorkPool(); @@ -149,6 +153,9 @@ private: RwLock m_RequestFilterLock; std::atomic m_HttpRequestFilter = nullptr; + + std::atomic m_TotalBytesReceived{0}; + std::atomic m_TotalBytesSent{0}; }; } // namespace zen @@ -591,7 +598,7 @@ HttpMessageResponseRequest::SuppressResponseBody() HttpSysRequestHandler* HttpMessageResponseRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTransferred) { - ZEN_UNUSED(NumberOfBytesTransferred); + Transaction().Server().m_TotalBytesSent.fetch_add(NumberOfBytesTransferred, std::memory_order_relaxed); if (IoResult != NO_ERROR) { @@ -2123,6 +2130,8 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT break; } + Transaction().Server().m_TotalBytesReceived.fetch_add(NumberOfBytesTransferred, std::memory_order_relaxed); + ZEN_TRACE_CPU("httpsys::HandleCompletion"); // Route request @@ -2401,6 +2410,18 @@ HttpSysServer::OnGetExternalHost() const } } +uint64_t +HttpSysServer::GetTotalBytesReceived() const +{ + return m_TotalBytesReceived.load(std::memory_order_relaxed); +} + +uint64_t +HttpSysServer::GetTotalBytesSent() const +{ + return m_TotalBytesSent.load(std::memory_order_relaxed); +} + void HttpSysServer::OnRegisterService(HttpService& Service) { -- cgit v1.2.3 From 6e51634c31cfbe6ad99e27bcefe7ec3bd06dd5c5 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 Mar 2026 13:58:26 +0100 Subject: IterateChunks callback is multithreaded - make sure AttachmentsSize can handle it (#804) --- CHANGELOG.md | 1 + src/zenserver/storage/cache/httpstructuredcache.cpp | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fc60ee14..c4b484d53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly - Bugfix: Time spent indexing local and remote state during `zen builds download` now show the correct time - Bugfix: ObjectStore failed to correctly parse urls with sub folder paths causing 404 to be returned +- Bugfix: Total attachment size when querying for bucket sizes could end up incorrect ## 5.7.21 - Feature: Added `--security-config-path` option to zenserver to configure security settings diff --git a/src/zenserver/storage/cache/httpstructuredcache.cpp b/src/zenserver/storage/cache/httpstructuredcache.cpp index 72f29d14e..00151f79e 100644 --- a/src/zenserver/storage/cache/httpstructuredcache.cpp +++ b/src/zenserver/storage/cache/httpstructuredcache.cpp @@ -654,7 +654,7 @@ HttpStructuredCacheService::HandleCacheNamespaceRequest(HttpServerRequest& Reque auto NewEnd = std::unique(AllAttachments.begin(), AllAttachments.end()); AllAttachments.erase(NewEnd, AllAttachments.end()); - uint64_t AttachmentsSize = 0; + std::atomic AttachmentsSize = 0; m_CidStore.IterateChunks( AllAttachments, @@ -746,7 +746,7 @@ HttpStructuredCacheService::HandleCacheBucketRequest(HttpServerRequest& Request, ResponseWriter << "Size" << ValuesSize; ResponseWriter << "AttachmentCount" << ContentStats.Attachments.size(); - uint64_t AttachmentsSize = 0; + std::atomic AttachmentsSize = 0; WorkerThreadPool& WorkerPool = GetMediumWorkerPool(EWorkloadType::Background); -- cgit v1.2.3 From 794f093057c58c4909a0653edb54fdf869560596 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 14:00:34 +0100 Subject: native xmake toolchain definition for UE-clang (#805) This change is meant to provide a smoother experience when working on Linux. After this change, the toolchain setup process is now simply ```bash $ scripts/ue_build_linux/get_ue_toolchain.sh ``` and then at config time the toolchain is automatically detected if you downloaded it to the default location or have the `UE_TOOLCHAIN_DIR` environment variable set ```bash xmake config --mode=debug ``` Compared to the old script-based approach this configures the toolchain more precisely, avoiding leakage into unrelated build processes such as when a package manager decides to build something like Ninja locally etc. --- .github/workflows/create_release.yml | 4 +- .github/workflows/validate.yml | 6 +-- README.md | 68 ++++++++++++++---------------- scripts/bundle.lua | 10 +++++ scripts/ue_build_linux/README.md | 42 +++++++++--------- scripts/ue_build_linux/clang | 2 - scripts/ue_build_linux/clang++ | 2 - scripts/ue_build_linux/get_ue_toolchain.sh | 2 +- scripts/ue_build_linux/ue_build.sh | 33 --------------- toolchains/ue_clang.lua | 60 ++++++++++++++++++++++++++ toolchains/xmake.lua | 3 ++ xmake.lua | 42 +++++++++++++----- 12 files changed, 164 insertions(+), 110 deletions(-) delete mode 100755 scripts/ue_build_linux/clang delete mode 100755 scripts/ue_build_linux/clang++ delete mode 100755 scripts/ue_build_linux/ue_build.sh create mode 100644 toolchains/ue_clang.lua create mode 100644 toolchains/xmake.lua diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 163c0cf85..2eeedc0c2 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -60,11 +60,11 @@ jobs: - name: Config run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake config -v -y -m release --zensentry=yes + xmake config -v -y -m release --zensentry=yes --toolchain=ue-clang --sdk=${{ github.workspace }}/.tmp-ue-toolchain - name: Bundle run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake bundle -v -y + xmake bundle -v -y - name: Get Sentry CLI run: | diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 32b75fe5e..d96645ac9 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -145,7 +145,7 @@ jobs: - name: Config run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} --zensentry=yes + xmake config -v -y -m ${{ matrix.config }} --arch=${{ matrix.arch }} --zensentry=yes --toolchain=ue-clang --sdk=${{ github.workspace }}/.tmp-ue-toolchain - name: Clean reports if: ${{ matrix.config == 'debug' }} @@ -155,7 +155,7 @@ jobs: - name: Build & Test if: ${{ matrix.config == 'debug' }} run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake test -v -y --junit + xmake test -v -y --junit - name: Upload report if: ${{ (failure() || success()) && (matrix.config == 'debug') }} @@ -176,7 +176,7 @@ jobs: - name: Bundle if: ${{ matrix.config == 'release' }} run: | - ./scripts/ue_build_linux/ue_build.sh ./.tmp-ue-toolchain xmake bundle -v -y + xmake bundle -v -y - name: Upload zenserver-linux if: ${{ github.ref_name == 'main' && matrix.config == 'release' }} diff --git a/README.md b/README.md index 36e02d213..2146738ec 100644 --- a/README.md +++ b/README.md @@ -75,60 +75,56 @@ line. ## Building on Linux -The following instructions have been collated using Ubuntu 20.04. - -Zen makes use of C++20 features which at the time of writing has limited -toolchain and C++ library support. A minimum compiler version of GCC-11 or -Clang-12 is required, along with GNU's libstdc+++-11 or newer. - -The first step is to acquire a suitable compiler and C++ library. On Ubuntu 20.04 -GCC-11 is not available in the standard package repositories so Ubuntu's tool- -chain test respository needs to be added to Apt (note that this is _not required_ -for Ubuntu 21.04 onwards); - -``` -sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test -``` - -Now GCC-11 can be installed via Apt. This will also install a suitable version of -the C++ library. - -``` -sudo apt install -y --no-install-recommends g++-11 -g++-11 --version -``` - -The easiest way to tell `xmake` to use the correct compiler version is to set -the `CXX` environment variable. +The following instructions have been collated using Ubuntu 24.04. Other distributions will +likely need a slightly different set of packages installed. -``` -export CXX=g++-11 +```bash +sudo apt install -y build-essential pkg-config unzip ``` Install [xmake](https://xmake.io/#/getting_started). -``` +It's important to use the correct version when installing, pay attention to the last argument in the command line below: + +```bash curl -fsSL https://xmake.io/shget.text | bash -s v2.9.9 ``` -Clone the Zen project and tell `xmake` to use the correct GCC version. +Clone the Zen project. -``` +```bash git clone https://github.com/EpicGames/zen.git ~/zen/main cd ~/zen/main ``` -Now we are ready to build Zen. The `-y` skips `xmake` from prompting about -building packages. +We recommend building using the UE Linux toolchain, to match the Unreal Editor +setup as closely as possible. It is also the most thoroughly tested toolchain +since this is what is used in the CI environment which produces the binaries +which end up in the UE tree. + +Using other toolchains may or may not work depending on which version and +compiler you happen to be using. +### Building with the UE Linux toolchain + +To build with the Unreal Engine cross-compile toolchain (producing binaries +compatible with the VFX Reference Platform), first download the toolchain: + +```bash +scripts/ue_build_linux/get_ue_toolchain.sh ``` -xmake config -y --mode=debug + +This downloads to `~/.ue-toolchain` by default. xmake automatically detects the +toolchain at this location, so no extra flags are needed: + +```bash +xmake config -y -m debug xmake build ``` -Note that the command above to set the build variant to debug is optional. Tests -are only built in debug.The `xmake` flags `-vD` can be useful to diagnose -`xmake` issues. +The toolchain can also be selected explicitly with `--toolchain=ue-clang`, and +the SDK location can be overridden with `--sdk=` or the +`UE_TOOLCHAIN_DIR` environment variable. ## Building on Mac diff --git a/scripts/bundle.lua b/scripts/bundle.lua index 07e120d04..b2a5e1e08 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -17,7 +17,15 @@ end -------------------------------------------------------------------------------- local function _build(arch, debug, config_args) + import("core.project.config") + config.load() + variant = debug and "debug" or "release" + + -- Preserve toolchain/sdk from current config so --clean doesn't lose them + local toolchain_arg = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or nil + local sdk_arg = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or nil + local ret = _exec( "xmake", "config", @@ -26,6 +34,8 @@ local function _build(arch, debug, config_args) "--mode="..variant, "--arch="..arch, "--zensentry=yes", + toolchain_arg, + sdk_arg, config_args) if ret > 0 then raise("Failed to configure xmake") diff --git a/scripts/ue_build_linux/README.md b/scripts/ue_build_linux/README.md index e93a234ae..afafcbe24 100755 --- a/scripts/ue_build_linux/README.md +++ b/scripts/ue_build_linux/README.md @@ -2,38 +2,38 @@ This folder contains scripts to build Zen using the UE Linux toolchain. This can be used to output binaries that meet the VFX Reference Platform versions. -It works by using the `--sysroot=` option to redirect compilers and linkers to -find headers and libraries. There are a few components involved; -1) get_ue_toolchain.sh +## Setup + +Download the toolchain using `get_ue_toolchain.sh`: ``` -$ scripts/ue_build_linux/get_ue_toolchain.sh ./.tmp-ue-toolchain +$ scripts/ue_build_linux/get_ue_toolchain.sh ``` -This will download the required components from cdn.unrealengine.com and -structure them in such a way that they can be used by both vcpkg and xmake -when building Zen. +By default this downloads to `~/.ue-toolchain`. A custom path can be given as +the first argument, or via the `UE_TOOLCHAIN_DIR` environment variable. -2) ue_build.sh [args...] +This will download the required components from cdn.unrealengine.com and +structure them in such a way that they can be used by xmake when building Zen. -Given the toolchain location downloaded in step (1) and the `VCPKG_ROOT` -environment variable is properly configured, this script sets up a suitable -environment and execs the "prog [args...]". +## Building -It is expected that this is used to invoke xmake to build Zen; +xmake automatically detects the toolchain at `~/.ue-toolchain`, so no extra +flags are needed: ``` -$ scripts/ue_build_linux/ue_build.sh .tmp-ue-toolchain xmake config --mode=debug -$ scripts/ue_build_linux/ue_build.sh .tmp-ue-toolchain xmake build +$ xmake config -y -m debug +$ xmake build -y ``` -It is possible that `--toolchain=clang` may be required as a configuration -option. The `ue_build.sh` script can also be sourced into the current shell, -although it is worth noting that this has never been tried. +To build a release bundle: -3) `scripts/ue_build_linux/clang` / `scripts/ue_build_linux/clang++` +``` +$ xmake config -y -m release +$ xmake bundle -y +``` -These acts as shims to the binaries in `toolchain_dir`, adding in the required -command line arguments to use the correct headers and libraries. -The `ue_build.sh` script adjusts `$PATH` appropriately. +The toolchain can also be selected explicitly with `--toolchain=ue-clang`, and +the SDK location can be overridden with `--sdk=` (must be absolute) or +the `UE_TOOLCHAIN_DIR` environment variable. diff --git a/scripts/ue_build_linux/clang b/scripts/ue_build_linux/clang deleted file mode 100755 index 9666ba4ba..000000000 --- a/scripts/ue_build_linux/clang +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec $UE_TOOLCHAIN_DIR/bin/clang --sysroot=$UE_TOOLCHAIN_DIR $CFLAGS "$@" diff --git a/scripts/ue_build_linux/clang++ b/scripts/ue_build_linux/clang++ deleted file mode 100755 index be106ae87..000000000 --- a/scripts/ue_build_linux/clang++ +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -exec $UE_TOOLCHAIN_DIR/bin/clang++ --sysroot=$UE_TOOLCHAIN_DIR -stdlib=libc++ $CXXFLAGS "$@" diff --git a/scripts/ue_build_linux/get_ue_toolchain.sh b/scripts/ue_build_linux/get_ue_toolchain.sh index c2538b09a..0afd40fe3 100755 --- a/scripts/ue_build_linux/get_ue_toolchain.sh +++ b/scripts/ue_build_linux/get_ue_toolchain.sh @@ -5,7 +5,7 @@ ZEN_ROOT=$(realpath $SCRIPT_DIR/../..) die() { echo "ERROR: $1"; exit; } -toolchain_dir="${1:-${ZEN_ROOT}/.tmp-ue-toolchain}" +toolchain_dir="${1:-${UE_TOOLCHAIN_DIR:-${HOME}/.ue-toolchain}}" if [[ $toolchain_dir == "--help" ]]; then echo "usage: $(basename ${BASH_SOURCE[0]}) " diff --git a/scripts/ue_build_linux/ue_build.sh b/scripts/ue_build_linux/ue_build.sh deleted file mode 100755 index 690f9f661..000000000 --- a/scripts/ue_build_linux/ue_build.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -ZEN_ROOT=$(realpath $SCRIPT_DIR/../..) - -die() { echo ERROR: $1; exit 1; } - -if [[ $1 == "--help" ]]; then - echo "usage: $0 (defaults to ${ZEN_ROOT}/.tmp-ue-toolchain)" - exit -fi - -toolchain_dir="${1:-${ZEN_ROOT}/.tmp-ue-toolchain}" - -# Validate input - -if ! [ -d $toolchain_dir ]; then - die "$1 is not a directory" -fi - -if ! [ -e $toolchain_dir/bin/clang++ ]; then - die "$1/bin/clang++ does not exist" -fi - -export UE_TOOLCHAIN_DIR=$(realpath $toolchain_dir) -export CC="clang" -export CXX="clang++" -export LD="clang++" - -export PATH="$(realpath $(dirname ${BASH_SOURCE[0]})):$PATH" - -shift -exec "$@" diff --git a/toolchains/ue_clang.lua b/toolchains/ue_clang.lua new file mode 100644 index 000000000..8a1efcc04 --- /dev/null +++ b/toolchains/ue_clang.lua @@ -0,0 +1,60 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +-- Custom toolchain for building with the Unreal Engine Linux clang toolchain. +-- +-- Usage: +-- xmake config --toolchain=ue-clang [--sdk=/path/to/ue-toolchain] +-- +-- If --sdk is not given, the toolchain is resolved in order: +-- 1. $UE_TOOLCHAIN_DIR environment variable +-- 2. ~/.ue-toolchain (default get_ue_toolchain.sh download location) + +toolchain("ue-clang") + set_kind("standalone") + + on_load(function (toolchain) + -- Resolve SDK directory: --sdk flag > $UE_TOOLCHAIN_DIR > ~/.ue-toolchain + local sdkdir = toolchain:sdkdir() + if not sdkdir or sdkdir == "" then + sdkdir = os.getenv("UE_TOOLCHAIN_DIR") + end + if not sdkdir or sdkdir == "" then + local default_path = path.join(os.getenv("HOME"), ".ue-toolchain") + if os.isdir(default_path) then + sdkdir = default_path + end + end + if not sdkdir or sdkdir == "" then + return + end + + toolchain:config_set("sdkdir", sdkdir) + + local bindir = path.join(sdkdir, "bin") + + -- Compiler and linker tools + toolchain:set("toolset", "cc", path.join(bindir, "clang")) + toolchain:set("toolset", "cxx", path.join(bindir, "clang++")) + toolchain:set("toolset", "ld", path.join(bindir, "clang++")) + toolchain:set("toolset", "sh", path.join(bindir, "clang++")) + toolchain:set("toolset", "ar", path.join(bindir, "llvm-ar")) + toolchain:set("toolset", "strip", path.join(bindir, "llvm-strip")) + toolchain:set("toolset", "objcopy", path.join(bindir, "llvm-objcopy")) + toolchain:set("toolset", "as", path.join(bindir, "clang")) + toolchain:set("toolset", "ranlib", path.join(bindir, "llvm-ranlib")) + + -- Sysroot and stdlib flags for both C and C++ + local sysroot_flag = "--sysroot=" .. sdkdir + toolchain:add("cxflags", sysroot_flag, {force = true}) + toolchain:add("cxflags", "-stdlib=libc++", {force = true}) + + -- Linker flags: sysroot and stdlib selection only. + -- Static libc++ overrides are applied at project level (xmake.lua) so they + -- don't leak into cmake package builds (e.g. sentry-native/crashpad). + toolchain:add("ldflags", sysroot_flag, {force = true}) + toolchain:add("ldflags", "-stdlib=libc++", {force = true}) + + toolchain:add("shflags", sysroot_flag, {force = true}) + toolchain:add("shflags", "-stdlib=libc++", {force = true}) + end) +toolchain_end() diff --git a/toolchains/xmake.lua b/toolchains/xmake.lua new file mode 100644 index 000000000..2817a1586 --- /dev/null +++ b/toolchains/xmake.lua @@ -0,0 +1,3 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +includes("ue_clang.lua") diff --git a/xmake.lua b/xmake.lua index 0d8f53a16..7b9fee594 100644 --- a/xmake.lua +++ b/xmake.lua @@ -101,16 +101,17 @@ if is_plat("windows") then add_requires("7z") end --- If we're using the UE cross-compile toolchain, we need to ensure we link statically --- against the toolchain libc++ and libc++abi, as the system ones can differ in ABI --- leading to nasty crashes - -if is_plat("linux") and os.getenv("UE_TOOLCHAIN_DIR") then - add_ldflags("-static-libstdc++") - add_ldflags("$(projectdir)/thirdparty/ue-libcxx/lib64/libc++.a") - add_ldflags("$(projectdir)/thirdparty/ue-libcxx/lib64/libc++abi.a") - set_toolset("objcopy", "$(env UE_TOOLCHAIN_DIR)/bin/llvm-objcopy") +-- When using the UE clang toolchain, statically link the toolchain's libc++ and +-- libc++abi to avoid ABI mismatches with system libraries at runtime. +-- These are project-level flags (not in the toolchain definition) so they don't +-- propagate into cmake package builds. +if is_plat("linux") and get_config("toolchain") == "ue-clang" then + add_ldflags("-static-libstdc++", {force = true}) + add_ldflags("$(projectdir)/thirdparty/ue-libcxx/lib64/libc++.a", {force = true}) + add_ldflags("$(projectdir)/thirdparty/ue-libcxx/lib64/libc++abi.a", {force = true}) + add_ldflags("-lpthread", {force = true}) end + if has_config("zensentry") and not use_asan then if is_plat("linux") then add_requires("sentry-native 0.12.1", {configs = {backend = "crashpad"}}) @@ -276,6 +277,25 @@ set_languages("cxx20") -- always generate debug information set_symbols("debug") +includes("toolchains") + +-- Auto-select the UE clang toolchain on Linux when the SDK is available +if is_plat("linux") and not get_config("toolchain") then + local ue_sdk = os.getenv("UE_TOOLCHAIN_DIR") + if not ue_sdk or ue_sdk == "" then + local home = os.getenv("HOME") + if home then + local default_path = path.join(home, ".ue-toolchain") + if os.isdir(default_path) then + ue_sdk = default_path + end + end + end + if ue_sdk and ue_sdk ~= "" and os.isdir(ue_sdk) then + set_toolchains("ue-clang") + end +end + includes("thirdparty") includes("src/transports") includes("src/zenbase") @@ -459,7 +479,9 @@ task("test") -- Only reconfigure if current config doesn't already match if config.get("mode") ~= "debug" or config.get("plat") ~= plat or config.get("arch") ~= arch then - os.exec("xmake config -c -m debug -p %s -a %s", plat, arch) + local toolchain_flag = config.get("toolchain") and ("--toolchain=" .. config.get("toolchain")) or "" + local sdk_flag = config.get("sdk") and ("--sdk=" .. config.get("sdk")) or "" + os.exec("xmake config -c -m debug -p %s -a %s %s %s", plat, arch, toolchain_flag, sdk_flag) end -- Build targets we're going to run -- cgit v1.2.3 From 0763d09a81e5a1d3df11763a7ec75e7860c9510a Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 14:13:46 +0100 Subject: compute orchestration (#763) - Added local process runners for Linux/Wine, Mac with some sandboxing support - Horde & Nomad provisioning for development and testing - Client session queues with lifecycle management (active/draining/cancelled), automatic retry with configurable limits, and manual reschedule API - Improved web UI for orchestrator, compute, and hub dashboards with WebSocket push updates - Some security hardening - Improved scalability and `zen exec` command Still experimental - compute support is disabled by default --- docs/compute.md | 79 +- repo/packages/n/nomad/xmake.lua | 37 + src/zen/cmds/exec_cmd.cpp | 753 ++++++- src/zen/cmds/exec_cmd.h | 8 +- src/zencompute/CLAUDE.md | 232 ++ src/zencompute/actionrecorder.cpp | 258 --- src/zencompute/actionrecorder.h | 91 - src/zencompute/cloudmetadata.cpp | 1010 +++++++++ src/zencompute/computeservice.cpp | 2236 ++++++++++++++++++++ src/zencompute/functionrunner.cpp | 112 - src/zencompute/functionrunner.h | 207 -- src/zencompute/functionservice.cpp | 957 --------- src/zencompute/httpcomputeservice.cpp | 1643 ++++++++++++++ src/zencompute/httpfunctionservice.cpp | 709 ------- src/zencompute/httporchestrator.cpp | 621 +++++- src/zencompute/include/zencompute/cloudmetadata.h | 151 ++ src/zencompute/include/zencompute/computeservice.h | 262 +++ .../include/zencompute/functionservice.h | 132 -- .../include/zencompute/httpcomputeservice.h | 54 + .../include/zencompute/httpfunctionservice.h | 73 - .../include/zencompute/httporchestrator.h | 81 +- src/zencompute/include/zencompute/mockimds.h | 102 + .../include/zencompute/orchestratorservice.h | 177 ++ .../include/zencompute/recordingreader.h | 4 +- src/zencompute/include/zencompute/zencompute.h | 4 + src/zencompute/localrunner.cpp | 722 ------- src/zencompute/localrunner.h | 100 - src/zencompute/orchestratorservice.cpp | 710 +++++++ src/zencompute/recording/actionrecorder.cpp | 258 +++ src/zencompute/recording/actionrecorder.h | 91 + src/zencompute/recording/recordingreader.cpp | 335 +++ src/zencompute/recordingreader.cpp | 335 --- src/zencompute/remotehttprunner.cpp | 457 ---- src/zencompute/remotehttprunner.h | 80 - src/zencompute/runners/deferreddeleter.cpp | 336 +++ src/zencompute/runners/deferreddeleter.h | 68 + src/zencompute/runners/functionrunner.cpp | 365 ++++ src/zencompute/runners/functionrunner.h | 214 ++ src/zencompute/runners/linuxrunner.cpp | 734 +++++++ src/zencompute/runners/linuxrunner.h | 44 + src/zencompute/runners/localrunner.cpp | 674 ++++++ src/zencompute/runners/localrunner.h | 138 ++ src/zencompute/runners/macrunner.cpp | 491 +++++ src/zencompute/runners/macrunner.h | 43 + src/zencompute/runners/remotehttprunner.cpp | 618 ++++++ src/zencompute/runners/remotehttprunner.h | 100 + src/zencompute/runners/windowsrunner.cpp | 460 ++++ src/zencompute/runners/windowsrunner.h | 53 + src/zencompute/runners/winerunner.cpp | 237 +++ src/zencompute/runners/winerunner.h | 37 + src/zencompute/testing/mockimds.cpp | 205 ++ src/zencompute/timeline/workertimeline.cpp | 430 ++++ src/zencompute/timeline/workertimeline.h | 169 ++ src/zencompute/xmake.lua | 10 + src/zencompute/zencompute.cpp | 9 + src/zencore/include/zencore/system.h | 36 +- src/zencore/system.cpp | 336 ++- src/zenhorde/hordeagent.cpp | 297 +++ src/zenhorde/hordeagent.h | 77 + src/zenhorde/hordeagentmessage.cpp | 340 +++ src/zenhorde/hordeagentmessage.h | 161 ++ src/zenhorde/hordebundle.cpp | 619 ++++++ src/zenhorde/hordebundle.h | 49 + src/zenhorde/hordeclient.cpp | 382 ++++ src/zenhorde/hordecomputebuffer.cpp | 454 ++++ src/zenhorde/hordecomputebuffer.h | 136 ++ src/zenhorde/hordecomputechannel.cpp | 37 + src/zenhorde/hordecomputechannel.h | 32 + src/zenhorde/hordecomputesocket.cpp | 204 ++ src/zenhorde/hordecomputesocket.h | 79 + src/zenhorde/hordeconfig.cpp | 89 + src/zenhorde/hordeprovisioner.cpp | 367 ++++ src/zenhorde/hordetransport.cpp | 169 ++ src/zenhorde/hordetransport.h | 71 + src/zenhorde/hordetransportaes.cpp | 425 ++++ src/zenhorde/hordetransportaes.h | 52 + src/zenhorde/include/zenhorde/hordeclient.h | 116 + src/zenhorde/include/zenhorde/hordeconfig.h | 62 + src/zenhorde/include/zenhorde/hordeprovisioner.h | 110 + src/zenhorde/include/zenhorde/zenhorde.h | 9 + src/zenhorde/xmake.lua | 22 + src/zenhttp/include/zenhttp/httpserver.h | 1 + src/zenhttp/servers/httpasio.cpp | 25 +- src/zenhttp/servers/httpsys.cpp | 25 +- src/zennomad/include/zennomad/nomadclient.h | 77 + src/zennomad/include/zennomad/nomadconfig.h | 65 + src/zennomad/include/zennomad/nomadprocess.h | 78 + src/zennomad/include/zennomad/nomadprovisioner.h | 107 + src/zennomad/include/zennomad/zennomad.h | 9 + src/zennomad/nomadclient.cpp | 366 ++++ src/zennomad/nomadconfig.cpp | 91 + src/zennomad/nomadprocess.cpp | 354 ++++ src/zennomad/nomadprovisioner.cpp | 264 +++ src/zennomad/xmake.lua | 10 + .../builds/buildstorageoperations.cpp | 2 +- src/zenremotestore/chunking/chunkingcache.cpp | 8 +- src/zenserver-test/compute-tests.cpp | 1700 +++++++++++++++ src/zenserver-test/function-tests.cpp | 38 - src/zenserver-test/logging-tests.cpp | 257 +++ src/zenserver-test/nomad-tests.cpp | 126 ++ src/zenserver-test/xmake.lua | 7 +- src/zenserver/compute/computeserver.cpp | 725 ++++++- src/zenserver/compute/computeserver.h | 111 +- src/zenserver/compute/computeservice.cpp | 100 - src/zenserver/compute/computeservice.h | 36 - src/zenserver/frontend/html.zip | Bin 238188 -> 319315 bytes src/zenserver/frontend/html/404.html | 486 +++++ src/zenserver/frontend/html/compute.html | 991 --------- src/zenserver/frontend/html/compute/banner.js | 321 +++ src/zenserver/frontend/html/compute/compute.html | 1072 ++++++++++ src/zenserver/frontend/html/compute/hub.html | 310 +++ src/zenserver/frontend/html/compute/index.html | 1 + src/zenserver/frontend/html/compute/nav.js | 79 + .../frontend/html/compute/orchestrator.html | 831 ++++++++ src/zenserver/frontend/html/pages/page.js | 36 + src/zenserver/frontend/html/zen.css | 27 + src/zenserver/hub/hubservice.cpp | 2 +- src/zenserver/hub/zenhubserver.cpp | 7 + src/zenserver/hub/zenhubserver.h | 6 +- src/zenserver/storage/zenstorageserver.cpp | 17 +- src/zenserver/storage/zenstorageserver.h | 4 +- src/zenserver/trace/tracerecorder.cpp | 565 +++++ src/zenserver/trace/tracerecorder.h | 46 + src/zenserver/xmake.lua | 19 + src/zentest-appstub/zentest-appstub.cpp | 11 + src/zenutil/include/zenutil/consoletui.h | 1 + src/zenutil/include/zenutil/zenserverprocess.h | 1 + src/zenutil/zenserverprocess.cpp | 6 + xmake.lua | 25 +- 129 files changed, 27082 insertions(+), 5611 deletions(-) create mode 100644 repo/packages/n/nomad/xmake.lua create mode 100644 src/zencompute/CLAUDE.md delete mode 100644 src/zencompute/actionrecorder.cpp delete mode 100644 src/zencompute/actionrecorder.h create mode 100644 src/zencompute/cloudmetadata.cpp create mode 100644 src/zencompute/computeservice.cpp delete mode 100644 src/zencompute/functionrunner.cpp delete mode 100644 src/zencompute/functionrunner.h delete mode 100644 src/zencompute/functionservice.cpp create mode 100644 src/zencompute/httpcomputeservice.cpp delete mode 100644 src/zencompute/httpfunctionservice.cpp create mode 100644 src/zencompute/include/zencompute/cloudmetadata.h create mode 100644 src/zencompute/include/zencompute/computeservice.h delete mode 100644 src/zencompute/include/zencompute/functionservice.h create mode 100644 src/zencompute/include/zencompute/httpcomputeservice.h delete mode 100644 src/zencompute/include/zencompute/httpfunctionservice.h create mode 100644 src/zencompute/include/zencompute/mockimds.h create mode 100644 src/zencompute/include/zencompute/orchestratorservice.h delete mode 100644 src/zencompute/localrunner.cpp delete mode 100644 src/zencompute/localrunner.h create mode 100644 src/zencompute/orchestratorservice.cpp create mode 100644 src/zencompute/recording/actionrecorder.cpp create mode 100644 src/zencompute/recording/actionrecorder.h create mode 100644 src/zencompute/recording/recordingreader.cpp delete mode 100644 src/zencompute/recordingreader.cpp delete mode 100644 src/zencompute/remotehttprunner.cpp delete mode 100644 src/zencompute/remotehttprunner.h create mode 100644 src/zencompute/runners/deferreddeleter.cpp create mode 100644 src/zencompute/runners/deferreddeleter.h create mode 100644 src/zencompute/runners/functionrunner.cpp create mode 100644 src/zencompute/runners/functionrunner.h create mode 100644 src/zencompute/runners/linuxrunner.cpp create mode 100644 src/zencompute/runners/linuxrunner.h create mode 100644 src/zencompute/runners/localrunner.cpp create mode 100644 src/zencompute/runners/localrunner.h create mode 100644 src/zencompute/runners/macrunner.cpp create mode 100644 src/zencompute/runners/macrunner.h create mode 100644 src/zencompute/runners/remotehttprunner.cpp create mode 100644 src/zencompute/runners/remotehttprunner.h create mode 100644 src/zencompute/runners/windowsrunner.cpp create mode 100644 src/zencompute/runners/windowsrunner.h create mode 100644 src/zencompute/runners/winerunner.cpp create mode 100644 src/zencompute/runners/winerunner.h create mode 100644 src/zencompute/testing/mockimds.cpp create mode 100644 src/zencompute/timeline/workertimeline.cpp create mode 100644 src/zencompute/timeline/workertimeline.h create mode 100644 src/zenhorde/hordeagent.cpp create mode 100644 src/zenhorde/hordeagent.h create mode 100644 src/zenhorde/hordeagentmessage.cpp create mode 100644 src/zenhorde/hordeagentmessage.h create mode 100644 src/zenhorde/hordebundle.cpp create mode 100644 src/zenhorde/hordebundle.h create mode 100644 src/zenhorde/hordeclient.cpp create mode 100644 src/zenhorde/hordecomputebuffer.cpp create mode 100644 src/zenhorde/hordecomputebuffer.h create mode 100644 src/zenhorde/hordecomputechannel.cpp create mode 100644 src/zenhorde/hordecomputechannel.h create mode 100644 src/zenhorde/hordecomputesocket.cpp create mode 100644 src/zenhorde/hordecomputesocket.h create mode 100644 src/zenhorde/hordeconfig.cpp create mode 100644 src/zenhorde/hordeprovisioner.cpp create mode 100644 src/zenhorde/hordetransport.cpp create mode 100644 src/zenhorde/hordetransport.h create mode 100644 src/zenhorde/hordetransportaes.cpp create mode 100644 src/zenhorde/hordetransportaes.h create mode 100644 src/zenhorde/include/zenhorde/hordeclient.h create mode 100644 src/zenhorde/include/zenhorde/hordeconfig.h create mode 100644 src/zenhorde/include/zenhorde/hordeprovisioner.h create mode 100644 src/zenhorde/include/zenhorde/zenhorde.h create mode 100644 src/zenhorde/xmake.lua create mode 100644 src/zennomad/include/zennomad/nomadclient.h create mode 100644 src/zennomad/include/zennomad/nomadconfig.h create mode 100644 src/zennomad/include/zennomad/nomadprocess.h create mode 100644 src/zennomad/include/zennomad/nomadprovisioner.h create mode 100644 src/zennomad/include/zennomad/zennomad.h create mode 100644 src/zennomad/nomadclient.cpp create mode 100644 src/zennomad/nomadconfig.cpp create mode 100644 src/zennomad/nomadprocess.cpp create mode 100644 src/zennomad/nomadprovisioner.cpp create mode 100644 src/zennomad/xmake.lua create mode 100644 src/zenserver-test/compute-tests.cpp delete mode 100644 src/zenserver-test/function-tests.cpp create mode 100644 src/zenserver-test/logging-tests.cpp create mode 100644 src/zenserver-test/nomad-tests.cpp delete mode 100644 src/zenserver/compute/computeservice.cpp delete mode 100644 src/zenserver/compute/computeservice.h create mode 100644 src/zenserver/frontend/html/404.html delete mode 100644 src/zenserver/frontend/html/compute.html create mode 100644 src/zenserver/frontend/html/compute/banner.js create mode 100644 src/zenserver/frontend/html/compute/compute.html create mode 100644 src/zenserver/frontend/html/compute/hub.html create mode 100644 src/zenserver/frontend/html/compute/index.html create mode 100644 src/zenserver/frontend/html/compute/nav.js create mode 100644 src/zenserver/frontend/html/compute/orchestrator.html create mode 100644 src/zenserver/trace/tracerecorder.cpp create mode 100644 src/zenserver/trace/tracerecorder.h diff --git a/docs/compute.md b/docs/compute.md index 417622f94..df8a22870 100644 --- a/docs/compute.md +++ b/docs/compute.md @@ -122,31 +122,82 @@ functions: version: '83027356-2cf7-41ca-aba5-c81ab0ff2129' ``` -## API (WIP not final) +## API -The compute interfaces are currently exposed on the `/apply` endpoint but this -will be subject to change as we adapt the interfaces during development. The LSN +The compute interfaces are exposed on the `/compute` endpoint. The LSN APIs below are intended to replace the action ID oriented APIs. The POST APIs typically involve a two-step dance where a descriptor is POSTed and -the service responds with a list of `needs` chunks (identified via `IoHash`) which -it does not have yet. The client can then follow up with a POST of a Compact Binary +the service responds with a list of `needs` chunks (identified via `IoHash`) which +it does not have yet. The client can then follow up with a POST of a Compact Binary Package containing the descriptor along with the needed chunks. -`/apply/ready` - health check endpoint returns HTTP 200 OK or HTTP 503 +`/compute/ready` - health check endpoint returns HTTP 200 OK or HTTP 503 -`/apply/sysinfo` - system information endpoint +`/compute/sysinfo` - system information endpoint -`/apply/record/start`, `/apply/record/stop` - start/stop action recording +`/compute/record/start`, `/compute/record/stop` - start/stop action recording -`/apply/workers/{worker}` - GET/POST worker descriptors and payloads +`/compute/workers/{worker}` - GET/POST worker descriptors and payloads -`/apply/jobs/completed` - GET list of completed actions +`/compute/jobs/completed` - GET list of completed actions -`/apply/jobs/{lsn}` - GET completed action results from LSN, POST action cancellation by LSN, priority changes by LSN +`/compute/jobs/{lsn}` - GET completed action results from LSN, POST action cancellation by LSN, priority changes by LSN -`/apply/jobs/{worker}/{action}` - GET completed action (job) results by action ID +`/compute/jobs/{worker}/{action}` - GET completed action (job) results by action ID -`/apply/jobs/{worker}` - GET pending/running jobs for worker, POST requests to schedule action as a job +`/compute/jobs/{worker}` - GET pending/running jobs for worker, POST requests to schedule action as a job -`/apply/jobs` - POST request to schedule action as a job +`/compute/jobs` - POST request to schedule action as a job + +### Queues + +Queues provide a way to logically group actions submitted by a client session. This enables +per-session cancellation and completion polling without affecting actions submitted by other +sessions. + +#### Local access (integer ID routes) + +These routes use sequential integer queue IDs and are restricted to local (loopback) +connections only. Remote requests receive HTTP 403 Forbidden. + +`/compute/queues` - POST to create a new queue. Returns a `queue_id` which is used to +reference the queue in subsequent requests. + +`/compute/queues/{queue}` - GET queue status (active, completed, failed, and cancelled +action counts, plus `is_complete` flag indicating all actions have finished). DELETE to +cancel all pending and running actions in the queue. + +`/compute/queues/{queue}/completed` - GET list of completed action LSNs for this queue +whose results have not yet been retired. A queue-scoped alternative to `/compute/jobs/completed`. + +`/compute/queues/{queue}/jobs` - POST to submit an action to a queue with automatic worker +resolution. Accepts an optional `priority` query parameter. + +`/compute/queues/{queue}/jobs/{worker}` - POST to submit an action to a queue targeting a +specific worker. Accepts an optional `priority` query parameter. + +`/compute/queues/{queue}/jobs/{lsn}` - GET action result by LSN, scoped to the queue + +#### Remote access (OID token routes) + +These routes use cryptographically generated 24-character hex tokens (OIDs) instead of +integer queue IDs. Tokens are unguessable and safe to use over the network. The token +mapping lives entirely in the HTTP service layer; the underlying compute service only +knows about integer queue IDs. + +`/compute/queues/remote` - POST to create a new queue with token-based access. Returns +`queue_token` (24-char hex string) and `queue_id` (integer, for internal visibility). + +`/compute/queues/{oidtoken}` - GET queue status or DELETE to cancel, same semantics as +the integer ID variant but using the OID token for identification. + +`/compute/queues/{oidtoken}/completed` - GET list of completed action LSNs for this queue. + +`/compute/queues/{oidtoken}/jobs` - POST to submit an action to a queue with automatic +worker resolution. + +`/compute/queues/{oidtoken}/jobs/{worker}` - POST to submit an action targeting a specific +worker. + +`/compute/queues/{oidtoken}/jobs/{lsn}` - GET action result by LSN, scoped to the queue diff --git a/repo/packages/n/nomad/xmake.lua b/repo/packages/n/nomad/xmake.lua new file mode 100644 index 000000000..85ea10985 --- /dev/null +++ b/repo/packages/n/nomad/xmake.lua @@ -0,0 +1,37 @@ +-- this package only provides the nomad binary, to be used for testing nomad provisioning + +package("nomad") + set_homepage("https://www.nomadproject.io/") + set_description("Nomad is a workload orchestrator that deploys and manages containers and non-containerized applications.") + + if is_plat("windows") then + add_urls("https://releases.hashicorp.com/nomad/$(version)/nomad_$(version)_windows_amd64.zip") + add_versions("1.9.7", "419e417d33f94888e176f2cccf1a101a16fc283bf721021f2a11f1b74570db97") + elseif is_plat("linux") then + add_urls("https://releases.hashicorp.com/nomad/$(version)/nomad_$(version)_linux_amd64.zip") + add_versions("1.9.7", "e9c7337893eceb549557ef9ad341b3ae64f5f43e29ff1fb167b70cfd16748d2d") + elseif is_plat("macosx") then + if is_arch("arm64") then + add_urls("https://releases.hashicorp.com/nomad/$(version)/nomad_$(version)_darwin_arm64.zip") + add_versions("1.9.7", "90f87dffb3669a842a8428899088f3a0ec5a0d204e5278dbb0c1ac16ab295935") + else + add_urls("https://releases.hashicorp.com/nomad/$(version)/nomad_$(version)_darwin_amd64.zip") + add_versions("1.9.7", "8f5befe1e11ef5664c0c212053aa3fc3e095e52a86e90c1315d7580f19ad7997") + end + end + + on_install(function (package) + if is_plat("windows") then + os.cp("nomad.exe", package:installdir("bin")) + else + os.cp("nomad", package:installdir("bin")) + end + end) + + on_test(function (package) + if is_plat("windows") then + os.run("%s version", package:installdir("bin", "nomad.exe")) + elseif is_plat("linux") then + os.run("%s version", package:installdir("bin", "nomad")) + end + end) diff --git a/src/zen/cmds/exec_cmd.cpp b/src/zen/cmds/exec_cmd.cpp index 407f42ee3..42c7119e7 100644 --- a/src/zen/cmds/exec_cmd.cpp +++ b/src/zen/cmds/exec_cmd.cpp @@ -2,7 +2,7 @@ #include "exec_cmd.h" -#include +#include #include #include #include @@ -14,9 +14,13 @@ #include #include #include +#include #include #include +#include #include +#include +#include #include #include @@ -47,12 +51,17 @@ ExecCommand::ExecCommand() m_Options.add_option("", "", "stride", "Recording replay stride", cxxopts::value(m_Stride), ""); m_Options.add_option("", "", "limit", "Recording replay limit", cxxopts::value(m_Limit), ""); m_Options.add_option("", "", "beacon", "Beacon path", cxxopts::value(m_BeaconPath), ""); + m_Options.add_option("", "", "orch", "Orchestrator URL for worker discovery", cxxopts::value(m_OrchestratorUrl), ""); m_Options.add_option("", "", "mode", "Select execution mode (http,inproc,dump,direct,beacon,buildlog)", cxxopts::value(m_Mode)->default_value("http"), ""); + m_Options + .add_option("", "", "dump-actions", "Dump each action to console as it is dispatched", cxxopts::value(m_DumpActions), ""); + m_Options.add_option("", "o", "output", "Save action results to directory", cxxopts::value(m_OutputPath), ""); + m_Options.add_option("", "", "binary", "Write output as binary packages instead of YAML", cxxopts::value(m_Binary), ""); m_Options.add_option("", "", "quiet", "Quiet mode (less logging)", cxxopts::value(m_Quiet), ""); m_Options.parse_positional("mode"); } @@ -236,16 +245,16 @@ ExecCommand::InProcessExecute() ZEN_ASSERT(m_ChunkResolver); ChunkResolver& Resolver = *m_ChunkResolver; - zen::compute::FunctionServiceSession FunctionSession(Resolver); + zen::compute::ComputeServiceSession ComputeSession(Resolver); std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); - FunctionSession.AddLocalRunner(Resolver, TempPath); + ComputeSession.AddLocalRunner(Resolver, TempPath); - return ExecUsingSession(FunctionSession); + return ExecUsingSession(ComputeSession); } int -ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSession) +ExecCommand::ExecUsingSession(zen::compute::ComputeServiceSession& ComputeSession) { struct JobTracker { @@ -281,6 +290,117 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess JobTracker PendingJobs; + struct ActionSummaryEntry + { + int32_t Lsn = 0; + int RecordingIndex = 0; + IoHash ActionId; + std::string FunctionName; + int InputAttachments = 0; + uint64_t InputBytes = 0; + int OutputAttachments = 0; + uint64_t OutputBytes = 0; + float WallSeconds = 0.0f; + float CpuSeconds = 0.0f; + uint64_t SubmittedTicks = 0; + uint64_t StartedTicks = 0; + std::string ExecutionLocation; + }; + + std::mutex SummaryLock; + std::unordered_map SummaryEntries; + + ComputeSession.WaitUntilReady(); + + // Register as a client with the orchestrator (best-effort) + + std::string OrchestratorClientId; + + if (!m_OrchestratorUrl.empty()) + { + try + { + HttpClient OrchestratorClient(m_OrchestratorUrl); + + CbObjectWriter Ann; + Ann << "session_id"sv << GetSessionId(); + Ann << "hostname"sv << std::string_view(GetMachineName()); + + CbObjectWriter Meta; + Meta << "source"sv + << "zen-exec"sv; + Ann << "metadata"sv << Meta.Save(); + + auto Resp = OrchestratorClient.Post("/orch/clients", Ann.Save()); + if (Resp.IsSuccess()) + { + OrchestratorClientId = std::string(Resp.AsObject()["id"].AsString()); + ZEN_CONSOLE_INFO("registered with orchestrator as {}", OrchestratorClientId); + } + else + { + ZEN_WARN("failed to register with orchestrator (status {})", static_cast(Resp.StatusCode)); + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("failed to register with orchestrator: {}", Ex.what()); + } + } + + Stopwatch OrchestratorHeartbeatTimer; + + auto SendOrchestratorHeartbeat = [&] { + if (OrchestratorClientId.empty() || OrchestratorHeartbeatTimer.GetElapsedTimeMs() < 30'000) + { + return; + } + OrchestratorHeartbeatTimer.Reset(); + try + { + HttpClient OrchestratorClient(m_OrchestratorUrl); + std::ignore = OrchestratorClient.Post(fmt::format("/orch/clients/{}/update", OrchestratorClientId)); + } + catch (...) + { + } + }; + + auto ClientCleanup = MakeGuard([&] { + if (!OrchestratorClientId.empty()) + { + try + { + HttpClient OrchestratorClient(m_OrchestratorUrl); + std::ignore = OrchestratorClient.Post(fmt::format("/orch/clients/{}/complete", OrchestratorClientId)); + } + catch (...) + { + } + } + }); + + // Create a queue to group all actions from this exec session + + CbObjectWriter Metadata; + Metadata << "source"sv + << "zen-exec"sv; + + auto QueueResult = ComputeSession.CreateQueue("zen-exec", Metadata.Save()); + const int QueueId = QueueResult.QueueId; + if (!QueueId) + { + ZEN_ERROR("failed to create compute queue"); + return 1; + } + + auto QueueCleanup = MakeGuard([&] { ComputeSession.DeleteQueue(QueueId); }); + + if (!m_OutputPath.empty()) + { + zen::CreateDirectories(m_OutputPath); + } + std::atomic IsDraining{0}; auto DrainCompletedJobs = [&] { @@ -292,7 +412,7 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess auto _ = MakeGuard([&] { IsDraining.store(0, std::memory_order_release); }); CbObjectWriter Cbo; - FunctionSession.GetCompleted(Cbo); + ComputeSession.GetQueueCompleted(QueueId, Cbo); if (CbObject Completed = Cbo.Save()) { @@ -301,10 +421,89 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess int32_t CompleteLsn = It.AsInt32(); CbPackage ResultPackage; - HttpResponseCode Response = FunctionSession.GetActionResult(CompleteLsn, /* out */ ResultPackage); + HttpResponseCode Response = ComputeSession.GetActionResult(CompleteLsn, /* out */ ResultPackage); if (Response == HttpResponseCode::OK) { + if (!m_OutputPath.empty() && ResultPackage) + { + int OutputAttachments = 0; + uint64_t OutputBytes = 0; + + if (!m_Binary) + { + // Write the root object as YAML + ExtendableStringBuilder<4096> YamlStr; + CompactBinaryToYaml(ResultPackage.GetObject(), YamlStr); + + std::string_view Yaml = YamlStr; + zen::WriteFile(m_OutputPath / fmt::format("{}.result.yaml", CompleteLsn), + IoBuffer(IoBuffer::Clone, Yaml.data(), Yaml.size())); + + // Write decompressed attachments + auto Attachments = ResultPackage.GetAttachments(); + + if (!Attachments.empty()) + { + std::filesystem::path AttDir = m_OutputPath / fmt::format("{}.result.attachments", CompleteLsn); + zen::CreateDirectories(AttDir); + + for (const CbAttachment& Att : Attachments) + { + ++OutputAttachments; + + IoHash AttHash = Att.GetHash(); + + if (Att.IsCompressedBinary()) + { + SharedBuffer Decompressed = Att.AsCompressedBinary().Decompress(); + OutputBytes += Decompressed.GetSize(); + zen::WriteFile(AttDir / AttHash.ToHexString(), + IoBuffer(IoBuffer::Clone, Decompressed.GetData(), Decompressed.GetSize())); + } + else + { + SharedBuffer Binary = Att.AsBinary(); + OutputBytes += Binary.GetSize(); + zen::WriteFile(AttDir / AttHash.ToHexString(), + IoBuffer(IoBuffer::Clone, Binary.GetData(), Binary.GetSize())); + } + } + } + + if (!m_QuietLogging) + { + ZEN_CONSOLE("saved result: {}/{}.result.yaml ({} attachments)", + m_OutputPath.string(), + CompleteLsn, + OutputAttachments); + } + } + else + { + CompositeBuffer Serialized = FormatPackageMessageBuffer(ResultPackage); + zen::WriteFile(m_OutputPath / fmt::format("{}.result.pkg", CompleteLsn), std::move(Serialized)); + + for (const CbAttachment& Att : ResultPackage.GetAttachments()) + { + ++OutputAttachments; + OutputBytes += Att.AsBinary().GetSize(); + } + + if (!m_QuietLogging) + { + ZEN_CONSOLE("saved result: {}/{}.result.pkg", m_OutputPath.string(), CompleteLsn); + } + } + + std::lock_guard Lock(SummaryLock); + if (auto It2 = SummaryEntries.find(CompleteLsn); It2 != SummaryEntries.end()) + { + It2->second.OutputAttachments = OutputAttachments; + It2->second.OutputBytes = OutputBytes; + } + } + PendingJobs.Remove(CompleteLsn); ZEN_CONSOLE("completed: LSN {} ({} still pending)", CompleteLsn, PendingJobs.GetSize()); @@ -321,7 +520,7 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess { CbPackage WorkerDesc = Kv.second; - FunctionSession.RegisterWorker(WorkerDesc); + ComputeSession.RegisterWorker(WorkerDesc); } // Then submit work items @@ -367,10 +566,14 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess TargetParallelism = 1; } + std::atomic RecordingIndex{0}; + m_RecordingReader->IterateActions( [&](CbObject ActionObject, const IoHash& ActionId) { // Enqueue job + const int CurrentRecordingIndex = RecordingIndex++; + Stopwatch SubmitTimer; const int Priority = 0; @@ -404,8 +607,29 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess ObjStr); } - if (zen::compute::FunctionServiceSession::EnqueueResult EnqueueResult = - FunctionSession.EnqueueAction(ActionObject, Priority)) + if (m_DumpActions) + { + int AttachmentCount = 0; + uint64_t AttachmentBytes = 0; + + ActionObject.IterateAttachments([&](CbFieldView Field) { + IoHash AttachData = Field.AsAttachment(); + + ++AttachmentCount; + + if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachData)) + { + AttachmentBytes += ChunkData.GetSize(); + } + }); + + zen::ExtendableStringBuilder<1024> ObjStr; + zen::CompactBinaryToYaml(ActionObject, ObjStr); + ZEN_CONSOLE("action {} ({} attachments, {}):\n{}", ActionId, AttachmentCount, NiceBytes(AttachmentBytes), ObjStr); + } + + if (zen::compute::ComputeServiceSession::EnqueueResult EnqueueResult = + ComputeSession.EnqueueActionToQueue(QueueId, ActionObject, Priority)) { const int32_t LsnField = EnqueueResult.Lsn; @@ -421,6 +645,96 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess RemainingWorkItems); } + if (!m_OutputPath.empty()) + { + ActionSummaryEntry Entry; + Entry.Lsn = LsnField; + Entry.RecordingIndex = CurrentRecordingIndex; + Entry.ActionId = ActionId; + Entry.FunctionName = std::string(ActionObject["Function"sv].AsString()); + + if (!m_Binary) + { + // Write action object as YAML + ExtendableStringBuilder<4096> YamlStr; + CompactBinaryToYaml(ActionObject, YamlStr); + + std::string_view Yaml = YamlStr; + zen::WriteFile(m_OutputPath / fmt::format("{}.action.yaml", LsnField), + IoBuffer(IoBuffer::Clone, Yaml.data(), Yaml.size())); + + // Write decompressed input attachments + std::filesystem::path AttDir = m_OutputPath / fmt::format("{}.action.attachments", LsnField); + bool AttDirCreated = false; + + ActionObject.IterateAttachments([&](CbFieldView Field) { + IoHash AttachCid = Field.AsAttachment(); + ++Entry.InputAttachments; + + if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachCid)) + { + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer(ChunkData), RawHash, RawSize); + SharedBuffer Decompressed = Compressed.Decompress(); + + Entry.InputBytes += Decompressed.GetSize(); + + if (!AttDirCreated) + { + zen::CreateDirectories(AttDir); + AttDirCreated = true; + } + + zen::WriteFile(AttDir / AttachCid.ToHexString(), + IoBuffer(IoBuffer::Clone, Decompressed.GetData(), Decompressed.GetSize())); + } + }); + + if (!m_QuietLogging) + { + ZEN_CONSOLE("saved action: {}/{}.action.yaml ({} attachments)", + m_OutputPath.string(), + LsnField, + Entry.InputAttachments); + } + } + else + { + // Build a CbPackage from the action and write as .pkg + CbPackage ActionPackage; + ActionPackage.SetObject(ActionObject); + + ActionObject.IterateAttachments([&](CbFieldView Field) { + IoHash AttachCid = Field.AsAttachment(); + ++Entry.InputAttachments; + + if (IoBuffer ChunkData = m_ChunkResolver->FindChunkByCid(AttachCid)) + { + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer(ChunkData), RawHash, RawSize); + + Entry.InputBytes += ChunkData.GetSize(); + ActionPackage.AddAttachment(CbAttachment(std::move(Compressed), RawHash)); + } + }); + + CompositeBuffer Serialized = FormatPackageMessageBuffer(ActionPackage); + zen::WriteFile(m_OutputPath / fmt::format("{}.action.pkg", LsnField), std::move(Serialized)); + + if (!m_QuietLogging) + { + ZEN_CONSOLE("saved action: {}/{}.action.pkg", m_OutputPath.string(), LsnField); + } + } + + std::lock_guard Lock(SummaryLock); + SummaryEntries.emplace(LsnField, std::move(Entry)); + } + PendingJobs.Insert(LsnField); } else @@ -450,6 +764,7 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess // Check for completed work DrainCompletedJobs(); + SendOrchestratorHeartbeat(); }, TargetParallelism); @@ -461,6 +776,394 @@ ExecCommand::ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSess zen::Sleep(500); DrainCompletedJobs(); + SendOrchestratorHeartbeat(); + } + + // Merge timing data from queue history into summary entries + + if (!SummaryEntries.empty()) + { + // RunnerAction::State indices (can't include functionrunner.h from here) + constexpr int kStateNew = 0; + constexpr int kStatePending = 1; + constexpr int kStateRunning = 3; + constexpr int kStateCompleted = 4; // first terminal state + constexpr int kStateCount = 8; + + for (const auto& HistEntry : ComputeSession.GetQueueHistory(QueueId, 0)) + { + std::lock_guard Lock(SummaryLock); + if (auto It = SummaryEntries.find(HistEntry.Lsn); It != SummaryEntries.end()) + { + // Find terminal state timestamp (Completed, Failed, Abandoned, or Cancelled) + uint64_t EndTick = 0; + for (int S = kStateCompleted; S < kStateCount; ++S) + { + if (HistEntry.Timestamps[S] != 0) + { + EndTick = HistEntry.Timestamps[S]; + break; + } + } + uint64_t StartTick = HistEntry.Timestamps[kStateNew]; + if (EndTick > StartTick) + { + It->second.WallSeconds = float(double(EndTick - StartTick) / double(TimeSpan::TicksPerSecond)); + } + It->second.CpuSeconds = HistEntry.CpuSeconds; + It->second.SubmittedTicks = HistEntry.Timestamps[kStatePending]; + It->second.StartedTicks = HistEntry.Timestamps[kStateRunning]; + It->second.ExecutionLocation = HistEntry.ExecutionLocation; + } + } + } + + // Write summary file if output path is set + + if (!m_OutputPath.empty() && !SummaryEntries.empty()) + { + std::vector Sorted; + Sorted.reserve(SummaryEntries.size()); + for (auto& [_, Entry] : SummaryEntries) + { + Sorted.push_back(std::move(Entry)); + } + + std::sort(Sorted.begin(), Sorted.end(), [](const ActionSummaryEntry& A, const ActionSummaryEntry& B) { + return A.RecordingIndex < B.RecordingIndex; + }); + + auto FormatTimestamp = [](uint64_t Ticks) -> std::string { + if (Ticks == 0) + { + return "-"; + } + return DateTime(Ticks).ToString("%H:%M:%S.%s"); + }; + + ExtendableStringBuilder<4096> Summary; + Summary.Append(fmt::format("{:<8} {:<8} {:<40} {:<40} {:>8} {:>12} {:>8} {:>12} {:>8} {:>8} {:>12} {:>12} {:<24}\n", + "LSN", + "Index", + "ActionId", + "Function", + "InAtt", + "InBytes", + "OutAtt", + "OutBytes", + "Wall(s)", + "CPU(s)", + "Submitted", + "Started", + "Location")); + Summary.Append(fmt::format("{:-<8} {:-<8} {:-<40} {:-<40} {:-<8} {:-<12} {:-<8} {:-<12} {:-<8} {:-<8} {:-<12} {:-<12} {:-<24}\n", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "")); + + for (const ActionSummaryEntry& Entry : Sorted) + { + Summary.Append(fmt::format("{:<8} {:<8} {:<40} {:<40} {:>8} {:>12} {:>8} {:>12} {:>8.2f} {:>8.2f} {:>12} {:>12} {:<24}\n", + Entry.Lsn, + Entry.RecordingIndex, + Entry.ActionId, + Entry.FunctionName, + Entry.InputAttachments, + NiceBytes(Entry.InputBytes), + Entry.OutputAttachments, + NiceBytes(Entry.OutputBytes), + Entry.WallSeconds, + Entry.CpuSeconds, + FormatTimestamp(Entry.SubmittedTicks), + FormatTimestamp(Entry.StartedTicks), + Entry.ExecutionLocation)); + } + + std::filesystem::path SummaryPath = m_OutputPath / "summary.txt"; + std::string_view SummaryStr = Summary; + zen::WriteFile(SummaryPath, IoBuffer(IoBuffer::Clone, SummaryStr.data(), SummaryStr.size())); + + ZEN_CONSOLE("wrote summary to {}", SummaryPath.string()); + + if (!m_Binary) + { + auto EscapeHtml = [](std::string_view Input) -> std::string { + std::string Out; + Out.reserve(Input.size()); + for (char C : Input) + { + switch (C) + { + case '&': + Out += "&"; + break; + case '<': + Out += "<"; + break; + case '>': + Out += ">"; + break; + case '"': + Out += """; + break; + case '\'': + Out += "'"; + break; + default: + Out += C; + } + } + return Out; + }; + + auto EscapeJson = [](std::string_view Input) -> std::string { + std::string Out; + Out.reserve(Input.size()); + for (char C : Input) + { + switch (C) + { + case '"': + Out += "\\\""; + break; + case '\\': + Out += "\\\\"; + break; + case '\n': + Out += "\\n"; + break; + case '\r': + Out += "\\r"; + break; + case '\t': + Out += "\\t"; + break; + default: + if (static_cast(C) < 0x20) + { + Out += fmt::format("\\u{:04x}", static_cast(static_cast(C))); + } + else + { + Out += C; + } + } + } + return Out; + }; + + ExtendableStringBuilder<8192> Html; + + Html.Append(std::string_view(R"( +Exec Summary + +

      Exec Summary

      + +
      + + + + + + + + + + + + + + + + + +
      LSN Index Action ID Function In Attachments In Bytes Out Attachments Out Bytes Wall(s) CPU(s) Submitted Started Location
      + +)JS")); + + std::filesystem::path HtmlPath = m_OutputPath / "summary.html"; + std::string_view HtmlStr = Html; + zen::WriteFile(HtmlPath, IoBuffer(IoBuffer::Clone, HtmlStr.data(), HtmlStr.size())); + + ZEN_CONSOLE("wrote HTML summary to {}", HtmlPath.string()); + } } if (FailedWorkCounter) @@ -491,10 +1194,10 @@ ExecCommand::HttpExecute() std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); - zen::compute::FunctionServiceSession FunctionSession(Resolver); - FunctionSession.AddRemoteRunner(Resolver, TempPath, m_HostName); + zen::compute::ComputeServiceSession ComputeSession(Resolver); + ComputeSession.AddRemoteRunner(Resolver, TempPath, m_HostName); - return ExecUsingSession(FunctionSession); + return ExecUsingSession(ComputeSession); } int @@ -504,11 +1207,21 @@ ExecCommand::BeaconExecute() ChunkResolver& Resolver = *m_ChunkResolver; std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); - zen::compute::FunctionServiceSession FunctionSession(Resolver); - FunctionSession.AddRemoteRunner(Resolver, TempPath, "http://localhost:8558"); - // FunctionSession.AddRemoteRunner(Resolver, TempPath, "http://10.99.9.246:8558"); + zen::compute::ComputeServiceSession ComputeSession(Resolver); + + if (!m_OrchestratorUrl.empty()) + { + ZEN_CONSOLE_INFO("using orchestrator at {}", m_OrchestratorUrl); + ComputeSession.SetOrchestratorEndpoint(m_OrchestratorUrl); + ComputeSession.SetOrchestratorBasePath(TempPath); + } + else + { + ZEN_CONSOLE_INFO("note: using hard-coded local worker path"); + ComputeSession.AddRemoteRunner(Resolver, TempPath, "http://localhost:8558"); + } - return ExecUsingSession(FunctionSession); + return ExecUsingSession(ComputeSession); } ////////////////////////////////////////////////////////////////////////// @@ -635,10 +1348,10 @@ ExecCommand::BuildActionsLog() std::filesystem::path TempPath = std::filesystem::absolute(".zen_temp"); - zen::compute::FunctionServiceSession FunctionSession(Resolver); - FunctionSession.StartRecording(Resolver, m_RecordingLogPath); + zen::compute::ComputeServiceSession ComputeSession(Resolver); + ComputeSession.StartRecording(Resolver, m_RecordingLogPath); - return ExecUsingSession(FunctionSession); + return ExecUsingSession(ComputeSession); } void diff --git a/src/zen/cmds/exec_cmd.h b/src/zen/cmds/exec_cmd.h index 43d092144..6311354c0 100644 --- a/src/zen/cmds/exec_cmd.h +++ b/src/zen/cmds/exec_cmd.h @@ -23,7 +23,7 @@ class ChunkResolver; #if ZEN_WITH_COMPUTE_SERVICES namespace zen::compute { -class FunctionServiceSession; +class ComputeServiceSession; } namespace zen { @@ -49,6 +49,7 @@ public: private: cxxopts::Options m_Options{Name, Description}; std::string m_HostName; + std::string m_OrchestratorUrl; std::filesystem::path m_BeaconPath; std::filesystem::path m_RecordingPath; std::filesystem::path m_RecordingLogPath; @@ -57,6 +58,8 @@ private: int m_Limit = 0; bool m_Quiet = false; std::string m_Mode{"http"}; + std::filesystem::path m_OutputPath; + bool m_Binary = false; struct FunctionDefinition { @@ -74,13 +77,14 @@ private: std::vector m_FunctionList; bool m_VerboseLogging = false; bool m_QuietLogging = false; + bool m_DumpActions = false; zen::ChunkResolver* m_ChunkResolver = nullptr; zen::compute::RecordingReaderBase* m_RecordingReader = nullptr; void RegisterWorkerFunctionsFromDescription(const zen::CbObject& WorkerDesc, const zen::IoHash& WorkerId); - int ExecUsingSession(zen::compute::FunctionServiceSession& FunctionSession); + int ExecUsingSession(zen::compute::ComputeServiceSession& ComputeSession); // Execution modes diff --git a/src/zencompute/CLAUDE.md b/src/zencompute/CLAUDE.md new file mode 100644 index 000000000..f5188123f --- /dev/null +++ b/src/zencompute/CLAUDE.md @@ -0,0 +1,232 @@ +# zencompute Module + +Lambda-style compute function service. Accepts execution requests from HTTP clients, schedules them across local and remote runners, and tracks results. + +## Directory Structure + +``` +src/zencompute/ +├── include/zencompute/ # Public headers +│ ├── computeservice.h # ComputeServiceSession public API +│ ├── httpcomputeservice.h # HTTP service wrapper +│ ├── orchestratorservice.h # Worker registry and orchestration +│ ├── httporchestrator.h # HTTP orchestrator with WebSocket push +│ ├── recordingreader.h # Recording/replay reader API +│ ├── cloudmetadata.h # Cloud provider detection (AWS/Azure/GCP) +│ └── mockimds.h # Test helper for cloud metadata +├── runners/ # Execution backends +│ ├── functionrunner.h/.cpp # Abstract base + BaseRunnerGroup/RunnerGroup +│ ├── localrunner.h/.cpp # LocalProcessRunner (sandbox, monitoring, CPU sampling) +│ ├── windowsrunner.h/.cpp # Windows AppContainer sandboxing + CreateProcessW +│ ├── linuxrunner.h/.cpp # Linux user/mount/network namespace isolation +│ ├── macrunner.h/.cpp # macOS Seatbelt sandboxing +│ ├── winerunner.h/.cpp # Wine runner for Windows executables on Linux +│ ├── remotehttprunner.h/.cpp # Remote HTTP submission to other zenserver instances +│ └── deferreddeleter.h/.cpp # Background deletion of sandbox directories +├── recording/ # Recording/replay subsystem +│ ├── actionrecorder.h/.cpp # Write actions+attachments to disk +│ └── recordingreader.cpp # Read recordings back +├── timeline/ +│ └── workertimeline.h/.cpp # Per-worker action lifecycle event tracking +├── testing/ +│ └── mockimds.cpp # Mock IMDS for cloud metadata tests +├── computeservice.cpp # ComputeServiceSession::Impl (~1700 lines) +├── httpcomputeservice.cpp # HTTP route registration and handlers (~900 lines) +├── httporchestrator.cpp # Orchestrator HTTP API + WebSocket push +├── orchestratorservice.cpp # Worker registry, health probing +└── cloudmetadata.cpp # IMDS probing, termination monitoring +``` + +## Key Classes + +### `ComputeServiceSession` (computeservice.h) +Public API entry point. Uses PIMPL (`struct Impl` in computeservice.cpp). Owns: +- Two `RunnerGroup`s: `m_LocalRunnerGroup`, `m_RemoteRunnerGroup` +- Scheduler thread that drains `m_UpdatedActions` and drives state transitions +- Action maps: `m_PendingActions`, `m_RunningMap`, `m_ResultsMap` +- Queue map: `m_Queues` (QueueEntry objects) +- Action history ring: `m_ActionHistory` (bounded deque, default 1000) + +**Session states:** Created → Ready → Draining → Paused → Abandoned → Sunset. Both Abandoned and Sunset can be jumped to from any earlier state. Abandoned is used for spot instance termination grace periods — on entry, all pending and running actions are immediately marked as `RunnerAction::State::Abandoned` and running processes are best-effort cancelled. Auto-retry is suppressed while the session is Abandoned. `IsHealthy()` returns false for Abandoned and Sunset. + +### `RunnerAction` (runners/functionrunner.h) +Shared ref-counted struct representing one action through its lifecycle. + +**Key fields:** +- `ActionLsn` — global unique sequence number +- `QueueId` — 0 for implicit/unqueued actions +- `Worker` — descriptor + content hash +- `ActionObj` — CbObject with the action spec +- `CpuUsagePercent` / `CpuSeconds` — atomics updated by monitor thread +- `RetryCount` — atomic int tracking how many times the action has been rescheduled +- `Timestamps[State::_Count]` — timestamp of each state transition + +**State machine (forward-only under normal flow, atomic):** +``` +New → Pending → Submitting → Running → Completed + → Failed + → Abandoned + → Cancelled +``` +`SetActionState()` rejects non-forward transitions. The one exception is `ResetActionStateToPending()`, which uses CAS to atomically transition from Failed/Abandoned back to Pending for rescheduling. It clears timestamps from Submitting onward, resets execution fields, increments `RetryCount`, and calls `PostUpdate()` to re-enter the scheduler pipeline. + +### `LocalProcessRunner` (runners/localrunner.h) +Base for all local execution. Platform runners subclass this and override: +- `SubmitAction()` — fork/exec the worker process +- `SweepRunningActions()` — poll for process exit (waitpid / WaitForSingleObject) +- `CancelRunningActions()` — signal all processes during shutdown +- `SampleProcessCpu(RunningAction&)` — read platform CPU usage (no-op default) + +**Infrastructure owned by LocalProcessRunner:** +- Monitor thread — calls `SweepRunningActions()` then `SampleRunningProcessCpu()` in a loop +- `m_RunningMap` — `RwLock`-guarded map of `Lsn → RunningAction` +- `DeferredDirectoryDeleter` — sandbox directories are queued for async deletion +- `PrepareActionSubmission()` — shared preamble (capacity check, sandbox creation, worker manifesting, input decompression) +- `ProcessCompletedActions()` — shared post-processing (gather outputs, set state, enqueue deletion) + +**CPU sampling:** `SampleRunningProcessCpu()` iterates `m_RunningMap` under shared lock, calls `SampleProcessCpu()` per entry, throttled to every 5 seconds per action. Platform implementations: +- Linux: `/proc/{pid}/stat` utime+stime jiffies ÷ `_SC_CLK_TCK` +- Windows: `GetProcessTimes()` in 100ns intervals ÷ 10,000,000 +- macOS: `proc_pidinfo(PROC_PIDTASKINFO)` pti_total_user+system nanoseconds ÷ 1,000,000,000 + +### `FunctionRunner` / `RunnerGroup` (runners/functionrunner.h) +Abstract base for runners. `RunnerGroup` holds a vector of runners and load-balances across them using a round-robin atomic index. `BaseRunnerGroup::SubmitActions()` distributes a batch proportionally based on per-runner capacity. + +### `HttpComputeService` (include/zencompute/httpcomputeservice.h) +Wraps `ComputeServiceSession` as an HTTP service. All routes are registered in the constructor. Handles CbPackage attachment ingestion from `CidStore` before forwarding to the service. + +## Action Lifecycle (End to End) + +1. **HTTP POST** → `HttpComputeService` ingests attachments, calls `EnqueueAction()` +2. **Enqueue** → creates `RunnerAction` (New → Pending), calls `PostUpdate()` +3. **PostUpdate** → appends to `m_UpdatedActions`, signals scheduler thread event +4. **Scheduler thread** → drains `m_UpdatedActions`, drives pending actions to runners +5. **Runner `SubmitAction()`** → Pending → Submitting (on runner's worker pool thread) +6. **Process launch** → Submitting → Running, added to `m_RunningMap` +7. **Monitor thread `SweepRunningActions()`** → detects exit, gathers outputs +8. **`ProcessCompletedActions()`** → Running → Completed/Failed/Abandoned, `PostUpdate()` +9. **Scheduler thread `HandleActionUpdates()`** — for Failed/Abandoned actions, checks retry limit; if retries remain, calls `ResetActionStateToPending()` which loops back to step 3. Otherwise moves to `m_ResultsMap`, records history, notifies queue. +10. **Client `GET /jobs/{lsn}`** → returns result from `m_ResultsMap`, schedules retirement + +### Action Rescheduling + +Actions that fail or are abandoned can be automatically retried or manually rescheduled via the API. + +**Automatic retry (scheduler path):** In `HandleActionUpdates()`, when a Failed or Abandoned state is detected, the scheduler checks `RetryCount < GetMaxRetriesForQueue(QueueId)`. If retries remain, the action is removed from active maps and `ResetActionStateToPending()` is called, which re-enters it into the scheduler pipeline. The action keeps its original LSN so clients can continue polling with the same identifier. + +**Manual retry (API path):** `POST /compute/jobs/{lsn}` calls `RescheduleAction()`, which finds the action in `m_ResultsMap`, validates state (must be Failed or Abandoned), checks the retry limit, reverses queue counters (moving the LSN from `FinishedLsns` back to `ActiveLsns`), removes from results, and calls `ResetActionStateToPending()`. Returns 200 with `{lsn, retry_count}` on success, 409 Conflict with `{error}` on failure. + +**Retry limit:** Default of 3, overridable per-queue via the `max_retries` integer field in the queue's `Config` CbObject (set at `CreateQueue` time). Both automatic and manual paths respect this limit. + +**Cancelled actions are never retried** — cancellation is an intentional user action, not a transient failure. + +## Queue System + +Queues group actions from a single client session. A `QueueEntry` (internal) tracks: +- `State` — `std::atomic` lifecycle state (Active → Draining → Cancelled) +- `ActiveCount` — pending + running actions (atomic) +- `CompletedCount / FailedCount / AbandonedCount / CancelledCount` (atomics) +- `ActiveLsns` — for cancellation lookup (under `m_Lock`) +- `FinishedLsns` — moved here when actions complete +- `IdleSince` — used for 15-minute automatic expiry +- `Config` — CbObject set at creation; supports `max_retries` (int) to override the default retry limit + +**Queue state machine (`QueueState` enum):** +``` +Active → Draining → Cancelled + \ ↑ + ─────────────────────/ +``` +- **Active** — accepts new work, schedules pending work, finishes running work (initial state) +- **Draining** — rejects new work, finishes existing work (one-way via CAS from Active; cannot override Cancelled) +- **Cancelled** — rejects new work, actively cancels in-flight work (reachable from Active or Draining) + +Key operations: +- `CreateQueue(Tag)` → returns `QueueId` +- `EnqueueActionToQueue(QueueId, ...)` → action's `QueueId` field is set at creation +- `CancelQueue(QueueId)` → marks all active LSNs for cancellation +- `DrainQueue(QueueId)` → stops accepting new submissions; existing work finishes naturally (irreversible) +- `GetQueueCompleted(QueueId)` → CbWriter output of finished results +- Queue references in HTTP routes accept either a decimal ID or an Oid token (24-hex), resolved by `ResolveQueueRef()` + +## HTTP API + +All routes registered in `HttpComputeService` constructor. Prefix is configured externally (typically `/compute`). + +### Global endpoints +| Method | Path | Description | +|--------|------|-------------| +| POST | `abandon` | Transition session to Abandoned state (409 if invalid) | +| GET | `jobs/history` | Action history (last N, with timestamps per state) | +| GET | `jobs/running` | In-flight actions with CPU metrics | +| GET | `jobs/completed` | Actions with results available | +| GET/POST/DELETE | `jobs/{lsn}` | GET: result; POST: reschedule failed action; DELETE: retire | +| POST | `jobs/{worker}` | Submit action for specific worker | +| POST | `jobs` | Submit action (worker resolved from descriptor) | +| GET | `workers` | List worker IDs | +| GET | `workers/all` | All workers with full descriptors | +| GET/POST | `workers/{worker}` | Get/register worker | + +### Queue-scoped endpoints +Queue ref is capture(1) in all `queues/{queueref}/...` routes. + +| Method | Path | Description | +|--------|------|-------------| +| GET | `queues` | List queue IDs | +| POST | `queues` | Create queue | +| GET/DELETE | `queues/{queueref}` | Status / delete | +| POST | `queues/{queueref}/drain` | Drain queue (irreversible; rejects new submissions) | +| GET | `queues/{queueref}/completed` | Queue's completed results | +| GET | `queues/{queueref}/history` | Queue's action history | +| GET | `queues/{queueref}/running` | Queue's running actions | +| POST | `queues/{queueref}/jobs` | Submit to queue | +| GET/POST | `queues/{queueref}/jobs/{lsn}` | GET: result; POST: reschedule | +| GET/POST | `queues/{queueref}/workers/...` | Worker endpoints (same as global) | + +Worker handler logic is extracted into private helpers (`HandleWorkersGet`, `HandleWorkersAllGet`, `HandleWorkerRequest`) shared by top-level and queue-scoped routes. + +## Concurrency Model + +**Locking discipline:** When multiple locks must be held simultaneously, always acquire in this order to prevent deadlocks: +1. `m_ResultsLock` +2. `m_RunningLock` (comment in localrunner.h: "must be taken *after* m_ResultsLock") +3. `m_PendingLock` +4. `m_QueueLock` + +**Atomic fields** for counters and simple state: queue counts, `CpuUsagePercent`, `CpuSeconds`, `RetryCount`, `RunnerAction::m_ActionState`. + +**Update decoupling:** Runners call `PostUpdate(RunnerAction*)` rather than directly mutating service state. The scheduler thread batches and deduplicates updates. + +**Thread ownership:** +- Scheduler thread — drives state transitions, owns `m_PendingActions` +- Monitor thread (per runner) — polls process completion, owns `m_RunningMap` via shared lock +- Worker pool threads — async submission, brief `SubmitAction()` calls +- HTTP threads — read-only access to results, queue status + +## Sandbox Layout + +Each action gets a unique numbered directory under `m_SandboxPath`: +``` +scratch/{counter}/ + worker/ ← worker binaries (or bind-mounted on Linux) + inputs/ ← decompressed action inputs + outputs/ ← written by worker process +``` + +On Linux with sandboxing enabled, the process runs in a pivot-rooted namespace with `/usr`, `/lib`, `/etc`, `/worker` bind-mounted read-only and a tmpfs `/dev`. + +## Adding a New HTTP Endpoint + +1. Register the route in the `HttpComputeService` constructor in `httpcomputeservice.cpp` +2. If the handler is shared between top-level and a `queues/{queueref}/...` variant, extract it as a private helper method declared in `httpcomputeservice.h` +3. Queue-scoped routes validate the queue ref with `ResolveQueueRef(HttpReq, Req.GetCapture(1))` which writes an error response and returns 0 on failure +4. Use `CbObjectWriter` for response bodies; emit via `HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save())` +5. Conditional fields (e.g., optional CPU metrics): emit inside `if (value > 0.0f)` / `if (value >= 0.0f)` guards to omit absent values rather than emitting sentinel values + +## Adding a New Runner Platform + +1. Subclass `LocalProcessRunner`, add `h`/`cpp` files in `runners/` +2. Override `SubmitAction()`, `SweepRunningActions()`, `CancelRunningActions()`, and optionally `CancelAction(int)` and `SampleProcessCpu(RunningAction&)` +3. `SampleProcessCpu()` must update both `Running.Action->CpuSeconds` (unconditionally from the absolute OS value) and `Running.Action->CpuUsagePercent` (delta-based, only after second sample) +4. `ProcessHandle` convention: store pid as `reinterpret_cast(static_cast(pid))` for consistency with the base class +5. Register in `ComputeServiceSession::AddLocalRunner()` in `computeservice.cpp` diff --git a/src/zencompute/actionrecorder.cpp b/src/zencompute/actionrecorder.cpp deleted file mode 100644 index 04c4b5141..000000000 --- a/src/zencompute/actionrecorder.cpp +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "actionrecorder.h" - -#include "functionrunner.h" - -#include -#include -#include -#include -#include -#include - -#if ZEN_PLATFORM_WINDOWS -# include -# define ZEN_CONCRT_AVAILABLE 1 -#else -# define ZEN_CONCRT_AVAILABLE 0 -#endif - -#if ZEN_WITH_COMPUTE_SERVICES - -namespace zen::compute { - -using namespace std::literals; - -////////////////////////////////////////////////////////////////////////// - -RecordingFileWriter::RecordingFileWriter() -{ -} - -RecordingFileWriter::~RecordingFileWriter() -{ - Close(); -} - -void -RecordingFileWriter::Open(std::filesystem::path FilePath) -{ - using namespace std::literals; - - m_File.Open(FilePath, BasicFile::Mode::kTruncate); - m_File.Write("----DDC2----DATA", 16, 0); - m_FileOffset = 16; - - std::filesystem::path TocPath = FilePath.replace_extension(".ztoc"); - m_TocFile.Open(TocPath, BasicFile::Mode::kTruncate); - - m_TocWriter << "version"sv << 1; - m_TocWriter.BeginArray("toc"sv); -} - -void -RecordingFileWriter::Close() -{ - m_TocWriter.EndArray(); - CbObject Toc = m_TocWriter.Save(); - - std::error_code Ec; - m_TocFile.WriteAll(Toc.GetBuffer().AsIoBuffer(), Ec); -} - -void -RecordingFileWriter::AppendObject(const CbObject& Object, const IoHash& ObjectHash) -{ - RwLock::ExclusiveLockScope _(m_FileLock); - - MemoryView ObjectView = Object.GetBuffer().GetView(); - - std::error_code Ec; - m_File.Write(ObjectView, m_FileOffset, Ec); - - if (Ec) - { - throw std::system_error(Ec, "failed writing to archive"); - } - - m_TocWriter.BeginArray(); - m_TocWriter.AddHash(ObjectHash); - m_TocWriter.AddInteger(m_FileOffset); - m_TocWriter.AddInteger(gsl::narrow(ObjectView.GetSize())); - m_TocWriter.EndArray(); - - m_FileOffset += ObjectView.GetSize(); -} - -////////////////////////////////////////////////////////////////////////// - -ActionRecorder::ActionRecorder(ChunkResolver& InChunkResolver, const std::filesystem::path& RecordingLogPath) -: m_ChunkResolver(InChunkResolver) -, m_RecordingLogDir(RecordingLogPath) -{ - std::error_code Ec; - CreateDirectories(m_RecordingLogDir, Ec); - - if (Ec) - { - ZEN_WARN("Could not create directory '{}': {}", m_RecordingLogDir, Ec.message()); - } - - CleanDirectory(m_RecordingLogDir, /* ForceRemoveReadOnlyFiles */ true, Ec); - - if (Ec) - { - ZEN_WARN("Could not clean directory '{}': {}", m_RecordingLogDir, Ec.message()); - } - - m_WorkersFile.Open(m_RecordingLogDir / "workers.zdat"); - m_ActionsFile.Open(m_RecordingLogDir / "actions.zdat"); - - CidStoreConfiguration CidConfig; - CidConfig.RootDirectory = m_RecordingLogDir / "cid"; - CidConfig.HugeValueThreshold = 128 * 1024 * 1024; - - m_CidStore.Initialize(CidConfig); -} - -ActionRecorder::~ActionRecorder() -{ - Shutdown(); -} - -void -ActionRecorder::Shutdown() -{ - m_CidStore.Flush(); -} - -void -ActionRecorder::RegisterWorker(const CbPackage& WorkerPackage) -{ - const IoHash WorkerId = WorkerPackage.GetObjectHash(); - - m_WorkersFile.AppendObject(WorkerPackage.GetObject(), WorkerId); - - std::unordered_set AddedChunks; - uint64_t AddedBytes = 0; - - // First add all attachments from the worker package itself - - for (const CbAttachment& Attachment : WorkerPackage.GetAttachments()) - { - CompressedBuffer Buffer = Attachment.AsCompressedBinary(); - IoBuffer Data = Buffer.GetCompressed().Flatten().AsIoBuffer(); - - const IoHash ChunkHash = Buffer.DecodeRawHash(); - - CidStore::InsertResult Result = m_CidStore.AddChunk(Data, ChunkHash, CidStore::InsertMode::kCopyOnly); - - AddedChunks.insert(ChunkHash); - - if (Result.New) - { - AddedBytes += Data.GetSize(); - } - } - - // Not all attachments will be present in the worker package, so we need to add - // all referenced chunks to ensure that the recording is self-contained and not - // referencing data in the main CID store - - CbObject WorkerDescriptor = WorkerPackage.GetObject(); - - WorkerDescriptor.IterateAttachments([&](const CbFieldView AttachmentField) { - const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); - - if (!AddedChunks.contains(AttachmentCid)) - { - IoBuffer AttachmentData = m_ChunkResolver.FindChunkByCid(AttachmentCid); - - if (AttachmentData) - { - CidStore::InsertResult Result = m_CidStore.AddChunk(AttachmentData, AttachmentCid, CidStore::InsertMode::kCopyOnly); - - if (Result.New) - { - AddedBytes += AttachmentData.GetSize(); - } - } - else - { - ZEN_WARN("RegisterWorker: could not resolve attachment chunk {} for worker {}", AttachmentCid, WorkerId); - } - - AddedChunks.insert(AttachmentCid); - } - }); - - ZEN_INFO("recorded worker {} with {} attachments ({} bytes)", WorkerId, AddedChunks.size(), AddedBytes); -} - -bool -ActionRecorder::RecordAction(Ref Action) -{ - bool AllGood = true; - - Action->ActionObj.IterateAttachments([&](CbFieldView Field) { - IoHash AttachData = Field.AsHash(); - IoBuffer ChunkData = m_ChunkResolver.FindChunkByCid(AttachData); - - if (ChunkData) - { - if (ChunkData.GetContentType() == ZenContentType::kCompressedBinary) - { - IoHash DecompressedHash; - uint64_t RawSize = 0; - CompressedBuffer Compressed = - CompressedBuffer::FromCompressed(SharedBuffer(ChunkData), /* out */ DecompressedHash, /* out*/ RawSize); - - OodleCompressor Compressor; - OodleCompressionLevel CompressionLevel; - uint64_t BlockSize = 0; - if (Compressed.TryGetCompressParameters(/* out */ Compressor, /* out */ CompressionLevel, /* out */ BlockSize)) - { - if (Compressor == OodleCompressor::NotSet) - { - CompositeBuffer Decompressed = Compressed.DecompressToComposite(); - CompressedBuffer NewCompressed = CompressedBuffer::Compress(std::move(Decompressed), - OodleCompressor::Mermaid, - OodleCompressionLevel::Fast, - BlockSize); - - ChunkData = NewCompressed.GetCompressed().Flatten().AsIoBuffer(); - } - } - } - - const uint64_t ChunkSize = ChunkData.GetSize(); - - m_CidStore.AddChunk(ChunkData, AttachData, CidStore::InsertMode::kCopyOnly); - ++m_ChunkCounter; - m_ChunkBytesCounter.fetch_add(ChunkSize); - } - else - { - AllGood = false; - - ZEN_WARN("could not resolve chunk {}", AttachData); - } - }); - - if (AllGood) - { - m_ActionsFile.AppendObject(Action->ActionObj, Action->ActionId); - ++m_ActionsCounter; - - return true; - } - else - { - return false; - } -} - -} // namespace zen::compute - -#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/actionrecorder.h b/src/zencompute/actionrecorder.h deleted file mode 100644 index 9cc2b44a2..000000000 --- a/src/zencompute/actionrecorder.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace zen { -class CbObject; -class CbPackage; -struct IoHash; -} // namespace zen - -#if ZEN_WITH_COMPUTE_SERVICES - -namespace zen::compute { - -////////////////////////////////////////////////////////////////////////// - -struct RecordingFileWriter -{ - RecordingFileWriter(RecordingFileWriter&&) = delete; - RecordingFileWriter& operator=(RecordingFileWriter&&) = delete; - - RwLock m_FileLock; - BasicFile m_File; - uint64_t m_FileOffset = 0; - CbObjectWriter m_TocWriter; - BasicFile m_TocFile; - - RecordingFileWriter(); - ~RecordingFileWriter(); - - void Open(std::filesystem::path FilePath); - void Close(); - void AppendObject(const CbObject& Object, const IoHash& ObjectHash); -}; - -////////////////////////////////////////////////////////////////////////// - -/** - * Recording "runner" implementation - * - * This class writes out all actions and their attachments to a recording directory - * in a format that can be read back by the RecordingReader. - * - * The contents of the recording directory will be self-contained, with all referenced - * attachments stored in the recording directory itself, so that the recording can be - * moved or shared without needing to maintain references to the main CID store. - * - */ - -class ActionRecorder -{ -public: - ActionRecorder(ChunkResolver& InChunkResolver, const std::filesystem::path& RecordingLogPath); - ~ActionRecorder(); - - ActionRecorder(const ActionRecorder&) = delete; - ActionRecorder& operator=(const ActionRecorder&) = delete; - - void Shutdown(); - void RegisterWorker(const CbPackage& WorkerPackage); - bool RecordAction(Ref Action); - -private: - ChunkResolver& m_ChunkResolver; - std::filesystem::path m_RecordingLogDir; - - RecordingFileWriter m_WorkersFile; - RecordingFileWriter m_ActionsFile; - GcManager m_Gc; - CidStore m_CidStore{m_Gc}; - std::atomic m_ChunkCounter{0}; - std::atomic m_ChunkBytesCounter{0}; - std::atomic m_ActionsCounter{0}; -}; - -} // namespace zen::compute - -#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/cloudmetadata.cpp b/src/zencompute/cloudmetadata.cpp new file mode 100644 index 000000000..b3b3210d9 --- /dev/null +++ b/src/zencompute/cloudmetadata.cpp @@ -0,0 +1,1010 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::compute { + +// All major cloud providers expose instance metadata at this link-local address. +// It is only routable from within a cloud VM; on bare-metal the TCP connect will +// fail, which is how we distinguish cloud from non-cloud environments. +static constexpr std::string_view kImdsEndpoint = "http://169.254.169.254"; + +// Short connect timeout so that detection on non-cloud machines is fast. The IMDS +// is a local service on the hypervisor so 200ms is generous for actual cloud VMs. +static constexpr auto kImdsTimeout = std::chrono::milliseconds{200}; + +std::string_view +ToString(CloudProvider Provider) +{ + switch (Provider) + { + case CloudProvider::AWS: + return "AWS"; + case CloudProvider::Azure: + return "Azure"; + case CloudProvider::GCP: + return "GCP"; + default: + return "None"; + } +} + +CloudMetadata::CloudMetadata(std::filesystem::path DataDir) : CloudMetadata(std::move(DataDir), std::string(kImdsEndpoint)) +{ +} + +CloudMetadata::CloudMetadata(std::filesystem::path DataDir, std::string ImdsEndpoint) +: m_Log(logging::Get("cloud")) +, m_DataDir(std::move(DataDir)) +, m_ImdsEndpoint(std::move(ImdsEndpoint)) +{ + ZEN_TRACE_CPU("CloudMetadata::CloudMetadata"); + + std::error_code Ec; + std::filesystem::create_directories(m_DataDir, Ec); + + DetectProvider(); + + if (m_Info.Provider != CloudProvider::None) + { + StartTerminationMonitor(); + } +} + +CloudMetadata::~CloudMetadata() +{ + ZEN_TRACE_CPU("CloudMetadata::~CloudMetadata"); + m_MonitorEnabled = false; + m_MonitorEvent.Set(); + if (m_MonitorThread.joinable()) + { + m_MonitorThread.join(); + } +} + +CloudProvider +CloudMetadata::GetProvider() const +{ + return m_InfoLock.WithSharedLock([&] { return m_Info.Provider; }); +} + +CloudInstanceInfo +CloudMetadata::GetInstanceInfo() const +{ + return m_InfoLock.WithSharedLock([&] { return m_Info; }); +} + +bool +CloudMetadata::IsTerminationPending() const +{ + return m_TerminationPending.load(std::memory_order_relaxed); +} + +std::string +CloudMetadata::GetTerminationReason() const +{ + return m_ReasonLock.WithSharedLock([&] { return m_TerminationReason; }); +} + +void +CloudMetadata::Describe(CbWriter& Writer) const +{ + ZEN_TRACE_CPU("CloudMetadata::Describe"); + CloudInstanceInfo Info = GetInstanceInfo(); + + if (Info.Provider == CloudProvider::None) + { + return; + } + + Writer.BeginObject("cloud"); + Writer << "provider" << ToString(Info.Provider); + Writer << "instance_id" << Info.InstanceId; + Writer << "availability_zone" << Info.AvailabilityZone; + Writer << "is_spot" << Info.IsSpot; + Writer << "is_autoscaling" << Info.IsAutoscaling; + Writer << "termination_pending" << IsTerminationPending(); + + if (IsTerminationPending()) + { + Writer << "termination_reason" << GetTerminationReason(); + } + + Writer.EndObject(); +} + +void +CloudMetadata::DetectProvider() +{ + ZEN_TRACE_CPU("CloudMetadata::DetectProvider"); + + if (TryDetectAWS()) + { + return; + } + + if (TryDetectAzure()) + { + return; + } + + if (TryDetectGCP()) + { + return; + } + + ZEN_DEBUG("no cloud provider detected"); +} + +// AWS detection uses IMDSv2 which requires a session token obtained via PUT before +// any GET requests are allowed. This is more secure than IMDSv1 (which allowed +// unauthenticated GETs) and is the default on modern EC2 instances. The token has +// a 300-second TTL and is reused for termination polling. +bool +CloudMetadata::TryDetectAWS() +{ + ZEN_TRACE_CPU("CloudMetadata::TryDetectAWS"); + + std::filesystem::path SentinelPath = m_DataDir / ".isNotAWS"; + + if (HasSentinelFile(SentinelPath)) + { + ZEN_DEBUG("skipping AWS detection - negative cache hit"); + return false; + } + + ZEN_DEBUG("probing AWS IMDS"); + + try + { + HttpClient ImdsClient(m_ImdsEndpoint, + {.LogCategory = "cloud-aws", .ConnectTimeout = kImdsTimeout, .Timeout = std::chrono::milliseconds{1000}}); + + // IMDSv2: acquire session token. The TTL header is mandatory; we request + // 300s which is sufficient for the detection phase. The token is also + // stored in m_AwsToken for reuse by the termination polling thread. + HttpClient::KeyValueMap TokenHeaders(std::pair{"X-aws-ec2-metadata-token-ttl-seconds", "300"}); + HttpClient::Response TokenResponse = ImdsClient.Put("/latest/api/token", IoBuffer{}, TokenHeaders); + + if (!TokenResponse.IsSuccess()) + { + ZEN_DEBUG("AWS IMDS token request failed ({}), not on AWS", static_cast(TokenResponse.StatusCode)); + WriteSentinelFile(SentinelPath); + return false; + } + + m_AwsToken = std::string(TokenResponse.AsText()); + + HttpClient::KeyValueMap AuthHeaders(std::pair{"X-aws-ec2-metadata-token", m_AwsToken}); + + HttpClient::Response IdResponse = ImdsClient.Get("/latest/meta-data/instance-id", AuthHeaders); + if (IdResponse.IsSuccess()) + { + m_Info.InstanceId = std::string(IdResponse.AsText()); + } + + HttpClient::Response AzResponse = ImdsClient.Get("/latest/meta-data/placement/availability-zone", AuthHeaders); + if (AzResponse.IsSuccess()) + { + m_Info.AvailabilityZone = std::string(AzResponse.AsText()); + } + + // "spot" vs "on-demand" — determines whether the instance can be + // reclaimed by AWS with a 2-minute warning + HttpClient::Response LifecycleResponse = ImdsClient.Get("/latest/meta-data/instance-life-cycle", AuthHeaders); + if (LifecycleResponse.IsSuccess()) + { + m_Info.IsSpot = (LifecycleResponse.AsText() == "spot"); + } + + // This endpoint only exists on instances managed by an Auto Scaling + // Group. A successful response (regardless of value) means autoscaling. + HttpClient::Response AutoscaleResponse = ImdsClient.Get("/latest/meta-data/autoscaling/target-lifecycle-state", AuthHeaders); + if (AutoscaleResponse.IsSuccess()) + { + m_Info.IsAutoscaling = true; + } + + m_Info.Provider = CloudProvider::AWS; + + ZEN_INFO("detected AWS instance: id={}, az={}, spot={}, autoscaling={}", + m_Info.InstanceId, + m_Info.AvailabilityZone, + m_Info.IsSpot, + m_Info.IsAutoscaling); + + return true; + } + catch (const std::exception& Ex) + { + ZEN_DEBUG("AWS IMDS probe failed: {}", Ex.what()); + WriteSentinelFile(SentinelPath); + return false; + } +} + +// Azure IMDS returns a single JSON document for the entire instance metadata, +// unlike AWS and GCP which use separate plain-text endpoints per field. The +// "Metadata: true" header is required; requests without it are rejected. +// The api-version parameter is mandatory and pins the response schema. +bool +CloudMetadata::TryDetectAzure() +{ + ZEN_TRACE_CPU("CloudMetadata::TryDetectAzure"); + + std::filesystem::path SentinelPath = m_DataDir / ".isNotAzure"; + + if (HasSentinelFile(SentinelPath)) + { + ZEN_DEBUG("skipping Azure detection - negative cache hit"); + return false; + } + + ZEN_DEBUG("probing Azure IMDS"); + + try + { + HttpClient ImdsClient(m_ImdsEndpoint, + {.LogCategory = "cloud-azure", .ConnectTimeout = kImdsTimeout, .Timeout = std::chrono::milliseconds{1000}}); + + HttpClient::KeyValueMap MetadataHeaders({ + std::pair{"Metadata", "true"}, + }); + + HttpClient::Response InstanceResponse = ImdsClient.Get("/metadata/instance?api-version=2021-02-01", MetadataHeaders); + + if (!InstanceResponse.IsSuccess()) + { + ZEN_DEBUG("Azure IMDS request failed ({}), not on Azure", static_cast(InstanceResponse.StatusCode)); + WriteSentinelFile(SentinelPath); + return false; + } + + std::string JsonError; + const json11::Json Json = json11::Json::parse(std::string(InstanceResponse.AsText()), JsonError); + + if (!JsonError.empty()) + { + ZEN_DEBUG("Azure IMDS returned invalid JSON: {}", JsonError); + WriteSentinelFile(SentinelPath); + return false; + } + + const json11::Json& Compute = Json["compute"]; + + m_Info.InstanceId = Compute["vmId"].string_value(); + m_Info.AvailabilityZone = Compute["location"].string_value(); + + // Azure spot VMs have priority "Spot"; regular VMs have "Regular" + std::string Priority = Compute["priority"].string_value(); + m_Info.IsSpot = (Priority == "Spot"); + + // Check if part of a VMSS (Virtual Machine Scale Set) — indicates autoscaling + std::string VmssName = Compute["vmScaleSetName"].string_value(); + m_Info.IsAutoscaling = !VmssName.empty(); + + m_Info.Provider = CloudProvider::Azure; + + ZEN_INFO("detected Azure instance: id={}, location={}, spot={}, vmss={}", + m_Info.InstanceId, + m_Info.AvailabilityZone, + m_Info.IsSpot, + m_Info.IsAutoscaling); + + return true; + } + catch (const std::exception& Ex) + { + ZEN_DEBUG("Azure IMDS probe failed: {}", Ex.what()); + WriteSentinelFile(SentinelPath); + return false; + } +} + +// GCP requires the "Metadata-Flavor: Google" header on all IMDS requests. +// Unlike AWS, there is no session token; the header itself is the auth mechanism +// (it prevents SSRF attacks since browsers won't send custom headers to the +// metadata endpoint). Each metadata field is fetched from a separate URL. +bool +CloudMetadata::TryDetectGCP() +{ + ZEN_TRACE_CPU("CloudMetadata::TryDetectGCP"); + + std::filesystem::path SentinelPath = m_DataDir / ".isNotGCP"; + + if (HasSentinelFile(SentinelPath)) + { + ZEN_DEBUG("skipping GCP detection - negative cache hit"); + return false; + } + + ZEN_DEBUG("probing GCP metadata service"); + + try + { + HttpClient ImdsClient(m_ImdsEndpoint, + {.LogCategory = "cloud-gcp", .ConnectTimeout = kImdsTimeout, .Timeout = std::chrono::milliseconds{1000}}); + + HttpClient::KeyValueMap MetadataHeaders(std::pair{"Metadata-Flavor", "Google"}); + + // Fetch instance ID + HttpClient::Response IdResponse = ImdsClient.Get("/computeMetadata/v1/instance/id", MetadataHeaders); + + if (!IdResponse.IsSuccess()) + { + ZEN_DEBUG("GCP metadata request failed ({}), not on GCP", static_cast(IdResponse.StatusCode)); + WriteSentinelFile(SentinelPath); + return false; + } + + m_Info.InstanceId = std::string(IdResponse.AsText()); + + // GCP returns the fully-qualified zone path "projects//zones/". + // Strip the prefix to get just the zone name (e.g. "us-central1-a"). + HttpClient::Response ZoneResponse = ImdsClient.Get("/computeMetadata/v1/instance/zone", MetadataHeaders); + if (ZoneResponse.IsSuccess()) + { + std::string_view Zone = ZoneResponse.AsText(); + if (auto Pos = Zone.rfind('/'); Pos != std::string_view::npos) + { + Zone = Zone.substr(Pos + 1); + } + m_Info.AvailabilityZone = std::string(Zone); + } + + // Check for preemptible/spot (scheduling/preemptible returns "TRUE" or "FALSE") + HttpClient::Response PreemptibleResponse = ImdsClient.Get("/computeMetadata/v1/instance/scheduling/preemptible", MetadataHeaders); + if (PreemptibleResponse.IsSuccess()) + { + m_Info.IsSpot = (PreemptibleResponse.AsText() == "TRUE"); + } + + // Check for maintenance event + HttpClient::Response MaintenanceResponse = ImdsClient.Get("/computeMetadata/v1/instance/maintenance-event", MetadataHeaders); + if (MaintenanceResponse.IsSuccess()) + { + std::string_view Event = MaintenanceResponse.AsText(); + if (!Event.empty() && Event != "NONE") + { + m_TerminationPending = true; + m_ReasonLock.WithExclusiveLock([&] { m_TerminationReason = fmt::format("GCP maintenance event: {}", Event); }); + } + } + + m_Info.Provider = CloudProvider::GCP; + + ZEN_INFO("detected GCP instance: id={}, az={}, spot={}", m_Info.InstanceId, m_Info.AvailabilityZone, m_Info.IsSpot); + + return true; + } + catch (const std::exception& Ex) + { + ZEN_DEBUG("GCP metadata probe failed: {}", Ex.what()); + WriteSentinelFile(SentinelPath); + return false; + } +} + +// Sentinel files are empty marker files whose mere existence signals that a +// previous detection attempt for a given provider failed. This avoids paying +// the connect-timeout cost on every startup for providers that are known to +// be absent. The files persist across process restarts; delete them manually +// (or remove the DataDir) to force re-detection. +void +CloudMetadata::WriteSentinelFile(const std::filesystem::path& Path) +{ + try + { + BasicFile File; + File.Open(Path, BasicFile::Mode::kTruncate); + } + catch (const std::exception& Ex) + { + ZEN_WARN("failed to write sentinel file '{}': {}", Path.string(), Ex.what()); + } +} + +bool +CloudMetadata::HasSentinelFile(const std::filesystem::path& Path) const +{ + return zen::IsFile(Path); +} + +void +CloudMetadata::ClearSentinelFiles() +{ + std::error_code Ec; + std::filesystem::remove(m_DataDir / ".isNotAWS", Ec); + std::filesystem::remove(m_DataDir / ".isNotAzure", Ec); + std::filesystem::remove(m_DataDir / ".isNotGCP", Ec); +} + +void +CloudMetadata::StartTerminationMonitor() +{ + ZEN_INFO("starting cloud termination monitor for {} instance {}", ToString(m_Info.Provider), m_Info.InstanceId); + + m_MonitorThread = std::thread{&CloudMetadata::TerminationMonitorThread, this}; +} + +void +CloudMetadata::TerminationMonitorThread() +{ + SetCurrentThreadName("cloud_term_mon"); + + // Poll every 5 seconds. The Event is used as an interruptible sleep so + // that the destructor can wake us up immediately for a clean shutdown. + while (m_MonitorEnabled) + { + m_MonitorEvent.Wait(5000); + m_MonitorEvent.Reset(); + + if (!m_MonitorEnabled) + { + return; + } + + PollTermination(); + } +} + +void +CloudMetadata::PollTermination() +{ + try + { + CloudProvider Provider = m_InfoLock.WithSharedLock([&] { return m_Info.Provider; }); + + if (Provider == CloudProvider::AWS) + { + PollAWSTermination(); + } + else if (Provider == CloudProvider::Azure) + { + PollAzureTermination(); + } + else if (Provider == CloudProvider::GCP) + { + PollGCPTermination(); + } + } + catch (const std::exception& Ex) + { + ZEN_DEBUG("termination poll error: {}", Ex.what()); + } +} + +// AWS termination signals: +// - /spot/instance-action: returns 200 with a JSON body ~2 minutes before +// a spot instance is reclaimed. Returns 404 when no action is pending. +// - /autoscaling/target-lifecycle-state: returns the ASG lifecycle state. +// "InService" is normal; anything else (e.g. "Terminated:Wait") means +// the instance is being cycled out. +void +CloudMetadata::PollAWSTermination() +{ + ZEN_TRACE_CPU("CloudMetadata::PollAWSTermination"); + + HttpClient ImdsClient(m_ImdsEndpoint, + {.LogCategory = "cloud-aws", .ConnectTimeout = kImdsTimeout, .Timeout = std::chrono::milliseconds{2000}}); + + HttpClient::KeyValueMap AuthHeaders(std::pair{"X-aws-ec2-metadata-token", m_AwsToken}); + + HttpClient::Response SpotResponse = ImdsClient.Get("/latest/meta-data/spot/instance-action", AuthHeaders); + if (SpotResponse.IsSuccess()) + { + if (!m_TerminationPending.exchange(true)) + { + m_ReasonLock.WithExclusiveLock([&] { m_TerminationReason = fmt::format("AWS spot interruption: {}", SpotResponse.AsText()); }); + ZEN_WARN("AWS spot interruption detected: {}", SpotResponse.AsText()); + } + return; + } + + HttpClient::Response AutoscaleResponse = ImdsClient.Get("/latest/meta-data/autoscaling/target-lifecycle-state", AuthHeaders); + if (AutoscaleResponse.IsSuccess()) + { + std::string_view State = AutoscaleResponse.AsText(); + if (State.find("InService") == std::string_view::npos) + { + if (!m_TerminationPending.exchange(true)) + { + m_ReasonLock.WithExclusiveLock([&] { m_TerminationReason = fmt::format("AWS autoscaling lifecycle: {}", State); }); + ZEN_WARN("AWS autoscaling termination detected: {}", State); + } + } + } +} + +// Azure Scheduled Events API returns a JSON array of upcoming platform events. +// We care about "Preempt" (spot eviction), "Terminate", and "Reboot" events. +// Other event types like "Freeze" (live migration) are non-destructive and +// ignored. The Events array is empty when nothing is pending. +void +CloudMetadata::PollAzureTermination() +{ + ZEN_TRACE_CPU("CloudMetadata::PollAzureTermination"); + + HttpClient ImdsClient(m_ImdsEndpoint, + {.LogCategory = "cloud-azure", .ConnectTimeout = kImdsTimeout, .Timeout = std::chrono::milliseconds{2000}}); + + HttpClient::KeyValueMap MetadataHeaders({ + std::pair{"Metadata", "true"}, + }); + + HttpClient::Response EventsResponse = ImdsClient.Get("/metadata/scheduledevents?api-version=2020-07-01", MetadataHeaders); + + if (!EventsResponse.IsSuccess()) + { + return; + } + + std::string JsonError; + const json11::Json Json = json11::Json::parse(std::string(EventsResponse.AsText()), JsonError); + + if (!JsonError.empty()) + { + return; + } + + const json11::Json::array& Events = Json["Events"].array_items(); + for (const auto& Evt : Events) + { + std::string EventType = Evt["EventType"].string_value(); + if (EventType == "Preempt" || EventType == "Terminate" || EventType == "Reboot") + { + if (!m_TerminationPending.exchange(true)) + { + std::string EventStatus = Evt["EventStatus"].string_value(); + m_ReasonLock.WithExclusiveLock( + [&] { m_TerminationReason = fmt::format("Azure scheduled event: {} ({})", EventType, EventStatus); }); + ZEN_WARN("Azure termination event detected: {} ({})", EventType, EventStatus); + } + return; + } + } +} + +// GCP maintenance-event returns "NONE" when nothing is pending, and a +// descriptive string like "TERMINATE_ON_HOST_MAINTENANCE" when the VM is +// about to be live-migrated or terminated. Preemptible/spot VMs get a +// 30-second warning before termination. +void +CloudMetadata::PollGCPTermination() +{ + ZEN_TRACE_CPU("CloudMetadata::PollGCPTermination"); + + HttpClient ImdsClient(m_ImdsEndpoint, + {.LogCategory = "cloud-gcp", .ConnectTimeout = kImdsTimeout, .Timeout = std::chrono::milliseconds{2000}}); + + HttpClient::KeyValueMap MetadataHeaders(std::pair{"Metadata-Flavor", "Google"}); + + HttpClient::Response MaintenanceResponse = ImdsClient.Get("/computeMetadata/v1/instance/maintenance-event", MetadataHeaders); + if (MaintenanceResponse.IsSuccess()) + { + std::string_view Event = MaintenanceResponse.AsText(); + if (!Event.empty() && Event != "NONE") + { + if (!m_TerminationPending.exchange(true)) + { + m_ReasonLock.WithExclusiveLock([&] { m_TerminationReason = fmt::format("GCP maintenance event: {}", Event); }); + ZEN_WARN("GCP maintenance event detected: {}", Event); + } + } + } +} + +} // namespace zen::compute + +////////////////////////////////////////////////////////////////////////// + +#if ZEN_WITH_TESTS + +# include + +# include +# include +# include +# include + +# include +# include + +namespace zen::compute { + +// --------------------------------------------------------------------------- +// Test helper — spins up a local ASIO HTTP server hosting a MockImdsService +// --------------------------------------------------------------------------- + +struct TestImdsServer +{ + MockImdsService Mock; + + void Start() + { + m_TmpDir.emplace(); + m_Server = CreateHttpServer(HttpServerConfig{.ServerClass = "asio"}); + m_Port = m_Server->Initialize(7575, m_TmpDir->Path() / "http"); + REQUIRE(m_Port != -1); + m_Server->RegisterService(Mock); + m_ServerThread = std::thread([this]() { m_Server->Run(false); }); + } + + std::string Endpoint() const { return fmt::format("http://127.0.0.1:{}", m_Port); } + + std::filesystem::path DataDir() const { return m_TmpDir->Path() / "cloud"; } + + std::unique_ptr CreateCloud() { return std::make_unique(DataDir(), Endpoint()); } + + ~TestImdsServer() + { + if (m_Server) + { + m_Server->RequestExit(); + } + if (m_ServerThread.joinable()) + { + m_ServerThread.join(); + } + if (m_Server) + { + m_Server->Close(); + } + } + +private: + std::optional m_TmpDir; + Ref m_Server; + std::thread m_ServerThread; + int m_Port = -1; +}; + +// --------------------------------------------------------------------------- +// AWS +// --------------------------------------------------------------------------- + +TEST_CASE("cloudmetadata.aws") +{ + TestImdsServer Imds; + Imds.Mock.ActiveProvider = CloudProvider::AWS; + + SUBCASE("detection basics") + { + Imds.Mock.Aws.InstanceId = "i-abc123"; + Imds.Mock.Aws.AvailabilityZone = "us-west-2b"; + Imds.Mock.Aws.LifeCycle = "on-demand"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + + CHECK(Cloud->GetProvider() == CloudProvider::AWS); + + CloudInstanceInfo Info = Cloud->GetInstanceInfo(); + CHECK(Info.InstanceId == "i-abc123"); + CHECK(Info.AvailabilityZone == "us-west-2b"); + CHECK(Info.IsSpot == false); + CHECK(Info.IsAutoscaling == false); + CHECK(Cloud->IsTerminationPending() == false); + } + + SUBCASE("spot instance") + { + Imds.Mock.Aws.LifeCycle = "spot"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CloudInstanceInfo Info = Cloud->GetInstanceInfo(); + CHECK(Info.IsSpot == true); + } + + SUBCASE("autoscaling instance") + { + Imds.Mock.Aws.AutoscalingState = "InService"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CloudInstanceInfo Info = Cloud->GetInstanceInfo(); + CHECK(Info.IsAutoscaling == true); + } + + SUBCASE("spot termination") + { + Imds.Mock.Aws.LifeCycle = "spot"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CHECK(Cloud->IsTerminationPending() == false); + + // Simulate a spot interruption notice appearing + Imds.Mock.Aws.SpotAction = R"({"action":"terminate","time":"2025-01-01T00:00:00Z"})"; + Cloud->PollTermination(); + + CHECK(Cloud->IsTerminationPending() == true); + CHECK(Cloud->GetTerminationReason().find("spot interruption") != std::string::npos); + } + + SUBCASE("autoscaling termination") + { + Imds.Mock.Aws.AutoscalingState = "InService"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CHECK(Cloud->IsTerminationPending() == false); + + // Simulate ASG cycling the instance out + Imds.Mock.Aws.AutoscalingState = "Terminated:Wait"; + Cloud->PollTermination(); + + CHECK(Cloud->IsTerminationPending() == true); + CHECK(Cloud->GetTerminationReason().find("autoscaling") != std::string::npos); + } + + SUBCASE("no termination when InService") + { + Imds.Mock.Aws.AutoscalingState = "InService"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + Cloud->PollTermination(); + + CHECK(Cloud->IsTerminationPending() == false); + } +} + +// --------------------------------------------------------------------------- +// Azure +// --------------------------------------------------------------------------- + +TEST_CASE("cloudmetadata.azure") +{ + TestImdsServer Imds; + Imds.Mock.ActiveProvider = CloudProvider::Azure; + + SUBCASE("detection basics") + { + Imds.Mock.Azure.VmId = "vm-test-1234"; + Imds.Mock.Azure.Location = "westeurope"; + Imds.Mock.Azure.Priority = "Regular"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + + CHECK(Cloud->GetProvider() == CloudProvider::Azure); + + CloudInstanceInfo Info = Cloud->GetInstanceInfo(); + CHECK(Info.InstanceId == "vm-test-1234"); + CHECK(Info.AvailabilityZone == "westeurope"); + CHECK(Info.IsSpot == false); + CHECK(Info.IsAutoscaling == false); + CHECK(Cloud->IsTerminationPending() == false); + } + + SUBCASE("spot instance") + { + Imds.Mock.Azure.Priority = "Spot"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CloudInstanceInfo Info = Cloud->GetInstanceInfo(); + CHECK(Info.IsSpot == true); + } + + SUBCASE("vmss instance") + { + Imds.Mock.Azure.VmScaleSetName = "my-vmss"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CloudInstanceInfo Info = Cloud->GetInstanceInfo(); + CHECK(Info.IsAutoscaling == true); + } + + SUBCASE("preempt termination") + { + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CHECK(Cloud->IsTerminationPending() == false); + + Imds.Mock.Azure.ScheduledEventType = "Preempt"; + Imds.Mock.Azure.ScheduledEventStatus = "Scheduled"; + Cloud->PollTermination(); + + CHECK(Cloud->IsTerminationPending() == true); + CHECK(Cloud->GetTerminationReason().find("Preempt") != std::string::npos); + } + + SUBCASE("terminate event") + { + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CHECK(Cloud->IsTerminationPending() == false); + + Imds.Mock.Azure.ScheduledEventType = "Terminate"; + Cloud->PollTermination(); + + CHECK(Cloud->IsTerminationPending() == true); + CHECK(Cloud->GetTerminationReason().find("Terminate") != std::string::npos); + } + + SUBCASE("no termination when events empty") + { + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + Cloud->PollTermination(); + + CHECK(Cloud->IsTerminationPending() == false); + } +} + +// --------------------------------------------------------------------------- +// GCP +// --------------------------------------------------------------------------- + +TEST_CASE("cloudmetadata.gcp") +{ + TestImdsServer Imds; + Imds.Mock.ActiveProvider = CloudProvider::GCP; + + SUBCASE("detection basics") + { + Imds.Mock.Gcp.InstanceId = "9876543210"; + Imds.Mock.Gcp.Zone = "projects/123/zones/europe-west1-b"; + Imds.Mock.Gcp.Preemptible = "FALSE"; + Imds.Mock.Gcp.MaintenanceEvent = "NONE"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + + CHECK(Cloud->GetProvider() == CloudProvider::GCP); + + CloudInstanceInfo Info = Cloud->GetInstanceInfo(); + CHECK(Info.InstanceId == "9876543210"); + CHECK(Info.AvailabilityZone == "europe-west1-b"); // zone prefix stripped + CHECK(Info.IsSpot == false); + CHECK(Cloud->IsTerminationPending() == false); + } + + SUBCASE("preemptible instance") + { + Imds.Mock.Gcp.Preemptible = "TRUE"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CloudInstanceInfo Info = Cloud->GetInstanceInfo(); + CHECK(Info.IsSpot == true); + } + + SUBCASE("maintenance event during detection") + { + Imds.Mock.Gcp.MaintenanceEvent = "TERMINATE_ON_HOST_MAINTENANCE"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + + // GCP sets termination pending immediately during detection if a + // maintenance event is active + CHECK(Cloud->IsTerminationPending() == true); + CHECK(Cloud->GetTerminationReason().find("maintenance") != std::string::npos); + } + + SUBCASE("maintenance event during polling") + { + Imds.Mock.Gcp.MaintenanceEvent = "NONE"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + CHECK(Cloud->IsTerminationPending() == false); + + Imds.Mock.Gcp.MaintenanceEvent = "TERMINATE_ON_HOST_MAINTENANCE"; + Cloud->PollTermination(); + + CHECK(Cloud->IsTerminationPending() == true); + CHECK(Cloud->GetTerminationReason().find("maintenance") != std::string::npos); + } + + SUBCASE("no termination when NONE") + { + Imds.Mock.Gcp.MaintenanceEvent = "NONE"; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + Cloud->PollTermination(); + + CHECK(Cloud->IsTerminationPending() == false); + } +} + +// --------------------------------------------------------------------------- +// No provider +// --------------------------------------------------------------------------- + +TEST_CASE("cloudmetadata.no_provider") +{ + TestImdsServer Imds; + Imds.Mock.ActiveProvider = CloudProvider::None; + Imds.Start(); + + auto Cloud = Imds.CreateCloud(); + + CHECK(Cloud->GetProvider() == CloudProvider::None); + + CloudInstanceInfo Info = Cloud->GetInstanceInfo(); + CHECK(Info.InstanceId.empty()); + CHECK(Info.AvailabilityZone.empty()); + CHECK(Info.IsSpot == false); + CHECK(Info.IsAutoscaling == false); + CHECK(Cloud->IsTerminationPending() == false); +} + +// --------------------------------------------------------------------------- +// Sentinel file management +// --------------------------------------------------------------------------- + +TEST_CASE("cloudmetadata.sentinel_files") +{ + TestImdsServer Imds; + Imds.Mock.ActiveProvider = CloudProvider::None; + Imds.Start(); + + auto DataDir = Imds.DataDir(); + + SUBCASE("sentinels are written on failed detection") + { + auto Cloud = Imds.CreateCloud(); + + CHECK(Cloud->GetProvider() == CloudProvider::None); + CHECK(zen::IsFile(DataDir / ".isNotAWS")); + CHECK(zen::IsFile(DataDir / ".isNotAzure")); + CHECK(zen::IsFile(DataDir / ".isNotGCP")); + } + + SUBCASE("ClearSentinelFiles removes sentinels") + { + auto Cloud = Imds.CreateCloud(); + + CHECK(zen::IsFile(DataDir / ".isNotAWS")); + CHECK(zen::IsFile(DataDir / ".isNotAzure")); + CHECK(zen::IsFile(DataDir / ".isNotGCP")); + + Cloud->ClearSentinelFiles(); + + CHECK_FALSE(zen::IsFile(DataDir / ".isNotAWS")); + CHECK_FALSE(zen::IsFile(DataDir / ".isNotAzure")); + CHECK_FALSE(zen::IsFile(DataDir / ".isNotGCP")); + } + + SUBCASE("only failed providers get sentinels") + { + // Switch to AWS — Azure and GCP never probed, so no sentinels for them + Imds.Mock.ActiveProvider = CloudProvider::AWS; + + auto Cloud = Imds.CreateCloud(); + + CHECK(Cloud->GetProvider() == CloudProvider::AWS); + CHECK_FALSE(zen::IsFile(DataDir / ".isNotAWS")); + CHECK_FALSE(zen::IsFile(DataDir / ".isNotAzure")); + CHECK_FALSE(zen::IsFile(DataDir / ".isNotGCP")); + } +} + +void +cloudmetadata_forcelink() +{ +} + +} // namespace zen::compute + +#endif // ZEN_WITH_TESTS diff --git a/src/zencompute/computeservice.cpp b/src/zencompute/computeservice.cpp new file mode 100644 index 000000000..838d741b6 --- /dev/null +++ b/src/zencompute/computeservice.cpp @@ -0,0 +1,2236 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencompute/computeservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "runners/functionrunner.h" +# include "recording/actionrecorder.h" +# include "runners/localrunner.h" +# include "runners/remotehttprunner.h" +# if ZEN_PLATFORM_LINUX +# include "runners/linuxrunner.h" +# elif ZEN_PLATFORM_WINDOWS +# include "runners/windowsrunner.h" +# elif ZEN_PLATFORM_MAC +# include "runners/macrunner.h" +# endif + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include +# include +# include +# include +# include +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +ZEN_THIRD_PARTY_INCLUDES_END + +using namespace std::literals; + +namespace zen { + +const char* +ToString(compute::ComputeServiceSession::SessionState State) +{ + using enum compute::ComputeServiceSession::SessionState; + switch (State) + { + case Created: + return "Created"; + case Ready: + return "Ready"; + case Draining: + return "Draining"; + case Paused: + return "Paused"; + case Abandoned: + return "Abandoned"; + case Sunset: + return "Sunset"; + } + return "Unknown"; +} + +const char* +ToString(compute::ComputeServiceSession::QueueState State) +{ + using enum compute::ComputeServiceSession::QueueState; + switch (State) + { + case Active: + return "active"; + case Draining: + return "draining"; + case Cancelled: + return "cancelled"; + } + return "unknown"; +} + +} // namespace zen + +namespace zen::compute { + +using SessionState = ComputeServiceSession::SessionState; + +static_assert(ZEN_ARRAY_COUNT(ComputeServiceSession::ActionHistoryEntry::Timestamps) == static_cast(RunnerAction::State::_Count)); + +////////////////////////////////////////////////////////////////////////// + +struct ComputeServiceSession::Impl +{ + ComputeServiceSession* m_ComputeServiceSession; + ChunkResolver& m_ChunkResolver; + LoggerRef m_Log{logging::Get("compute")}; + + Impl(ComputeServiceSession* InComputeServiceSession, ChunkResolver& InChunkResolver) + : m_ComputeServiceSession(InComputeServiceSession) + , m_ChunkResolver(InChunkResolver) + , m_LocalSubmitPool(GetLargeWorkerPool(EWorkloadType::Burst)) + , m_RemoteSubmitPool(GetLargeWorkerPool(EWorkloadType::Burst)) + { + // Create a non-expiring, non-deletable implicit queue for legacy endpoints + auto Result = CreateQueue("implicit"sv, {}, {}); + m_ImplicitQueueId = Result.QueueId; + m_QueueLock.WithSharedLock([&] { m_Queues[m_ImplicitQueueId]->Implicit = true; }); + + m_SchedulingThread = std::thread{&Impl::SchedulerThreadFunction, this}; + } + + void WaitUntilReady(); + void Shutdown(); + bool IsHealthy(); + + bool RequestStateTransition(SessionState NewState); + void AbandonAllActions(); + + LoggerRef Log() { return m_Log; } + + // Orchestration + + void SetOrchestratorEndpoint(std::string_view Endpoint); + void SetOrchestratorBasePath(std::filesystem::path BasePath); + + std::string m_OrchestratorEndpoint; + std::filesystem::path m_OrchestratorBasePath; + Stopwatch m_OrchestratorQueryTimer; + std::unordered_set m_KnownWorkerUris; + + void UpdateCoordinatorState(); + + // Worker registration and discovery + + struct FunctionDefinition + { + std::string FunctionName; + Guid FunctionVersion; + Guid BuildSystemVersion; + IoHash WorkerId; + }; + + void RegisterWorker(CbPackage Worker); + WorkerDesc GetWorkerDescriptor(const IoHash& WorkerId); + + // Action scheduling and tracking + + std::atomic m_SessionState{SessionState::Created}; + std::atomic m_ActionsCounter = 0; // sequence number + metrics::Meter m_ArrivalRate; + + RwLock m_PendingLock; + std::map> m_PendingActions; + + RwLock m_RunningLock; + std::unordered_map> m_RunningMap; + + RwLock m_ResultsLock; + std::unordered_map> m_ResultsMap; + metrics::Meter m_ResultRate; + std::atomic m_RetiredCount{0}; + + EnqueueResult EnqueueAction(int QueueId, CbObject ActionObject, int Priority); + EnqueueResult EnqueueResolvedAction(int QueueId, WorkerDesc Worker, CbObject ActionObj, int RequestPriority); + + void GetCompleted(CbWriter& Cbo); + + HttpResponseCode GetActionResult(int ActionLsn, CbPackage& OutResultPackage); + HttpResponseCode FindActionResult(const IoHash& ActionId, CbPackage& ResultPackage); + void RetireActionResult(int ActionLsn); + + std::thread m_SchedulingThread; + std::atomic m_SchedulingThreadEnabled{true}; + Event m_SchedulingThreadEvent; + + void SchedulerThreadFunction(); + void SchedulePendingActions(); + + // Workers + + RwLock m_WorkerLock; + std::unordered_map m_WorkerMap; + std::vector m_FunctionList; + std::vector GetKnownWorkerIds(); + void SyncWorkersToRunner(FunctionRunner& Runner); + + // Runners + + DeferredDirectoryDeleter m_DeferredDeleter; + WorkerThreadPool& m_LocalSubmitPool; + WorkerThreadPool& m_RemoteSubmitPool; + RunnerGroup m_LocalRunnerGroup; + RunnerGroup m_RemoteRunnerGroup; + + void ShutdownRunners(); + + // Recording + + void StartRecording(ChunkResolver& InCidStore, const std::filesystem::path& RecordingPath); + void StopRecording(); + + std::unique_ptr m_Recorder; + + // History tracking + + RwLock m_ActionHistoryLock; + std::deque m_ActionHistory; + size_t m_HistoryLimit = 1000; + + // Queue tracking + + using QueueState = ComputeServiceSession::QueueState; + + struct QueueEntry : RefCounted + { + int QueueId; + bool Implicit{false}; + std::atomic State{QueueState::Active}; + std::atomic ActiveCount{0}; // pending + running + std::atomic CompletedCount{0}; // successfully completed + std::atomic FailedCount{0}; // failed + std::atomic AbandonedCount{0}; // abandoned + std::atomic CancelledCount{0}; // cancelled + std::atomic IdleSince{0}; // hifreq tick when queue became idle; 0 = has active work + + RwLock m_Lock; + std::unordered_set ActiveLsns; // for cancellation lookup + std::unordered_set FinishedLsns; // completed/failed/cancelled LSNs + + std::string Tag; + CbObject Metadata; + CbObject Config; + }; + + int m_ImplicitQueueId{0}; + std::atomic m_QueueCounter{0}; + RwLock m_QueueLock; + std::unordered_map> m_Queues; + + Ref FindQueue(int QueueId) + { + Ref Queue; + m_QueueLock.WithSharedLock([&] { + if (auto It = m_Queues.find(QueueId); It != m_Queues.end()) + { + Queue = It->second; + } + }); + return Queue; + } + + ComputeServiceSession::CreateQueueResult CreateQueue(std::string_view Tag, CbObject Metadata, CbObject Config); + std::vector GetQueueIds(); + ComputeServiceSession::QueueStatus GetQueueStatus(int QueueId); + CbObject GetQueueMetadata(int QueueId); + CbObject GetQueueConfig(int QueueId); + void CancelQueue(int QueueId); + void DeleteQueue(int QueueId); + void DrainQueue(int QueueId); + ComputeServiceSession::EnqueueResult EnqueueActionToQueue(int QueueId, CbObject ActionObject, int Priority); + ComputeServiceSession::EnqueueResult EnqueueResolvedActionToQueue(int QueueId, WorkerDesc Worker, CbObject ActionObj, int Priority); + void GetQueueCompleted(int QueueId, CbWriter& Cbo); + void NotifyQueueActionComplete(int QueueId, int Lsn, RunnerAction::State ActionState); + void ExpireCompletedQueues(); + + Stopwatch m_QueueExpiryTimer; + + std::vector GetRunningActions(); + std::vector GetActionHistory(int Limit); + std::vector GetQueueHistory(int QueueId, int Limit); + + // Action submission + + [[nodiscard]] size_t QueryCapacity(); + + [[nodiscard]] SubmitResult SubmitAction(Ref Action); + [[nodiscard]] std::vector SubmitActions(const std::vector>& Actions); + [[nodiscard]] size_t GetSubmittedActionCount(); + + // Updates + + RwLock m_UpdatedActionsLock; + std::vector> m_UpdatedActions; + + void HandleActionUpdates(); + void PostUpdate(RunnerAction* Action); + + static constexpr int kDefaultMaxRetries = 3; + int GetMaxRetriesForQueue(int QueueId); + + ComputeServiceSession::RescheduleResult RescheduleAction(int ActionLsn); + + ActionCounts GetActionCounts() + { + ActionCounts Counts; + Counts.Pending = (int)m_PendingLock.WithSharedLock([&] { return m_PendingActions.size(); }); + Counts.Running = (int)m_RunningLock.WithSharedLock([&] { return m_RunningMap.size(); }); + Counts.Completed = (int)m_ResultsLock.WithSharedLock([&] { return m_ResultsMap.size(); }) + (int)m_RetiredCount.load(); + Counts.ActiveQueues = (int)m_QueueLock.WithSharedLock([&] { + size_t Count = 0; + for (const auto& [Id, Queue] : m_Queues) + { + if (!Queue->Implicit) + { + ++Count; + } + } + return Count; + }); + return Counts; + } + + void EmitStats(CbObjectWriter& Cbo) + { + Cbo << "session_state"sv << ToString(m_SessionState.load(std::memory_order_relaxed)); + m_WorkerLock.WithSharedLock([&] { Cbo << "worker_count"sv << m_WorkerMap.size(); }); + m_ResultsLock.WithSharedLock([&] { Cbo << "actions_complete"sv << m_ResultsMap.size(); }); + m_PendingLock.WithSharedLock([&] { Cbo << "actions_pending"sv << m_PendingActions.size(); }); + Cbo << "actions_submitted"sv << GetSubmittedActionCount(); + EmitSnapshot("actions_arrival"sv, m_ArrivalRate, Cbo); + EmitSnapshot("actions_retired"sv, m_ResultRate, Cbo); + } +}; + +bool +ComputeServiceSession::Impl::IsHealthy() +{ + return m_SessionState.load(std::memory_order_relaxed) < SessionState::Abandoned; +} + +bool +ComputeServiceSession::Impl::RequestStateTransition(SessionState NewState) +{ + SessionState Current = m_SessionState.load(std::memory_order_relaxed); + + for (;;) + { + if (Current == NewState) + { + return true; + } + + // Validate the transition + bool Valid = false; + + switch (Current) + { + case SessionState::Created: + Valid = (NewState == SessionState::Ready); + break; + case SessionState::Ready: + Valid = (NewState == SessionState::Draining); + break; + case SessionState::Draining: + Valid = (NewState == SessionState::Ready || NewState == SessionState::Paused); + break; + case SessionState::Paused: + Valid = (NewState == SessionState::Ready || NewState == SessionState::Sunset); + break; + case SessionState::Abandoned: + Valid = (NewState == SessionState::Sunset); + break; + case SessionState::Sunset: + Valid = false; + break; + } + + // Allow jumping directly to Abandoned or Sunset from any non-terminal state + if (NewState == SessionState::Abandoned && Current < SessionState::Abandoned) + { + Valid = true; + } + if (NewState == SessionState::Sunset && Current != SessionState::Sunset) + { + Valid = true; + } + + if (!Valid) + { + ZEN_WARN("invalid session state transition {} -> {}", ToString(Current), ToString(NewState)); + return false; + } + + if (m_SessionState.compare_exchange_strong(Current, NewState, std::memory_order_acq_rel)) + { + ZEN_INFO("session state: {} -> {}", ToString(Current), ToString(NewState)); + + if (NewState == SessionState::Abandoned) + { + AbandonAllActions(); + } + + return true; + } + + // CAS failed, Current was updated — retry with the new value + } +} + +void +ComputeServiceSession::Impl::AbandonAllActions() +{ + // Collect all pending actions and mark them as Abandoned + std::vector> PendingToAbandon; + + m_PendingLock.WithSharedLock([&] { + PendingToAbandon.reserve(m_PendingActions.size()); + for (auto& [Lsn, Action] : m_PendingActions) + { + PendingToAbandon.push_back(Action); + } + }); + + for (auto& Action : PendingToAbandon) + { + Action->SetActionState(RunnerAction::State::Abandoned); + } + + // Collect all running actions and mark them as Abandoned, then + // best-effort cancel via the local runner group + std::vector> RunningToAbandon; + + m_RunningLock.WithSharedLock([&] { + RunningToAbandon.reserve(m_RunningMap.size()); + for (auto& [Lsn, Action] : m_RunningMap) + { + RunningToAbandon.push_back(Action); + } + }); + + for (auto& Action : RunningToAbandon) + { + Action->SetActionState(RunnerAction::State::Abandoned); + m_LocalRunnerGroup.CancelAction(Action->ActionLsn); + } + + ZEN_INFO("abandoned all actions: {} pending, {} running", PendingToAbandon.size(), RunningToAbandon.size()); +} + +void +ComputeServiceSession::Impl::SetOrchestratorEndpoint(std::string_view Endpoint) +{ + m_OrchestratorEndpoint = Endpoint; +} + +void +ComputeServiceSession::Impl::SetOrchestratorBasePath(std::filesystem::path BasePath) +{ + m_OrchestratorBasePath = std::move(BasePath); +} + +void +ComputeServiceSession::Impl::UpdateCoordinatorState() +{ + ZEN_TRACE_CPU("ComputeServiceSession::UpdateCoordinatorState"); + if (m_OrchestratorEndpoint.empty()) + { + return; + } + + // Poll faster when we have no discovered workers yet so remote runners come online quickly + const uint64_t PollIntervalMs = m_KnownWorkerUris.empty() ? 500 : 5000; + if (m_OrchestratorQueryTimer.GetElapsedTimeMs() < PollIntervalMs) + { + return; + } + + m_OrchestratorQueryTimer.Reset(); + + try + { + HttpClient Client(m_OrchestratorEndpoint); + + HttpClient::Response Response = Client.Get("/orch/agents"); + + if (!Response.IsSuccess()) + { + ZEN_WARN("orchestrator query failed with status {}", static_cast(Response.StatusCode)); + return; + } + + CbObject WorkerList = Response.AsObject(); + + std::unordered_set ValidWorkerUris; + + for (auto& Item : WorkerList["workers"sv]) + { + CbObjectView Worker = Item.AsObjectView(); + + uint64_t Dt = Worker["dt"sv].AsUInt64(); + bool Reachable = Worker["reachable"sv].AsBool(); + std::string_view Uri = Worker["uri"sv].AsString(); + + // Skip stale workers (not seen in over 30 seconds) + if (Dt > 30000) + { + continue; + } + + // Skip workers that are not confirmed reachable + if (!Reachable) + { + continue; + } + + std::string UriStr{Uri}; + ValidWorkerUris.insert(UriStr); + + // Skip workers we already know about + if (m_KnownWorkerUris.contains(UriStr)) + { + continue; + } + + ZEN_INFO("discovered new worker at {}", UriStr); + + m_KnownWorkerUris.insert(UriStr); + + auto* NewRunner = new RemoteHttpRunner(m_ChunkResolver, m_OrchestratorBasePath, UriStr, m_RemoteSubmitPool); + SyncWorkersToRunner(*NewRunner); + m_RemoteRunnerGroup.AddRunner(NewRunner); + } + + // Remove workers that are no longer valid (stale or unreachable) + for (auto It = m_KnownWorkerUris.begin(); It != m_KnownWorkerUris.end();) + { + if (!ValidWorkerUris.contains(*It)) + { + const std::string& ExpiredUri = *It; + ZEN_INFO("removing expired worker at {}", ExpiredUri); + + m_RemoteRunnerGroup.RemoveRunnerIf([&](const RemoteHttpRunner& Runner) { return Runner.GetHostName() == ExpiredUri; }); + + It = m_KnownWorkerUris.erase(It); + } + else + { + ++It; + } + } + } + catch (const HttpClientError& Ex) + { + ZEN_WARN("orchestrator query error: {}", Ex.what()); + } + catch (const std::exception& Ex) + { + ZEN_WARN("orchestrator query unexpected error: {}", Ex.what()); + } +} + +void +ComputeServiceSession::Impl::WaitUntilReady() +{ + if (m_RemoteRunnerGroup.GetRunnerCount() || !m_OrchestratorEndpoint.empty()) + { + ZEN_INFO("waiting for remote runners..."); + + constexpr int MaxWaitSeconds = 120; + + for (int Elapsed = 0; Elapsed < MaxWaitSeconds; Elapsed++) + { + if (!m_SchedulingThreadEnabled.load(std::memory_order_relaxed)) + { + ZEN_WARN("shutdown requested while waiting for remote runners"); + return; + } + + const size_t Capacity = m_RemoteRunnerGroup.QueryCapacity(); + + if (Capacity > 0) + { + ZEN_INFO("found {} remote runners (capacity: {})", m_RemoteRunnerGroup.GetRunnerCount(), Capacity); + break; + } + + zen::Sleep(1000); + } + } + else + { + ZEN_ASSERT(m_LocalRunnerGroup.GetRunnerCount(), "no runners available"); + } + + RequestStateTransition(SessionState::Ready); +} + +void +ComputeServiceSession::Impl::Shutdown() +{ + RequestStateTransition(SessionState::Sunset); + + m_SchedulingThreadEnabled = false; + m_SchedulingThreadEvent.Set(); + if (m_SchedulingThread.joinable()) + { + m_SchedulingThread.join(); + } + + ShutdownRunners(); + + m_DeferredDeleter.Shutdown(); +} + +void +ComputeServiceSession::Impl::ShutdownRunners() +{ + m_LocalRunnerGroup.Shutdown(); + m_RemoteRunnerGroup.Shutdown(); +} + +void +ComputeServiceSession::Impl::StartRecording(ChunkResolver& InCidStore, const std::filesystem::path& RecordingPath) +{ + ZEN_INFO("starting recording to '{}'", RecordingPath); + + m_Recorder = std::make_unique(InCidStore, RecordingPath); + + ZEN_INFO("started recording to '{}'", RecordingPath); +} + +void +ComputeServiceSession::Impl::StopRecording() +{ + ZEN_INFO("stopping recording"); + + m_Recorder = nullptr; + + ZEN_INFO("stopped recording"); +} + +std::vector +ComputeServiceSession::Impl::GetRunningActions() +{ + std::vector Result; + m_RunningLock.WithSharedLock([&] { + Result.reserve(m_RunningMap.size()); + for (const auto& [Lsn, Action] : m_RunningMap) + { + Result.push_back({.Lsn = Lsn, + .QueueId = Action->QueueId, + .ActionId = Action->ActionId, + .CpuUsagePercent = Action->CpuUsagePercent.load(std::memory_order_relaxed), + .CpuSeconds = Action->CpuSeconds.load(std::memory_order_relaxed)}); + } + }); + return Result; +} + +std::vector +ComputeServiceSession::Impl::GetActionHistory(int Limit) +{ + RwLock::SharedLockScope _(m_ActionHistoryLock); + + if (Limit > 0 && static_cast(Limit) < m_ActionHistory.size()) + { + return std::vector(m_ActionHistory.end() - Limit, m_ActionHistory.end()); + } + + return std::vector(m_ActionHistory.begin(), m_ActionHistory.end()); +} + +std::vector +ComputeServiceSession::Impl::GetQueueHistory(int QueueId, int Limit) +{ + // Resolve the queue and snapshot its finished LSN set + Ref Queue = FindQueue(QueueId); + + if (!Queue) + { + return {}; + } + + std::unordered_set FinishedLsns; + + Queue->m_Lock.WithSharedLock([&] { FinishedLsns = Queue->FinishedLsns; }); + + // Filter the global history to entries belonging to this queue. + // m_ActionHistory is ordered oldest-first, so the filtered result keeps the same ordering. + std::vector Result; + + m_ActionHistoryLock.WithSharedLock([&] { + for (const auto& Entry : m_ActionHistory) + { + if (FinishedLsns.contains(Entry.Lsn)) + { + Result.push_back(Entry); + } + } + }); + + if (Limit > 0 && static_cast(Limit) < Result.size()) + { + Result.erase(Result.begin(), Result.end() - Limit); + } + + return Result; +} + +void +ComputeServiceSession::Impl::RegisterWorker(CbPackage Worker) +{ + ZEN_TRACE_CPU("ComputeServiceSession::RegisterWorker"); + RwLock::ExclusiveLockScope _(m_WorkerLock); + + const IoHash& WorkerId = Worker.GetObject().GetHash(); + + if (m_WorkerMap.insert_or_assign(WorkerId, Worker).second) + { + // Note that since the convention currently is that WorkerId is equal to the hash + // of the worker descriptor there is no chance that we get a second write with a + // different descriptor. Thus we only need to call this the first time, when the + // worker is added + + m_LocalRunnerGroup.RegisterWorker(Worker); + m_RemoteRunnerGroup.RegisterWorker(Worker); + + if (m_Recorder) + { + m_Recorder->RegisterWorker(Worker); + } + + CbObject WorkerObj = Worker.GetObject(); + + // Populate worker database + + const Guid WorkerBuildSystemVersion = WorkerObj["buildsystem_version"sv].AsUuid(); + + for (auto& Item : WorkerObj["functions"sv]) + { + CbObjectView Function = Item.AsObjectView(); + + std::string_view FunctionName = Function["name"sv].AsString(); + const Guid FunctionVersion = Function["version"sv].AsUuid(); + + m_FunctionList.emplace_back(FunctionDefinition{.FunctionName = std::string{FunctionName}, + .FunctionVersion = FunctionVersion, + .BuildSystemVersion = WorkerBuildSystemVersion, + .WorkerId = WorkerId}); + } + } +} + +void +ComputeServiceSession::Impl::SyncWorkersToRunner(FunctionRunner& Runner) +{ + ZEN_TRACE_CPU("SyncWorkersToRunner"); + + std::vector Workers; + + { + RwLock::SharedLockScope _(m_WorkerLock); + Workers.reserve(m_WorkerMap.size()); + for (const auto& [Id, Pkg] : m_WorkerMap) + { + Workers.push_back(Pkg); + } + } + + for (const CbPackage& Worker : Workers) + { + Runner.RegisterWorker(Worker); + } +} + +WorkerDesc +ComputeServiceSession::Impl::GetWorkerDescriptor(const IoHash& WorkerId) +{ + RwLock::SharedLockScope _(m_WorkerLock); + + if (auto It = m_WorkerMap.find(WorkerId); It != m_WorkerMap.end()) + { + const CbPackage& Desc = It->second; + return {Desc, WorkerId}; + } + + return {}; +} + +std::vector +ComputeServiceSession::Impl::GetKnownWorkerIds() +{ + std::vector WorkerIds; + + m_WorkerLock.WithSharedLock([&] { + WorkerIds.reserve(m_WorkerMap.size()); + for (const auto& [WorkerId, _] : m_WorkerMap) + { + WorkerIds.push_back(WorkerId); + } + }); + + return WorkerIds; +} + +ComputeServiceSession::EnqueueResult +ComputeServiceSession::Impl::EnqueueAction(int QueueId, CbObject ActionObject, int Priority) +{ + ZEN_TRACE_CPU("ComputeServiceSession::EnqueueAction"); + + // Resolve function to worker + + IoHash WorkerId{IoHash::Zero}; + CbPackage WorkerPackage; + + std::string_view FunctionName = ActionObject["Function"sv].AsString(); + const Guid FunctionVersion = ActionObject["FunctionVersion"sv].AsUuid(); + const Guid BuildSystemVersion = ActionObject["BuildSystemVersion"sv].AsUuid(); + + m_WorkerLock.WithSharedLock([&] { + for (const FunctionDefinition& FuncDef : m_FunctionList) + { + if (FuncDef.FunctionName == FunctionName && FuncDef.FunctionVersion == FunctionVersion && + FuncDef.BuildSystemVersion == BuildSystemVersion) + { + WorkerId = FuncDef.WorkerId; + + break; + } + } + + if (WorkerId != IoHash::Zero) + { + if (auto It = m_WorkerMap.find(WorkerId); It != m_WorkerMap.end()) + { + WorkerPackage = It->second; + } + } + }); + + if (WorkerId == IoHash::Zero) + { + CbObjectWriter Writer; + + Writer << "Function"sv << FunctionName << "FunctionVersion"sv << FunctionVersion << "BuildSystemVersion" << BuildSystemVersion; + Writer << "error" + << "no worker matches the action specification"; + + return {0, Writer.Save()}; + } + + if (WorkerPackage) + { + return EnqueueResolvedAction(QueueId, WorkerDesc{WorkerPackage, WorkerId}, ActionObject, Priority); + } + + CbObjectWriter Writer; + + Writer << "Function"sv << FunctionName << "FunctionVersion"sv << FunctionVersion << "BuildSystemVersion" << BuildSystemVersion; + Writer << "error" + << "no worker found despite match"; + + return {0, Writer.Save()}; +} + +ComputeServiceSession::EnqueueResult +ComputeServiceSession::Impl::EnqueueResolvedAction(int QueueId, WorkerDesc Worker, CbObject ActionObj, int RequestPriority) +{ + ZEN_TRACE_CPU("ComputeServiceSession::EnqueueResolvedAction"); + + if (m_SessionState.load(std::memory_order_relaxed) != SessionState::Ready) + { + CbObjectWriter Writer; + Writer << "error"sv << fmt::format("session is not accepting actions (state: {})", ToString(m_SessionState.load())); + return {0, Writer.Save()}; + } + + const int ActionLsn = ++m_ActionsCounter; + + m_ArrivalRate.Mark(); + + Ref Pending{new RunnerAction(m_ComputeServiceSession)}; + + Pending->ActionLsn = ActionLsn; + Pending->QueueId = QueueId; + Pending->Worker = Worker; + Pending->ActionId = ActionObj.GetHash(); + Pending->ActionObj = ActionObj; + Pending->Priority = RequestPriority; + + // For now simply put action into pending state, so we can do batch scheduling + + ZEN_DEBUG("action {} ({}) PENDING", Pending->ActionId, Pending->ActionLsn); + + Pending->SetActionState(RunnerAction::State::Pending); + + if (m_Recorder) + { + m_Recorder->RecordAction(Pending); + } + + CbObjectWriter Writer; + Writer << "lsn" << Pending->ActionLsn; + Writer << "worker" << Pending->Worker.WorkerId; + Writer << "action" << Pending->ActionId; + + return {Pending->ActionLsn, Writer.Save()}; +} + +SubmitResult +ComputeServiceSession::Impl::SubmitAction(Ref Action) +{ + // Loosely round-robin scheduling of actions across runners. + // + // It's not entirely clear what this means given that submits + // can come in across multiple threads, but it's probably better + // than always starting with the first runner. + // + // Longer term we should track the state of the individual + // runners and make decisions accordingly. + + SubmitResult Result = m_LocalRunnerGroup.SubmitAction(Action); + if (Result.IsAccepted) + { + return Result; + } + + return m_RemoteRunnerGroup.SubmitAction(Action); +} + +size_t +ComputeServiceSession::Impl::GetSubmittedActionCount() +{ + return m_LocalRunnerGroup.GetSubmittedActionCount() + m_RemoteRunnerGroup.GetSubmittedActionCount(); +} + +HttpResponseCode +ComputeServiceSession::Impl::GetActionResult(int ActionLsn, CbPackage& OutResultPackage) +{ + // This lock is held for the duration of the function since we need to + // be sure that the action doesn't change state while we are checking the + // different data structures + + RwLock::ExclusiveLockScope _(m_ResultsLock); + + if (auto It = m_ResultsMap.find(ActionLsn); It != m_ResultsMap.end()) + { + OutResultPackage = std::move(It->second->GetResult()); + + m_ResultsMap.erase(It); + + return HttpResponseCode::OK; + } + + { + RwLock::SharedLockScope __(m_PendingLock); + + if (auto FindIt = m_PendingActions.find(ActionLsn); FindIt != m_PendingActions.end()) + { + return HttpResponseCode::Accepted; + } + } + + // Lock order is important here to avoid deadlocks, RwLock m_RunningLock must + // always be taken after m_ResultsLock if both are needed + + { + RwLock::SharedLockScope __(m_RunningLock); + + if (m_RunningMap.find(ActionLsn) != m_RunningMap.end()) + { + return HttpResponseCode::Accepted; + } + } + + return HttpResponseCode::NotFound; +} + +HttpResponseCode +ComputeServiceSession::Impl::FindActionResult(const IoHash& ActionId, CbPackage& OutResultPackage) +{ + // This lock is held for the duration of the function since we need to + // be sure that the action doesn't change state while we are checking the + // different data structures + + RwLock::ExclusiveLockScope _(m_ResultsLock); + + for (auto It = begin(m_ResultsMap), End = end(m_ResultsMap); It != End; ++It) + { + if (It->second->ActionId == ActionId) + { + OutResultPackage = std::move(It->second->GetResult()); + + m_ResultsMap.erase(It); + + return HttpResponseCode::OK; + } + } + + { + RwLock::SharedLockScope __(m_PendingLock); + + for (const auto& [K, Pending] : m_PendingActions) + { + if (Pending->ActionId == ActionId) + { + return HttpResponseCode::Accepted; + } + } + } + + // Lock order is important here to avoid deadlocks, RwLock m_RunningLock must + // always be taken after m_ResultsLock if both are needed + + { + RwLock::SharedLockScope __(m_RunningLock); + + for (const auto& [K, v] : m_RunningMap) + { + if (v->ActionId == ActionId) + { + return HttpResponseCode::Accepted; + } + } + } + + return HttpResponseCode::NotFound; +} + +void +ComputeServiceSession::Impl::RetireActionResult(int ActionLsn) +{ + m_DeferredDeleter.MarkReady(ActionLsn); +} + +void +ComputeServiceSession::Impl::GetCompleted(CbWriter& Cbo) +{ + Cbo.BeginArray("completed"); + + m_ResultsLock.WithSharedLock([&] { + for (auto& [Lsn, Action] : m_ResultsMap) + { + Cbo.BeginObject(); + Cbo << "lsn"sv << Lsn; + Cbo << "state"sv << RunnerAction::ToString(Action->ActionState()); + Cbo.EndObject(); + } + }); + + Cbo.EndArray(); +} + +////////////////////////////////////////////////////////////////////////// +// Queue management + +ComputeServiceSession::CreateQueueResult +ComputeServiceSession::Impl::CreateQueue(std::string_view Tag, CbObject Metadata, CbObject Config) +{ + const int QueueId = ++m_QueueCounter; + + Ref Queue{new QueueEntry()}; + Queue->QueueId = QueueId; + Queue->Tag = Tag; + Queue->Metadata = std::move(Metadata); + Queue->Config = std::move(Config); + Queue->IdleSince = GetHifreqTimerValue(); + + m_QueueLock.WithExclusiveLock([&] { m_Queues[QueueId] = Queue; }); + + ZEN_DEBUG("created queue {}", QueueId); + + return {.QueueId = QueueId}; +} + +std::vector +ComputeServiceSession::Impl::GetQueueIds() +{ + std::vector Ids; + + m_QueueLock.WithSharedLock([&] { + Ids.reserve(m_Queues.size()); + for (const auto& [Id, Queue] : m_Queues) + { + if (!Queue->Implicit) + { + Ids.push_back(Id); + } + } + }); + + return Ids; +} + +ComputeServiceSession::QueueStatus +ComputeServiceSession::Impl::GetQueueStatus(int QueueId) +{ + Ref Queue = FindQueue(QueueId); + + if (!Queue) + { + return {}; + } + + const int Active = Queue->ActiveCount.load(std::memory_order_relaxed); + const int Completed = Queue->CompletedCount.load(std::memory_order_relaxed); + const int Failed = Queue->FailedCount.load(std::memory_order_relaxed); + const int AbandonedN = Queue->AbandonedCount.load(std::memory_order_relaxed); + const int CancelledN = Queue->CancelledCount.load(std::memory_order_relaxed); + const QueueState QState = Queue->State.load(); + + return { + .IsValid = true, + .QueueId = QueueId, + .ActiveCount = Active, + .CompletedCount = Completed, + .FailedCount = Failed, + .AbandonedCount = AbandonedN, + .CancelledCount = CancelledN, + .State = QState, + .IsComplete = (Active == 0), + }; +} + +CbObject +ComputeServiceSession::Impl::GetQueueMetadata(int QueueId) +{ + Ref Queue = FindQueue(QueueId); + + if (!Queue) + { + return {}; + } + + return Queue->Metadata; +} + +CbObject +ComputeServiceSession::Impl::GetQueueConfig(int QueueId) +{ + Ref Queue = FindQueue(QueueId); + + if (!Queue) + { + return {}; + } + + return Queue->Config; +} + +void +ComputeServiceSession::Impl::CancelQueue(int QueueId) +{ + Ref Queue = FindQueue(QueueId); + + if (!Queue || Queue->Implicit) + { + return; + } + + Queue->State.store(QueueState::Cancelled); + + // Collect active LSNs snapshot for cancellation + std::vector LsnsToCancel; + + Queue->m_Lock.WithSharedLock([&] { LsnsToCancel.assign(Queue->ActiveLsns.begin(), Queue->ActiveLsns.end()); }); + + // Identify which LSNs are still pending (not yet dispatched to a runner) + std::vector> PendingActionsToCancel; + std::vector RunningLsnsToCancel; + + m_PendingLock.WithSharedLock([&] { + for (int Lsn : LsnsToCancel) + { + if (auto It = m_PendingActions.find(Lsn); It != m_PendingActions.end()) + { + PendingActionsToCancel.push_back(It->second); + } + } + }); + + m_RunningLock.WithSharedLock([&] { + for (int Lsn : LsnsToCancel) + { + if (m_RunningMap.find(Lsn) != m_RunningMap.end()) + { + RunningLsnsToCancel.push_back(Lsn); + } + } + }); + + // Cancel pending actions by marking them as Cancelled; they will flow through + // HandleActionUpdates and eventually be removed from the pending map. + for (auto& Action : PendingActionsToCancel) + { + Action->SetActionState(RunnerAction::State::Cancelled); + } + + // Best-effort cancellation of running actions via the local runner group. + // Also set the action state to Cancelled directly so a subsequent Failed + // transition from the runner is blocked (Cancelled > Failed in the enum). + for (int Lsn : RunningLsnsToCancel) + { + m_RunningLock.WithSharedLock([&] { + if (auto It = m_RunningMap.find(Lsn); It != m_RunningMap.end()) + { + It->second->SetActionState(RunnerAction::State::Cancelled); + } + }); + m_LocalRunnerGroup.CancelAction(Lsn); + } + + m_RemoteRunnerGroup.CancelRemoteQueue(QueueId); + + ZEN_INFO("cancelled queue {}: {} pending, {} running actions cancelled", + QueueId, + PendingActionsToCancel.size(), + RunningLsnsToCancel.size()); + + // Wake up the scheduler to process the cancelled actions + m_SchedulingThreadEvent.Set(); +} + +void +ComputeServiceSession::Impl::DeleteQueue(int QueueId) +{ + // Never delete the implicit queue + { + Ref Queue = FindQueue(QueueId); + if (Queue && Queue->Implicit) + { + return; + } + } + + // Cancel any active work first + CancelQueue(QueueId); + + m_QueueLock.WithExclusiveLock([&] { + if (auto It = m_Queues.find(QueueId); It != m_Queues.end()) + { + m_Queues.erase(It); + } + }); +} + +void +ComputeServiceSession::Impl::DrainQueue(int QueueId) +{ + Ref Queue = FindQueue(QueueId); + + if (!Queue || Queue->Implicit) + { + return; + } + + QueueState Expected = QueueState::Active; + Queue->State.compare_exchange_strong(Expected, QueueState::Draining); + ZEN_INFO("draining queue {}", QueueId); +} + +ComputeServiceSession::EnqueueResult +ComputeServiceSession::Impl::EnqueueActionToQueue(int QueueId, CbObject ActionObject, int Priority) +{ + Ref Queue = FindQueue(QueueId); + + if (!Queue) + { + CbObjectWriter Writer; + Writer << "error"sv + << "queue not found"sv; + return {0, Writer.Save()}; + } + + QueueState QState = Queue->State.load(); + if (QState == QueueState::Cancelled) + { + CbObjectWriter Writer; + Writer << "error"sv + << "queue is cancelled"sv; + return {0, Writer.Save()}; + } + + if (QState == QueueState::Draining) + { + CbObjectWriter Writer; + Writer << "error"sv + << "queue is draining"sv; + return {0, Writer.Save()}; + } + + EnqueueResult Result = EnqueueAction(QueueId, ActionObject, Priority); + + if (Result.Lsn != 0) + { + Queue->m_Lock.WithExclusiveLock([&] { Queue->ActiveLsns.insert(Result.Lsn); }); + Queue->ActiveCount.fetch_add(1, std::memory_order_relaxed); + Queue->IdleSince.store(0, std::memory_order_relaxed); + } + + return Result; +} + +ComputeServiceSession::EnqueueResult +ComputeServiceSession::Impl::EnqueueResolvedActionToQueue(int QueueId, WorkerDesc Worker, CbObject ActionObj, int Priority) +{ + Ref Queue = FindQueue(QueueId); + + if (!Queue) + { + CbObjectWriter Writer; + Writer << "error"sv + << "queue not found"sv; + return {0, Writer.Save()}; + } + + QueueState QState = Queue->State.load(); + if (QState == QueueState::Cancelled) + { + CbObjectWriter Writer; + Writer << "error"sv + << "queue is cancelled"sv; + return {0, Writer.Save()}; + } + + if (QState == QueueState::Draining) + { + CbObjectWriter Writer; + Writer << "error"sv + << "queue is draining"sv; + return {0, Writer.Save()}; + } + + EnqueueResult Result = EnqueueResolvedAction(QueueId, Worker, ActionObj, Priority); + + if (Result.Lsn != 0) + { + Queue->m_Lock.WithExclusiveLock([&] { Queue->ActiveLsns.insert(Result.Lsn); }); + Queue->ActiveCount.fetch_add(1, std::memory_order_relaxed); + Queue->IdleSince.store(0, std::memory_order_relaxed); + } + + return Result; +} + +void +ComputeServiceSession::Impl::GetQueueCompleted(int QueueId, CbWriter& Cbo) +{ + Ref Queue = FindQueue(QueueId); + + Cbo.BeginArray("completed"); + + if (Queue) + { + Queue->m_Lock.WithSharedLock([&] { + m_ResultsLock.WithSharedLock([&] { + for (int Lsn : Queue->FinishedLsns) + { + if (m_ResultsMap.contains(Lsn)) + { + Cbo << Lsn; + } + } + }); + }); + } + + Cbo.EndArray(); +} + +void +ComputeServiceSession::Impl::NotifyQueueActionComplete(int QueueId, int Lsn, RunnerAction::State ActionState) +{ + if (QueueId == 0) + { + return; + } + + Ref Queue = FindQueue(QueueId); + + if (!Queue) + { + return; + } + + Queue->m_Lock.WithExclusiveLock([&] { + Queue->ActiveLsns.erase(Lsn); + Queue->FinishedLsns.insert(Lsn); + }); + + const int PreviousActive = Queue->ActiveCount.fetch_sub(1, std::memory_order_relaxed); + if (PreviousActive == 1) + { + Queue->IdleSince.store(GetHifreqTimerValue(), std::memory_order_relaxed); + } + + switch (ActionState) + { + case RunnerAction::State::Completed: + Queue->CompletedCount.fetch_add(1, std::memory_order_relaxed); + break; + case RunnerAction::State::Abandoned: + Queue->AbandonedCount.fetch_add(1, std::memory_order_relaxed); + break; + case RunnerAction::State::Cancelled: + Queue->CancelledCount.fetch_add(1, std::memory_order_relaxed); + break; + default: + Queue->FailedCount.fetch_add(1, std::memory_order_relaxed); + break; + } +} + +void +ComputeServiceSession::Impl::ExpireCompletedQueues() +{ + static constexpr uint64_t ExpiryTimeMs = 15 * 60 * 1000; + + std::vector ExpiredQueueIds; + + m_QueueLock.WithSharedLock([&] { + for (const auto& [Id, Queue] : m_Queues) + { + if (Queue->Implicit) + { + continue; + } + const uint64_t Idle = Queue->IdleSince.load(std::memory_order_relaxed); + if (Idle != 0 && Queue->ActiveCount.load(std::memory_order_relaxed) == 0) + { + const uint64_t ElapsedMs = Stopwatch::GetElapsedTimeMs(GetHifreqTimerValue() - Idle); + if (ElapsedMs >= ExpiryTimeMs) + { + ExpiredQueueIds.push_back(Id); + } + } + } + }); + + for (int QueueId : ExpiredQueueIds) + { + ZEN_INFO("expiring idle queue {}", QueueId); + DeleteQueue(QueueId); + } +} + +void +ComputeServiceSession::Impl::SchedulePendingActions() +{ + ZEN_TRACE_CPU("ComputeServiceSession::SchedulePendingActions"); + int ScheduledCount = 0; + size_t RunningCount = m_RunningLock.WithSharedLock([&] { return m_RunningMap.size(); }); + size_t PendingCount = m_PendingLock.WithSharedLock([&] { return m_PendingActions.size(); }); + size_t ResultCount = m_ResultsLock.WithSharedLock([&] { return m_ResultsMap.size(); }); + + static Stopwatch DumpRunningTimer; + + auto _ = MakeGuard([&] { + ZEN_INFO("scheduled {} pending actions. {} running ({} retired), {} still pending, {} results", + ScheduledCount, + RunningCount, + m_RetiredCount.load(), + PendingCount, + ResultCount); + + if (DumpRunningTimer.GetElapsedTimeMs() > 30000) + { + DumpRunningTimer.Reset(); + + std::set RunningList; + m_RunningLock.WithSharedLock([&] { + for (auto& [K, V] : m_RunningMap) + { + RunningList.insert(K); + } + }); + + ExtendableStringBuilder<1024> RunningString; + for (int i : RunningList) + { + if (RunningString.Size()) + { + RunningString << ", "; + } + + RunningString.Append(IntNum(i)); + } + + ZEN_INFO("running: {}", RunningString); + } + }); + + size_t Capacity = QueryCapacity(); + + if (!Capacity) + { + _.Dismiss(); + + return; + } + + std::vector> ActionsToSchedule; + + // Pull actions to schedule from the pending queue, we will + // try to submit these to the runner outside of the lock. Note + // that because of how the state transitions work it's not + // actually the case that all of these actions will still be + // pending by the time we try to submit them, but that's fine. + // + // Also note that the m_PendingActions list is not maintained + // here, that's done periodically in SchedulePendingActions() + + m_PendingLock.WithExclusiveLock([&] { + if (m_SessionState.load(std::memory_order_relaxed) >= SessionState::Paused) + { + return; + } + + if (m_PendingActions.empty()) + { + return; + } + + for (auto& [Lsn, Pending] : m_PendingActions) + { + switch (Pending->ActionState()) + { + case RunnerAction::State::Pending: + ActionsToSchedule.push_back(Pending); + break; + + case RunnerAction::State::Submitting: + break; // already claimed by async submission + + case RunnerAction::State::Running: + case RunnerAction::State::Completed: + case RunnerAction::State::Failed: + case RunnerAction::State::Abandoned: + case RunnerAction::State::Cancelled: + break; + + default: + case RunnerAction::State::New: + ZEN_WARN("unexpected state {} for pending action {}", static_cast(Pending->ActionState()), Pending->ActionLsn); + break; + } + } + + // Sort by priority descending, then by LSN ascending (FIFO within same priority) + std::sort(ActionsToSchedule.begin(), ActionsToSchedule.end(), [](const Ref& A, const Ref& B) { + if (A->Priority != B->Priority) + { + return A->Priority > B->Priority; + } + return A->ActionLsn < B->ActionLsn; + }); + + if (ActionsToSchedule.size() > Capacity) + { + ActionsToSchedule.resize(Capacity); + } + + PendingCount = m_PendingActions.size(); + }); + + if (ActionsToSchedule.empty()) + { + _.Dismiss(); + return; + } + + ZEN_INFO("attempting schedule of {} pending actions", ActionsToSchedule.size()); + + Stopwatch SubmitTimer; + std::vector SubmitResults = SubmitActions(ActionsToSchedule); + + int NotAcceptedCount = 0; + int ScheduledActionCount = 0; + + for (const SubmitResult& SubResult : SubmitResults) + { + if (SubResult.IsAccepted) + { + ++ScheduledActionCount; + } + else + { + ++NotAcceptedCount; + } + } + + ZEN_INFO("scheduled {} pending actions in {} ({} rejected)", + ScheduledActionCount, + NiceTimeSpanMs(SubmitTimer.GetElapsedTimeMs()), + NotAcceptedCount); + + ScheduledCount += ScheduledActionCount; + PendingCount -= ScheduledActionCount; +} + +void +ComputeServiceSession::Impl::SchedulerThreadFunction() +{ + SetCurrentThreadName("Function_Scheduler"); + + auto _ = MakeGuard([&] { ZEN_INFO("scheduler thread exiting"); }); + + do + { + int TimeoutMs = 500; + + auto PendingCount = m_PendingLock.WithSharedLock([&] { return m_PendingActions.size(); }); + + if (PendingCount) + { + TimeoutMs = 100; + } + + const bool WasSignaled = m_SchedulingThreadEvent.Wait(TimeoutMs); + + if (m_SchedulingThreadEnabled == false) + { + return; + } + + if (WasSignaled) + { + m_SchedulingThreadEvent.Reset(); + } + + ZEN_DEBUG("compute scheduler TICK (Pending: {} was {}, Running: {}, Results: {}) timeout: {}", + m_PendingLock.WithSharedLock([&] { return m_PendingActions.size(); }), + PendingCount, + m_RunningLock.WithSharedLock([&] { return m_RunningMap.size(); }), + m_ResultsLock.WithSharedLock([&] { return m_ResultsMap.size(); }), + TimeoutMs); + + HandleActionUpdates(); + + // Auto-transition Draining → Paused when all work is done + if (m_SessionState.load(std::memory_order_relaxed) == SessionState::Draining) + { + size_t Pending = m_PendingLock.WithSharedLock([&] { return m_PendingActions.size(); }); + size_t Running = m_RunningLock.WithSharedLock([&] { return m_RunningMap.size(); }); + + if (Pending == 0 && Running == 0) + { + SessionState Expected = SessionState::Draining; + if (m_SessionState.compare_exchange_strong(Expected, SessionState::Paused, std::memory_order_acq_rel)) + { + ZEN_INFO("session state: Draining -> Paused (all work completed)"); + } + } + } + + UpdateCoordinatorState(); + SchedulePendingActions(); + + static constexpr uint64_t QueueExpirySweepIntervalMs = 30000; + if (m_QueueExpiryTimer.GetElapsedTimeMs() >= QueueExpirySweepIntervalMs) + { + m_QueueExpiryTimer.Reset(); + ExpireCompletedQueues(); + } + } while (m_SchedulingThreadEnabled); +} + +void +ComputeServiceSession::Impl::PostUpdate(RunnerAction* Action) +{ + m_UpdatedActionsLock.WithExclusiveLock([&] { m_UpdatedActions.emplace_back(Action); }); + m_SchedulingThreadEvent.Set(); +} + +int +ComputeServiceSession::Impl::GetMaxRetriesForQueue(int QueueId) +{ + if (QueueId == 0) + { + return kDefaultMaxRetries; + } + + CbObject Config = GetQueueConfig(QueueId); + + if (Config) + { + int Value = Config["max_retries"].AsInt32(0); + + if (Value > 0) + { + return Value; + } + } + + return kDefaultMaxRetries; +} + +ComputeServiceSession::RescheduleResult +ComputeServiceSession::Impl::RescheduleAction(int ActionLsn) +{ + Ref Action; + RunnerAction::State State; + RescheduleResult ValidationError; + bool Removed = false; + + // Find, validate, and remove atomically under a single lock scope to prevent + // concurrent RescheduleAction calls from double-removing the same action. + m_ResultsLock.WithExclusiveLock([&] { + auto It = m_ResultsMap.find(ActionLsn); + if (It == m_ResultsMap.end()) + { + ValidationError = {.Success = false, .Error = "Action not found in results"}; + return; + } + + Action = It->second; + State = Action->ActionState(); + + if (State != RunnerAction::State::Failed && State != RunnerAction::State::Abandoned) + { + ValidationError = {.Success = false, .Error = "Action is not in a failed or abandoned state"}; + return; + } + + int MaxRetries = GetMaxRetriesForQueue(Action->QueueId); + if (Action->RetryCount.load(std::memory_order_relaxed) >= MaxRetries) + { + ValidationError = {.Success = false, .Error = "Retry limit reached"}; + return; + } + + m_ResultsMap.erase(It); + Removed = true; + }); + + if (!Removed) + { + return ValidationError; + } + + if (Action->QueueId != 0) + { + Ref Queue = FindQueue(Action->QueueId); + + if (Queue) + { + Queue->m_Lock.WithExclusiveLock([&] { + Queue->FinishedLsns.erase(ActionLsn); + Queue->ActiveLsns.insert(ActionLsn); + }); + + Queue->ActiveCount.fetch_add(1, std::memory_order_relaxed); + Queue->IdleSince.store(0, std::memory_order_relaxed); + + if (State == RunnerAction::State::Failed) + { + Queue->FailedCount.fetch_sub(1, std::memory_order_relaxed); + } + else + { + Queue->AbandonedCount.fetch_sub(1, std::memory_order_relaxed); + } + } + } + + // Reset action state — this calls PostUpdate() internally + Action->ResetActionStateToPending(); + + int NewRetryCount = Action->RetryCount.load(std::memory_order_relaxed); + ZEN_INFO("action {} ({}) manually rescheduled (retry {})", Action->ActionId, ActionLsn, NewRetryCount); + + return {.Success = true, .RetryCount = NewRetryCount}; +} + +void +ComputeServiceSession::Impl::HandleActionUpdates() +{ + ZEN_TRACE_CPU("ComputeServiceSession::HandleActionUpdates"); + + // Drain the update queue atomically + std::vector> UpdatedActions; + m_UpdatedActionsLock.WithExclusiveLock([&] { std::swap(UpdatedActions, m_UpdatedActions); }); + + std::unordered_set SeenLsn; + + // Process each action's latest state, deduplicating by LSN. + // + // This is safe because state transitions are monotonically increasing by enum + // rank (Pending < Submitting < Running < Completed/Failed/Cancelled), so + // SetActionState rejects any transition to a lower-ranked state. By the time + // we read ActionState() here, it reflects the highest state reached — making + // the first occurrence per LSN authoritative and duplicates redundant. + for (Ref& Action : UpdatedActions) + { + const int ActionLsn = Action->ActionLsn; + + if (auto [It, Inserted] = SeenLsn.insert(ActionLsn); Inserted) + { + switch (Action->ActionState()) + { + // Newly enqueued — add to pending map for scheduling + case RunnerAction::State::Pending: + m_PendingLock.WithExclusiveLock([&] { m_PendingActions.insert({ActionLsn, Action}); }); + break; + + // Async submission in progress — remains in pending map + case RunnerAction::State::Submitting: + break; + + // Dispatched to a runner — move from pending to running + case RunnerAction::State::Running: + m_RunningLock.WithExclusiveLock([&] { + m_PendingLock.WithExclusiveLock([&] { + m_RunningMap.insert({ActionLsn, Action}); + m_PendingActions.erase(ActionLsn); + }); + }); + ZEN_DEBUG("action {} ({}) RUNNING", Action->ActionId, ActionLsn); + break; + + // Terminal states — move to results, record history, notify queue + case RunnerAction::State::Completed: + case RunnerAction::State::Failed: + case RunnerAction::State::Abandoned: + case RunnerAction::State::Cancelled: + { + auto TerminalState = Action->ActionState(); + + // Automatic retry for Failed/Abandoned actions with retries remaining. + // Skip retries when the session itself is abandoned — those actions + // were intentionally abandoned and should not be rescheduled. + if ((TerminalState == RunnerAction::State::Failed || TerminalState == RunnerAction::State::Abandoned) && + m_SessionState.load(std::memory_order_relaxed) < SessionState::Abandoned) + { + int MaxRetries = GetMaxRetriesForQueue(Action->QueueId); + + if (Action->RetryCount.load(std::memory_order_relaxed) < MaxRetries) + { + // Remove from whichever active map the action is in before resetting + m_RunningLock.WithExclusiveLock([&] { + m_PendingLock.WithExclusiveLock([&] { + if (auto FindIt = m_RunningMap.find(ActionLsn); FindIt == m_RunningMap.end()) + { + m_PendingActions.erase(ActionLsn); + } + else + { + m_RunningMap.erase(FindIt); + } + }); + }); + + // Reset triggers PostUpdate() which re-enters the action as Pending + Action->ResetActionStateToPending(); + int NewRetryCount = Action->RetryCount.load(std::memory_order_relaxed); + + ZEN_INFO("action {} ({}) auto-rescheduled (retry {}/{})", + Action->ActionId, + ActionLsn, + NewRetryCount, + MaxRetries); + break; + } + } + + // Remove from whichever active map the action is in + m_RunningLock.WithExclusiveLock([&] { + m_PendingLock.WithExclusiveLock([&] { + if (auto FindIt = m_RunningMap.find(ActionLsn); FindIt == m_RunningMap.end()) + { + m_PendingActions.erase(ActionLsn); + } + else + { + m_RunningMap.erase(FindIt); + } + }); + }); + + m_ResultsLock.WithExclusiveLock([&] { + m_ResultsMap[ActionLsn] = Action; + + // Append to bounded action history ring + m_ActionHistoryLock.WithExclusiveLock([&] { + ActionHistoryEntry Entry{.Lsn = ActionLsn, + .QueueId = Action->QueueId, + .ActionId = Action->ActionId, + .WorkerId = Action->Worker.WorkerId, + .ActionDescriptor = Action->ActionObj, + .ExecutionLocation = std::move(Action->ExecutionLocation), + .Succeeded = TerminalState == RunnerAction::State::Completed, + .CpuSeconds = Action->CpuSeconds.load(std::memory_order_relaxed), + .RetryCount = Action->RetryCount.load(std::memory_order_relaxed)}; + + std::copy(std::begin(Action->Timestamps), std::end(Action->Timestamps), std::begin(Entry.Timestamps)); + + m_ActionHistory.push_back(std::move(Entry)); + + if (m_ActionHistory.size() > m_HistoryLimit) + { + m_ActionHistory.pop_front(); + } + }); + }); + m_RetiredCount.fetch_add(1); + m_ResultRate.Mark(1); + ZEN_DEBUG("action {} ({}) RUNNING -> COMPLETED with {}", + Action->ActionId, + ActionLsn, + TerminalState == RunnerAction::State::Completed ? "SUCCESS" : "FAILURE"); + NotifyQueueActionComplete(Action->QueueId, ActionLsn, TerminalState); + break; + } + } + } + } +} + +size_t +ComputeServiceSession::Impl::QueryCapacity() +{ + return m_LocalRunnerGroup.QueryCapacity() + m_RemoteRunnerGroup.QueryCapacity(); +} + +std::vector +ComputeServiceSession::Impl::SubmitActions(const std::vector>& Actions) +{ + ZEN_TRACE_CPU("ComputeServiceSession::SubmitActions"); + std::vector Results(Actions.size()); + + // First try submitting the batch to local runners in parallel + + std::vector LocalResults = m_LocalRunnerGroup.SubmitActions(Actions); + std::vector RemoteIndices; + std::vector> RemoteActions; + + for (size_t i = 0; i < Actions.size(); ++i) + { + if (LocalResults[i].IsAccepted) + { + Results[i] = std::move(LocalResults[i]); + } + else + { + RemoteIndices.push_back(i); + RemoteActions.push_back(Actions[i]); + } + } + + // Submit remaining actions to remote runners in parallel + if (!RemoteActions.empty()) + { + std::vector RemoteResults = m_RemoteRunnerGroup.SubmitActions(RemoteActions); + + for (size_t j = 0; j < RemoteIndices.size(); ++j) + { + Results[RemoteIndices[j]] = std::move(RemoteResults[j]); + } + } + + return Results; +} + +////////////////////////////////////////////////////////////////////////// + +ComputeServiceSession::ComputeServiceSession(ChunkResolver& InChunkResolver) +{ + m_Impl = std::make_unique(this, InChunkResolver); +} + +ComputeServiceSession::~ComputeServiceSession() +{ + Shutdown(); +} + +bool +ComputeServiceSession::IsHealthy() +{ + return m_Impl->IsHealthy(); +} + +void +ComputeServiceSession::WaitUntilReady() +{ + m_Impl->WaitUntilReady(); +} + +void +ComputeServiceSession::Shutdown() +{ + m_Impl->Shutdown(); +} + +ComputeServiceSession::SessionState +ComputeServiceSession::GetSessionState() const +{ + return m_Impl->m_SessionState.load(std::memory_order_relaxed); +} + +bool +ComputeServiceSession::RequestStateTransition(SessionState NewState) +{ + return m_Impl->RequestStateTransition(NewState); +} + +void +ComputeServiceSession::SetOrchestratorEndpoint(std::string_view Endpoint) +{ + m_Impl->SetOrchestratorEndpoint(Endpoint); +} + +void +ComputeServiceSession::SetOrchestratorBasePath(std::filesystem::path BasePath) +{ + m_Impl->SetOrchestratorBasePath(std::move(BasePath)); +} + +void +ComputeServiceSession::StartRecording(ChunkResolver& InResolver, const std::filesystem::path& RecordingPath) +{ + m_Impl->StartRecording(InResolver, RecordingPath); +} + +void +ComputeServiceSession::StopRecording() +{ + m_Impl->StopRecording(); +} + +ComputeServiceSession::ActionCounts +ComputeServiceSession::GetActionCounts() +{ + return m_Impl->GetActionCounts(); +} + +void +ComputeServiceSession::EmitStats(CbObjectWriter& Cbo) +{ + m_Impl->EmitStats(Cbo); +} + +std::vector +ComputeServiceSession::GetKnownWorkerIds() +{ + return m_Impl->GetKnownWorkerIds(); +} + +WorkerDesc +ComputeServiceSession::GetWorkerDescriptor(const IoHash& WorkerId) +{ + return m_Impl->GetWorkerDescriptor(WorkerId); +} + +void +ComputeServiceSession::AddLocalRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath, int32_t MaxConcurrentActions) +{ + ZEN_TRACE_CPU("ComputeServiceSession::AddLocalRunner"); + +# if ZEN_PLATFORM_LINUX + auto* NewRunner = new LinuxProcessRunner(InChunkResolver, + BasePath, + m_Impl->m_DeferredDeleter, + m_Impl->m_LocalSubmitPool, + false, + MaxConcurrentActions); +# elif ZEN_PLATFORM_WINDOWS + auto* NewRunner = new WindowsProcessRunner(InChunkResolver, + BasePath, + m_Impl->m_DeferredDeleter, + m_Impl->m_LocalSubmitPool, + false, + MaxConcurrentActions); +# elif ZEN_PLATFORM_MAC + auto* NewRunner = + new MacProcessRunner(InChunkResolver, BasePath, m_Impl->m_DeferredDeleter, m_Impl->m_LocalSubmitPool, false, MaxConcurrentActions); +# endif + + m_Impl->SyncWorkersToRunner(*NewRunner); + m_Impl->m_LocalRunnerGroup.AddRunner(NewRunner); +} + +void +ComputeServiceSession::AddRemoteRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath, std::string_view HostName) +{ + ZEN_TRACE_CPU("ComputeServiceSession::AddRemoteRunner"); + + auto* NewRunner = new RemoteHttpRunner(InChunkResolver, BasePath, HostName, m_Impl->m_RemoteSubmitPool); + m_Impl->SyncWorkersToRunner(*NewRunner); + m_Impl->m_RemoteRunnerGroup.AddRunner(NewRunner); +} + +ComputeServiceSession::EnqueueResult +ComputeServiceSession::EnqueueAction(CbObject ActionObject, int Priority) +{ + return m_Impl->EnqueueActionToQueue(m_Impl->m_ImplicitQueueId, ActionObject, Priority); +} + +ComputeServiceSession::EnqueueResult +ComputeServiceSession::EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int RequestPriority) +{ + return m_Impl->EnqueueResolvedActionToQueue(m_Impl->m_ImplicitQueueId, Worker, ActionObj, RequestPriority); +} +ComputeServiceSession::CreateQueueResult +ComputeServiceSession::CreateQueue(std::string_view Tag, CbObject Metadata, CbObject Config) +{ + return m_Impl->CreateQueue(Tag, std::move(Metadata), std::move(Config)); +} + +CbObject +ComputeServiceSession::GetQueueMetadata(int QueueId) +{ + return m_Impl->GetQueueMetadata(QueueId); +} + +CbObject +ComputeServiceSession::GetQueueConfig(int QueueId) +{ + return m_Impl->GetQueueConfig(QueueId); +} + +std::vector +ComputeServiceSession::GetQueueIds() +{ + return m_Impl->GetQueueIds(); +} + +ComputeServiceSession::QueueStatus +ComputeServiceSession::GetQueueStatus(int QueueId) +{ + return m_Impl->GetQueueStatus(QueueId); +} + +void +ComputeServiceSession::CancelQueue(int QueueId) +{ + m_Impl->CancelQueue(QueueId); +} + +void +ComputeServiceSession::DrainQueue(int QueueId) +{ + m_Impl->DrainQueue(QueueId); +} + +void +ComputeServiceSession::DeleteQueue(int QueueId) +{ + m_Impl->DeleteQueue(QueueId); +} + +void +ComputeServiceSession::GetQueueCompleted(int QueueId, CbWriter& Cbo) +{ + m_Impl->GetQueueCompleted(QueueId, Cbo); +} + +ComputeServiceSession::EnqueueResult +ComputeServiceSession::EnqueueActionToQueue(int QueueId, CbObject ActionObject, int Priority) +{ + return m_Impl->EnqueueActionToQueue(QueueId, ActionObject, Priority); +} + +ComputeServiceSession::EnqueueResult +ComputeServiceSession::EnqueueResolvedActionToQueue(int QueueId, WorkerDesc Worker, CbObject ActionObj, int RequestPriority) +{ + return m_Impl->EnqueueResolvedActionToQueue(QueueId, Worker, ActionObj, RequestPriority); +} + +void +ComputeServiceSession::RegisterWorker(CbPackage Worker) +{ + m_Impl->RegisterWorker(Worker); +} + +HttpResponseCode +ComputeServiceSession::GetActionResult(int ActionLsn, CbPackage& OutResultPackage) +{ + return m_Impl->GetActionResult(ActionLsn, OutResultPackage); +} + +HttpResponseCode +ComputeServiceSession::FindActionResult(const IoHash& ActionId, CbPackage& OutResultPackage) +{ + return m_Impl->FindActionResult(ActionId, OutResultPackage); +} + +void +ComputeServiceSession::RetireActionResult(int ActionLsn) +{ + m_Impl->RetireActionResult(ActionLsn); +} + +ComputeServiceSession::RescheduleResult +ComputeServiceSession::RescheduleAction(int ActionLsn) +{ + return m_Impl->RescheduleAction(ActionLsn); +} + +std::vector +ComputeServiceSession::GetRunningActions() +{ + return m_Impl->GetRunningActions(); +} + +std::vector +ComputeServiceSession::GetActionHistory(int Limit) +{ + return m_Impl->GetActionHistory(Limit); +} + +std::vector +ComputeServiceSession::GetQueueHistory(int QueueId, int Limit) +{ + return m_Impl->GetQueueHistory(QueueId, Limit); +} + +void +ComputeServiceSession::GetCompleted(CbWriter& Cbo) +{ + m_Impl->GetCompleted(Cbo); +} + +void +ComputeServiceSession::PostUpdate(RunnerAction* Action) +{ + m_Impl->PostUpdate(Action); +} + +////////////////////////////////////////////////////////////////////////// + +void +computeservice_forcelink() +{ +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/functionrunner.cpp b/src/zencompute/functionrunner.cpp deleted file mode 100644 index 8e7c12b2b..000000000 --- a/src/zencompute/functionrunner.cpp +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "functionrunner.h" - -#if ZEN_WITH_COMPUTE_SERVICES - -# include -# include - -# include -# include - -namespace zen::compute { - -FunctionRunner::FunctionRunner(std::filesystem::path BasePath) : m_ActionsPath(BasePath / "actions") -{ -} - -FunctionRunner::~FunctionRunner() = default; - -size_t -FunctionRunner::QueryCapacity() -{ - return 1; -} - -std::vector -FunctionRunner::SubmitActions(const std::vector>& Actions) -{ - std::vector Results; - Results.reserve(Actions.size()); - - for (const Ref& Action : Actions) - { - Results.push_back(SubmitAction(Action)); - } - - return Results; -} - -void -FunctionRunner::MaybeDumpAction(int ActionLsn, const CbObject& ActionObject) -{ - if (m_DumpActions) - { - std::string UniqueId = fmt::format("{}.ddb", ActionLsn); - std::filesystem::path Path = m_ActionsPath / UniqueId; - - zen::WriteFile(Path, IoBuffer(ActionObject.GetBuffer().AsIoBuffer())); - } -} - -////////////////////////////////////////////////////////////////////////// - -RunnerAction::RunnerAction(FunctionServiceSession* OwnerSession) : m_OwnerSession(OwnerSession) -{ - this->Timestamps[static_cast(State::New)] = DateTime::Now().GetTicks(); -} - -RunnerAction::~RunnerAction() -{ -} - -void -RunnerAction::SetActionState(State NewState) -{ - ZEN_ASSERT(NewState < State::_Count); - this->Timestamps[static_cast(NewState)] = DateTime::Now().GetTicks(); - - do - { - if (State CurrentState = m_ActionState.load(); CurrentState == NewState) - { - // No state change - return; - } - else - { - if (NewState <= CurrentState) - { - // Cannot transition to an earlier or same state - return; - } - - if (m_ActionState.compare_exchange_strong(CurrentState, NewState)) - { - // Successful state change - - m_OwnerSession->PostUpdate(this); - - return; - } - } - } while (true); -} - -void -RunnerAction::SetResult(CbPackage&& Result) -{ - m_Result = std::move(Result); -} - -CbPackage& -RunnerAction::GetResult() -{ - ZEN_ASSERT(IsCompleted()); - return m_Result; -} - -} // namespace zen::compute - -#endif // ZEN_WITH_COMPUTE_SERVICES \ No newline at end of file diff --git a/src/zencompute/functionrunner.h b/src/zencompute/functionrunner.h deleted file mode 100644 index 6fd0d84cc..000000000 --- a/src/zencompute/functionrunner.h +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include - -#if ZEN_WITH_COMPUTE_SERVICES - -# include -# include - -namespace zen::compute { - -struct SubmitResult -{ - bool IsAccepted = false; - std::string Reason; -}; - -/** Base interface for classes implementing a remote execution "runner" - */ -class FunctionRunner : public RefCounted -{ - FunctionRunner(FunctionRunner&&) = delete; - FunctionRunner& operator=(FunctionRunner&&) = delete; - -public: - FunctionRunner(std::filesystem::path BasePath); - virtual ~FunctionRunner() = 0; - - virtual void Shutdown() = 0; - virtual void RegisterWorker(const CbPackage& WorkerPackage) = 0; - - [[nodiscard]] virtual SubmitResult SubmitAction(Ref Action) = 0; - [[nodiscard]] virtual size_t GetSubmittedActionCount() = 0; - [[nodiscard]] virtual bool IsHealthy() = 0; - [[nodiscard]] virtual size_t QueryCapacity(); - [[nodiscard]] virtual std::vector SubmitActions(const std::vector>& Actions); - -protected: - std::filesystem::path m_ActionsPath; - bool m_DumpActions = false; - void MaybeDumpAction(int ActionLsn, const CbObject& ActionObject); -}; - -template -struct RunnerGroup -{ - void AddRunner(RunnerType* Runner) - { - m_RunnersLock.WithExclusiveLock([&] { m_Runners.emplace_back(Runner); }); - } - size_t QueryCapacity() - { - size_t TotalCapacity = 0; - m_RunnersLock.WithSharedLock([&] { - for (const auto& Runner : m_Runners) - { - TotalCapacity += Runner->QueryCapacity(); - } - }); - return TotalCapacity; - } - - SubmitResult SubmitAction(Ref Action) - { - RwLock::SharedLockScope _(m_RunnersLock); - - const int InitialIndex = m_NextSubmitIndex.load(std::memory_order_acquire); - int Index = InitialIndex; - const int RunnerCount = gsl::narrow(m_Runners.size()); - - if (RunnerCount == 0) - { - return {.IsAccepted = false, .Reason = "No runners available"}; - } - - do - { - while (Index >= RunnerCount) - { - Index -= RunnerCount; - } - - auto& Runner = m_Runners[Index++]; - - SubmitResult Result = Runner->SubmitAction(Action); - - if (Result.IsAccepted == true) - { - m_NextSubmitIndex = Index % RunnerCount; - - return Result; - } - - while (Index >= RunnerCount) - { - Index -= RunnerCount; - } - } while (Index != InitialIndex); - - return {.IsAccepted = false}; - } - - size_t GetSubmittedActionCount() - { - RwLock::SharedLockScope _(m_RunnersLock); - - size_t TotalCount = 0; - - for (const auto& Runner : m_Runners) - { - TotalCount += Runner->GetSubmittedActionCount(); - } - - return TotalCount; - } - - void RegisterWorker(CbPackage Worker) - { - RwLock::SharedLockScope _(m_RunnersLock); - - for (auto& Runner : m_Runners) - { - Runner->RegisterWorker(Worker); - } - } - - void Shutdown() - { - RwLock::SharedLockScope _(m_RunnersLock); - - for (auto& Runner : m_Runners) - { - Runner->Shutdown(); - } - } - -private: - RwLock m_RunnersLock; - std::vector> m_Runners; - std::atomic m_NextSubmitIndex{0}; -}; - -/** - * This represents an action going through different stages of scheduling and execution. - */ -struct RunnerAction : public RefCounted -{ - explicit RunnerAction(FunctionServiceSession* OwnerSession); - ~RunnerAction(); - - int ActionLsn = 0; - WorkerDesc Worker; - IoHash ActionId; - CbObject ActionObj; - int Priority = 0; - - enum class State - { - New, - Pending, - Running, - Completed, - Failed, - _Count - }; - - static const char* ToString(State _) - { - switch (_) - { - case State::New: - return "New"; - case State::Pending: - return "Pending"; - case State::Running: - return "Running"; - case State::Completed: - return "Completed"; - case State::Failed: - return "Failed"; - default: - return "Unknown"; - } - } - - uint64_t Timestamps[static_cast(State::_Count)] = {}; - - State ActionState() const { return m_ActionState; } - void SetActionState(State NewState); - - bool IsSuccess() const { return ActionState() == State::Completed; } - bool IsCompleted() const { return ActionState() == State::Completed || ActionState() == State::Failed; } - - void SetResult(CbPackage&& Result); - CbPackage& GetResult(); - -private: - std::atomic m_ActionState = State::New; - FunctionServiceSession* m_OwnerSession = nullptr; - CbPackage m_Result; -}; - -} // namespace zen::compute - -#endif // ZEN_WITH_COMPUTE_SERVICES \ No newline at end of file diff --git a/src/zencompute/functionservice.cpp b/src/zencompute/functionservice.cpp deleted file mode 100644 index 0698449e9..000000000 --- a/src/zencompute/functionservice.cpp +++ /dev/null @@ -1,957 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencompute/functionservice.h" - -#if ZEN_WITH_COMPUTE_SERVICES - -# include "functionrunner.h" -# include "actionrecorder.h" -# include "localrunner.h" -# include "remotehttprunner.h" - -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include -# include -# include -# include -# include - -ZEN_THIRD_PARTY_INCLUDES_START -# include -ZEN_THIRD_PARTY_INCLUDES_END - -using namespace std::literals; - -namespace zen::compute { - -////////////////////////////////////////////////////////////////////////// - -struct FunctionServiceSession::Impl -{ - FunctionServiceSession* m_FunctionServiceSession; - ChunkResolver& m_ChunkResolver; - LoggerRef m_Log{logging::Get("apply")}; - - Impl(FunctionServiceSession* InFunctionServiceSession, ChunkResolver& InChunkResolver) - : m_FunctionServiceSession(InFunctionServiceSession) - , m_ChunkResolver(InChunkResolver) - { - m_SchedulingThread = std::thread{&Impl::MonitorThreadFunction, this}; - } - - void Shutdown(); - bool IsHealthy(); - - LoggerRef Log() { return m_Log; } - - std::atomic_bool m_AcceptActions = true; - - struct FunctionDefinition - { - std::string FunctionName; - Guid FunctionVersion; - Guid BuildSystemVersion; - IoHash WorkerId; - }; - - void EmitStats(CbObjectWriter& Cbo) - { - m_WorkerLock.WithSharedLock([&] { Cbo << "worker_count"sv << m_WorkerMap.size(); }); - m_ResultsLock.WithSharedLock([&] { Cbo << "actions_complete"sv << m_ResultsMap.size(); }); - m_PendingLock.WithSharedLock([&] { Cbo << "actions_pending"sv << m_PendingActions.size(); }); - Cbo << "actions_submitted"sv << GetSubmittedActionCount(); - EmitSnapshot("actions_retired"sv, m_ResultRate, Cbo); - } - - void RegisterWorker(CbPackage Worker); - WorkerDesc GetWorkerDescriptor(const IoHash& WorkerId); - - std::atomic m_ActionsCounter = 0; // sequence number - - RwLock m_PendingLock; - std::map> m_PendingActions; - - RwLock m_RunningLock; - std::unordered_map> m_RunningMap; - - RwLock m_ResultsLock; - std::unordered_map> m_ResultsMap; - metrics::Meter m_ResultRate; - std::atomic m_RetiredCount{0}; - - HttpResponseCode GetActionResult(int ActionLsn, CbPackage& OutResultPackage); - HttpResponseCode FindActionResult(const IoHash& ActionId, CbPackage& ResultPackage); - - std::atomic m_ShutdownRequested{false}; - - std::thread m_SchedulingThread; - std::atomic m_SchedulingThreadEnabled{true}; - Event m_SchedulingThreadEvent; - - void MonitorThreadFunction(); - void SchedulePendingActions(); - - // Workers - - RwLock m_WorkerLock; - std::unordered_map m_WorkerMap; - std::vector m_FunctionList; - std::vector GetKnownWorkerIds(); - - // Runners - - RunnerGroup m_LocalRunnerGroup; - RunnerGroup m_RemoteRunnerGroup; - - EnqueueResult EnqueueAction(CbObject ActionObject, int Priority); - EnqueueResult EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int RequestPriority); - - void GetCompleted(CbWriter& Cbo); - - // Recording - - void StartRecording(ChunkResolver& InCidStore, const std::filesystem::path& RecordingPath); - void StopRecording(); - - std::unique_ptr m_Recorder; - - // History tracking - - RwLock m_ActionHistoryLock; - std::deque m_ActionHistory; - size_t m_HistoryLimit = 1000; - - std::vector GetActionHistory(int Limit); - - // - - [[nodiscard]] size_t QueryCapacity(); - - [[nodiscard]] SubmitResult SubmitAction(Ref Action); - [[nodiscard]] std::vector SubmitActions(const std::vector>& Actions); - [[nodiscard]] size_t GetSubmittedActionCount(); - - // Updates - - RwLock m_UpdatedActionsLock; - std::vector> m_UpdatedActions; - - void HandleActionUpdates(); - void PostUpdate(RunnerAction* Action); - - void ShutdownRunners(); -}; - -bool -FunctionServiceSession::Impl::IsHealthy() -{ - return true; -} - -void -FunctionServiceSession::Impl::Shutdown() -{ - m_AcceptActions = false; - m_ShutdownRequested = true; - - m_SchedulingThreadEnabled = false; - m_SchedulingThreadEvent.Set(); - if (m_SchedulingThread.joinable()) - { - m_SchedulingThread.join(); - } - - ShutdownRunners(); -} - -void -FunctionServiceSession::Impl::ShutdownRunners() -{ - m_LocalRunnerGroup.Shutdown(); - m_RemoteRunnerGroup.Shutdown(); -} - -void -FunctionServiceSession::Impl::StartRecording(ChunkResolver& InCidStore, const std::filesystem::path& RecordingPath) -{ - ZEN_INFO("starting recording to '{}'", RecordingPath); - - m_Recorder = std::make_unique(InCidStore, RecordingPath); - - ZEN_INFO("started recording to '{}'", RecordingPath); -} - -void -FunctionServiceSession::Impl::StopRecording() -{ - ZEN_INFO("stopping recording"); - - m_Recorder = nullptr; - - ZEN_INFO("stopped recording"); -} - -std::vector -FunctionServiceSession::Impl::GetActionHistory(int Limit) -{ - RwLock::SharedLockScope _(m_ActionHistoryLock); - - if (Limit > 0 && static_cast(Limit) < m_ActionHistory.size()) - { - return std::vector(m_ActionHistory.end() - Limit, m_ActionHistory.end()); - } - - return std::vector(m_ActionHistory.begin(), m_ActionHistory.end()); -} - -void -FunctionServiceSession::Impl::RegisterWorker(CbPackage Worker) -{ - RwLock::ExclusiveLockScope _(m_WorkerLock); - - const IoHash& WorkerId = Worker.GetObject().GetHash(); - - if (m_WorkerMap.insert_or_assign(WorkerId, Worker).second) - { - // Note that since the convention currently is that WorkerId is equal to the hash - // of the worker descriptor there is no chance that we get a second write with a - // different descriptor. Thus we only need to call this the first time, when the - // worker is added - - m_LocalRunnerGroup.RegisterWorker(Worker); - m_RemoteRunnerGroup.RegisterWorker(Worker); - - if (m_Recorder) - { - m_Recorder->RegisterWorker(Worker); - } - - CbObject WorkerObj = Worker.GetObject(); - - // Populate worker database - - const Guid WorkerBuildSystemVersion = WorkerObj["buildsystem_version"sv].AsUuid(); - - for (auto& Item : WorkerObj["functions"sv]) - { - CbObjectView Function = Item.AsObjectView(); - - std::string_view FunctionName = Function["name"sv].AsString(); - const Guid FunctionVersion = Function["version"sv].AsUuid(); - - m_FunctionList.emplace_back(FunctionDefinition{.FunctionName = std::string{FunctionName}, - .FunctionVersion = FunctionVersion, - .BuildSystemVersion = WorkerBuildSystemVersion, - .WorkerId = WorkerId}); - } - } -} - -WorkerDesc -FunctionServiceSession::Impl::GetWorkerDescriptor(const IoHash& WorkerId) -{ - RwLock::SharedLockScope _(m_WorkerLock); - - if (auto It = m_WorkerMap.find(WorkerId); It != m_WorkerMap.end()) - { - const CbPackage& Desc = It->second; - return {Desc, WorkerId}; - } - - return {}; -} - -std::vector -FunctionServiceSession::Impl::GetKnownWorkerIds() -{ - std::vector WorkerIds; - WorkerIds.reserve(m_WorkerMap.size()); - - m_WorkerLock.WithSharedLock([&] { - for (const auto& [WorkerId, _] : m_WorkerMap) - { - WorkerIds.push_back(WorkerId); - } - }); - - return WorkerIds; -} - -FunctionServiceSession::EnqueueResult -FunctionServiceSession::Impl::EnqueueAction(CbObject ActionObject, int Priority) -{ - // Resolve function to worker - - IoHash WorkerId{IoHash::Zero}; - - std::string_view FunctionName = ActionObject["Function"sv].AsString(); - const Guid FunctionVersion = ActionObject["FunctionVersion"sv].AsUuid(); - const Guid BuildSystemVersion = ActionObject["BuildSystemVersion"sv].AsUuid(); - - for (const FunctionDefinition& FuncDef : m_FunctionList) - { - if (FuncDef.FunctionName == FunctionName && FuncDef.FunctionVersion == FunctionVersion && - FuncDef.BuildSystemVersion == BuildSystemVersion) - { - WorkerId = FuncDef.WorkerId; - - break; - } - } - - if (WorkerId == IoHash::Zero) - { - CbObjectWriter Writer; - - Writer << "Function"sv << FunctionName << "FunctionVersion"sv << FunctionVersion << "BuildSystemVersion" << BuildSystemVersion; - Writer << "error" - << "no worker matches the action specification"; - - return {0, Writer.Save()}; - } - - if (auto It = m_WorkerMap.find(WorkerId); It != m_WorkerMap.end()) - { - CbPackage WorkerPackage = It->second; - - return EnqueueResolvedAction(WorkerDesc{WorkerPackage, WorkerId}, ActionObject, Priority); - } - - CbObjectWriter Writer; - - Writer << "Function"sv << FunctionName << "FunctionVersion"sv << FunctionVersion << "BuildSystemVersion" << BuildSystemVersion; - Writer << "error" - << "no worker found despite match"; - - return {0, Writer.Save()}; -} - -FunctionServiceSession::EnqueueResult -FunctionServiceSession::Impl::EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int RequestPriority) -{ - const int ActionLsn = ++m_ActionsCounter; - - Ref Pending{new RunnerAction(m_FunctionServiceSession)}; - - Pending->ActionLsn = ActionLsn; - Pending->Worker = Worker; - Pending->ActionId = ActionObj.GetHash(); - Pending->ActionObj = ActionObj; - Pending->Priority = RequestPriority; - - SubmitResult SubResult = SubmitAction(Pending); - - if (SubResult.IsAccepted) - { - // Great, the job is being taken care of by the runner - ZEN_DEBUG("direct schedule LSN {}", Pending->ActionLsn); - } - else - { - ZEN_DEBUG("action {} ({}) PENDING", Pending->ActionId, Pending->ActionLsn); - - Pending->SetActionState(RunnerAction::State::Pending); - } - - if (m_Recorder) - { - m_Recorder->RecordAction(Pending); - } - - CbObjectWriter Writer; - Writer << "lsn" << Pending->ActionLsn; - Writer << "worker" << Pending->Worker.WorkerId; - Writer << "action" << Pending->ActionId; - - return {Pending->ActionLsn, Writer.Save()}; -} - -SubmitResult -FunctionServiceSession::Impl::SubmitAction(Ref Action) -{ - // Loosely round-robin scheduling of actions across runners. - // - // It's not entirely clear what this means given that submits - // can come in across multiple threads, but it's probably better - // than always starting with the first runner. - // - // Longer term we should track the state of the individual - // runners and make decisions accordingly. - - SubmitResult Result = m_LocalRunnerGroup.SubmitAction(Action); - if (Result.IsAccepted) - { - return Result; - } - - return m_RemoteRunnerGroup.SubmitAction(Action); -} - -size_t -FunctionServiceSession::Impl::GetSubmittedActionCount() -{ - return m_LocalRunnerGroup.GetSubmittedActionCount() + m_RemoteRunnerGroup.GetSubmittedActionCount(); -} - -HttpResponseCode -FunctionServiceSession::Impl::GetActionResult(int ActionLsn, CbPackage& OutResultPackage) -{ - // This lock is held for the duration of the function since we need to - // be sure that the action doesn't change state while we are checking the - // different data structures - - RwLock::ExclusiveLockScope _(m_ResultsLock); - - if (auto It = m_ResultsMap.find(ActionLsn); It != m_ResultsMap.end()) - { - OutResultPackage = std::move(It->second->GetResult()); - - m_ResultsMap.erase(It); - - return HttpResponseCode::OK; - } - - { - RwLock::SharedLockScope __(m_PendingLock); - - if (auto FindIt = m_PendingActions.find(ActionLsn); FindIt != m_PendingActions.end()) - { - return HttpResponseCode::Accepted; - } - } - - // Lock order is important here to avoid deadlocks, RwLock m_RunningLock must - // always be taken after m_ResultsLock if both are needed - - { - RwLock::SharedLockScope __(m_RunningLock); - - if (m_RunningMap.find(ActionLsn) != m_RunningMap.end()) - { - return HttpResponseCode::Accepted; - } - } - - return HttpResponseCode::NotFound; -} - -HttpResponseCode -FunctionServiceSession::Impl::FindActionResult(const IoHash& ActionId, CbPackage& OutResultPackage) -{ - // This lock is held for the duration of the function since we need to - // be sure that the action doesn't change state while we are checking the - // different data structures - - RwLock::ExclusiveLockScope _(m_ResultsLock); - - for (auto It = begin(m_ResultsMap), End = end(m_ResultsMap); It != End; ++It) - { - if (It->second->ActionId == ActionId) - { - OutResultPackage = std::move(It->second->GetResult()); - - m_ResultsMap.erase(It); - - return HttpResponseCode::OK; - } - } - - { - RwLock::SharedLockScope __(m_PendingLock); - - for (const auto& [K, Pending] : m_PendingActions) - { - if (Pending->ActionId == ActionId) - { - return HttpResponseCode::Accepted; - } - } - } - - // Lock order is important here to avoid deadlocks, RwLock m_RunningLock must - // always be taken after m_ResultsLock if both are needed - - { - RwLock::SharedLockScope __(m_RunningLock); - - for (const auto& [K, v] : m_RunningMap) - { - if (v->ActionId == ActionId) - { - return HttpResponseCode::Accepted; - } - } - } - - return HttpResponseCode::NotFound; -} - -void -FunctionServiceSession::Impl::GetCompleted(CbWriter& Cbo) -{ - Cbo.BeginArray("completed"); - - m_ResultsLock.WithSharedLock([&] { - for (auto& Kv : m_ResultsMap) - { - Cbo << Kv.first; - } - }); - - Cbo.EndArray(); -} - -# define ZEN_BATCH_SCHEDULER 1 - -void -FunctionServiceSession::Impl::SchedulePendingActions() -{ - int ScheduledCount = 0; - size_t RunningCount = m_RunningLock.WithSharedLock([&] { return m_RunningMap.size(); }); - size_t PendingCount = m_PendingLock.WithSharedLock([&] { return m_PendingActions.size(); }); - size_t ResultCount = m_ResultsLock.WithSharedLock([&] { return m_ResultsMap.size(); }); - - static Stopwatch DumpRunningTimer; - - auto _ = MakeGuard([&] { - ZEN_INFO("scheduled {} pending actions. {} running ({} retired), {} still pending, {} results", - ScheduledCount, - RunningCount, - m_RetiredCount.load(), - PendingCount, - ResultCount); - - if (DumpRunningTimer.GetElapsedTimeMs() > 30000) - { - DumpRunningTimer.Reset(); - - std::set RunningList; - m_RunningLock.WithSharedLock([&] { - for (auto& [K, V] : m_RunningMap) - { - RunningList.insert(K); - } - }); - - ExtendableStringBuilder<1024> RunningString; - for (int i : RunningList) - { - if (RunningString.Size()) - { - RunningString << ", "; - } - - RunningString.Append(IntNum(i)); - } - - ZEN_INFO("running: {}", RunningString); - } - }); - -# if ZEN_BATCH_SCHEDULER - size_t Capacity = QueryCapacity(); - - if (!Capacity) - { - _.Dismiss(); - - return; - } - - std::vector> ActionsToSchedule; - - // Pull actions to schedule from the pending queue, we will try to submit these to the runner outside of the lock - - m_PendingLock.WithExclusiveLock([&] { - if (m_ShutdownRequested) - { - return; - } - - if (m_PendingActions.empty()) - { - return; - } - - size_t NumActionsToSchedule = std::min(Capacity, m_PendingActions.size()); - - auto PendingIt = m_PendingActions.begin(); - const auto PendingEnd = m_PendingActions.end(); - - while (NumActionsToSchedule && PendingIt != PendingEnd) - { - const Ref& Pending = PendingIt->second; - - switch (Pending->ActionState()) - { - case RunnerAction::State::Pending: - ActionsToSchedule.push_back(Pending); - break; - - case RunnerAction::State::Running: - case RunnerAction::State::Completed: - case RunnerAction::State::Failed: - break; - - default: - case RunnerAction::State::New: - ZEN_WARN("unexpected state {} for pending action {}", static_cast(Pending->ActionState()), Pending->ActionLsn); - break; - } - - ++PendingIt; - --NumActionsToSchedule; - } - - PendingCount = m_PendingActions.size(); - }); - - if (ActionsToSchedule.empty()) - { - _.Dismiss(); - return; - } - - ZEN_INFO("attempting schedule of {} pending actions", ActionsToSchedule.size()); - - auto SubmitResults = SubmitActions(ActionsToSchedule); - - // Move successfully scheduled actions to the running map and remove - // from pending queue. It's actually possible that by the time we get - // to this stage some of the actions may have already completed, so - // they should not always be added to the running map - - eastl::hash_set ScheduledActions; - - for (size_t i = 0; i < ActionsToSchedule.size(); ++i) - { - const Ref& Pending = ActionsToSchedule[i]; - const SubmitResult& SubResult = SubmitResults[i]; - - if (SubResult.IsAccepted) - { - ScheduledActions.insert(Pending->ActionLsn); - } - } - - ScheduledCount += (int)ActionsToSchedule.size(); - -# else - m_PendingLock.WithExclusiveLock([&] { - while (!m_PendingActions.empty()) - { - if (m_ShutdownRequested) - { - return; - } - - // Here it would be good if we could decide to pop immediately to avoid - // holding the lock while creating processes etc - const Ref& Pending = m_PendingActions.begin()->second; - FunctionRunner::SubmitResult SubResult = SubmitAction(Pending); - - if (SubResult.IsAccepted) - { - // Great, the job is being taken care of by the runner - - ZEN_DEBUG("action {} ({}) PENDING -> RUNNING", Pending->ActionId, Pending->ActionLsn); - - m_RunningLock.WithExclusiveLock([&] { - m_RunningMap.insert({Pending->ActionLsn, Pending}); - - RunningCount = m_RunningMap.size(); - }); - - m_PendingActions.pop_front(); - - PendingCount = m_PendingActions.size(); - ++ScheduledCount; - } - else - { - // Runner could not accept the job, leave it on the pending queue - - return; - } - } - }); -# endif -} - -void -FunctionServiceSession::Impl::MonitorThreadFunction() -{ - SetCurrentThreadName("FunctionServiceSession_Monitor"); - - auto _ = MakeGuard([&] { ZEN_INFO("monitor thread exiting"); }); - - do - { - int TimeoutMs = 1000; - - if (m_PendingLock.WithSharedLock([&] { return m_PendingActions.size(); })) - { - TimeoutMs = 100; - } - - const bool Timedout = m_SchedulingThreadEvent.Wait(TimeoutMs); - - if (m_SchedulingThreadEnabled == false) - { - return; - } - - HandleActionUpdates(); - - // Schedule pending actions - - SchedulePendingActions(); - - if (!Timedout) - { - m_SchedulingThreadEvent.Reset(); - } - } while (m_SchedulingThreadEnabled); -} - -void -FunctionServiceSession::Impl::PostUpdate(RunnerAction* Action) -{ - m_UpdatedActionsLock.WithExclusiveLock([&] { m_UpdatedActions.emplace_back(Action); }); -} - -void -FunctionServiceSession::Impl::HandleActionUpdates() -{ - std::vector> UpdatedActions; - - m_UpdatedActionsLock.WithExclusiveLock([&] { std::swap(UpdatedActions, m_UpdatedActions); }); - - std::unordered_set SeenLsn; - std::unordered_set RunningLsn; - - for (Ref& Action : UpdatedActions) - { - const int ActionLsn = Action->ActionLsn; - - if (auto [It, Inserted] = SeenLsn.insert(ActionLsn); Inserted) - { - switch (Action->ActionState()) - { - case RunnerAction::State::Pending: - m_PendingLock.WithExclusiveLock([&] { m_PendingActions.insert({ActionLsn, Action}); }); - break; - - case RunnerAction::State::Running: - m_PendingLock.WithExclusiveLock([&] { - m_RunningLock.WithExclusiveLock([&] { - m_RunningMap.insert({ActionLsn, Action}); - m_PendingActions.erase(ActionLsn); - }); - }); - ZEN_DEBUG("action {} ({}) RUNNING", Action->ActionId, ActionLsn); - break; - - case RunnerAction::State::Completed: - case RunnerAction::State::Failed: - m_ResultsLock.WithExclusiveLock([&] { - m_ResultsMap[ActionLsn] = Action; - - m_PendingLock.WithExclusiveLock([&] { - m_RunningLock.WithExclusiveLock([&] { - if (auto FindIt = m_RunningMap.find(ActionLsn); FindIt == m_RunningMap.end()) - { - m_PendingActions.erase(ActionLsn); - } - else - { - m_RunningMap.erase(FindIt); - } - }); - }); - - m_ActionHistoryLock.WithExclusiveLock([&] { - ActionHistoryEntry Entry{.Lsn = ActionLsn, - .ActionId = Action->ActionId, - .WorkerId = Action->Worker.WorkerId, - .ActionDescriptor = Action->ActionObj, - .Succeeded = Action->ActionState() == RunnerAction::State::Completed}; - - std::copy(std::begin(Action->Timestamps), std::end(Action->Timestamps), std::begin(Entry.Timestamps)); - - m_ActionHistory.push_back(std::move(Entry)); - - if (m_ActionHistory.size() > m_HistoryLimit) - { - m_ActionHistory.pop_front(); - } - }); - }); - m_RetiredCount.fetch_add(1); - m_ResultRate.Mark(1); - ZEN_DEBUG("action {} ({}) RUNNING -> COMPLETED with {}", - Action->ActionId, - ActionLsn, - Action->ActionState() == RunnerAction::State::Completed ? "SUCCESS" : "FAILURE"); - break; - } - } - } -} - -size_t -FunctionServiceSession::Impl::QueryCapacity() -{ - return m_LocalRunnerGroup.QueryCapacity() + m_RemoteRunnerGroup.QueryCapacity(); -} - -std::vector -FunctionServiceSession::Impl::SubmitActions(const std::vector>& Actions) -{ - std::vector Results; - - for (const Ref& Action : Actions) - { - Results.push_back(SubmitAction(Action)); - } - - return Results; -} - -////////////////////////////////////////////////////////////////////////// - -FunctionServiceSession::FunctionServiceSession(ChunkResolver& InChunkResolver) -{ - m_Impl = std::make_unique(this, InChunkResolver); -} - -FunctionServiceSession::~FunctionServiceSession() -{ - Shutdown(); -} - -bool -FunctionServiceSession::IsHealthy() -{ - return m_Impl->IsHealthy(); -} - -void -FunctionServiceSession::Shutdown() -{ - m_Impl->Shutdown(); -} - -void -FunctionServiceSession::StartRecording(ChunkResolver& InResolver, const std::filesystem::path& RecordingPath) -{ - m_Impl->StartRecording(InResolver, RecordingPath); -} - -void -FunctionServiceSession::StopRecording() -{ - m_Impl->StopRecording(); -} - -void -FunctionServiceSession::EmitStats(CbObjectWriter& Cbo) -{ - m_Impl->EmitStats(Cbo); -} - -std::vector -FunctionServiceSession::GetKnownWorkerIds() -{ - return m_Impl->GetKnownWorkerIds(); -} - -WorkerDesc -FunctionServiceSession::GetWorkerDescriptor(const IoHash& WorkerId) -{ - return m_Impl->GetWorkerDescriptor(WorkerId); -} - -void -FunctionServiceSession::AddLocalRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath) -{ - m_Impl->m_LocalRunnerGroup.AddRunner(new LocalProcessRunner(InChunkResolver, BasePath)); -} - -void -FunctionServiceSession::AddRemoteRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath, std::string_view HostName) -{ - m_Impl->m_RemoteRunnerGroup.AddRunner(new RemoteHttpRunner(InChunkResolver, BasePath, HostName)); -} - -FunctionServiceSession::EnqueueResult -FunctionServiceSession::EnqueueAction(CbObject ActionObject, int Priority) -{ - return m_Impl->EnqueueAction(ActionObject, Priority); -} - -FunctionServiceSession::EnqueueResult -FunctionServiceSession::EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int RequestPriority) -{ - return m_Impl->EnqueueResolvedAction(Worker, ActionObj, RequestPriority); -} - -void -FunctionServiceSession::RegisterWorker(CbPackage Worker) -{ - m_Impl->RegisterWorker(Worker); -} - -HttpResponseCode -FunctionServiceSession::GetActionResult(int ActionLsn, CbPackage& OutResultPackage) -{ - return m_Impl->GetActionResult(ActionLsn, OutResultPackage); -} - -HttpResponseCode -FunctionServiceSession::FindActionResult(const IoHash& ActionId, CbPackage& OutResultPackage) -{ - return m_Impl->FindActionResult(ActionId, OutResultPackage); -} - -std::vector -FunctionServiceSession::GetActionHistory(int Limit) -{ - return m_Impl->GetActionHistory(Limit); -} - -void -FunctionServiceSession::GetCompleted(CbWriter& Cbo) -{ - m_Impl->GetCompleted(Cbo); -} - -void -FunctionServiceSession::PostUpdate(RunnerAction* Action) -{ - m_Impl->PostUpdate(Action); -} - -////////////////////////////////////////////////////////////////////////// - -void -function_forcelink() -{ -} - -} // namespace zen::compute - -#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/httpcomputeservice.cpp b/src/zencompute/httpcomputeservice.cpp new file mode 100644 index 000000000..e82a40781 --- /dev/null +++ b/src/zencompute/httpcomputeservice.cpp @@ -0,0 +1,1643 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencompute/httpcomputeservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "runners/functionrunner.h" + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include +# include + +using namespace std::literals; + +namespace zen::compute { + +constinit AsciiSet g_DecimalSet("0123456789"); +constinit AsciiSet g_HexSet("0123456789abcdefABCDEF"); + +auto DecimalMatcher = [](std::string_view Str) { return AsciiSet::HasOnly(Str, g_DecimalSet); }; +auto IoHashMatcher = [](std::string_view Str) { return Str.size() == 40 && AsciiSet::HasOnly(Str, g_HexSet); }; +auto OidMatcher = [](std::string_view Str) { return Str.size() == 24 && AsciiSet::HasOnly(Str, g_HexSet); }; + +////////////////////////////////////////////////////////////////////////// + +struct HttpComputeService::Impl +{ + HttpComputeService* m_Self; + CidStore& m_CidStore; + IHttpStatsService& m_StatsService; + LoggerRef m_Log; + std::filesystem::path m_BaseDir; + HttpRequestRouter m_Router; + ComputeServiceSession m_ComputeService; + SystemMetricsTracker m_MetricsTracker; + + // Metrics + + metrics::OperationTiming m_HttpRequests; + + // Per-remote-queue metadata, shared across all lookup maps below. + + struct RemoteQueueInfo : RefCounted + { + int QueueId = 0; + Oid Token; + std::string IdempotencyKey; // empty if no idempotency key was provided + std::string ClientHostname; // empty if no hostname was provided + }; + + // Remote queue registry — all three maps share the same RemoteQueueInfo objects. + // All maps are guarded by m_RemoteQueueLock. + + RwLock m_RemoteQueueLock; + std::unordered_map, Oid::Hasher> m_RemoteQueuesByToken; // Token → info + std::unordered_map> m_RemoteQueuesByQueueId; // QueueId → info + std::unordered_map> m_RemoteQueuesByTag; // idempotency key → info + + LoggerRef Log() { return m_Log; } + + int ResolveQueueToken(const Oid& Token); + int ResolveQueueRef(HttpServerRequest& HttpReq, std::string_view Capture); + + struct IngestStats + { + int Count = 0; + int NewCount = 0; + uint64_t Bytes = 0; + uint64_t NewBytes = 0; + }; + + IngestStats IngestPackageAttachments(const CbPackage& Package); + bool CheckAttachments(const CbObject& ActionObj, std::vector& NeedList); + void HandleWorkersGet(HttpServerRequest& HttpReq); + void HandleWorkersAllGet(HttpServerRequest& HttpReq); + void WriteQueueDescription(CbWriter& Cbo, int QueueId, const ComputeServiceSession::QueueStatus& Status); + void HandleWorkerRequest(HttpServerRequest& HttpReq, const IoHash& WorkerId); + + void RegisterRoutes(); + + Impl(HttpComputeService* Self, + CidStore& InCidStore, + IHttpStatsService& StatsService, + const std::filesystem::path& BaseDir, + int32_t MaxConcurrentActions) + : m_Self(Self) + , m_CidStore(InCidStore) + , m_StatsService(StatsService) + , m_Log(logging::Get("compute")) + , m_BaseDir(BaseDir) + , m_ComputeService(InCidStore) + { + m_ComputeService.AddLocalRunner(InCidStore, m_BaseDir / "local", MaxConcurrentActions); + m_ComputeService.WaitUntilReady(); + m_StatsService.RegisterHandler("compute", *m_Self); + RegisterRoutes(); + } +}; + +////////////////////////////////////////////////////////////////////////// + +void +HttpComputeService::Impl::RegisterRoutes() +{ + m_Router.AddMatcher("lsn", DecimalMatcher); + m_Router.AddMatcher("worker", IoHashMatcher); + m_Router.AddMatcher("action", IoHashMatcher); + m_Router.AddMatcher("queue", DecimalMatcher); + m_Router.AddMatcher("oidtoken", OidMatcher); + m_Router.AddMatcher("queueref", [](std::string_view Str) { return DecimalMatcher(Str) || OidMatcher(Str); }); + + m_Router.RegisterRoute( + "ready", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (m_ComputeService.IsHealthy()) + { + return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "ok"); + } + + return HttpReq.WriteResponse(HttpResponseCode::ServiceUnavailable); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "abandon", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!HttpReq.IsLocalMachineRequest()) + { + return HttpReq.WriteResponse(HttpResponseCode::Forbidden); + } + + bool Success = m_ComputeService.RequestStateTransition(ComputeServiceSession::SessionState::Abandoned); + + if (Success) + { + CbObjectWriter Cbo; + Cbo << "state"sv + << "Abandoned"sv; + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } + + CbObjectWriter Cbo; + Cbo << "error"sv + << "Cannot transition to Abandoned from current state"sv; + return HttpReq.WriteResponse(HttpResponseCode::Conflict, Cbo.Save()); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "workers", + [this](HttpRouterRequest& Req) { HandleWorkersGet(Req.ServerRequest()); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "workers/{worker}", + [this](HttpRouterRequest& Req) { HandleWorkerRequest(Req.ServerRequest(), IoHash::FromHexString(Req.GetCapture(1))); }, + HttpVerb::kGet | HttpVerb::kPost); + + m_Router.RegisterRoute( + "jobs/completed", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + CbObjectWriter Cbo; + m_ComputeService.GetCompleted(Cbo); + + ExtendedSystemMetrics Sm = ApplyReportingOverrides(m_MetricsTracker.Query()); + Cbo.BeginObject("metrics"); + Describe(Sm, Cbo); + Cbo.EndObject(); + + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/history", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const auto QueryParams = HttpReq.GetQueryParams(); + + int QueryLimit = 50; + + if (auto LimitParam = QueryParams.GetValue("limit"); LimitParam.empty() == false) + { + QueryLimit = ParseInt(LimitParam).value_or(50); + } + + CbObjectWriter Cbo; + Cbo.BeginArray("history"); + for (const auto& Entry : m_ComputeService.GetActionHistory(QueryLimit)) + { + Cbo.BeginObject(); + Cbo << "lsn"sv << Entry.Lsn; + Cbo << "queueId"sv << Entry.QueueId; + Cbo << "actionId"sv << Entry.ActionId; + Cbo << "workerId"sv << Entry.WorkerId; + Cbo << "succeeded"sv << Entry.Succeeded; + Cbo << "actionDescriptor"sv << Entry.ActionDescriptor; + if (Entry.CpuSeconds > 0.0f) + { + Cbo.AddFloat("cpuSeconds"sv, Entry.CpuSeconds); + } + if (Entry.RetryCount > 0) + { + Cbo << "retry_count"sv << Entry.RetryCount; + } + + for (const auto& Timestamp : Entry.Timestamps) + { + Cbo.AddInteger( + fmt::format("time_{}"sv, RunnerAction::ToString(static_cast(&Timestamp - Entry.Timestamps))), + Timestamp); + } + Cbo.EndObject(); + } + Cbo.EndArray(); + + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/running", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + auto Running = m_ComputeService.GetRunningActions(); + CbObjectWriter Cbo; + Cbo.BeginArray("running"); + for (const auto& Info : Running) + { + Cbo.BeginObject(); + Cbo << "lsn"sv << Info.Lsn; + Cbo << "queueId"sv << Info.QueueId; + Cbo << "actionId"sv << Info.ActionId; + if (Info.CpuUsagePercent >= 0.0f) + { + Cbo.AddFloat("cpuUsage"sv, Info.CpuUsagePercent); + } + if (Info.CpuSeconds > 0.0f) + { + Cbo.AddFloat("cpuSeconds"sv, Info.CpuSeconds); + } + Cbo.EndObject(); + } + Cbo.EndArray(); + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/{lsn}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int ActionLsn = ParseInt(Req.GetCapture(1)).value_or(0); + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + CbPackage Output; + HttpResponseCode ResponseCode = m_ComputeService.GetActionResult(ActionLsn, Output); + + if (ResponseCode == HttpResponseCode::OK) + { + HttpReq.WriteResponse(HttpResponseCode::OK, Output); + } + else + { + HttpReq.WriteResponse(ResponseCode); + } + + // Once we've initiated the response we can mark the result + // as retired, allowing the service to free any associated + // resources. Note that there still needs to be a delay + // to allow the transmission to complete, it would be better + // if we could issue this once the response is fully sent... + m_ComputeService.RetireActionResult(ActionLsn); + } + break; + + case HttpVerb::kPost: + { + auto Result = m_ComputeService.RescheduleAction(ActionLsn); + + CbObjectWriter Cbo; + if (Result.Success) + { + Cbo << "lsn"sv << ActionLsn; + Cbo << "retry_count"sv << Result.RetryCount; + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } + else + { + Cbo << "error"sv << Result.Error; + HttpReq.WriteResponse(HttpResponseCode::Conflict, Cbo.Save()); + } + } + break; + + default: + break; + } + }, + HttpVerb::kGet | HttpVerb::kPost); + + m_Router.RegisterRoute( + "jobs/{worker}/{action}", // This route is inefficient, and is only here for backwards compatibility. The preferred path is the + // one which uses the scheduled action lsn for lookups + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const IoHash ActionId = IoHash::FromHexString(Req.GetCapture(2)); + + CbPackage Output; + if (HttpResponseCode ResponseCode = m_ComputeService.FindActionResult(ActionId, /* out */ Output); + ResponseCode != HttpResponseCode::OK) + { + ZEN_TRACE("jobs/{}/{}: {}", Req.GetCapture(1), Req.GetCapture(2), ToString(ResponseCode)) + + if (ResponseCode == HttpResponseCode::NotFound) + { + return HttpReq.WriteResponse(ResponseCode); + } + + return HttpReq.WriteResponse(ResponseCode); + } + + ZEN_DEBUG("jobs/{}/{}: OK", Req.GetCapture(1), Req.GetCapture(2)) + + return HttpReq.WriteResponse(HttpResponseCode::OK, Output); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "jobs/{worker}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const IoHash WorkerId = IoHash::FromHexString(Req.GetCapture(1)); + + WorkerDesc Worker = m_ComputeService.GetWorkerDescriptor(WorkerId); + + if (!Worker) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + const auto QueryParams = Req.ServerRequest().GetQueryParams(); + + int RequestPriority = -1; + + if (auto PriorityParam = QueryParams.GetValue("priority"); PriorityParam.empty() == false) + { + RequestPriority = ParseInt(PriorityParam).value_or(-1); + } + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + // TODO: return status of all pending or executing jobs + break; + + case HttpVerb::kPost: + switch (HttpReq.RequestContentType()) + { + case HttpContentType::kCbObject: + { + // This operation takes the proposed job spec and identifies which + // chunks are not present on this server. This list is then returned in + // the "need" list in the response + + IoBuffer Payload = HttpReq.ReadPayload(); + CbObject ActionObj = LoadCompactBinaryObject(Payload); + + std::vector NeedList; + + ActionObj.IterateAttachments([&](CbFieldView Field) { + const IoHash FileHash = Field.AsHash(); + + if (!m_CidStore.ContainsChunk(FileHash)) + { + NeedList.push_back(FileHash); + } + }); + + if (NeedList.empty()) + { + // We already have everything, enqueue the action for execution + + if (ComputeServiceSession::EnqueueResult Result = + m_ComputeService.EnqueueResolvedAction(Worker, ActionObj, RequestPriority)) + { + ZEN_DEBUG("action {} accepted (lsn {})", ActionObj.GetHash(), Result.Lsn); + + HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + + return; + } + + CbObjectWriter Cbo; + Cbo.BeginArray("need"); + + for (const IoHash& Hash : NeedList) + { + Cbo << Hash; + } + + Cbo.EndArray(); + CbObject Response = Cbo.Save(); + + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Response); + } + break; + + case HttpContentType::kCbPackage: + { + CbPackage Action = HttpReq.ReadPayloadPackage(); + CbObject ActionObj = Action.GetObject(); + + std::span Attachments = Action.GetAttachments(); + + int AttachmentCount = 0; + int NewAttachmentCount = 0; + uint64_t TotalAttachmentBytes = 0; + uint64_t TotalNewBytes = 0; + + for (const CbAttachment& Attachment : Attachments) + { + ZEN_ASSERT(Attachment.IsCompressedBinary()); + + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer DataView = Attachment.AsCompressedBinary(); + + ZEN_UNUSED(DataHash); + + const uint64_t CompressedSize = DataView.GetCompressedSize(); + + TotalAttachmentBytes += CompressedSize; + ++AttachmentCount; + + const CidStore::InsertResult InsertResult = + m_CidStore.AddChunk(DataView.GetCompressed().Flatten().AsIoBuffer(), DataHash); + + if (InsertResult.New) + { + TotalNewBytes += CompressedSize; + ++NewAttachmentCount; + } + } + + if (ComputeServiceSession::EnqueueResult Result = + m_ComputeService.EnqueueResolvedAction(Worker, ActionObj, RequestPriority)) + { + ZEN_DEBUG("accepted action {} (lsn {}): {} in {} attachments. {} new ({} attachments)", + ActionObj.GetHash(), + Result.Lsn, + zen::NiceBytes(TotalAttachmentBytes), + AttachmentCount, + zen::NiceBytes(TotalNewBytes), + NewAttachmentCount); + + HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + + return; + } + break; + + default: + break; + } + break; + + default: + break; + } + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "jobs", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + const auto QueryParams = HttpReq.GetQueryParams(); + + int RequestPriority = -1; + + if (auto PriorityParam = QueryParams.GetValue("priority"); PriorityParam.empty() == false) + { + RequestPriority = ParseInt(PriorityParam).value_or(-1); + } + + // Resolve worker + + // + + switch (HttpReq.RequestContentType()) + { + case HttpContentType::kCbObject: + { + // This operation takes the proposed job spec and identifies which + // chunks are not present on this server. This list is then returned in + // the "need" list in the response + + IoBuffer Payload = HttpReq.ReadPayload(); + CbObject ActionObj = LoadCompactBinaryObject(Payload); + + std::vector NeedList; + + ActionObj.IterateAttachments([&](CbFieldView Field) { + const IoHash FileHash = Field.AsHash(); + + if (!m_CidStore.ContainsChunk(FileHash)) + { + NeedList.push_back(FileHash); + } + }); + + if (NeedList.empty()) + { + // We already have everything, enqueue the action for execution + + if (ComputeServiceSession::EnqueueResult Result = m_ComputeService.EnqueueAction(ActionObj, RequestPriority)) + { + ZEN_DEBUG("action accepted (lsn {})", Result.Lsn); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + else + { + // Could not resolve? + return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); + } + } + + CbObjectWriter Cbo; + Cbo.BeginArray("need"); + + for (const IoHash& Hash : NeedList) + { + Cbo << Hash; + } + + Cbo.EndArray(); + CbObject Response = Cbo.Save(); + + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Response); + } + + case HttpContentType::kCbPackage: + { + CbPackage Action = HttpReq.ReadPayloadPackage(); + CbObject ActionObj = Action.GetObject(); + + std::span Attachments = Action.GetAttachments(); + + int AttachmentCount = 0; + int NewAttachmentCount = 0; + uint64_t TotalAttachmentBytes = 0; + uint64_t TotalNewBytes = 0; + + for (const CbAttachment& Attachment : Attachments) + { + ZEN_ASSERT(Attachment.IsCompressedBinary()); + + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer DataView = Attachment.AsCompressedBinary(); + + ZEN_UNUSED(DataHash); + + const uint64_t CompressedSize = DataView.GetCompressedSize(); + + TotalAttachmentBytes += CompressedSize; + ++AttachmentCount; + + const CidStore::InsertResult InsertResult = + m_CidStore.AddChunk(DataView.GetCompressed().Flatten().AsIoBuffer(), DataHash); + + if (InsertResult.New) + { + TotalNewBytes += CompressedSize; + ++NewAttachmentCount; + } + } + + if (ComputeServiceSession::EnqueueResult Result = m_ComputeService.EnqueueAction(ActionObj, RequestPriority)) + { + ZEN_DEBUG("accepted action (lsn {}): {} in {} attachments. {} new ({} attachments)", + Result.Lsn, + zen::NiceBytes(TotalAttachmentBytes), + AttachmentCount, + zen::NiceBytes(TotalNewBytes), + NewAttachmentCount); + + HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + else + { + // Could not resolve? + return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); + } + } + return; + } + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "workers/all", + [this](HttpRouterRequest& Req) { HandleWorkersAllGet(Req.ServerRequest()); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "queues/{queueref}/workers", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + if (ResolveQueueRef(HttpReq, Req.GetCapture(1)) == 0) + return; + HandleWorkersGet(HttpReq); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "queues/{queueref}/workers/all", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + if (ResolveQueueRef(HttpReq, Req.GetCapture(1)) == 0) + return; + HandleWorkersAllGet(HttpReq); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "queues/{queueref}/workers/{worker}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + if (ResolveQueueRef(HttpReq, Req.GetCapture(1)) == 0) + return; + HandleWorkerRequest(HttpReq, IoHash::FromHexString(Req.GetCapture(2))); + }, + HttpVerb::kGet | HttpVerb::kPost); + + m_Router.RegisterRoute( + "sysinfo", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + ExtendedSystemMetrics Sm = ApplyReportingOverrides(m_MetricsTracker.Query()); + + CbObjectWriter Cbo; + Describe(Sm, Cbo); + + Cbo << "cpu_usage" << Sm.CpuUsagePercent; + Cbo << "memory_total" << Sm.SystemMemoryMiB * 1024 * 1024; + Cbo << "memory_used" << (Sm.SystemMemoryMiB - Sm.AvailSystemMemoryMiB) * 1024 * 1024; + Cbo << "disk_used" << 100 * 1024; + Cbo << "disk_total" << 100 * 1024 * 1024; + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "record/start", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!HttpReq.IsLocalMachineRequest()) + { + return HttpReq.WriteResponse(HttpResponseCode::Forbidden); + } + + m_ComputeService.StartRecording(m_CidStore, m_BaseDir / "recording"); + + return HttpReq.WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "record/stop", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!HttpReq.IsLocalMachineRequest()) + { + return HttpReq.WriteResponse(HttpResponseCode::Forbidden); + } + + m_ComputeService.StopRecording(); + + return HttpReq.WriteResponse(HttpResponseCode::OK); + }, + HttpVerb::kPost); + + // Local-only queue listing and creation + + m_Router.RegisterRoute( + "queues", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + if (!HttpReq.IsLocalMachineRequest()) + { + return HttpReq.WriteResponse(HttpResponseCode::Forbidden); + } + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + CbObjectWriter Cbo; + Cbo.BeginArray("queues"sv); + + for (const int QueueId : m_ComputeService.GetQueueIds()) + { + ComputeServiceSession::QueueStatus Status = m_ComputeService.GetQueueStatus(QueueId); + + if (!Status.IsValid) + { + continue; + } + + Cbo.BeginObject(); + WriteQueueDescription(Cbo, QueueId, Status); + Cbo.EndObject(); + } + + Cbo.EndArray(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } + + case HttpVerb::kPost: + { + CbObject Metadata; + CbObject Config; + if (const CbObject Body = HttpReq.ReadPayloadObject()) + { + Metadata = Body.Find("metadata"sv).AsObject(); + Config = Body.Find("config"sv).AsObject(); + } + + ComputeServiceSession::CreateQueueResult Result = + m_ComputeService.CreateQueue({}, std::move(Metadata), std::move(Config)); + + CbObjectWriter Cbo; + Cbo << "queue_id"sv << Result.QueueId; + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } + + default: + break; + } + }, + HttpVerb::kGet | HttpVerb::kPost); + + // Queue creation routes — these remain separate since local creates a plain queue + // while remote additionally generates an OID token for external access. + + m_Router.RegisterRoute( + "queues/remote", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + // Extract optional fields from the request body. + // idempotency_key: when present, we return the existing remote queue token for this + // key rather than creating a new queue, making the endpoint safe to call concurrently. + // hostname: human-readable origin context stored alongside the queue for diagnostics. + // metadata: arbitrary CbObject metadata propagated from the originating queue. + // config: arbitrary CbObject config propagated from the originating queue. + std::string IdempotencyKey; + std::string ClientHostname; + CbObject Metadata; + CbObject Config; + if (const CbObject Body = HttpReq.ReadPayloadObject()) + { + IdempotencyKey = std::string(Body["idempotency_key"sv].AsString()); + ClientHostname = std::string(Body["hostname"sv].AsString()); + Metadata = Body.Find("metadata"sv).AsObject(); + Config = Body.Find("config"sv).AsObject(); + } + + // Stamp the forwarding node's hostname into the metadata so that the + // remote side knows which node originated the queue. + if (!ClientHostname.empty()) + { + CbObjectWriter MetaWriter; + for (auto Field : Metadata) + { + MetaWriter.AddField(Field.GetName(), Field); + } + MetaWriter << "via"sv << ClientHostname; + Metadata = MetaWriter.Save(); + } + + RwLock::ExclusiveLockScope _(m_RemoteQueueLock); + + if (!IdempotencyKey.empty()) + { + if (auto It = m_RemoteQueuesByTag.find(IdempotencyKey); It != m_RemoteQueuesByTag.end()) + { + Ref Existing = It->second; + if (m_ComputeService.GetQueueStatus(Existing->QueueId).IsValid) + { + CbObjectWriter Cbo; + Cbo << "queue_token"sv << Existing->Token.ToString(); + Cbo << "queue_id"sv << Existing->QueueId; + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } + + // Queue has since expired — clean up stale entries and fall through to create a new one + m_RemoteQueuesByToken.erase(Existing->Token); + m_RemoteQueuesByQueueId.erase(Existing->QueueId); + m_RemoteQueuesByTag.erase(It); + } + } + + ComputeServiceSession::CreateQueueResult Result = m_ComputeService.CreateQueue({}, std::move(Metadata), std::move(Config)); + Ref InfoRef(new RemoteQueueInfo()); + InfoRef->QueueId = Result.QueueId; + InfoRef->Token = Oid::NewOid(); + InfoRef->IdempotencyKey = std::move(IdempotencyKey); + InfoRef->ClientHostname = std::move(ClientHostname); + + m_RemoteQueuesByToken[InfoRef->Token] = InfoRef; + m_RemoteQueuesByQueueId[InfoRef->QueueId] = InfoRef; + if (!InfoRef->IdempotencyKey.empty()) + { + m_RemoteQueuesByTag[InfoRef->IdempotencyKey] = InfoRef; + } + + CbObjectWriter Cbo; + Cbo << "queue_token"sv << InfoRef->Token.ToString(); + Cbo << "queue_id"sv << InfoRef->QueueId; + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kPost); + + // Unified queue routes — {queueref} accepts both local integer IDs and remote OID tokens. + // ResolveQueueRef() handles access control (local-only for integer IDs) and token resolution. + + m_Router.RegisterRoute( + "queues/{queueref}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int QueueId = ResolveQueueRef(HttpReq, Req.GetCapture(1)); + + if (QueueId == 0) + { + return; + } + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + ComputeServiceSession::QueueStatus Status = m_ComputeService.GetQueueStatus(QueueId); + + if (!Status.IsValid) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter Cbo; + WriteQueueDescription(Cbo, QueueId, Status); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } + + case HttpVerb::kDelete: + { + ComputeServiceSession::QueueStatus Status = m_ComputeService.GetQueueStatus(QueueId); + + if (!Status.IsValid) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + m_ComputeService.CancelQueue(QueueId); + + return HttpReq.WriteResponse(HttpResponseCode::NoContent); + } + + default: + break; + } + }, + HttpVerb::kGet | HttpVerb::kDelete); + + m_Router.RegisterRoute( + "queues/{queueref}/drain", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int QueueId = ResolveQueueRef(HttpReq, Req.GetCapture(1)); + + if (QueueId == 0) + { + return; + } + + ComputeServiceSession::QueueStatus Status = m_ComputeService.GetQueueStatus(QueueId); + + if (!Status.IsValid) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + m_ComputeService.DrainQueue(QueueId); + + // Return updated queue status + Status = m_ComputeService.GetQueueStatus(QueueId); + + CbObjectWriter Cbo; + WriteQueueDescription(Cbo, QueueId, Status); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "queues/{queueref}/completed", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int QueueId = ResolveQueueRef(HttpReq, Req.GetCapture(1)); + + if (QueueId == 0) + { + return; + } + + ComputeServiceSession::QueueStatus Status = m_ComputeService.GetQueueStatus(QueueId); + + if (!Status.IsValid) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + CbObjectWriter Cbo; + m_ComputeService.GetQueueCompleted(QueueId, Cbo); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "queues/{queueref}/history", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int QueueId = ResolveQueueRef(HttpReq, Req.GetCapture(1)); + + if (QueueId == 0) + { + return; + } + + ComputeServiceSession::QueueStatus Status = m_ComputeService.GetQueueStatus(QueueId); + + if (!Status.IsValid) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + const auto QueryParams = HttpReq.GetQueryParams(); + + int QueryLimit = 50; + + if (auto LimitParam = QueryParams.GetValue("limit"); LimitParam.empty() == false) + { + QueryLimit = ParseInt(LimitParam).value_or(50); + } + + CbObjectWriter Cbo; + Cbo.BeginArray("history"); + for (const auto& Entry : m_ComputeService.GetQueueHistory(QueueId, QueryLimit)) + { + Cbo.BeginObject(); + Cbo << "lsn"sv << Entry.Lsn; + Cbo << "queueId"sv << Entry.QueueId; + Cbo << "actionId"sv << Entry.ActionId; + Cbo << "workerId"sv << Entry.WorkerId; + Cbo << "succeeded"sv << Entry.Succeeded; + if (Entry.CpuSeconds > 0.0f) + { + Cbo.AddFloat("cpuSeconds"sv, Entry.CpuSeconds); + } + if (Entry.RetryCount > 0) + { + Cbo << "retry_count"sv << Entry.RetryCount; + } + + for (const auto& Timestamp : Entry.Timestamps) + { + Cbo.AddInteger( + fmt::format("time_{}"sv, RunnerAction::ToString(static_cast(&Timestamp - Entry.Timestamps))), + Timestamp); + } + Cbo.EndObject(); + } + Cbo.EndArray(); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "queues/{queueref}/running", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int QueueId = ResolveQueueRef(HttpReq, Req.GetCapture(1)); + if (QueueId == 0) + { + return; + } + // Filter global running list to this queue + auto AllRunning = m_ComputeService.GetRunningActions(); + std::vector Running; + for (auto& Info : AllRunning) + if (Info.QueueId == QueueId) + Running.push_back(Info); + CbObjectWriter Cbo; + Cbo.BeginArray("running"); + for (const auto& Info : Running) + { + Cbo.BeginObject(); + Cbo << "lsn"sv << Info.Lsn; + Cbo << "queueId"sv << Info.QueueId; + Cbo << "actionId"sv << Info.ActionId; + if (Info.CpuUsagePercent >= 0.0f) + { + Cbo.AddFloat("cpuUsage"sv, Info.CpuUsagePercent); + } + if (Info.CpuSeconds > 0.0f) + { + Cbo.AddFloat("cpuSeconds"sv, Info.CpuSeconds); + } + Cbo.EndObject(); + } + Cbo.EndArray(); + return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "queues/{queueref}/jobs/{worker}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int QueueId = ResolveQueueRef(HttpReq, Req.GetCapture(1)); + + if (QueueId == 0) + { + return; + } + + const IoHash WorkerId = IoHash::FromHexString(Req.GetCapture(2)); + WorkerDesc Worker = m_ComputeService.GetWorkerDescriptor(WorkerId); + + if (!Worker) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + const auto QueryParams = Req.ServerRequest().GetQueryParams(); + int RequestPriority = -1; + + if (auto PriorityParam = QueryParams.GetValue("priority"); PriorityParam.empty() == false) + { + RequestPriority = ParseInt(PriorityParam).value_or(-1); + } + + switch (HttpReq.RequestContentType()) + { + case HttpContentType::kCbObject: + { + IoBuffer Payload = HttpReq.ReadPayload(); + CbObject ActionObj = LoadCompactBinaryObject(Payload); + + std::vector NeedList; + + if (!CheckAttachments(ActionObj, NeedList)) + { + CbObjectWriter Cbo; + Cbo.BeginArray("need"); + + for (const IoHash& Hash : NeedList) + { + Cbo << Hash; + } + + Cbo.EndArray(); + + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Cbo.Save()); + } + + if (ComputeServiceSession::EnqueueResult Result = + m_ComputeService.EnqueueResolvedActionToQueue(QueueId, Worker, ActionObj, RequestPriority)) + { + ZEN_DEBUG("queue {}: action {} accepted (lsn {})", QueueId, ActionObj.GetHash(), Result.Lsn); + return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); + } + } + + case HttpContentType::kCbPackage: + { + CbPackage Action = HttpReq.ReadPayloadPackage(); + CbObject ActionObj = Action.GetObject(); + + IngestStats Stats = IngestPackageAttachments(Action); + + if (ComputeServiceSession::EnqueueResult Result = + m_ComputeService.EnqueueResolvedActionToQueue(QueueId, Worker, ActionObj, RequestPriority)) + { + ZEN_DEBUG("queue {}: accepted action {} (lsn {}): {} in {} attachments. {} new ({} attachments)", + QueueId, + ActionObj.GetHash(), + Result.Lsn, + zen::NiceBytes(Stats.Bytes), + Stats.Count, + zen::NiceBytes(Stats.NewBytes), + Stats.NewCount); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); + } + } + + default: + break; + } + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "queues/{queueref}/jobs", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int QueueId = ResolveQueueRef(HttpReq, Req.GetCapture(1)); + + if (QueueId == 0) + { + return; + } + + const auto QueryParams = Req.ServerRequest().GetQueryParams(); + int RequestPriority = -1; + + if (auto PriorityParam = QueryParams.GetValue("priority"); PriorityParam.empty() == false) + { + RequestPriority = ParseInt(PriorityParam).value_or(-1); + } + + switch (HttpReq.RequestContentType()) + { + case HttpContentType::kCbObject: + { + IoBuffer Payload = HttpReq.ReadPayload(); + CbObject ActionObj = LoadCompactBinaryObject(Payload); + + std::vector NeedList; + + if (!CheckAttachments(ActionObj, NeedList)) + { + CbObjectWriter Cbo; + Cbo.BeginArray("need"); + + for (const IoHash& Hash : NeedList) + { + Cbo << Hash; + } + + Cbo.EndArray(); + + return HttpReq.WriteResponse(HttpResponseCode::NotFound, Cbo.Save()); + } + + if (ComputeServiceSession::EnqueueResult Result = + m_ComputeService.EnqueueActionToQueue(QueueId, ActionObj, RequestPriority)) + { + ZEN_DEBUG("queue {}: action accepted (lsn {})", QueueId, Result.Lsn); + return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); + } + } + + case HttpContentType::kCbPackage: + { + CbPackage Action = HttpReq.ReadPayloadPackage(); + CbObject ActionObj = Action.GetObject(); + + IngestStats Stats = IngestPackageAttachments(Action); + + if (ComputeServiceSession::EnqueueResult Result = + m_ComputeService.EnqueueActionToQueue(QueueId, ActionObj, RequestPriority)) + { + ZEN_DEBUG("queue {}: accepted action (lsn {}): {} in {} attachments. {} new ({} attachments)", + QueueId, + Result.Lsn, + zen::NiceBytes(Stats.Bytes), + Stats.Count, + zen::NiceBytes(Stats.NewBytes), + Stats.NewCount); + + return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); + } + else + { + return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); + } + } + + default: + break; + } + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "queues/{queueref}/jobs/{lsn}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + const int QueueId = ResolveQueueRef(HttpReq, Req.GetCapture(1)); + const int ActionLsn = ParseInt(Req.GetCapture(2)).value_or(0); + + if (QueueId == 0) + { + return; + } + + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + { + ZEN_UNUSED(QueueId); + + CbPackage Output; + HttpResponseCode ResponseCode = m_ComputeService.GetActionResult(ActionLsn, Output); + + if (ResponseCode == HttpResponseCode::OK) + { + HttpReq.WriteResponse(HttpResponseCode::OK, Output); + } + else + { + HttpReq.WriteResponse(ResponseCode); + } + + m_ComputeService.RetireActionResult(ActionLsn); + } + break; + + case HttpVerb::kPost: + { + ZEN_UNUSED(QueueId); + + auto Result = m_ComputeService.RescheduleAction(ActionLsn); + + CbObjectWriter Cbo; + if (Result.Success) + { + Cbo << "lsn"sv << ActionLsn; + Cbo << "retry_count"sv << Result.RetryCount; + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + } + else + { + Cbo << "error"sv << Result.Error; + HttpReq.WriteResponse(HttpResponseCode::Conflict, Cbo.Save()); + } + } + break; + + default: + break; + } + }, + HttpVerb::kGet | HttpVerb::kPost); +} + +////////////////////////////////////////////////////////////////////////// + +HttpComputeService::HttpComputeService(CidStore& InCidStore, + IHttpStatsService& StatsService, + const std::filesystem::path& BaseDir, + int32_t MaxConcurrentActions) +: m_Impl(std::make_unique(this, InCidStore, StatsService, BaseDir, MaxConcurrentActions)) +{ +} + +HttpComputeService::~HttpComputeService() +{ + m_Impl->m_StatsService.UnregisterHandler("compute", *this); +} + +void +HttpComputeService::Shutdown() +{ + m_Impl->m_ComputeService.Shutdown(); +} + +ComputeServiceSession::ActionCounts +HttpComputeService::GetActionCounts() +{ + return m_Impl->m_ComputeService.GetActionCounts(); +} + +const char* +HttpComputeService::BaseUri() const +{ + return "/compute/"; +} + +void +HttpComputeService::HandleRequest(HttpServerRequest& Request) +{ + ZEN_TRACE_CPU("HttpComputeService::HandleRequest"); + metrics::OperationTiming::Scope $(m_Impl->m_HttpRequests); + + if (m_Impl->m_Router.HandleRequest(Request) == false) + { + ZEN_WARN("No route found for {0}", Request.RelativeUri()); + } +} + +void +HttpComputeService::HandleStatsRequest(HttpServerRequest& Request) +{ + CbObjectWriter Cbo; + m_Impl->m_ComputeService.EmitStats(Cbo); + + Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +////////////////////////////////////////////////////////////////////////// + +void +HttpComputeService::Impl::WriteQueueDescription(CbWriter& Cbo, int QueueId, const ComputeServiceSession::QueueStatus& Status) +{ + Cbo << "queue_id"sv << Status.QueueId; + Cbo << "active_count"sv << Status.ActiveCount; + Cbo << "completed_count"sv << Status.CompletedCount; + Cbo << "failed_count"sv << Status.FailedCount; + Cbo << "abandoned_count"sv << Status.AbandonedCount; + Cbo << "cancelled_count"sv << Status.CancelledCount; + Cbo << "state"sv << ToString(Status.State); + Cbo << "cancelled"sv << (Status.State == ComputeServiceSession::QueueState::Cancelled); + Cbo << "draining"sv << (Status.State == ComputeServiceSession::QueueState::Draining); + Cbo << "is_complete"sv << Status.IsComplete; + + if (CbObject Meta = m_ComputeService.GetQueueMetadata(QueueId)) + { + Cbo << "metadata"sv << Meta; + } + + if (CbObject Cfg = m_ComputeService.GetQueueConfig(QueueId)) + { + Cbo << "config"sv << Cfg; + } + + { + RwLock::SharedLockScope $(m_RemoteQueueLock); + if (auto It = m_RemoteQueuesByQueueId.find(QueueId); It != m_RemoteQueuesByQueueId.end()) + { + Cbo << "queue_token"sv << It->second->Token.ToString(); + if (!It->second->ClientHostname.empty()) + { + Cbo << "hostname"sv << It->second->ClientHostname; + } + } + } +} + +////////////////////////////////////////////////////////////////////////// + +int +HttpComputeService::Impl::ResolveQueueToken(const Oid& Token) +{ + RwLock::SharedLockScope $(m_RemoteQueueLock); + + auto It = m_RemoteQueuesByToken.find(Token); + + if (It != m_RemoteQueuesByToken.end()) + { + return It->second->QueueId; + } + + return 0; +} + +int +HttpComputeService::Impl::ResolveQueueRef(HttpServerRequest& HttpReq, std::string_view Capture) +{ + if (OidMatcher(Capture)) + { + // Remote OID token — accessible from any client + const Oid Token = Oid::FromHexString(Capture); + const int QueueId = ResolveQueueToken(Token); + + if (QueueId == 0) + { + HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + return QueueId; + } + + // Local integer queue ID — restricted to local machine requests + if (!HttpReq.IsLocalMachineRequest()) + { + HttpReq.WriteResponse(HttpResponseCode::Forbidden); + return 0; + } + + return ParseInt(Capture).value_or(0); +} + +HttpComputeService::Impl::IngestStats +HttpComputeService::Impl::IngestPackageAttachments(const CbPackage& Package) +{ + IngestStats Stats; + + for (const CbAttachment& Attachment : Package.GetAttachments()) + { + ZEN_ASSERT(Attachment.IsCompressedBinary()); + + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer DataView = Attachment.AsCompressedBinary(); + + ZEN_UNUSED(DataHash); + + const uint64_t CompressedSize = DataView.GetCompressedSize(); + + Stats.Bytes += CompressedSize; + ++Stats.Count; + + const CidStore::InsertResult InsertResult = m_CidStore.AddChunk(DataView.GetCompressed().Flatten().AsIoBuffer(), DataHash); + + if (InsertResult.New) + { + Stats.NewBytes += CompressedSize; + ++Stats.NewCount; + } + } + + return Stats; +} + +bool +HttpComputeService::Impl::CheckAttachments(const CbObject& ActionObj, std::vector& NeedList) +{ + ActionObj.IterateAttachments([&](CbFieldView Field) { + const IoHash FileHash = Field.AsHash(); + + if (!m_CidStore.ContainsChunk(FileHash)) + { + NeedList.push_back(FileHash); + } + }); + + return NeedList.empty(); +} + +void +HttpComputeService::Impl::HandleWorkersGet(HttpServerRequest& HttpReq) +{ + CbObjectWriter Cbo; + Cbo.BeginArray("workers"sv); + for (const IoHash& WorkerId : m_ComputeService.GetKnownWorkerIds()) + { + Cbo << WorkerId; + } + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpComputeService::Impl::HandleWorkersAllGet(HttpServerRequest& HttpReq) +{ + std::vector WorkerIds = m_ComputeService.GetKnownWorkerIds(); + + CbObjectWriter Cbo; + Cbo.BeginArray("workers"); + + for (const IoHash& WorkerId : WorkerIds) + { + Cbo.BeginObject(); + Cbo << "id" << WorkerId; + Cbo << "descriptor" << m_ComputeService.GetWorkerDescriptor(WorkerId).Descriptor.GetObject(); + Cbo.EndObject(); + } + + Cbo.EndArray(); + HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); +} + +void +HttpComputeService::Impl::HandleWorkerRequest(HttpServerRequest& HttpReq, const IoHash& WorkerId) +{ + switch (HttpReq.RequestVerb()) + { + case HttpVerb::kGet: + if (WorkerDesc Desc = m_ComputeService.GetWorkerDescriptor(WorkerId)) + { + return HttpReq.WriteResponse(HttpResponseCode::OK, Desc.Descriptor.GetObject()); + } + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + + case HttpVerb::kPost: + { + switch (HttpReq.RequestContentType()) + { + case HttpContentType::kCbObject: + { + CbObject WorkerSpec = HttpReq.ReadPayloadObject(); + + HashKeySet ChunkSet; + WorkerSpec.IterateAttachments([&](CbFieldView Field) { + const IoHash Hash = Field.AsHash(); + ChunkSet.AddHashToSet(Hash); + }); + + CbPackage WorkerPackage; + WorkerPackage.SetObject(WorkerSpec); + + m_CidStore.FilterChunks(ChunkSet); + + if (ChunkSet.IsEmpty()) + { + ZEN_DEBUG("worker {}: all attachments already available", WorkerId); + m_ComputeService.RegisterWorker(WorkerPackage); + return HttpReq.WriteResponse(HttpResponseCode::NoContent); + } + + CbObjectWriter ResponseWriter; + ResponseWriter.BeginArray("need"); + ChunkSet.IterateHashes([&](const IoHash& Hash) { + ZEN_DEBUG("worker {}: need chunk {}", WorkerId, Hash); + ResponseWriter.AddHash(Hash); + }); + ResponseWriter.EndArray(); + + ZEN_DEBUG("worker {}: need {} attachments", WorkerId, ChunkSet.GetSize()); + return HttpReq.WriteResponse(HttpResponseCode::NotFound, ResponseWriter.Save()); + } + break; + + case HttpContentType::kCbPackage: + { + CbPackage WorkerSpecPackage = HttpReq.ReadPayloadPackage(); + CbObject WorkerSpec = WorkerSpecPackage.GetObject(); + + std::span Attachments = WorkerSpecPackage.GetAttachments(); + + int AttachmentCount = 0; + int NewAttachmentCount = 0; + uint64_t TotalAttachmentBytes = 0; + uint64_t TotalNewBytes = 0; + + for (const CbAttachment& Attachment : Attachments) + { + ZEN_ASSERT(Attachment.IsCompressedBinary()); + + const IoHash DataHash = Attachment.GetHash(); + CompressedBuffer Buffer = Attachment.AsCompressedBinary(); + + ZEN_UNUSED(DataHash); + TotalAttachmentBytes += Buffer.GetCompressedSize(); + ++AttachmentCount; + + const CidStore::InsertResult InsertResult = + m_CidStore.AddChunk(Buffer.GetCompressed().Flatten().AsIoBuffer(), DataHash); + + if (InsertResult.New) + { + TotalNewBytes += Buffer.GetCompressedSize(); + ++NewAttachmentCount; + } + } + + ZEN_DEBUG("worker {}: {} in {} attachments, {} in {} new attachments", + WorkerId, + zen::NiceBytes(TotalAttachmentBytes), + AttachmentCount, + zen::NiceBytes(TotalNewBytes), + NewAttachmentCount); + + m_ComputeService.RegisterWorker(WorkerSpecPackage); + return HttpReq.WriteResponse(HttpResponseCode::NoContent); + } + break; + + default: + break; + } + } + break; + + default: + break; + } +} + +////////////////////////////////////////////////////////////////////////// + +void +httpcomputeservice_forcelink() +{ +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/httpfunctionservice.cpp b/src/zencompute/httpfunctionservice.cpp deleted file mode 100644 index 09a9684a7..000000000 --- a/src/zencompute/httpfunctionservice.cpp +++ /dev/null @@ -1,709 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencompute/httpfunctionservice.h" - -#if ZEN_WITH_COMPUTE_SERVICES - -# include "functionrunner.h" - -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include - -using namespace std::literals; - -namespace zen::compute { - -constinit AsciiSet g_DecimalSet("0123456789"); -auto DecimalMatcher = [](std::string_view Str) { return AsciiSet::HasOnly(Str, g_DecimalSet); }; - -constinit AsciiSet g_HexSet("0123456789abcdefABCDEF"); -auto IoHashMatcher = [](std::string_view Str) { return Str.size() == 40 && AsciiSet::HasOnly(Str, g_HexSet); }; - -HttpFunctionService::HttpFunctionService(CidStore& InCidStore, - IHttpStatsService& StatsService, - [[maybe_unused]] const std::filesystem::path& BaseDir) -: m_CidStore(InCidStore) -, m_StatsService(StatsService) -, m_Log(logging::Get("apply")) -, m_BaseDir(BaseDir) -, m_FunctionService(InCidStore) -{ - m_FunctionService.AddLocalRunner(InCidStore, m_BaseDir / "local"); - - m_StatsService.RegisterHandler("apply", *this); - - m_Router.AddMatcher("lsn", DecimalMatcher); - m_Router.AddMatcher("worker", IoHashMatcher); - m_Router.AddMatcher("action", IoHashMatcher); - - m_Router.RegisterRoute( - "ready", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - if (m_FunctionService.IsHealthy()) - { - return HttpReq.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, "ok"); - } - - return HttpReq.WriteResponse(HttpResponseCode::ServiceUnavailable); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "workers", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - CbObjectWriter Cbo; - Cbo.BeginArray("workers"sv); - for (const IoHash& WorkerId : m_FunctionService.GetKnownWorkerIds()) - { - Cbo << WorkerId; - } - Cbo.EndArray(); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "workers/{worker}", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const IoHash WorkerId = IoHash::FromHexString(Req.GetCapture(1)); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kGet: - if (WorkerDesc Desc = m_FunctionService.GetWorkerDescriptor(WorkerId)) - { - return HttpReq.WriteResponse(HttpResponseCode::OK, Desc.Descriptor.GetObject()); - } - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - - case HttpVerb::kPost: - { - switch (HttpReq.RequestContentType()) - { - case HttpContentType::kCbObject: - { - CbObject WorkerSpec = HttpReq.ReadPayloadObject(); - - // Determine which pieces are missing and need to be transmitted - - HashKeySet ChunkSet; - - WorkerSpec.IterateAttachments([&](CbFieldView Field) { - const IoHash Hash = Field.AsHash(); - ChunkSet.AddHashToSet(Hash); - }); - - CbPackage WorkerPackage; - WorkerPackage.SetObject(WorkerSpec); - - m_CidStore.FilterChunks(ChunkSet); - - if (ChunkSet.IsEmpty()) - { - ZEN_DEBUG("worker {}: all attachments already available", WorkerId); - m_FunctionService.RegisterWorker(WorkerPackage); - return HttpReq.WriteResponse(HttpResponseCode::NoContent); - } - - CbObjectWriter ResponseWriter; - ResponseWriter.BeginArray("need"); - - ChunkSet.IterateHashes([&](const IoHash& Hash) { - ZEN_DEBUG("worker {}: need chunk {}", WorkerId, Hash); - ResponseWriter.AddHash(Hash); - }); - - ResponseWriter.EndArray(); - - ZEN_DEBUG("worker {}: need {} attachments", WorkerId, ChunkSet.GetSize()); - - return HttpReq.WriteResponse(HttpResponseCode::NotFound, ResponseWriter.Save()); - } - break; - - case HttpContentType::kCbPackage: - { - CbPackage WorkerSpecPackage = HttpReq.ReadPayloadPackage(); - CbObject WorkerSpec = WorkerSpecPackage.GetObject(); - - std::span Attachments = WorkerSpecPackage.GetAttachments(); - - int AttachmentCount = 0; - int NewAttachmentCount = 0; - uint64_t TotalAttachmentBytes = 0; - uint64_t TotalNewBytes = 0; - - for (const CbAttachment& Attachment : Attachments) - { - ZEN_ASSERT(Attachment.IsCompressedBinary()); - - const IoHash DataHash = Attachment.GetHash(); - CompressedBuffer Buffer = Attachment.AsCompressedBinary(); - - ZEN_UNUSED(DataHash); - TotalAttachmentBytes += Buffer.GetCompressedSize(); - ++AttachmentCount; - - const CidStore::InsertResult InsertResult = - m_CidStore.AddChunk(Buffer.GetCompressed().Flatten().AsIoBuffer(), DataHash); - - if (InsertResult.New) - { - TotalNewBytes += Buffer.GetCompressedSize(); - ++NewAttachmentCount; - } - } - - ZEN_DEBUG("worker {}: {} in {} attachments, {} in {} new attachments", - WorkerId, - zen::NiceBytes(TotalAttachmentBytes), - AttachmentCount, - zen::NiceBytes(TotalNewBytes), - NewAttachmentCount); - - m_FunctionService.RegisterWorker(WorkerSpecPackage); - - return HttpReq.WriteResponse(HttpResponseCode::NoContent); - } - break; - - default: - break; - } - } - break; - - default: - break; - } - }, - HttpVerb::kGet | HttpVerb::kPost); - - m_Router.RegisterRoute( - "jobs/completed", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - CbObjectWriter Cbo; - m_FunctionService.GetCompleted(Cbo); - - SystemMetrics Sm = GetSystemMetricsForReporting(); - Cbo.BeginObject("metrics"); - Describe(Sm, Cbo); - Cbo.EndObject(); - - HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "jobs/history", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - const auto QueryParams = HttpReq.GetQueryParams(); - - int QueryLimit = 50; - - if (auto LimitParam = QueryParams.GetValue("limit"); LimitParam.empty() == false) - { - QueryLimit = ParseInt(LimitParam).value_or(50); - } - - CbObjectWriter Cbo; - Cbo.BeginArray("history"); - for (const auto& Entry : m_FunctionService.GetActionHistory(QueryLimit)) - { - Cbo.BeginObject(); - Cbo << "lsn"sv << Entry.Lsn; - Cbo << "actionId"sv << Entry.ActionId; - Cbo << "workerId"sv << Entry.WorkerId; - Cbo << "succeeded"sv << Entry.Succeeded; - Cbo << "actionDescriptor"sv << Entry.ActionDescriptor; - - for (const auto& Timestamp : Entry.Timestamps) - { - Cbo.AddInteger( - fmt::format("time_{}"sv, RunnerAction::ToString(static_cast(&Timestamp - Entry.Timestamps))), - Timestamp); - } - Cbo.EndObject(); - } - Cbo.EndArray(); - - HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "jobs/{lsn}", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - const int ActionLsn = std::stoi(std::string{Req.GetCapture(1)}); - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kGet: - { - CbPackage Output; - HttpResponseCode ResponseCode = m_FunctionService.GetActionResult(ActionLsn, Output); - - if (ResponseCode == HttpResponseCode::OK) - { - return HttpReq.WriteResponse(HttpResponseCode::OK, Output); - } - - return HttpReq.WriteResponse(ResponseCode); - } - break; - - case HttpVerb::kPost: - { - // Add support for cancellation, priority changes - } - break; - - default: - break; - } - }, - HttpVerb::kGet | HttpVerb::kPost); - - m_Router.RegisterRoute( - "jobs/{worker}/{action}", // This route is inefficient, and is only here for backwards compatibility. The preferred path is the - // one which uses the scheduled action lsn for lookups - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - const IoHash ActionId = IoHash::FromHexString(Req.GetCapture(2)); - - CbPackage Output; - if (HttpResponseCode ResponseCode = m_FunctionService.FindActionResult(ActionId, /* out */ Output); - ResponseCode != HttpResponseCode::OK) - { - ZEN_TRACE("jobs/{}/{}: {}", Req.GetCapture(1), Req.GetCapture(2), ToString(ResponseCode)) - - if (ResponseCode == HttpResponseCode::NotFound) - { - return HttpReq.WriteResponse(ResponseCode); - } - - return HttpReq.WriteResponse(ResponseCode); - } - - ZEN_DEBUG("jobs/{}/{}: OK", Req.GetCapture(1), Req.GetCapture(2)) - - return HttpReq.WriteResponse(HttpResponseCode::OK, Output); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "jobs/{worker}", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - const IoHash WorkerId = IoHash::FromHexString(Req.GetCapture(1)); - - WorkerDesc Worker = m_FunctionService.GetWorkerDescriptor(WorkerId); - - if (!Worker) - { - return HttpReq.WriteResponse(HttpResponseCode::NotFound); - } - - const auto QueryParams = Req.ServerRequest().GetQueryParams(); - - int RequestPriority = -1; - - if (auto PriorityParam = QueryParams.GetValue("priority"); PriorityParam.empty() == false) - { - RequestPriority = ParseInt(PriorityParam).value_or(-1); - } - - switch (HttpReq.RequestVerb()) - { - case HttpVerb::kGet: - // TODO: return status of all pending or executing jobs - break; - - case HttpVerb::kPost: - switch (HttpReq.RequestContentType()) - { - case HttpContentType::kCbObject: - { - // This operation takes the proposed job spec and identifies which - // chunks are not present on this server. This list is then returned in - // the "need" list in the response - - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject ActionObj = LoadCompactBinaryObject(Payload); - - std::vector NeedList; - - ActionObj.IterateAttachments([&](CbFieldView Field) { - const IoHash FileHash = Field.AsHash(); - - if (!m_CidStore.ContainsChunk(FileHash)) - { - NeedList.push_back(FileHash); - } - }); - - if (NeedList.empty()) - { - // We already have everything, enqueue the action for execution - - if (FunctionServiceSession::EnqueueResult Result = - m_FunctionService.EnqueueResolvedAction(Worker, ActionObj, RequestPriority)) - { - ZEN_DEBUG("action {} accepted (lsn {})", ActionObj.GetHash(), Result.Lsn); - - HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); - } - - return; - } - - CbObjectWriter Cbo; - Cbo.BeginArray("need"); - - for (const IoHash& Hash : NeedList) - { - Cbo << Hash; - } - - Cbo.EndArray(); - CbObject Response = Cbo.Save(); - - return HttpReq.WriteResponse(HttpResponseCode::NotFound, Response); - } - break; - - case HttpContentType::kCbPackage: - { - CbPackage Action = HttpReq.ReadPayloadPackage(); - CbObject ActionObj = Action.GetObject(); - - std::span Attachments = Action.GetAttachments(); - - int AttachmentCount = 0; - int NewAttachmentCount = 0; - uint64_t TotalAttachmentBytes = 0; - uint64_t TotalNewBytes = 0; - - for (const CbAttachment& Attachment : Attachments) - { - ZEN_ASSERT(Attachment.IsCompressedBinary()); - - const IoHash DataHash = Attachment.GetHash(); - CompressedBuffer DataView = Attachment.AsCompressedBinary(); - - ZEN_UNUSED(DataHash); - - const uint64_t CompressedSize = DataView.GetCompressedSize(); - - TotalAttachmentBytes += CompressedSize; - ++AttachmentCount; - - const CidStore::InsertResult InsertResult = - m_CidStore.AddChunk(DataView.GetCompressed().Flatten().AsIoBuffer(), DataHash); - - if (InsertResult.New) - { - TotalNewBytes += CompressedSize; - ++NewAttachmentCount; - } - } - - if (FunctionServiceSession::EnqueueResult Result = - m_FunctionService.EnqueueResolvedAction(Worker, ActionObj, RequestPriority)) - { - ZEN_DEBUG("accepted action {} (lsn {}): {} in {} attachments. {} new ({} attachments)", - ActionObj.GetHash(), - Result.Lsn, - zen::NiceBytes(TotalAttachmentBytes), - AttachmentCount, - zen::NiceBytes(TotalNewBytes), - NewAttachmentCount); - - HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); - } - - return; - } - break; - - default: - break; - } - break; - - default: - break; - } - }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "jobs", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - const auto QueryParams = HttpReq.GetQueryParams(); - - int RequestPriority = -1; - - if (auto PriorityParam = QueryParams.GetValue("priority"); PriorityParam.empty() == false) - { - RequestPriority = ParseInt(PriorityParam).value_or(-1); - } - - // Resolve worker - - // - - switch (HttpReq.RequestContentType()) - { - case HttpContentType::kCbObject: - { - // This operation takes the proposed job spec and identifies which - // chunks are not present on this server. This list is then returned in - // the "need" list in the response - - IoBuffer Payload = HttpReq.ReadPayload(); - CbObject ActionObj = LoadCompactBinaryObject(Payload); - - std::vector NeedList; - - ActionObj.IterateAttachments([&](CbFieldView Field) { - const IoHash FileHash = Field.AsHash(); - - if (!m_CidStore.ContainsChunk(FileHash)) - { - NeedList.push_back(FileHash); - } - }); - - if (NeedList.empty()) - { - // We already have everything, enqueue the action for execution - - if (FunctionServiceSession::EnqueueResult Result = m_FunctionService.EnqueueAction(ActionObj, RequestPriority)) - { - ZEN_DEBUG("action accepted (lsn {})", Result.Lsn); - - return HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); - } - else - { - // Could not resolve? - return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); - } - } - - CbObjectWriter Cbo; - Cbo.BeginArray("need"); - - for (const IoHash& Hash : NeedList) - { - Cbo << Hash; - } - - Cbo.EndArray(); - CbObject Response = Cbo.Save(); - - return HttpReq.WriteResponse(HttpResponseCode::NotFound, Response); - } - - case HttpContentType::kCbPackage: - { - CbPackage Action = HttpReq.ReadPayloadPackage(); - CbObject ActionObj = Action.GetObject(); - - std::span Attachments = Action.GetAttachments(); - - int AttachmentCount = 0; - int NewAttachmentCount = 0; - uint64_t TotalAttachmentBytes = 0; - uint64_t TotalNewBytes = 0; - - for (const CbAttachment& Attachment : Attachments) - { - ZEN_ASSERT(Attachment.IsCompressedBinary()); - - const IoHash DataHash = Attachment.GetHash(); - CompressedBuffer DataView = Attachment.AsCompressedBinary(); - - ZEN_UNUSED(DataHash); - - const uint64_t CompressedSize = DataView.GetCompressedSize(); - - TotalAttachmentBytes += CompressedSize; - ++AttachmentCount; - - const CidStore::InsertResult InsertResult = - m_CidStore.AddChunk(DataView.GetCompressed().Flatten().AsIoBuffer(), DataHash); - - if (InsertResult.New) - { - TotalNewBytes += CompressedSize; - ++NewAttachmentCount; - } - } - - if (FunctionServiceSession::EnqueueResult Result = m_FunctionService.EnqueueAction(ActionObj, RequestPriority)) - { - ZEN_DEBUG("accepted action (lsn {}): {} in {} attachments. {} new ({} attachments)", - Result.Lsn, - zen::NiceBytes(TotalAttachmentBytes), - AttachmentCount, - zen::NiceBytes(TotalNewBytes), - NewAttachmentCount); - - HttpReq.WriteResponse(HttpResponseCode::OK, Result.ResponseMessage); - } - else - { - // Could not resolve? - return HttpReq.WriteResponse(HttpResponseCode::FailedDependency, Result.ResponseMessage); - } - } - return; - } - }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "workers/all", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - std::vector WorkerIds = m_FunctionService.GetKnownWorkerIds(); - - CbObjectWriter Cbo; - Cbo.BeginArray("workers"); - - for (const IoHash& WorkerId : WorkerIds) - { - Cbo.BeginObject(); - - Cbo << "id" << WorkerId; - - const auto& Descriptor = m_FunctionService.GetWorkerDescriptor(WorkerId); - - Cbo << "descriptor" << Descriptor.Descriptor.GetObject(); - - Cbo.EndObject(); - } - - Cbo.EndArray(); - - HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "sysinfo", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - SystemMetrics Sm = GetSystemMetricsForReporting(); - - CbObjectWriter Cbo; - Describe(Sm, Cbo); - - Cbo << "cpu_usage" << Sm.CpuUsagePercent; - Cbo << "memory_total" << Sm.SystemMemoryMiB * 1024 * 1024; - Cbo << "memory_used" << (Sm.SystemMemoryMiB - Sm.AvailSystemMemoryMiB) * 1024 * 1024; - Cbo << "disk_used" << 100 * 1024; - Cbo << "disk_total" << 100 * 1024 * 1024; - - return HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "record/start", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - m_FunctionService.StartRecording(m_CidStore, m_BaseDir / "recording"); - - return HttpReq.WriteResponse(HttpResponseCode::OK); - }, - HttpVerb::kPost); - - m_Router.RegisterRoute( - "record/stop", - [this](HttpRouterRequest& Req) { - HttpServerRequest& HttpReq = Req.ServerRequest(); - - m_FunctionService.StopRecording(); - - return HttpReq.WriteResponse(HttpResponseCode::OK); - }, - HttpVerb::kPost); -} - -HttpFunctionService::~HttpFunctionService() -{ - m_StatsService.UnregisterHandler("apply", *this); -} - -void -HttpFunctionService::Shutdown() -{ - m_FunctionService.Shutdown(); -} - -const char* -HttpFunctionService::BaseUri() const -{ - return "/apply/"; -} - -void -HttpFunctionService::HandleRequest(HttpServerRequest& Request) -{ - metrics::OperationTiming::Scope $(m_HttpRequests); - - if (m_Router.HandleRequest(Request) == false) - { - ZEN_WARN("No route found for {0}", Request.RelativeUri()); - } -} - -void -HttpFunctionService::HandleStatsRequest(HttpServerRequest& Request) -{ - CbObjectWriter Cbo; - m_FunctionService.EmitStats(Cbo); - - Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); -} - -////////////////////////////////////////////////////////////////////////// - -void -httpfunction_forcelink() -{ -} - -} // namespace zen::compute - -#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/httporchestrator.cpp b/src/zencompute/httporchestrator.cpp index 39e7e60d7..6cbe01e04 100644 --- a/src/zencompute/httporchestrator.cpp +++ b/src/zencompute/httporchestrator.cpp @@ -2,65 +2,398 @@ #include "zencompute/httporchestrator.h" -#include -#include +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include namespace zen::compute { -HttpOrchestratorService::HttpOrchestratorService() : m_Log(logging::Get("orch")) +// Worker IDs must be 3-64 characters and can only contain letters, numbers, underscores, and dashes +static bool +IsValidWorkerId(std::string_view Id) +{ + if (Id.size() < 3 || Id.size() > 64) + { + return false; + } + for (char c : Id) + { + if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-') + { + continue; + } + return false; + } + return true; +} + +// Shared announce payload parser used by both the HTTP POST route and the +// WebSocket message handler. Returns the worker ID on success (empty on +// validation failure). The returned WorkerAnnouncement has string_view +// fields that reference the supplied CbObjectView, so the CbObject must +// outlive the returned announcement. +static std::string_view +ParseWorkerAnnouncement(const CbObjectView& Data, OrchestratorService::WorkerAnnouncement& Ann) { + Ann.Id = Data["id"].AsString(""); + Ann.Uri = Data["uri"].AsString(""); + + if (!IsValidWorkerId(Ann.Id)) + { + return {}; + } + + if (!Ann.Uri.starts_with("http://") && !Ann.Uri.starts_with("https://")) + { + return {}; + } + + Ann.Hostname = Data["hostname"].AsString(""); + Ann.Platform = Data["platform"].AsString(""); + Ann.CpuUsagePercent = Data["cpu_usage"].AsFloat(0.0f); + Ann.MemoryTotalBytes = Data["memory_total"].AsUInt64(0); + Ann.MemoryUsedBytes = Data["memory_used"].AsUInt64(0); + Ann.BytesReceived = Data["bytes_received"].AsUInt64(0); + Ann.BytesSent = Data["bytes_sent"].AsUInt64(0); + Ann.ActionsPending = Data["actions_pending"].AsInt32(0); + Ann.ActionsRunning = Data["actions_running"].AsInt32(0); + Ann.ActionsCompleted = Data["actions_completed"].AsInt32(0); + Ann.ActiveQueues = Data["active_queues"].AsInt32(0); + Ann.Provisioner = Data["provisioner"].AsString(""); + + if (auto Metrics = Data["metrics"].AsObjectView()) + { + Ann.Cpus = Metrics["lp_count"].AsInt32(0); + if (Ann.Cpus <= 0) + { + Ann.Cpus = 1; + } + } + + return Ann.Id; +} + +HttpOrchestratorService::HttpOrchestratorService(std::filesystem::path DataDir, bool EnableWorkerWebSocket) +: m_Service(std::make_unique(std::move(DataDir), EnableWorkerWebSocket)) +, m_Hostname(GetMachineName()) +{ + m_Router.AddMatcher("workerid", [](std::string_view Segment) { return IsValidWorkerId(Segment); }); + m_Router.AddMatcher("clientid", [](std::string_view Segment) { return IsValidWorkerId(Segment); }); + + // dummy endpoint for websocket clients + m_Router.RegisterRoute( + "ws", + [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "status", + [this](HttpRouterRequest& Req) { + CbObjectWriter Cbo; + Cbo << "hostname" << std::string_view(m_Hostname); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Cbo.Save()); + }, + HttpVerb::kGet); + m_Router.RegisterRoute( "provision", + [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, m_Service->GetWorkerList()); }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "announce", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); - CbObjectWriter Cbo; - Cbo.BeginArray("workers"); + CbObject Data = HttpReq.ReadPayloadObject(); + + OrchestratorService::WorkerAnnouncement Ann; + std::string_view WorkerId = ParseWorkerAnnouncement(Data, Ann); - m_KnownWorkersLock.WithSharedLock([&] { - for (const auto& [WorkerId, Worker] : m_KnownWorkers) + if (WorkerId.empty()) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + "Invalid worker announcement: id must be 3-64 alphanumeric/underscore/dash " + "characters and uri must start with http:// or https://"); + } + + m_Service->AnnounceWorker(Ann); + + HttpReq.WriteResponse(HttpResponseCode::OK); + +# if ZEN_WITH_WEBSOCKETS + // Notify push thread that state may have changed + m_PushEvent.Set(); +# endif + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "agents", + [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, m_Service->GetWorkerList()); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "history", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + auto Params = HttpReq.GetQueryParams(); + + int Limit = 100; + auto LimitStr = Params.GetValue("limit"); + if (!LimitStr.empty()) + { + Limit = std::atoi(std::string(LimitStr).c_str()); + } + + HttpReq.WriteResponse(HttpResponseCode::OK, m_Service->GetProvisioningHistory(Limit)); + }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "timeline/{workerid}", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + + std::string_view WorkerId = Req.GetCapture(1); + auto Params = HttpReq.GetQueryParams(); + + auto FromStr = Params.GetValue("from"); + auto ToStr = Params.GetValue("to"); + auto LimitStr = Params.GetValue("limit"); + + std::optional From; + std::optional To; + + if (!FromStr.empty()) + { + auto Val = zen::ParseInt(FromStr); + if (!Val) { - Cbo.BeginObject(); - Cbo << "uri" << Worker.BaseUri; - Cbo << "dt" << Worker.LastSeen.GetElapsedTimeMs(); - Cbo.EndObject(); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); } - }); + From = DateTime(*Val); + } + + if (!ToStr.empty()) + { + auto Val = zen::ParseInt(ToStr); + if (!Val) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + To = DateTime(*Val); + } + + int Limit = !LimitStr.empty() ? zen::ParseInt(LimitStr).value_or(0) : 0; - Cbo.EndArray(); + CbObject Result = m_Service->GetWorkerTimeline(WorkerId, From, To, Limit); - HttpReq.WriteResponse(HttpResponseCode::OK, Cbo.Save()); + if (!Result) + { + return HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + + HttpReq.WriteResponse(HttpResponseCode::OK, std::move(Result)); }, - HttpVerb::kPost); + HttpVerb::kGet); m_Router.RegisterRoute( - "announce", + "timeline", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + auto Params = HttpReq.GetQueryParams(); + + auto FromStr = Params.GetValue("from"); + auto ToStr = Params.GetValue("to"); + + DateTime From = DateTime(0); + DateTime To = DateTime::Now(); + + if (!FromStr.empty()) + { + auto Val = zen::ParseInt(FromStr); + if (!Val) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + From = DateTime(*Val); + } + + if (!ToStr.empty()) + { + auto Val = zen::ParseInt(ToStr); + if (!Val) + { + return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + } + To = DateTime(*Val); + } + + CbObject Result = m_Service->GetAllTimelines(From, To); + + HttpReq.WriteResponse(HttpResponseCode::OK, std::move(Result)); + }, + HttpVerb::kGet); + + // Client tracking endpoints + + m_Router.RegisterRoute( + "clients", [this](HttpRouterRequest& Req) { HttpServerRequest& HttpReq = Req.ServerRequest(); CbObject Data = HttpReq.ReadPayloadObject(); - std::string_view WorkerId = Data["id"].AsString(""); - std::string_view WorkerUri = Data["uri"].AsString(""); + OrchestratorService::ClientAnnouncement Ann; + Ann.SessionId = Data["session_id"].AsObjectId(Oid::Zero); + Ann.Hostname = Data["hostname"].AsString(""); + Ann.Address = HttpReq.GetRemoteAddress(); - if (WorkerId.empty() || WorkerUri.empty()) + auto MetadataView = Data["metadata"].AsObjectView(); + if (MetadataView) { - return HttpReq.WriteResponse(HttpResponseCode::BadRequest); + Ann.Metadata = CbObject::Clone(MetadataView); } - m_KnownWorkersLock.WithExclusiveLock([&] { - auto& Worker = m_KnownWorkers[std::string(WorkerId)]; - Worker.BaseUri = WorkerUri; - Worker.LastSeen.Reset(); - }); + std::string ClientId = m_Service->AnnounceClient(Ann); - HttpReq.WriteResponse(HttpResponseCode::OK); + CbObjectWriter ResponseObj; + ResponseObj << "id" << std::string_view(ClientId); + HttpReq.WriteResponse(HttpResponseCode::OK, ResponseObj.Save()); + +# if ZEN_WITH_WEBSOCKETS + m_PushEvent.Set(); +# endif }, HttpVerb::kPost); + + m_Router.RegisterRoute( + "clients/{clientid}/update", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + std::string_view ClientId = Req.GetCapture(1); + + CbObject MetadataObj; + CbObject Data = HttpReq.ReadPayloadObject(); + if (Data) + { + auto MetadataView = Data["metadata"].AsObjectView(); + if (MetadataView) + { + MetadataObj = CbObject::Clone(MetadataView); + } + } + + if (m_Service->UpdateClient(ClientId, std::move(MetadataObj))) + { + HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "clients/{clientid}/complete", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + std::string_view ClientId = Req.GetCapture(1); + + if (m_Service->CompleteClient(ClientId)) + { + HttpReq.WriteResponse(HttpResponseCode::OK); + } + else + { + HttpReq.WriteResponse(HttpResponseCode::NotFound); + } + +# if ZEN_WITH_WEBSOCKETS + m_PushEvent.Set(); +# endif + }, + HttpVerb::kPost); + + m_Router.RegisterRoute( + "clients", + [this](HttpRouterRequest& Req) { Req.ServerRequest().WriteResponse(HttpResponseCode::OK, m_Service->GetClientList()); }, + HttpVerb::kGet); + + m_Router.RegisterRoute( + "clients/history", + [this](HttpRouterRequest& Req) { + HttpServerRequest& HttpReq = Req.ServerRequest(); + auto Params = HttpReq.GetQueryParams(); + + int Limit = 100; + auto LimitStr = Params.GetValue("limit"); + if (!LimitStr.empty()) + { + Limit = std::atoi(std::string(LimitStr).c_str()); + } + + HttpReq.WriteResponse(HttpResponseCode::OK, m_Service->GetClientHistory(Limit)); + }, + HttpVerb::kGet); + +# if ZEN_WITH_WEBSOCKETS + + // Start the WebSocket push thread + m_PushEnabled.store(true); + m_PushThread = std::thread([this] { PushThreadFunction(); }); +# endif } HttpOrchestratorService::~HttpOrchestratorService() { + Shutdown(); +} + +void +HttpOrchestratorService::Shutdown() +{ +# if ZEN_WITH_WEBSOCKETS + if (!m_PushEnabled.exchange(false)) + { + return; + } + + // Stop the push thread first, before touching connections. This ensures + // the push thread is no longer reading m_WsConnections or calling into + // m_Service when we start tearing things down. + m_PushEvent.Set(); + if (m_PushThread.joinable()) + { + m_PushThread.join(); + } + + // Clean up worker WebSocket connections — collect IDs under lock, then + // notify the service outside the lock to avoid lock-order inversions. + std::vector WorkerIds; + m_WorkerWsLock.WithExclusiveLock([&] { + WorkerIds.reserve(m_WorkerWsMap.size()); + for (const auto& [Conn, Id] : m_WorkerWsMap) + { + WorkerIds.push_back(Id); + } + m_WorkerWsMap.clear(); + }); + for (const auto& Id : WorkerIds) + { + m_Service->SetWorkerWebSocketConnected(Id, false); + } + + // Now that the push thread is gone, release all dashboard connections. + m_WsConnectionsLock.WithExclusiveLock([&] { m_WsConnections.clear(); }); +# endif } const char* @@ -78,4 +411,240 @@ HttpOrchestratorService::HandleRequest(HttpServerRequest& Request) } } +////////////////////////////////////////////////////////////////////////// +// +// IWebSocketHandler +// + +# if ZEN_WITH_WEBSOCKETS +void +HttpOrchestratorService::OnWebSocketOpen(Ref Connection) +{ + if (!m_PushEnabled.load()) + { + return; + } + + ZEN_INFO("WebSocket client connected"); + + m_WsConnectionsLock.WithExclusiveLock([&] { m_WsConnections.push_back(std::move(Connection)); }); + + // Wake push thread to send initial state immediately + m_PushEvent.Set(); +} + +void +HttpOrchestratorService::OnWebSocketMessage(WebSocketConnection& Conn, const WebSocketMessage& Msg) +{ + // Only handle binary messages from workers when the feature is enabled. + if (!m_Service->IsWorkerWebSocketEnabled() || Msg.Opcode != WebSocketOpcode::kBinary) + { + return; + } + + std::string WorkerId = HandleWorkerWebSocketMessage(Msg); + if (WorkerId.empty()) + { + return; + } + + // Check if this is a new worker WebSocket connection + bool IsNewWorkerWs = false; + m_WorkerWsLock.WithExclusiveLock([&] { + auto It = m_WorkerWsMap.find(&Conn); + if (It == m_WorkerWsMap.end()) + { + m_WorkerWsMap[&Conn] = WorkerId; + IsNewWorkerWs = true; + } + }); + + if (IsNewWorkerWs) + { + m_Service->SetWorkerWebSocketConnected(WorkerId, true); + } + + m_PushEvent.Set(); +} + +std::string +HttpOrchestratorService::HandleWorkerWebSocketMessage(const WebSocketMessage& Msg) +{ + // Workers send CbObject in native binary format over the WebSocket to + // avoid the lossy CbObject↔JSON round-trip. + CbObject Data = CbObject::MakeView(Msg.Payload.GetData()); + if (!Data) + { + ZEN_WARN("worker WebSocket message is not a valid CbObject"); + return {}; + } + + OrchestratorService::WorkerAnnouncement Ann; + std::string_view WorkerId = ParseWorkerAnnouncement(Data, Ann); + if (WorkerId.empty()) + { + ZEN_WARN("invalid worker announcement via WebSocket"); + return {}; + } + + m_Service->AnnounceWorker(Ann); + return std::string(WorkerId); +} + +void +HttpOrchestratorService::OnWebSocketClose(WebSocketConnection& Conn, + [[maybe_unused]] uint16_t Code, + [[maybe_unused]] std::string_view Reason) +{ + ZEN_INFO("WebSocket client disconnected (code {})", Code); + + // Check if this was a worker WebSocket connection; collect the ID under + // the worker lock, then notify the service outside the lock. + std::string DisconnectedWorkerId; + m_WorkerWsLock.WithExclusiveLock([&] { + auto It = m_WorkerWsMap.find(&Conn); + if (It != m_WorkerWsMap.end()) + { + DisconnectedWorkerId = std::move(It->second); + m_WorkerWsMap.erase(It); + } + }); + + if (!DisconnectedWorkerId.empty()) + { + m_Service->SetWorkerWebSocketConnected(DisconnectedWorkerId, false); + m_PushEvent.Set(); + } + + if (!m_PushEnabled.load()) + { + return; + } + + // Remove from dashboard connections + m_WsConnectionsLock.WithExclusiveLock([&] { + auto It = std::remove_if(m_WsConnections.begin(), m_WsConnections.end(), [&Conn](const Ref& C) { + return C.Get() == &Conn; + }); + m_WsConnections.erase(It, m_WsConnections.end()); + }); +} +# endif + +////////////////////////////////////////////////////////////////////////// +// +// Push thread +// + +# if ZEN_WITH_WEBSOCKETS +void +HttpOrchestratorService::PushThreadFunction() +{ + SetCurrentThreadName("orch_ws_push"); + + while (m_PushEnabled.load()) + { + m_PushEvent.Wait(2000); + m_PushEvent.Reset(); + + if (!m_PushEnabled.load()) + { + break; + } + + // Snapshot current connections + std::vector> Connections; + m_WsConnectionsLock.WithSharedLock([&] { Connections = m_WsConnections; }); + + if (Connections.empty()) + { + continue; + } + + // Build combined JSON with worker list, provisioning history, clients, and client history + CbObject WorkerList = m_Service->GetWorkerList(); + CbObject History = m_Service->GetProvisioningHistory(50); + CbObject ClientList = m_Service->GetClientList(); + CbObject ClientHistory = m_Service->GetClientHistory(50); + + ExtendableStringBuilder<4096> JsonBuilder; + JsonBuilder.Append("{"); + JsonBuilder.Append(fmt::format("\"hostname\":\"{}\",", m_Hostname)); + + // Emit workers array from worker list + ExtendableStringBuilder<2048> WorkerJson; + WorkerList.ToJson(WorkerJson); + std::string_view WorkerJsonView = WorkerJson.ToView(); + // Strip outer braces: {"workers":[...]} -> "workers":[...] + if (WorkerJsonView.size() >= 2) + { + JsonBuilder.Append(WorkerJsonView.substr(1, WorkerJsonView.size() - 2)); + } + + JsonBuilder.Append(","); + + // Emit events array from history + ExtendableStringBuilder<2048> HistoryJson; + History.ToJson(HistoryJson); + std::string_view HistoryJsonView = HistoryJson.ToView(); + if (HistoryJsonView.size() >= 2) + { + JsonBuilder.Append(HistoryJsonView.substr(1, HistoryJsonView.size() - 2)); + } + + JsonBuilder.Append(","); + + // Emit clients array from client list + ExtendableStringBuilder<2048> ClientJson; + ClientList.ToJson(ClientJson); + std::string_view ClientJsonView = ClientJson.ToView(); + if (ClientJsonView.size() >= 2) + { + JsonBuilder.Append(ClientJsonView.substr(1, ClientJsonView.size() - 2)); + } + + JsonBuilder.Append(","); + + // Emit client_events array from client history + ExtendableStringBuilder<2048> ClientHistoryJson; + ClientHistory.ToJson(ClientHistoryJson); + std::string_view ClientHistoryJsonView = ClientHistoryJson.ToView(); + if (ClientHistoryJsonView.size() >= 2) + { + JsonBuilder.Append(ClientHistoryJsonView.substr(1, ClientHistoryJsonView.size() - 2)); + } + + JsonBuilder.Append("}"); + std::string_view Json = JsonBuilder.ToView(); + + // Broadcast to all connected clients, prune closed ones + bool HadClosedConnections = false; + + for (auto& Conn : Connections) + { + if (Conn->IsOpen()) + { + Conn->SendText(Json); + } + else + { + HadClosedConnections = true; + } + } + + if (HadClosedConnections) + { + m_WsConnectionsLock.WithExclusiveLock([&] { + auto It = std::remove_if(m_WsConnections.begin(), m_WsConnections.end(), [](const Ref& C) { + return !C->IsOpen(); + }); + m_WsConnections.erase(It, m_WsConnections.end()); + }); + } + } +} +# endif + } // namespace zen::compute + +#endif diff --git a/src/zencompute/include/zencompute/cloudmetadata.h b/src/zencompute/include/zencompute/cloudmetadata.h new file mode 100644 index 000000000..a5bc5a34d --- /dev/null +++ b/src/zencompute/include/zencompute/cloudmetadata.h @@ -0,0 +1,151 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +namespace zen::compute { + +enum class CloudProvider +{ + None, + AWS, + Azure, + GCP +}; + +std::string_view ToString(CloudProvider Provider); + +/** Snapshot of detected cloud instance properties. */ +struct CloudInstanceInfo +{ + CloudProvider Provider = CloudProvider::None; + std::string InstanceId; + std::string AvailabilityZone; + bool IsSpot = false; + bool IsAutoscaling = false; +}; + +/** + * Detects whether the process is running on a cloud VM (AWS, Azure, or GCP) + * and monitors for impending termination signals. + * + * Detection works by querying the Instance Metadata Service (IMDS) at the + * well-known link-local address 169.254.169.254, which is only routable from + * within a cloud VM. Each provider is probed in sequence (AWS -> Azure -> GCP); + * the first successful response wins. + * + * To avoid a ~200ms connect timeout penalty on every startup when running on + * bare-metal or non-cloud machines, failed probes write sentinel files + * (e.g. ".isNotAWS") to DataDir. Subsequent startups skip providers that have + * a sentinel present. Delete the sentinel files to force re-detection. + * + * When a provider is detected, a background thread polls for termination + * signals every 5 seconds (spot interruption, autoscaling lifecycle changes, + * scheduled maintenance). The termination state is exposed as an atomic bool + * so the compute server can include it in coordinator announcements and react + * to imminent shutdown. + * + * Thread safety: GetInstanceInfo() and GetTerminationReason() acquire a + * shared RwLock; the background monitor thread acquires the exclusive lock + * only when writing the termination reason (a one-time transition). The + * termination-pending flag itself is a lock-free atomic. + * + * Usage: + * auto Cloud = std::make_unique(DataDir / "cloud"); + * if (Cloud->IsTerminationPending()) { ... } + * Cloud->Describe(AnnounceBody); // writes "cloud" sub-object into CB + */ +class CloudMetadata +{ +public: + /** Synchronously probes cloud providers and starts the termination monitor + * if a provider is detected. Creates DataDir if it does not exist. + */ + explicit CloudMetadata(std::filesystem::path DataDir); + + /** Synchronously probes cloud providers at the given IMDS endpoint. + * Intended for testing — allows redirecting all IMDS queries to a local + * mock HTTP server instead of the real 169.254.169.254 endpoint. + */ + CloudMetadata(std::filesystem::path DataDir, std::string ImdsEndpoint); + + /** Stops the termination monitor thread and joins it. */ + ~CloudMetadata(); + + CloudMetadata(const CloudMetadata&) = delete; + CloudMetadata& operator=(const CloudMetadata&) = delete; + + CloudProvider GetProvider() const; + CloudInstanceInfo GetInstanceInfo() const; + bool IsTerminationPending() const; + std::string GetTerminationReason() const; + + /** Writes a "cloud" sub-object into the compact binary writer if a provider + * was detected. No-op when running on bare metal. + */ + void Describe(CbWriter& Writer) const; + + /** Executes a single termination-poll cycle for the detected provider. + * Public so tests can drive poll cycles synchronously without relying on + * the background thread's 5-second timer. + */ + void PollTermination(); + + /** Removes the negative-cache sentinel files (.isNotAWS, .isNotAzure, + * .isNotGCP) from DataDir so subsequent detection probes are not skipped. + * Primarily intended for tests that need to reset state between sub-cases. + */ + void ClearSentinelFiles(); + +private: + /** Tries each provider in order, stops on first successful detection. */ + void DetectProvider(); + bool TryDetectAWS(); + bool TryDetectAzure(); + bool TryDetectGCP(); + + void WriteSentinelFile(const std::filesystem::path& Path); + bool HasSentinelFile(const std::filesystem::path& Path) const; + + void StartTerminationMonitor(); + void TerminationMonitorThread(); + void PollAWSTermination(); + void PollAzureTermination(); + void PollGCPTermination(); + + LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + std::filesystem::path m_DataDir; + std::string m_ImdsEndpoint; + + mutable RwLock m_InfoLock; + CloudInstanceInfo m_Info; + + std::atomic m_TerminationPending{false}; + + mutable RwLock m_ReasonLock; + std::string m_TerminationReason; + + // IMDSv2 session token, acquired during AWS detection and reused for + // subsequent termination polling. Has a 300s TTL on the AWS side; if it + // expires mid-run the poll requests will get 401s which we treat as + // non-terminal (the monitor simply retries next cycle). + std::string m_AwsToken; + + std::thread m_MonitorThread; + std::atomic m_MonitorEnabled{true}; + Event m_MonitorEvent; +}; + +void cloudmetadata_forcelink(); // internal + +} // namespace zen::compute diff --git a/src/zencompute/include/zencompute/computeservice.h b/src/zencompute/include/zencompute/computeservice.h new file mode 100644 index 000000000..65ec5f9ee --- /dev/null +++ b/src/zencompute/include/zencompute/computeservice.h @@ -0,0 +1,262 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include + +# include + +namespace zen { +class ChunkResolver; +class CbObjectWriter; +} // namespace zen + +namespace zen::compute { + +class ActionRecorder; +class ComputeServiceSession; +class IActionResultHandler; +class LocalProcessRunner; +class RemoteHttpRunner; +struct RunnerAction; +struct SubmitResult; + +struct WorkerDesc +{ + CbPackage Descriptor; + IoHash WorkerId{IoHash::Zero}; + + inline operator bool() const { return WorkerId != IoHash::Zero; } +}; + +/** + * Lambda style compute function service + * + * The responsibility of this class is to accept function execution requests, and + * schedule them using one or more FunctionRunner instances. It will basically always + * accept requests, queueing them if necessary, and then hand them off to runners + * as they become available. + * + * This is typically fronted by an API service that handles communication with clients. + */ +class ComputeServiceSession final +{ +public: + /** + * Session lifecycle state machine. + * + * Forward transitions: Created -> Ready -> Draining -> Paused -> Sunset + * Backward transitions: Draining -> Ready, Paused -> Ready + * Automatic transition: Draining -> Paused (when pending + running reaches 0) + * Jump transitions: any non-terminal -> Abandoned, any non-terminal -> Sunset + * Terminal states: Abandoned (only Sunset out), Sunset (no transitions out) + * + * | State | Accept new actions | Schedule pending | Finish running | + * |-----------|-------------------|-----------------|----------------| + * | Created | No | No | N/A | + * | Ready | Yes | Yes | Yes | + * | Draining | No | Yes | Yes | + * | Paused | No | No | No | + * | Abandoned | No | No | No (all abandoned) | + * | Sunset | No | No | No | + */ + enum class SessionState + { + Created, // Initial state before WaitUntilReady completes + Ready, // Normal operating state; accepts and schedules work + Draining, // Stops accepting new work; finishes existing; auto-transitions to Paused when empty + Paused, // Idle; no work accepted or scheduled; can resume to Ready + Abandoned, // Spot termination grace period; all actions abandoned; only Sunset out + Sunset // Terminal; triggers full shutdown + }; + + ComputeServiceSession(ChunkResolver& InChunkResolver); + ~ComputeServiceSession(); + + void WaitUntilReady(); + void Shutdown(); + bool IsHealthy(); + + SessionState GetSessionState() const; + + // Request a state transition. Returns false if the transition is invalid. + // Sunset can be reached from any non-Sunset state. + bool RequestStateTransition(SessionState NewState); + + // Orchestration + + void SetOrchestratorEndpoint(std::string_view Endpoint); + void SetOrchestratorBasePath(std::filesystem::path BasePath); + + // Worker registration and discovery + + void RegisterWorker(CbPackage Worker); + [[nodiscard]] WorkerDesc GetWorkerDescriptor(const IoHash& WorkerId); + [[nodiscard]] std::vector GetKnownWorkerIds(); + + // Action runners + + void AddLocalRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath, int32_t MaxConcurrentActions = 0); + void AddRemoteRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath, std::string_view HostName); + + // Action submission + + struct EnqueueResult + { + int Lsn; + CbObject ResponseMessage; + + inline operator bool() const { return Lsn != 0; } + }; + + [[nodiscard]] EnqueueResult EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int Priority); + [[nodiscard]] EnqueueResult EnqueueAction(CbObject ActionObject, int Priority); + + // Queue management + // + // Queues group actions submitted by a single client session. They allow + // cancelling or polling completion of all actions in the group. + + struct CreateQueueResult + { + int QueueId = 0; // 0 if creation failed + }; + + enum class QueueState + { + Active, + Draining, + Cancelled, + }; + + struct QueueStatus + { + bool IsValid = false; + int QueueId = 0; + int ActiveCount = 0; // pending + running (not yet completed) + int CompletedCount = 0; // successfully completed + int FailedCount = 0; // failed + int AbandonedCount = 0; // abandoned + int CancelledCount = 0; // cancelled + QueueState State = QueueState::Active; + bool IsComplete = false; // ActiveCount == 0 + }; + + [[nodiscard]] CreateQueueResult CreateQueue(std::string_view Tag = {}, CbObject Metadata = {}, CbObject Config = {}); + [[nodiscard]] std::vector GetQueueIds(); + [[nodiscard]] QueueStatus GetQueueStatus(int QueueId); + [[nodiscard]] CbObject GetQueueMetadata(int QueueId); + [[nodiscard]] CbObject GetQueueConfig(int QueueId); + void CancelQueue(int QueueId); + void DrainQueue(int QueueId); + void DeleteQueue(int QueueId); + void GetQueueCompleted(int QueueId, CbWriter& Cbo); + + // Queue-scoped action submission. Actions submitted via these methods are + // tracked under the given queue in addition to the global LSN-based tracking. + + [[nodiscard]] EnqueueResult EnqueueActionToQueue(int QueueId, CbObject ActionObject, int Priority); + [[nodiscard]] EnqueueResult EnqueueResolvedActionToQueue(int QueueId, WorkerDesc Worker, CbObject ActionObj, int Priority); + + // Completed action tracking + + [[nodiscard]] HttpResponseCode GetActionResult(int ActionLsn, CbPackage& OutResultPackage); + [[nodiscard]] HttpResponseCode FindActionResult(const IoHash& ActionId, CbPackage& ResultPackage); + void RetireActionResult(int ActionLsn); + + // Action rescheduling + + struct RescheduleResult + { + bool Success = false; + std::string Error; + int RetryCount = 0; + }; + + [[nodiscard]] RescheduleResult RescheduleAction(int ActionLsn); + + void GetCompleted(CbWriter&); + + // Running action tracking + + struct RunningActionInfo + { + int Lsn; + int QueueId; + IoHash ActionId; + float CpuUsagePercent; // -1.0 if not yet sampled + float CpuSeconds; // 0.0 if not yet sampled + }; + + [[nodiscard]] std::vector GetRunningActions(); + + // Action history tracking (note that this is separate from completed action tracking, and + // will include actions which have been retired and no longer have their results available) + + struct ActionHistoryEntry + { + int Lsn; + int QueueId = 0; + IoHash ActionId; + IoHash WorkerId; + CbObject ActionDescriptor; + std::string ExecutionLocation; + bool Succeeded; + float CpuSeconds = 0.0f; // total CPU time at completion; 0.0 if not sampled + int RetryCount = 0; // number of times this action was rescheduled + // sized to match RunnerAction::State::_Count but we can't use the enum here + // for dependency reasons, so just use a fixed size array and static assert in + // the implementation file + uint64_t Timestamps[8] = {}; + }; + + [[nodiscard]] std::vector GetActionHistory(int Limit = 100); + [[nodiscard]] std::vector GetQueueHistory(int QueueId, int Limit = 100); + + // Stats reporting + + struct ActionCounts + { + int Pending = 0; + int Running = 0; + int Completed = 0; + int ActiveQueues = 0; + }; + + [[nodiscard]] ActionCounts GetActionCounts(); + + void EmitStats(CbObjectWriter& Cbo); + + // Recording + + void StartRecording(ChunkResolver& InResolver, const std::filesystem::path& RecordingPath); + void StopRecording(); + +private: + void PostUpdate(RunnerAction* Action); + + friend class FunctionRunner; + friend struct RunnerAction; + + struct Impl; + std::unique_ptr m_Impl; +}; + +void computeservice_forcelink(); + +} // namespace zen::compute + +namespace zen { +const char* ToString(compute::ComputeServiceSession::SessionState State); +const char* ToString(compute::ComputeServiceSession::QueueState State); +} // namespace zen + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/include/zencompute/functionservice.h b/src/zencompute/include/zencompute/functionservice.h deleted file mode 100644 index 1deb99fd5..000000000 --- a/src/zencompute/include/zencompute/functionservice.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include - -#if !defined(ZEN_WITH_COMPUTE_SERVICES) -# define ZEN_WITH_COMPUTE_SERVICES 1 -#endif - -#if ZEN_WITH_COMPUTE_SERVICES - -# include -# include -# include -# include -# include - -# include - -namespace zen { -class ChunkResolver; -class CbObjectWriter; -} // namespace zen - -namespace zen::compute { - -class ActionRecorder; -class FunctionServiceSession; -class IActionResultHandler; -class LocalProcessRunner; -class RemoteHttpRunner; -struct RunnerAction; -struct SubmitResult; - -struct WorkerDesc -{ - CbPackage Descriptor; - IoHash WorkerId{IoHash::Zero}; - - inline operator bool() const { return WorkerId != IoHash::Zero; } -}; - -/** - * Lambda style compute function service - * - * The responsibility of this class is to accept function execution requests, and - * schedule them using one or more FunctionRunner instances. It will basically always - * accept requests, queueing them if necessary, and then hand them off to runners - * as they become available. - * - * This is typically fronted by an API service that handles communication with clients. - */ -class FunctionServiceSession final -{ -public: - FunctionServiceSession(ChunkResolver& InChunkResolver); - ~FunctionServiceSession(); - - void Shutdown(); - bool IsHealthy(); - - // Worker registration and discovery - - void RegisterWorker(CbPackage Worker); - [[nodiscard]] WorkerDesc GetWorkerDescriptor(const IoHash& WorkerId); - [[nodiscard]] std::vector GetKnownWorkerIds(); - - // Action runners - - void AddLocalRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath); - void AddRemoteRunner(ChunkResolver& InChunkResolver, std::filesystem::path BasePath, std::string_view HostName); - - // Action submission - - struct EnqueueResult - { - int Lsn; - CbObject ResponseMessage; - - inline operator bool() const { return Lsn != 0; } - }; - - [[nodiscard]] EnqueueResult EnqueueResolvedAction(WorkerDesc Worker, CbObject ActionObj, int Priority); - [[nodiscard]] EnqueueResult EnqueueAction(CbObject ActionObject, int Priority); - - // Completed action tracking - - [[nodiscard]] HttpResponseCode GetActionResult(int ActionLsn, CbPackage& OutResultPackage); - [[nodiscard]] HttpResponseCode FindActionResult(const IoHash& ActionId, CbPackage& ResultPackage); - - void GetCompleted(CbWriter&); - - // Action history tracking (note that this is separate from completed action tracking, and - // will include actions which have been retired and no longer have their results available) - - struct ActionHistoryEntry - { - int Lsn; - IoHash ActionId; - IoHash WorkerId; - CbObject ActionDescriptor; - bool Succeeded; - uint64_t Timestamps[5] = {}; - }; - - [[nodiscard]] std::vector GetActionHistory(int Limit = 100); - - // Stats reporting - - void EmitStats(CbObjectWriter& Cbo); - - // Recording - - void StartRecording(ChunkResolver& InResolver, const std::filesystem::path& RecordingPath); - void StopRecording(); - -private: - void PostUpdate(RunnerAction* Action); - - friend class FunctionRunner; - friend struct RunnerAction; - - struct Impl; - std::unique_ptr m_Impl; -}; - -void function_forcelink(); - -} // namespace zen::compute - -#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/include/zencompute/httpcomputeservice.h b/src/zencompute/include/zencompute/httpcomputeservice.h new file mode 100644 index 000000000..ee1cd2614 --- /dev/null +++ b/src/zencompute/include/zencompute/httpcomputeservice.h @@ -0,0 +1,54 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "zencompute/computeservice.h" + +# include + +# include +# include + +namespace zen { +class CidStore; +} + +namespace zen::compute { + +/** + * HTTP interface for compute service + */ +class HttpComputeService : public HttpService, public IHttpStatsProvider +{ +public: + HttpComputeService(CidStore& InCidStore, + IHttpStatsService& StatsService, + const std::filesystem::path& BaseDir, + int32_t MaxConcurrentActions = 0); + ~HttpComputeService(); + + void Shutdown(); + + [[nodiscard]] ComputeServiceSession::ActionCounts GetActionCounts(); + + const char* BaseUri() const override; + void HandleRequest(HttpServerRequest& Request) override; + + // IHttpStatsProvider + + void HandleStatsRequest(HttpServerRequest& Request) override; + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +void httpcomputeservice_forcelink(); + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/include/zencompute/httpfunctionservice.h b/src/zencompute/include/zencompute/httpfunctionservice.h deleted file mode 100644 index 6e2344ae6..000000000 --- a/src/zencompute/include/zencompute/httpfunctionservice.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include - -#if !defined(ZEN_WITH_COMPUTE_SERVICES) -# define ZEN_WITH_COMPUTE_SERVICES 1 -#endif - -#if ZEN_WITH_COMPUTE_SERVICES - -# include "zencompute/functionservice.h" - -# include -# include -# include -# include -# include -# include - -# include -# include -# include - -namespace zen { -class CidStore; -} - -namespace zen::compute { - -class HttpFunctionService; -class FunctionService; - -/** - * HTTP interface for compute function service - */ -class HttpFunctionService : public HttpService, public IHttpStatsProvider -{ -public: - HttpFunctionService(CidStore& InCidStore, IHttpStatsService& StatsService, const std::filesystem::path& BaseDir); - ~HttpFunctionService(); - - void Shutdown(); - - virtual const char* BaseUri() const override; - virtual void HandleRequest(HttpServerRequest& Request) override; - - // IHttpStatsProvider - - virtual void HandleStatsRequest(HttpServerRequest& Request) override; - -protected: - CidStore& m_CidStore; - IHttpStatsService& m_StatsService; - LoggerRef Log() { return m_Log; } - -private: - LoggerRef m_Log; - std::filesystem ::path m_BaseDir; - HttpRequestRouter m_Router; - FunctionServiceSession m_FunctionService; - - // Metrics - - metrics::OperationTiming m_HttpRequests; -}; - -void httpfunction_forcelink(); - -} // namespace zen::compute - -#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/include/zencompute/httporchestrator.h b/src/zencompute/include/zencompute/httporchestrator.h index 168c6d7fe..da5c5dfc3 100644 --- a/src/zencompute/include/zencompute/httporchestrator.h +++ b/src/zencompute/include/zencompute/httporchestrator.h @@ -2,43 +2,100 @@ #pragma once +#include + #include #include -#include #include +#include +#include +#include +#include +#include +#include #include +#include + +#define ZEN_WITH_WEBSOCKETS 1 namespace zen::compute { +class OrchestratorService; + +// Experimental helper, to see if we can get rid of some error-prone +// boilerplate when declaring loggers as class members. + +class LoggerHelper +{ +public: + LoggerHelper(std::string_view Logger) : m_Log(logging::Get(Logger)) {} + + LoggerRef operator()() { return m_Log; } + +private: + LoggerRef m_Log; +}; + /** - * Mock orchestrator service, for testing dynamic provisioning + * Orchestrator HTTP service with WebSocket push support + * + * Normal HTTP requests are routed through the HttpRequestRouter as before. + * WebSocket clients connecting to /orch/ws receive periodic state broadcasts + * from a dedicated push thread, eliminating the need for polling. */ class HttpOrchestratorService : public HttpService +#if ZEN_WITH_WEBSOCKETS +, + public IWebSocketHandler +#endif { public: - HttpOrchestratorService(); + explicit HttpOrchestratorService(std::filesystem::path DataDir, bool EnableWorkerWebSocket = false); ~HttpOrchestratorService(); HttpOrchestratorService(const HttpOrchestratorService&) = delete; HttpOrchestratorService& operator=(const HttpOrchestratorService&) = delete; + /** + * Gracefully shut down the WebSocket push thread and release connections. + * Must be called while the ASIO io_context is still alive. The destructor + * also calls this, so it is safe (but not ideal) to omit the explicit call. + */ + void Shutdown(); + virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; + // IWebSocketHandler +#if ZEN_WITH_WEBSOCKETS + void OnWebSocketOpen(Ref Connection) override; + void OnWebSocketMessage(WebSocketConnection& Conn, const WebSocketMessage& Msg) override; + void OnWebSocketClose(WebSocketConnection& Conn, uint16_t Code, std::string_view Reason) override; +#endif + private: - HttpRequestRouter m_Router; - LoggerRef m_Log; + HttpRequestRouter m_Router; + LoggerHelper Log{"orch"}; + std::unique_ptr m_Service; + std::string m_Hostname; + + // WebSocket push - struct KnownWorker - { - std::string_view BaseUri; - Stopwatch LastSeen; - }; +#if ZEN_WITH_WEBSOCKETS + RwLock m_WsConnectionsLock; + std::vector> m_WsConnections; + std::thread m_PushThread; + std::atomic m_PushEnabled{false}; + Event m_PushEvent; + void PushThreadFunction(); - RwLock m_KnownWorkersLock; - std::unordered_map m_KnownWorkers; + // Worker WebSocket connections (worker→orchestrator persistent links) + RwLock m_WorkerWsLock; + std::unordered_map m_WorkerWsMap; // connection ptr → worker ID + std::string HandleWorkerWebSocketMessage(const WebSocketMessage& Msg); +#endif }; } // namespace zen::compute diff --git a/src/zencompute/include/zencompute/mockimds.h b/src/zencompute/include/zencompute/mockimds.h new file mode 100644 index 000000000..521722e63 --- /dev/null +++ b/src/zencompute/include/zencompute/mockimds.h @@ -0,0 +1,102 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include + +#if ZEN_WITH_TESTS + +namespace zen::compute { + +/** + * Mock IMDS (Instance Metadata Service) for testing CloudMetadata. + * + * Implements an HttpService that responds to the same URL paths as the real + * cloud provider metadata endpoints (AWS IMDSv2, Azure IMDS, GCP metadata). + * Tests configure which provider is "active" and set the desired response + * values, then pass the mock server's address as the ImdsEndpoint to the + * CloudMetadata constructor. + * + * When a request arrives for a provider that is not the ActiveProvider, the + * mock returns 404, causing CloudMetadata to write a sentinel file and move + * on to the next provider — exactly like a failed probe on bare metal. + * + * All config fields are public and can be mutated between poll cycles to + * simulate state changes (e.g. a spot interruption appearing mid-run). + * + * Usage: + * MockImdsService Mock; + * Mock.ActiveProvider = CloudProvider::AWS; + * Mock.Aws.InstanceId = "i-test"; + * // ... stand up ASIO server, register Mock, create CloudMetadata with endpoint + */ +class MockImdsService : public HttpService +{ +public: + /** AWS IMDSv2 response configuration. */ + struct AwsConfig + { + std::string Token = "mock-aws-token-v2"; + std::string InstanceId = "i-0123456789abcdef0"; + std::string AvailabilityZone = "us-east-1a"; + std::string LifeCycle = "on-demand"; // "spot" or "on-demand" + + // Empty string → endpoint returns 404 (instance not in an ASG). + // Non-empty → returned as the response body. "InService" means healthy; + // anything else (e.g. "Terminated:Wait") triggers termination detection. + std::string AutoscalingState; + + // Empty string → endpoint returns 404 (no spot interruption). + // Non-empty → returned as the response body, signalling a spot reclaim. + std::string SpotAction; + }; + + /** Azure IMDS response configuration. */ + struct AzureConfig + { + std::string VmId = "vm-12345678-1234-1234-1234-123456789abc"; + std::string Location = "eastus"; + std::string Priority = "Regular"; // "Spot" or "Regular" + + // Empty → instance is not in a VM Scale Set (no autoscaling). + std::string VmScaleSetName; + + // Empty → no scheduled events. Set to "Preempt", "Terminate", or + // "Reboot" to simulate a termination-class event. + std::string ScheduledEventType; + std::string ScheduledEventStatus = "Scheduled"; + }; + + /** GCP metadata response configuration. */ + struct GcpConfig + { + std::string InstanceId = "1234567890123456789"; + std::string Zone = "projects/123456/zones/us-central1-a"; + std::string Preemptible = "FALSE"; // "TRUE" or "FALSE" + std::string MaintenanceEvent = "NONE"; // "NONE" or event description + }; + + /** Which provider's endpoints respond successfully. + * Requests targeting other providers receive 404. + */ + CloudProvider ActiveProvider = CloudProvider::None; + + AwsConfig Aws; + AzureConfig Azure; + GcpConfig Gcp; + + const char* BaseUri() const override; + void HandleRequest(HttpServerRequest& Request) override; + +private: + void HandleAwsRequest(HttpServerRequest& Request); + void HandleAzureRequest(HttpServerRequest& Request); + void HandleGcpRequest(HttpServerRequest& Request); +}; + +} // namespace zen::compute + +#endif // ZEN_WITH_TESTS diff --git a/src/zencompute/include/zencompute/orchestratorservice.h b/src/zencompute/include/zencompute/orchestratorservice.h new file mode 100644 index 000000000..071e902b3 --- /dev/null +++ b/src/zencompute/include/zencompute/orchestratorservice.h @@ -0,0 +1,177 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include + +# include +# include +# include +# include +# include +# include +# include +# include + +namespace zen::compute { + +class WorkerTimelineStore; + +class OrchestratorService +{ +public: + explicit OrchestratorService(std::filesystem::path DataDir, bool EnableWorkerWebSocket = false); + ~OrchestratorService(); + + OrchestratorService(const OrchestratorService&) = delete; + OrchestratorService& operator=(const OrchestratorService&) = delete; + + struct WorkerAnnouncement + { + std::string_view Id; + std::string_view Uri; + std::string_view Hostname; + std::string_view Platform; // e.g. "windows", "wine", "linux", "macos" + int Cpus = 0; + float CpuUsagePercent = 0.0f; + uint64_t MemoryTotalBytes = 0; + uint64_t MemoryUsedBytes = 0; + uint64_t BytesReceived = 0; + uint64_t BytesSent = 0; + int ActionsPending = 0; + int ActionsRunning = 0; + int ActionsCompleted = 0; + int ActiveQueues = 0; + std::string_view Provisioner; // e.g. "horde", "nomad", or empty + }; + + struct ProvisioningEvent + { + enum class Type + { + Joined, + Left, + Returned + }; + Type EventType; + DateTime Timestamp; + std::string WorkerId; + std::string Hostname; + }; + + struct ClientAnnouncement + { + Oid SessionId; + std::string_view Hostname; + std::string_view Address; + CbObject Metadata; + }; + + struct ClientEvent + { + enum class Type + { + Connected, + Disconnected, + Updated + }; + Type EventType; + DateTime Timestamp; + std::string ClientId; + std::string Hostname; + }; + + CbObject GetWorkerList(); + void AnnounceWorker(const WorkerAnnouncement& Announcement); + + bool IsWorkerWebSocketEnabled() const; + void SetWorkerWebSocketConnected(std::string_view WorkerId, bool Connected); + + CbObject GetProvisioningHistory(int Limit = 100); + + CbObject GetWorkerTimeline(std::string_view WorkerId, std::optional From, std::optional To, int Limit); + + CbObject GetAllTimelines(DateTime From, DateTime To); + + std::string AnnounceClient(const ClientAnnouncement& Announcement); + bool UpdateClient(std::string_view ClientId, CbObject Metadata = {}); + bool CompleteClient(std::string_view ClientId); + CbObject GetClientList(); + CbObject GetClientHistory(int Limit = 100); + +private: + enum class ReachableState + { + Unknown, + Reachable, + Unreachable, + }; + + struct KnownWorker + { + std::string BaseUri; + Stopwatch LastSeen; + std::string Hostname; + std::string Platform; + int Cpus = 0; + float CpuUsagePercent = 0.0f; + uint64_t MemoryTotalBytes = 0; + uint64_t MemoryUsedBytes = 0; + uint64_t BytesReceived = 0; + uint64_t BytesSent = 0; + int ActionsPending = 0; + int ActionsRunning = 0; + int ActionsCompleted = 0; + int ActiveQueues = 0; + std::string Provisioner; + ReachableState Reachable = ReachableState::Unknown; + bool WsConnected = false; + Stopwatch LastProbed; + }; + + RwLock m_KnownWorkersLock; + std::unordered_map m_KnownWorkers; + std::unique_ptr m_TimelineStore; + + RwLock m_ProvisioningLogLock; + std::deque m_ProvisioningLog; + static constexpr size_t kMaxProvisioningEvents = 1000; + + void RecordProvisioningEvent(ProvisioningEvent::Type Type, std::string_view WorkerId, std::string_view Hostname); + + struct KnownClient + { + Oid SessionId; + std::string Hostname; + std::string Address; + Stopwatch LastSeen; + CbObject Metadata; + }; + + RwLock m_KnownClientsLock; + std::unordered_map m_KnownClients; + + RwLock m_ClientLogLock; + std::deque m_ClientLog; + static constexpr size_t kMaxClientEvents = 1000; + + void RecordClientEvent(ClientEvent::Type Type, std::string_view ClientId, std::string_view Hostname); + + bool m_EnableWorkerWebSocket = false; + + std::thread m_ProbeThread; + std::atomic m_ProbeThreadEnabled{true}; + Event m_ProbeThreadEvent; + void ProbeThreadFunction(); +}; + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/include/zencompute/recordingreader.h b/src/zencompute/include/zencompute/recordingreader.h index bf1aff125..3f233fae0 100644 --- a/src/zencompute/include/zencompute/recordingreader.h +++ b/src/zencompute/include/zencompute/recordingreader.h @@ -2,7 +2,9 @@ #pragma once -#include +#include + +#include #include #include #include diff --git a/src/zencompute/include/zencompute/zencompute.h b/src/zencompute/include/zencompute/zencompute.h index 6dc32eeea..00be4d4a0 100644 --- a/src/zencompute/include/zencompute/zencompute.h +++ b/src/zencompute/include/zencompute/zencompute.h @@ -4,6 +4,10 @@ #include +#if !defined(ZEN_WITH_COMPUTE_SERVICES) +# define ZEN_WITH_COMPUTE_SERVICES 1 +#endif + namespace zen { void zencompute_forcelinktests(); diff --git a/src/zencompute/localrunner.cpp b/src/zencompute/localrunner.cpp deleted file mode 100644 index 9a27f3f3d..000000000 --- a/src/zencompute/localrunner.cpp +++ /dev/null @@ -1,722 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "localrunner.h" - -#if ZEN_WITH_COMPUTE_SERVICES - -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include - -namespace zen::compute { - -using namespace std::literals; - -LocalProcessRunner::LocalProcessRunner(ChunkResolver& Resolver, const std::filesystem::path& BaseDir) -: FunctionRunner(BaseDir) -, m_Log(logging::Get("local_exec")) -, m_ChunkResolver(Resolver) -, m_WorkerPath(std::filesystem::weakly_canonical(BaseDir / "workers")) -, m_SandboxPath(std::filesystem::weakly_canonical(BaseDir / "scratch")) -{ - SystemMetrics Sm = GetSystemMetricsForReporting(); - - m_MaxRunningActions = Sm.LogicalProcessorCount * 2; - - ZEN_INFO("Max concurrent action count: {}", m_MaxRunningActions); - - bool DidCleanup = false; - - if (std::filesystem::is_directory(m_ActionsPath)) - { - ZEN_INFO("Cleaning '{}'", m_ActionsPath); - - std::error_code Ec; - CleanDirectory(m_ActionsPath, /* ForceRemoveReadOnlyFiles */ true, Ec); - - if (Ec) - { - ZEN_WARN("Unable to clean '{}': {}", m_ActionsPath, Ec.message()); - } - - DidCleanup = true; - } - - if (std::filesystem::is_directory(m_SandboxPath)) - { - ZEN_INFO("Cleaning '{}'", m_SandboxPath); - std::error_code Ec; - CleanDirectory(m_SandboxPath, /* ForceRemoveReadOnlyFiles */ true, Ec); - - if (Ec) - { - ZEN_WARN("Unable to clean '{}': {}", m_SandboxPath, Ec.message()); - } - - DidCleanup = true; - } - - // We clean out all workers on startup since we can't know they are good. They could be bad - // due to tampering, malware (which I also mean to include AV and antimalware software) or - // other processes we have no control over - if (std::filesystem::is_directory(m_WorkerPath)) - { - ZEN_INFO("Cleaning '{}'", m_WorkerPath); - std::error_code Ec; - CleanDirectory(m_WorkerPath, /* ForceRemoveReadOnlyFiles */ true, Ec); - - if (Ec) - { - ZEN_WARN("Unable to clean '{}': {}", m_WorkerPath, Ec.message()); - } - - DidCleanup = true; - } - - if (DidCleanup) - { - ZEN_INFO("Cleanup complete"); - } - - m_MonitorThread = std::thread{&LocalProcessRunner::MonitorThreadFunction, this}; - -# if ZEN_PLATFORM_WINDOWS - // Suppress any error dialogs caused by missing dependencies - UINT OldMode = ::SetErrorMode(0); - ::SetErrorMode(OldMode | SEM_FAILCRITICALERRORS); -# endif - - m_AcceptNewActions = true; -} - -LocalProcessRunner::~LocalProcessRunner() -{ - try - { - Shutdown(); - } - catch (std::exception& Ex) - { - ZEN_WARN("exception during local process runner shutdown: {}", Ex.what()); - } -} - -void -LocalProcessRunner::Shutdown() -{ - m_AcceptNewActions = false; - - m_MonitorThreadEnabled = false; - m_MonitorThreadEvent.Set(); - if (m_MonitorThread.joinable()) - { - m_MonitorThread.join(); - } - - CancelRunningActions(); -} - -std::filesystem::path -LocalProcessRunner::CreateNewSandbox() -{ - std::string UniqueId = std::to_string(++m_SandboxCounter); - std::filesystem::path Path = m_SandboxPath / UniqueId; - zen::CreateDirectories(Path); - - return Path; -} - -void -LocalProcessRunner::RegisterWorker(const CbPackage& WorkerPackage) -{ - if (m_DumpActions) - { - CbObject WorkerDescriptor = WorkerPackage.GetObject(); - const IoHash& WorkerId = WorkerPackage.GetObjectHash(); - - std::string UniqueId = fmt::format("worker_{}"sv, WorkerId); - std::filesystem::path Path = m_ActionsPath / UniqueId; - - zen::WriteFile(Path / "worker.ucb", WorkerDescriptor.GetBuffer().AsIoBuffer()); - - ManifestWorker(WorkerPackage, Path / "tree", [&](const IoHash& Cid, CompressedBuffer& ChunkBuffer) { - std::filesystem::path ChunkPath = Path / "chunks" / Cid.ToHexString(); - zen::WriteFile(ChunkPath, ChunkBuffer.GetCompressed()); - }); - - ZEN_INFO("dumped worker '{}' to 'file://{}'", WorkerId, Path); - } -} - -size_t -LocalProcessRunner::QueryCapacity() -{ - // Estimate how much more work we're ready to accept - - RwLock::SharedLockScope _{m_RunningLock}; - - if (!m_AcceptNewActions) - { - return 0; - } - - size_t RunningCount = m_RunningMap.size(); - - if (RunningCount >= size_t(m_MaxRunningActions)) - { - return 0; - } - - return m_MaxRunningActions - RunningCount; -} - -std::vector -LocalProcessRunner::SubmitActions(const std::vector>& Actions) -{ - std::vector Results; - - for (const Ref& Action : Actions) - { - Results.push_back(SubmitAction(Action)); - } - - return Results; -} - -SubmitResult -LocalProcessRunner::SubmitAction(Ref Action) -{ - // Verify whether we can accept more work - - { - RwLock::SharedLockScope _{m_RunningLock}; - - if (!m_AcceptNewActions) - { - return SubmitResult{.IsAccepted = false}; - } - - if (m_RunningMap.size() >= size_t(m_MaxRunningActions)) - { - return SubmitResult{.IsAccepted = false}; - } - } - - using namespace std::literals; - - // Each enqueued action is assigned an integer index (logical sequence number), - // which we use as a key for tracking data structures and as an opaque id which - // may be used by clients to reference the scheduled action - - const int32_t ActionLsn = Action->ActionLsn; - const CbObject& ActionObj = Action->ActionObj; - const IoHash ActionId = ActionObj.GetHash(); - - MaybeDumpAction(ActionLsn, ActionObj); - - std::filesystem::path SandboxPath = CreateNewSandbox(); - - CbPackage WorkerPackage = Action->Worker.Descriptor; - - std::filesystem::path WorkerPath = ManifestWorker(Action->Worker); - - // Write out action - - zen::WriteFile(SandboxPath / "build.action", ActionObj.GetBuffer().AsIoBuffer()); - - // Manifest inputs in sandbox - - ActionObj.IterateAttachments([&](CbFieldView Field) { - const IoHash Cid = Field.AsHash(); - std::filesystem::path FilePath{SandboxPath / "Inputs"sv / Cid.ToHexString()}; - IoBuffer DataBuffer = m_ChunkResolver.FindChunkByCid(Cid); - - if (!DataBuffer) - { - throw std::runtime_error(fmt::format("input CID chunk '{}' missing", Cid)); - } - - zen::WriteFile(FilePath, DataBuffer); - }); - -# if ZEN_PLATFORM_WINDOWS - // Set up environment variables - - StringBuilder<1024> EnvironmentBlock; - - CbObject WorkerDescription = WorkerPackage.GetObject(); - - for (auto& It : WorkerDescription["environment"sv]) - { - EnvironmentBlock.Append(It.AsString()); - EnvironmentBlock.Append('\0'); - } - EnvironmentBlock.Append('\0'); - EnvironmentBlock.Append('\0'); - - // Execute process - this spawns the child process immediately without waiting - // for completion - - std::string_view ExecPath = WorkerDescription["path"sv].AsString(); - std::filesystem::path ExePath = WorkerPath / std::filesystem::path(ExecPath).make_preferred(); - - ExtendableWideStringBuilder<512> CommandLine; - CommandLine.Append(L'"'); - CommandLine.Append(ExePath.c_str()); - CommandLine.Append(L'"'); - CommandLine.Append(L" -Build=build.action"); - - LPSECURITY_ATTRIBUTES lpProcessAttributes = nullptr; - LPSECURITY_ATTRIBUTES lpThreadAttributes = nullptr; - BOOL bInheritHandles = FALSE; - DWORD dwCreationFlags = 0; - - STARTUPINFO StartupInfo{}; - StartupInfo.cb = sizeof StartupInfo; - - PROCESS_INFORMATION ProcessInformation{}; - - ZEN_DEBUG("Executing: {}", WideToUtf8(CommandLine.c_str())); - - CommandLine.EnsureNulTerminated(); - - BOOL Success = CreateProcessW(nullptr, - CommandLine.Data(), - lpProcessAttributes, - lpThreadAttributes, - bInheritHandles, - dwCreationFlags, - (LPVOID)EnvironmentBlock.Data(), // Environment block - SandboxPath.c_str(), // Current directory - &StartupInfo, - /* out */ &ProcessInformation); - - if (!Success) - { - // TODO: this is probably not the best way to report failure. The return - // object should include a failure state and context - - zen::ThrowLastError("Unable to launch process" /* TODO: Add context */); - } - - CloseHandle(ProcessInformation.hThread); - - Ref NewAction{new RunningAction()}; - NewAction->Action = Action; - NewAction->ProcessHandle = ProcessInformation.hProcess; - NewAction->SandboxPath = std::move(SandboxPath); - - { - RwLock::ExclusiveLockScope _(m_RunningLock); - - m_RunningMap[ActionLsn] = std::move(NewAction); - } - - Action->SetActionState(RunnerAction::State::Running); -# else - ZEN_UNUSED(ActionId); - - ZEN_NOT_IMPLEMENTED(); - - int ExitCode = 0; -# endif - - return SubmitResult{.IsAccepted = true}; -} - -size_t -LocalProcessRunner::GetSubmittedActionCount() -{ - RwLock::SharedLockScope _(m_RunningLock); - return m_RunningMap.size(); -} - -std::filesystem::path -LocalProcessRunner::ManifestWorker(const WorkerDesc& Worker) -{ - RwLock::SharedLockScope _(m_WorkerLock); - - std::filesystem::path WorkerDir = m_WorkerPath / fmt::format("runner_{}", Worker.WorkerId); - - if (!std::filesystem::exists(WorkerDir)) - { - _.ReleaseNow(); - - RwLock::ExclusiveLockScope $(m_WorkerLock); - - if (!std::filesystem::exists(WorkerDir)) - { - ManifestWorker(Worker.Descriptor, WorkerDir, [](const IoHash&, CompressedBuffer&) {}); - } - } - - return WorkerDir; -} - -void -LocalProcessRunner::DecompressAttachmentToFile(const CbPackage& FromPackage, - CbObjectView FileEntry, - const std::filesystem::path& SandboxRootPath, - std::function& ChunkReferenceCallback) -{ - std::string_view Name = FileEntry["name"sv].AsString(); - const IoHash ChunkHash = FileEntry["hash"sv].AsHash(); - const uint64_t Size = FileEntry["size"sv].AsUInt64(); - - CompressedBuffer Compressed; - - if (const CbAttachment* Attachment = FromPackage.FindAttachment(ChunkHash)) - { - Compressed = Attachment->AsCompressedBinary(); - } - else - { - IoBuffer DataBuffer = m_ChunkResolver.FindChunkByCid(ChunkHash); - - if (!DataBuffer) - { - throw std::runtime_error(fmt::format("worker chunk '{}' missing", ChunkHash)); - } - - uint64_t DataRawSize = 0; - IoHash DataRawHash; - Compressed = CompressedBuffer::FromCompressed(SharedBuffer{DataBuffer}, DataRawHash, DataRawSize); - - if (DataRawSize != Size) - { - throw std::runtime_error( - fmt::format("worker chunk '{}' size: {}, action spec expected {}", ChunkHash, DataBuffer.Size(), Size)); - } - } - - ChunkReferenceCallback(ChunkHash, Compressed); - - std::filesystem::path FilePath{SandboxRootPath / std::filesystem::path(Name).make_preferred()}; - - SharedBuffer Decompressed = Compressed.Decompress(); - zen::WriteFile(FilePath, Decompressed.AsIoBuffer()); -} - -void -LocalProcessRunner::ManifestWorker(const CbPackage& WorkerPackage, - const std::filesystem::path& SandboxPath, - std::function&& ChunkReferenceCallback) -{ - CbObject WorkerDescription = WorkerPackage.GetObject(); - - // Manifest worker in Sandbox - - for (auto& It : WorkerDescription["executables"sv]) - { - DecompressAttachmentToFile(WorkerPackage, It.AsObjectView(), SandboxPath, ChunkReferenceCallback); - } - - for (auto& It : WorkerDescription["dirs"sv]) - { - std::string_view Name = It.AsString(); - std::filesystem::path DirPath{SandboxPath / std::filesystem::path(Name).make_preferred()}; - zen::CreateDirectories(DirPath); - } - - for (auto& It : WorkerDescription["files"sv]) - { - DecompressAttachmentToFile(WorkerPackage, It.AsObjectView(), SandboxPath, ChunkReferenceCallback); - } - - WriteFile(SandboxPath / "worker.zcb", WorkerDescription.GetBuffer().AsIoBuffer()); -} - -CbPackage -LocalProcessRunner::GatherActionOutputs(std::filesystem::path SandboxPath) -{ - std::filesystem::path OutputFile = SandboxPath / "build.output"; - FileContents OutputData = zen::ReadFile(OutputFile); - - if (OutputData.ErrorCode) - { - throw std::system_error(OutputData.ErrorCode, fmt::format("Failed to read build output file '{}'", OutputFile)); - } - - CbPackage OutputPackage; - CbObject Output = zen::LoadCompactBinaryObject(OutputData.Flatten()); - - uint64_t TotalAttachmentBytes = 0; - uint64_t TotalRawAttachmentBytes = 0; - - Output.IterateAttachments([&](CbFieldView Field) { - IoHash Hash = Field.AsHash(); - std::filesystem::path OutputPath{SandboxPath / "Outputs" / Hash.ToHexString()}; - FileContents ChunkData = zen::ReadFile(OutputPath); - - if (ChunkData.ErrorCode) - { - throw std::system_error(ChunkData.ErrorCode, fmt::format("Failed to read build output file '{}'", OutputPath)); - } - - uint64_t ChunkDataRawSize = 0; - IoHash ChunkDataHash; - CompressedBuffer AttachmentBuffer = - CompressedBuffer::FromCompressed(SharedBuffer(ChunkData.Flatten()), ChunkDataHash, ChunkDataRawSize); - - if (!AttachmentBuffer) - { - throw std::runtime_error("Invalid output encountered (not valid CompressedBuffer format)"); - } - - TotalAttachmentBytes += AttachmentBuffer.GetCompressedSize(); - TotalRawAttachmentBytes += ChunkDataRawSize; - - CbAttachment Attachment(std::move(AttachmentBuffer), ChunkDataHash); - OutputPackage.AddAttachment(Attachment); - }); - - OutputPackage.SetObject(Output); - - ZEN_DEBUG("Action completed with {} attachments ({} compressed, {} uncompressed)", - OutputPackage.GetAttachments().size(), - NiceBytes(TotalAttachmentBytes), - NiceBytes(TotalRawAttachmentBytes)); - - return OutputPackage; -} - -void -LocalProcessRunner::MonitorThreadFunction() -{ - SetCurrentThreadName("LocalProcessRunner_Monitor"); - - auto _ = MakeGuard([&] { ZEN_INFO("monitor thread exiting"); }); - - do - { - // On Windows it's possible to wait on process handles, so we wait for either a process to exit - // or for the monitor event to be signaled (which indicates we should check for cancellation - // or shutdown). This could be further improved by using a completion port and registering process - // handles with it, but this is a reasonable first implementation given that we shouldn't be dealing - // with an enormous number of concurrent processes. - // - // On other platforms we just wait on the monitor event and poll for process exits at intervals. -# if ZEN_PLATFORM_WINDOWS - auto WaitOnce = [&] { - HANDLE WaitHandles[MAXIMUM_WAIT_OBJECTS]; - - uint32_t NumHandles = 0; - - WaitHandles[NumHandles++] = m_MonitorThreadEvent.GetWindowsHandle(); - - m_RunningLock.WithSharedLock([&] { - for (auto It = begin(m_RunningMap), ItEnd = end(m_RunningMap); It != ItEnd && NumHandles < MAXIMUM_WAIT_OBJECTS; ++It) - { - Ref Action = It->second; - - WaitHandles[NumHandles++] = Action->ProcessHandle; - } - }); - - DWORD WaitResult = WaitForMultipleObjects(NumHandles, WaitHandles, FALSE, 1000); - - // return true if a handle was signaled - return (WaitResult <= NumHandles); - }; -# else - auto WaitOnce = [&] { return m_MonitorThreadEvent.Wait(1000); }; -# endif - - while (!WaitOnce()) - { - if (m_MonitorThreadEnabled == false) - { - return; - } - - SweepRunningActions(); - } - - // Signal received - - SweepRunningActions(); - } while (m_MonitorThreadEnabled); -} - -void -LocalProcessRunner::CancelRunningActions() -{ - Stopwatch Timer; - std::unordered_map> RunningMap; - - m_RunningLock.WithExclusiveLock([&] { std::swap(RunningMap, m_RunningMap); }); - - if (RunningMap.empty()) - { - return; - } - - ZEN_INFO("cancelling all running actions"); - - // For expedience we initiate the process termination for all known - // processes before attempting to wait for them to exit. - - std::vector TerminatedLsnList; - - for (const auto& Kv : RunningMap) - { - Ref Action = Kv.second; - - // Terminate running process - -# if ZEN_PLATFORM_WINDOWS - BOOL Success = TerminateProcess(Action->ProcessHandle, 222); - - if (Success) - { - TerminatedLsnList.push_back(Kv.first); - } - else - { - DWORD LastError = GetLastError(); - - if (LastError != ERROR_ACCESS_DENIED) - { - ZEN_WARN("TerminateProcess for LSN {} not successful: {}", Action->Action->ActionLsn, GetSystemErrorAsString(LastError)); - } - } -# else - ZEN_NOT_IMPLEMENTED("need to implement process termination"); -# endif - } - - // We only post results for processes we have terminated, in order - // to avoid multiple results getting posted for the same action - - for (int Lsn : TerminatedLsnList) - { - if (auto It = RunningMap.find(Lsn); It != RunningMap.end()) - { - Ref Running = It->second; - -# if ZEN_PLATFORM_WINDOWS - if (Running->ProcessHandle != INVALID_HANDLE_VALUE) - { - DWORD WaitResult = WaitForSingleObject(Running->ProcessHandle, 2000); - - if (WaitResult != WAIT_OBJECT_0) - { - ZEN_WARN("wait for LSN {}: process exit did not succeed, result = {}", Running->Action->ActionLsn, WaitResult); - } - else - { - ZEN_DEBUG("LSN {}: process exit OK", Running->Action->ActionLsn); - } - } -# endif - - // Clean up and post error result - - DeleteDirectories(Running->SandboxPath); - Running->Action->SetActionState(RunnerAction::State::Failed); - } - } - - ZEN_INFO("DONE - cancelled {} running processes (took {})", TerminatedLsnList.size(), NiceTimeSpanMs(Timer.GetElapsedTimeMs())); -} - -void -LocalProcessRunner::SweepRunningActions() -{ - std::vector> CompletedActions; - - m_RunningLock.WithExclusiveLock([&] { - // TODO: It would be good to not hold the exclusive lock while making - // system calls and other expensive operations. - - for (auto It = begin(m_RunningMap), ItEnd = end(m_RunningMap); It != ItEnd;) - { - Ref Action = It->second; - -# if ZEN_PLATFORM_WINDOWS - DWORD ExitCode = 0; - BOOL IsSuccess = GetExitCodeProcess(Action->ProcessHandle, &ExitCode); - - if (IsSuccess && ExitCode != STILL_ACTIVE) - { - CloseHandle(Action->ProcessHandle); - Action->ProcessHandle = INVALID_HANDLE_VALUE; - - CompletedActions.push_back(std::move(Action)); - It = m_RunningMap.erase(It); - } - else - { - ++It; - } -# else - // TODO: implement properly for Mac/Linux - - ZEN_UNUSED(Action); -# endif - } - }); - - // Notify outer. Note that this has to be done without holding any local locks - // otherwise we may end up with deadlocks. - - for (Ref Running : CompletedActions) - { - const int ActionLsn = Running->Action->ActionLsn; - - if (Running->ExitCode == 0) - { - try - { - // Gather outputs - - CbPackage OutputPackage = GatherActionOutputs(Running->SandboxPath); - - Running->Action->SetResult(std::move(OutputPackage)); - Running->Action->SetActionState(RunnerAction::State::Completed); - - // We can delete the files at this point - if (!DeleteDirectories(Running->SandboxPath)) - { - ZEN_WARN("Unable to delete directory '{}', this will continue to exist until service restart", Running->SandboxPath); - } - - // Success -- continue with next iteration of the loop - continue; - } - catch (std::exception& Ex) - { - ZEN_ERROR("Encountered failure while gathering outputs for action lsn {}, '{}'", ActionLsn, Ex.what()); - } - } - - // Failed - for now this is indicated with an empty package in - // the results map. We can clean out the sandbox directory immediately. - - std::error_code Ec; - DeleteDirectories(Running->SandboxPath, Ec); - - if (Ec) - { - ZEN_WARN("Unable to delete sandbox directory '{}': {}", Running->SandboxPath, Ec.message()); - } - - Running->Action->SetActionState(RunnerAction::State::Failed); - } -} - -} // namespace zen::compute - -#endif diff --git a/src/zencompute/localrunner.h b/src/zencompute/localrunner.h deleted file mode 100644 index 35f464805..000000000 --- a/src/zencompute/localrunner.h +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencompute/functionservice.h" - -#if ZEN_WITH_COMPUTE_SERVICES - -# include "functionrunner.h" - -# include -# include -# include -# include -# include - -# include -# include -# include - -namespace zen { -class CbPackage; -} - -namespace zen::compute { - -/** Direct process spawner - - This runner simply sets up a directory structure for each job and - creates a process to perform the computation in it. It is not very - efficient and is intended mostly for testing. - - */ - -class LocalProcessRunner : public FunctionRunner -{ - LocalProcessRunner(LocalProcessRunner&&) = delete; - LocalProcessRunner& operator=(LocalProcessRunner&&) = delete; - -public: - LocalProcessRunner(ChunkResolver& Resolver, const std::filesystem::path& BaseDir); - ~LocalProcessRunner(); - - virtual void Shutdown() override; - virtual void RegisterWorker(const CbPackage& WorkerPackage) override; - [[nodiscard]] virtual SubmitResult SubmitAction(Ref Action) override; - [[nodiscard]] virtual bool IsHealthy() override { return true; } - [[nodiscard]] virtual size_t GetSubmittedActionCount() override; - [[nodiscard]] virtual size_t QueryCapacity() override; - [[nodiscard]] virtual std::vector SubmitActions(const std::vector>& Actions) override; - -protected: - LoggerRef Log() { return m_Log; } - - LoggerRef m_Log; - - struct RunningAction : public RefCounted - { - Ref Action; - void* ProcessHandle = nullptr; - int ExitCode = 0; - std::filesystem::path SandboxPath; - }; - - std::atomic_bool m_AcceptNewActions; - ChunkResolver& m_ChunkResolver; - RwLock m_WorkerLock; - std::filesystem::path m_WorkerPath; - std::atomic m_SandboxCounter = 0; - std::filesystem::path m_SandboxPath; - int32_t m_MaxRunningActions = 64; // arbitrary limit for testing - - // if used in conjuction with m_ResultsLock, this lock must be taken *after* - // m_ResultsLock to avoid deadlocks - RwLock m_RunningLock; - std::unordered_map> m_RunningMap; - - std::thread m_MonitorThread; - std::atomic m_MonitorThreadEnabled{true}; - Event m_MonitorThreadEvent; - void MonitorThreadFunction(); - void SweepRunningActions(); - void CancelRunningActions(); - - std::filesystem::path CreateNewSandbox(); - void ManifestWorker(const CbPackage& WorkerPackage, - const std::filesystem::path& SandboxPath, - std::function&& ChunkReferenceCallback); - std::filesystem::path ManifestWorker(const WorkerDesc& Worker); - CbPackage GatherActionOutputs(std::filesystem::path SandboxPath); - - void DecompressAttachmentToFile(const CbPackage& FromPackage, - CbObjectView FileEntry, - const std::filesystem::path& SandboxRootPath, - std::function& ChunkReferenceCallback); -}; - -} // namespace zen::compute - -#endif diff --git a/src/zencompute/orchestratorservice.cpp b/src/zencompute/orchestratorservice.cpp new file mode 100644 index 000000000..9ea695305 --- /dev/null +++ b/src/zencompute/orchestratorservice.cpp @@ -0,0 +1,710 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include + +# include "timeline/workertimeline.h" + +namespace zen::compute { + +OrchestratorService::OrchestratorService(std::filesystem::path DataDir, bool EnableWorkerWebSocket) +: m_TimelineStore(std::make_unique(DataDir / "timelines")) +, m_EnableWorkerWebSocket(EnableWorkerWebSocket) +{ + m_ProbeThread = std::thread{&OrchestratorService::ProbeThreadFunction, this}; +} + +OrchestratorService::~OrchestratorService() +{ + m_ProbeThreadEnabled = false; + m_ProbeThreadEvent.Set(); + if (m_ProbeThread.joinable()) + { + m_ProbeThread.join(); + } +} + +CbObject +OrchestratorService::GetWorkerList() +{ + ZEN_TRACE_CPU("OrchestratorService::GetWorkerList"); + CbObjectWriter Cbo; + Cbo.BeginArray("workers"); + + m_KnownWorkersLock.WithSharedLock([&] { + for (const auto& [WorkerId, Worker] : m_KnownWorkers) + { + Cbo.BeginObject(); + Cbo << "id" << WorkerId; + Cbo << "uri" << Worker.BaseUri; + Cbo << "hostname" << Worker.Hostname; + if (!Worker.Platform.empty()) + { + Cbo << "platform" << std::string_view(Worker.Platform); + } + Cbo << "cpus" << Worker.Cpus; + Cbo << "cpu_usage" << Worker.CpuUsagePercent; + Cbo << "memory_total" << Worker.MemoryTotalBytes; + Cbo << "memory_used" << Worker.MemoryUsedBytes; + Cbo << "bytes_received" << Worker.BytesReceived; + Cbo << "bytes_sent" << Worker.BytesSent; + Cbo << "actions_pending" << Worker.ActionsPending; + Cbo << "actions_running" << Worker.ActionsRunning; + Cbo << "actions_completed" << Worker.ActionsCompleted; + Cbo << "active_queues" << Worker.ActiveQueues; + if (!Worker.Provisioner.empty()) + { + Cbo << "provisioner" << std::string_view(Worker.Provisioner); + } + if (Worker.Reachable != ReachableState::Unknown) + { + Cbo << "reachable" << (Worker.Reachable == ReachableState::Reachable); + } + if (Worker.WsConnected) + { + Cbo << "ws_connected" << true; + } + Cbo << "dt" << Worker.LastSeen.GetElapsedTimeMs(); + Cbo.EndObject(); + } + }); + + Cbo.EndArray(); + return Cbo.Save(); +} + +void +OrchestratorService::AnnounceWorker(const WorkerAnnouncement& Ann) +{ + ZEN_TRACE_CPU("OrchestratorService::AnnounceWorker"); + + bool IsNew = false; + std::string EvictedId; + std::string EvictedHostname; + + m_KnownWorkersLock.WithExclusiveLock([&] { + IsNew = (m_KnownWorkers.find(std::string(Ann.Id)) == m_KnownWorkers.end()); + + // If a different worker ID already maps to the same URI, the old entry + // is stale (e.g. a previous Horde lease on the same machine). Remove it + // so the dashboard doesn't show duplicates. + if (IsNew) + { + for (auto It = m_KnownWorkers.begin(); It != m_KnownWorkers.end(); ++It) + { + if (It->second.BaseUri == Ann.Uri && It->first != Ann.Id) + { + EvictedId = It->first; + EvictedHostname = It->second.Hostname; + m_KnownWorkers.erase(It); + break; + } + } + } + + auto& Worker = m_KnownWorkers[std::string(Ann.Id)]; + Worker.BaseUri = Ann.Uri; + Worker.Hostname = Ann.Hostname; + if (!Ann.Platform.empty()) + { + Worker.Platform = Ann.Platform; + } + Worker.Cpus = Ann.Cpus; + Worker.CpuUsagePercent = Ann.CpuUsagePercent; + Worker.MemoryTotalBytes = Ann.MemoryTotalBytes; + Worker.MemoryUsedBytes = Ann.MemoryUsedBytes; + Worker.BytesReceived = Ann.BytesReceived; + Worker.BytesSent = Ann.BytesSent; + Worker.ActionsPending = Ann.ActionsPending; + Worker.ActionsRunning = Ann.ActionsRunning; + Worker.ActionsCompleted = Ann.ActionsCompleted; + Worker.ActiveQueues = Ann.ActiveQueues; + if (!Ann.Provisioner.empty()) + { + Worker.Provisioner = Ann.Provisioner; + } + Worker.LastSeen.Reset(); + }); + + if (!EvictedId.empty()) + { + ZEN_INFO("worker {} superseded by {} (same endpoint)", EvictedId, Ann.Id); + RecordProvisioningEvent(ProvisioningEvent::Type::Left, EvictedId, EvictedHostname); + } + + if (IsNew) + { + RecordProvisioningEvent(ProvisioningEvent::Type::Joined, Ann.Id, Ann.Hostname); + } +} + +bool +OrchestratorService::IsWorkerWebSocketEnabled() const +{ + return m_EnableWorkerWebSocket; +} + +void +OrchestratorService::SetWorkerWebSocketConnected(std::string_view WorkerId, bool Connected) +{ + ReachableState PrevState = ReachableState::Unknown; + std::string WorkerHostname; + + m_KnownWorkersLock.WithExclusiveLock([&] { + auto It = m_KnownWorkers.find(std::string(WorkerId)); + if (It == m_KnownWorkers.end()) + { + return; + } + + PrevState = It->second.Reachable; + WorkerHostname = It->second.Hostname; + It->second.WsConnected = Connected; + It->second.Reachable = Connected ? ReachableState::Reachable : ReachableState::Unreachable; + + if (Connected) + { + ZEN_INFO("worker {} WebSocket connected — marking reachable", WorkerId); + } + else + { + ZEN_WARN("worker {} WebSocket disconnected — marking unreachable", WorkerId); + } + }); + + // Record provisioning events for state transitions outside the lock + if (Connected && PrevState == ReachableState::Unreachable) + { + RecordProvisioningEvent(ProvisioningEvent::Type::Returned, WorkerId, WorkerHostname); + } + else if (!Connected && PrevState == ReachableState::Reachable) + { + RecordProvisioningEvent(ProvisioningEvent::Type::Left, WorkerId, WorkerHostname); + } +} + +CbObject +OrchestratorService::GetWorkerTimeline(std::string_view WorkerId, std::optional From, std::optional To, int Limit) +{ + ZEN_TRACE_CPU("OrchestratorService::GetWorkerTimeline"); + + Ref Timeline = m_TimelineStore->Find(WorkerId); + if (!Timeline) + { + return {}; + } + + std::vector Events; + + if (From || To) + { + DateTime StartTime = From.value_or(DateTime(0)); + DateTime EndTime = To.value_or(DateTime::Now()); + Events = Timeline->QueryTimeline(StartTime, EndTime); + } + else if (Limit > 0) + { + Events = Timeline->QueryRecent(Limit); + } + else + { + Events = Timeline->QueryRecent(); + } + + WorkerTimeline::TimeRange Range = Timeline->GetTimeRange(); + + CbObjectWriter Cbo; + Cbo << "worker_id" << WorkerId; + Cbo << "event_count" << static_cast(Timeline->GetEventCount()); + + if (Range) + { + Cbo.AddDateTime("time_first", Range.First); + Cbo.AddDateTime("time_last", Range.Last); + } + + Cbo.BeginArray("events"); + for (const auto& Evt : Events) + { + Cbo.BeginObject(); + Cbo << "type" << WorkerTimeline::ToString(Evt.Type); + Cbo.AddDateTime("ts", Evt.Timestamp); + + if (Evt.ActionLsn != 0) + { + Cbo << "lsn" << Evt.ActionLsn; + Cbo << "action_id" << Evt.ActionId; + } + + if (Evt.Type == WorkerTimeline::EventType::ActionStateChanged) + { + Cbo << "prev_state" << RunnerAction::ToString(Evt.PreviousState); + Cbo << "state" << RunnerAction::ToString(Evt.ActionState); + } + + if (!Evt.Reason.empty()) + { + Cbo << "reason" << std::string_view(Evt.Reason); + } + + Cbo.EndObject(); + } + Cbo.EndArray(); + + return Cbo.Save(); +} + +CbObject +OrchestratorService::GetAllTimelines(DateTime From, DateTime To) +{ + ZEN_TRACE_CPU("OrchestratorService::GetAllTimelines"); + + DateTime StartTime = From; + DateTime EndTime = To; + + auto AllInfo = m_TimelineStore->GetAllWorkerInfo(); + + CbObjectWriter Cbo; + Cbo.AddDateTime("from", StartTime); + Cbo.AddDateTime("to", EndTime); + + Cbo.BeginArray("workers"); + for (const auto& Info : AllInfo) + { + if (!Info.Range || Info.Range.Last < StartTime || Info.Range.First > EndTime) + { + continue; + } + + Cbo.BeginObject(); + Cbo << "worker_id" << Info.WorkerId; + Cbo.AddDateTime("time_first", Info.Range.First); + Cbo.AddDateTime("time_last", Info.Range.Last); + Cbo.EndObject(); + } + Cbo.EndArray(); + + return Cbo.Save(); +} + +void +OrchestratorService::RecordProvisioningEvent(ProvisioningEvent::Type Type, std::string_view WorkerId, std::string_view Hostname) +{ + ProvisioningEvent Evt{ + .EventType = Type, + .Timestamp = DateTime::Now(), + .WorkerId = std::string(WorkerId), + .Hostname = std::string(Hostname), + }; + + m_ProvisioningLogLock.WithExclusiveLock([&] { + m_ProvisioningLog.push_back(std::move(Evt)); + while (m_ProvisioningLog.size() > kMaxProvisioningEvents) + { + m_ProvisioningLog.pop_front(); + } + }); +} + +CbObject +OrchestratorService::GetProvisioningHistory(int Limit) +{ + ZEN_TRACE_CPU("OrchestratorService::GetProvisioningHistory"); + + if (Limit <= 0) + { + Limit = 100; + } + + CbObjectWriter Cbo; + Cbo.BeginArray("events"); + + m_ProvisioningLogLock.WithSharedLock([&] { + // Return last N events, newest first + int Count = 0; + for (auto It = m_ProvisioningLog.rbegin(); It != m_ProvisioningLog.rend() && Count < Limit; ++It, ++Count) + { + const auto& Evt = *It; + Cbo.BeginObject(); + + switch (Evt.EventType) + { + case ProvisioningEvent::Type::Joined: + Cbo << "type" + << "joined"; + break; + case ProvisioningEvent::Type::Left: + Cbo << "type" + << "left"; + break; + case ProvisioningEvent::Type::Returned: + Cbo << "type" + << "returned"; + break; + } + + Cbo.AddDateTime("ts", Evt.Timestamp); + Cbo << "worker_id" << std::string_view(Evt.WorkerId); + Cbo << "hostname" << std::string_view(Evt.Hostname); + Cbo.EndObject(); + } + }); + + Cbo.EndArray(); + return Cbo.Save(); +} + +std::string +OrchestratorService::AnnounceClient(const ClientAnnouncement& Ann) +{ + ZEN_TRACE_CPU("OrchestratorService::AnnounceClient"); + + std::string ClientId = fmt::format("client-{}", Oid::NewOid().ToString()); + + bool IsNew = false; + + m_KnownClientsLock.WithExclusiveLock([&] { + auto It = m_KnownClients.find(ClientId); + IsNew = (It == m_KnownClients.end()); + + auto& Client = m_KnownClients[ClientId]; + Client.SessionId = Ann.SessionId; + Client.Hostname = Ann.Hostname; + if (!Ann.Address.empty()) + { + Client.Address = Ann.Address; + } + if (Ann.Metadata) + { + Client.Metadata = Ann.Metadata; + } + Client.LastSeen.Reset(); + }); + + if (IsNew) + { + RecordClientEvent(ClientEvent::Type::Connected, ClientId, Ann.Hostname); + } + else + { + RecordClientEvent(ClientEvent::Type::Updated, ClientId, Ann.Hostname); + } + + return ClientId; +} + +bool +OrchestratorService::UpdateClient(std::string_view ClientId, CbObject Metadata) +{ + ZEN_TRACE_CPU("OrchestratorService::UpdateClient"); + + bool Found = false; + + m_KnownClientsLock.WithExclusiveLock([&] { + auto It = m_KnownClients.find(std::string(ClientId)); + if (It != m_KnownClients.end()) + { + Found = true; + if (Metadata) + { + It->second.Metadata = std::move(Metadata); + } + It->second.LastSeen.Reset(); + } + }); + + return Found; +} + +bool +OrchestratorService::CompleteClient(std::string_view ClientId) +{ + ZEN_TRACE_CPU("OrchestratorService::CompleteClient"); + + std::string Hostname; + bool Found = false; + + m_KnownClientsLock.WithExclusiveLock([&] { + auto It = m_KnownClients.find(std::string(ClientId)); + if (It != m_KnownClients.end()) + { + Found = true; + Hostname = It->second.Hostname; + m_KnownClients.erase(It); + } + }); + + if (Found) + { + RecordClientEvent(ClientEvent::Type::Disconnected, ClientId, Hostname); + } + + return Found; +} + +CbObject +OrchestratorService::GetClientList() +{ + ZEN_TRACE_CPU("OrchestratorService::GetClientList"); + CbObjectWriter Cbo; + Cbo.BeginArray("clients"); + + m_KnownClientsLock.WithSharedLock([&] { + for (const auto& [ClientId, Client] : m_KnownClients) + { + Cbo.BeginObject(); + Cbo << "id" << ClientId; + if (Client.SessionId) + { + Cbo << "session_id" << Client.SessionId; + } + Cbo << "hostname" << std::string_view(Client.Hostname); + if (!Client.Address.empty()) + { + Cbo << "address" << std::string_view(Client.Address); + } + Cbo << "dt" << Client.LastSeen.GetElapsedTimeMs(); + if (Client.Metadata) + { + Cbo << "metadata" << Client.Metadata; + } + Cbo.EndObject(); + } + }); + + Cbo.EndArray(); + return Cbo.Save(); +} + +CbObject +OrchestratorService::GetClientHistory(int Limit) +{ + ZEN_TRACE_CPU("OrchestratorService::GetClientHistory"); + + if (Limit <= 0) + { + Limit = 100; + } + + CbObjectWriter Cbo; + Cbo.BeginArray("client_events"); + + m_ClientLogLock.WithSharedLock([&] { + int Count = 0; + for (auto It = m_ClientLog.rbegin(); It != m_ClientLog.rend() && Count < Limit; ++It, ++Count) + { + const auto& Evt = *It; + Cbo.BeginObject(); + + switch (Evt.EventType) + { + case ClientEvent::Type::Connected: + Cbo << "type" + << "connected"; + break; + case ClientEvent::Type::Disconnected: + Cbo << "type" + << "disconnected"; + break; + case ClientEvent::Type::Updated: + Cbo << "type" + << "updated"; + break; + } + + Cbo.AddDateTime("ts", Evt.Timestamp); + Cbo << "client_id" << std::string_view(Evt.ClientId); + Cbo << "hostname" << std::string_view(Evt.Hostname); + Cbo.EndObject(); + } + }); + + Cbo.EndArray(); + return Cbo.Save(); +} + +void +OrchestratorService::RecordClientEvent(ClientEvent::Type Type, std::string_view ClientId, std::string_view Hostname) +{ + ClientEvent Evt{ + .EventType = Type, + .Timestamp = DateTime::Now(), + .ClientId = std::string(ClientId), + .Hostname = std::string(Hostname), + }; + + m_ClientLogLock.WithExclusiveLock([&] { + m_ClientLog.push_back(std::move(Evt)); + while (m_ClientLog.size() > kMaxClientEvents) + { + m_ClientLog.pop_front(); + } + }); +} + +void +OrchestratorService::ProbeThreadFunction() +{ + ZEN_TRACE_CPU("OrchestratorService::ProbeThreadFunction"); + SetCurrentThreadName("orch_probe"); + + bool IsFirstProbe = true; + + do + { + if (!IsFirstProbe) + { + m_ProbeThreadEvent.Wait(5'000); + m_ProbeThreadEvent.Reset(); + } + else + { + IsFirstProbe = false; + } + + if (m_ProbeThreadEnabled == false) + { + return; + } + + m_ProbeThreadEvent.Reset(); + + // Snapshot worker IDs and URIs under shared lock + struct WorkerSnapshot + { + std::string Id; + std::string Uri; + bool WsConnected = false; + }; + std::vector Snapshots; + + m_KnownWorkersLock.WithSharedLock([&] { + Snapshots.reserve(m_KnownWorkers.size()); + for (const auto& [WorkerId, Worker] : m_KnownWorkers) + { + Snapshots.push_back({WorkerId, Worker.BaseUri, Worker.WsConnected}); + } + }); + + // Probe each worker outside the lock + for (const auto& Snap : Snapshots) + { + if (m_ProbeThreadEnabled == false) + { + return; + } + + // Workers with an active WebSocket connection are known-reachable; + // skip the HTTP health probe for them. + if (Snap.WsConnected) + { + continue; + } + + ReachableState NewState = ReachableState::Unreachable; + + try + { + HttpClient Client(Snap.Uri, + {.ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{5000}}); + HttpClient::Response Response = Client.Get("/health/"); + if (Response.IsSuccess()) + { + NewState = ReachableState::Reachable; + } + } + catch (const std::exception& Ex) + { + ZEN_WARN("probe failed for worker {} ({}): {}", Snap.Id, Snap.Uri, Ex.what()); + } + + ReachableState PrevState = ReachableState::Unknown; + std::string WorkerHostname; + + m_KnownWorkersLock.WithExclusiveLock([&] { + auto It = m_KnownWorkers.find(Snap.Id); + if (It != m_KnownWorkers.end()) + { + PrevState = It->second.Reachable; + WorkerHostname = It->second.Hostname; + It->second.Reachable = NewState; + It->second.LastProbed.Reset(); + + if (PrevState != NewState) + { + if (NewState == ReachableState::Reachable && PrevState == ReachableState::Unreachable) + { + ZEN_INFO("worker {} ({}) is reachable again", Snap.Id, Snap.Uri); + } + else if (NewState == ReachableState::Reachable) + { + ZEN_INFO("worker {} ({}) is now reachable", Snap.Id, Snap.Uri); + } + else if (PrevState == ReachableState::Reachable) + { + ZEN_WARN("worker {} ({}) is no longer reachable", Snap.Id, Snap.Uri); + } + else + { + ZEN_WARN("worker {} ({}) is not reachable", Snap.Id, Snap.Uri); + } + } + } + }); + + // Record provisioning events for state transitions outside the lock + if (PrevState != NewState) + { + if (NewState == ReachableState::Unreachable && PrevState == ReachableState::Reachable) + { + RecordProvisioningEvent(ProvisioningEvent::Type::Left, Snap.Id, WorkerHostname); + } + else if (NewState == ReachableState::Reachable && PrevState == ReachableState::Unreachable) + { + RecordProvisioningEvent(ProvisioningEvent::Type::Returned, Snap.Id, WorkerHostname); + } + } + } + + // Sweep expired clients (5-minute timeout) + static constexpr int64_t kClientTimeoutMs = 5 * 60 * 1000; + + struct ExpiredClient + { + std::string Id; + std::string Hostname; + }; + std::vector ExpiredClients; + + m_KnownClientsLock.WithExclusiveLock([&] { + for (auto It = m_KnownClients.begin(); It != m_KnownClients.end();) + { + if (It->second.LastSeen.GetElapsedTimeMs() > kClientTimeoutMs) + { + ExpiredClients.push_back({It->first, It->second.Hostname}); + It = m_KnownClients.erase(It); + } + else + { + ++It; + } + } + }); + + for (const auto& Expired : ExpiredClients) + { + ZEN_INFO("client {} timed out (no announcement for >5 minutes)", Expired.Id); + RecordClientEvent(ClientEvent::Type::Disconnected, Expired.Id, Expired.Hostname); + } + } while (m_ProbeThreadEnabled); +} + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/recording/actionrecorder.cpp b/src/zencompute/recording/actionrecorder.cpp new file mode 100644 index 000000000..90141ca55 --- /dev/null +++ b/src/zencompute/recording/actionrecorder.cpp @@ -0,0 +1,258 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "actionrecorder.h" + +#include "../runners/functionrunner.h" + +#include +#include +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +# define ZEN_CONCRT_AVAILABLE 1 +#else +# define ZEN_CONCRT_AVAILABLE 0 +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen::compute { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +RecordingFileWriter::RecordingFileWriter() +{ +} + +RecordingFileWriter::~RecordingFileWriter() +{ + Close(); +} + +void +RecordingFileWriter::Open(std::filesystem::path FilePath) +{ + using namespace std::literals; + + m_File.Open(FilePath, BasicFile::Mode::kTruncate); + m_File.Write("----DDC2----DATA", 16, 0); + m_FileOffset = 16; + + std::filesystem::path TocPath = FilePath.replace_extension(".ztoc"); + m_TocFile.Open(TocPath, BasicFile::Mode::kTruncate); + + m_TocWriter << "version"sv << 1; + m_TocWriter.BeginArray("toc"sv); +} + +void +RecordingFileWriter::Close() +{ + m_TocWriter.EndArray(); + CbObject Toc = m_TocWriter.Save(); + + std::error_code Ec; + m_TocFile.WriteAll(Toc.GetBuffer().AsIoBuffer(), Ec); +} + +void +RecordingFileWriter::AppendObject(const CbObject& Object, const IoHash& ObjectHash) +{ + RwLock::ExclusiveLockScope _(m_FileLock); + + MemoryView ObjectView = Object.GetBuffer().GetView(); + + std::error_code Ec; + m_File.Write(ObjectView, m_FileOffset, Ec); + + if (Ec) + { + throw std::system_error(Ec, "failed writing to archive"); + } + + m_TocWriter.BeginArray(); + m_TocWriter.AddHash(ObjectHash); + m_TocWriter.AddInteger(m_FileOffset); + m_TocWriter.AddInteger(gsl::narrow(ObjectView.GetSize())); + m_TocWriter.EndArray(); + + m_FileOffset += ObjectView.GetSize(); +} + +////////////////////////////////////////////////////////////////////////// + +ActionRecorder::ActionRecorder(ChunkResolver& InChunkResolver, const std::filesystem::path& RecordingLogPath) +: m_ChunkResolver(InChunkResolver) +, m_RecordingLogDir(RecordingLogPath) +{ + std::error_code Ec; + CreateDirectories(m_RecordingLogDir, Ec); + + if (Ec) + { + ZEN_WARN("Could not create directory '{}': {}", m_RecordingLogDir, Ec.message()); + } + + CleanDirectory(m_RecordingLogDir, /* ForceRemoveReadOnlyFiles */ true, Ec); + + if (Ec) + { + ZEN_WARN("Could not clean directory '{}': {}", m_RecordingLogDir, Ec.message()); + } + + m_WorkersFile.Open(m_RecordingLogDir / "workers.zdat"); + m_ActionsFile.Open(m_RecordingLogDir / "actions.zdat"); + + CidStoreConfiguration CidConfig; + CidConfig.RootDirectory = m_RecordingLogDir / "cid"; + CidConfig.HugeValueThreshold = 128 * 1024 * 1024; + + m_CidStore.Initialize(CidConfig); +} + +ActionRecorder::~ActionRecorder() +{ + Shutdown(); +} + +void +ActionRecorder::Shutdown() +{ + m_CidStore.Flush(); +} + +void +ActionRecorder::RegisterWorker(const CbPackage& WorkerPackage) +{ + const IoHash WorkerId = WorkerPackage.GetObjectHash(); + + m_WorkersFile.AppendObject(WorkerPackage.GetObject(), WorkerId); + + std::unordered_set AddedChunks; + uint64_t AddedBytes = 0; + + // First add all attachments from the worker package itself + + for (const CbAttachment& Attachment : WorkerPackage.GetAttachments()) + { + CompressedBuffer Buffer = Attachment.AsCompressedBinary(); + IoBuffer Data = Buffer.GetCompressed().Flatten().AsIoBuffer(); + + const IoHash ChunkHash = Buffer.DecodeRawHash(); + + CidStore::InsertResult Result = m_CidStore.AddChunk(Data, ChunkHash, CidStore::InsertMode::kCopyOnly); + + AddedChunks.insert(ChunkHash); + + if (Result.New) + { + AddedBytes += Data.GetSize(); + } + } + + // Not all attachments will be present in the worker package, so we need to add + // all referenced chunks to ensure that the recording is self-contained and not + // referencing data in the main CID store + + CbObject WorkerDescriptor = WorkerPackage.GetObject(); + + WorkerDescriptor.IterateAttachments([&](const CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + + if (!AddedChunks.contains(AttachmentCid)) + { + IoBuffer AttachmentData = m_ChunkResolver.FindChunkByCid(AttachmentCid); + + if (AttachmentData) + { + CidStore::InsertResult Result = m_CidStore.AddChunk(AttachmentData, AttachmentCid, CidStore::InsertMode::kCopyOnly); + + if (Result.New) + { + AddedBytes += AttachmentData.GetSize(); + } + } + else + { + ZEN_WARN("RegisterWorker: could not resolve attachment chunk {} for worker {}", AttachmentCid, WorkerId); + } + + AddedChunks.insert(AttachmentCid); + } + }); + + ZEN_INFO("recorded worker {} with {} attachments ({} bytes)", WorkerId, AddedChunks.size(), AddedBytes); +} + +bool +ActionRecorder::RecordAction(Ref Action) +{ + bool AllGood = true; + + Action->ActionObj.IterateAttachments([&](CbFieldView Field) { + IoHash AttachData = Field.AsHash(); + IoBuffer ChunkData = m_ChunkResolver.FindChunkByCid(AttachData); + + if (ChunkData) + { + if (ChunkData.GetContentType() == ZenContentType::kCompressedBinary) + { + IoHash DecompressedHash; + uint64_t RawSize = 0; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer(ChunkData), /* out */ DecompressedHash, /* out*/ RawSize); + + OodleCompressor Compressor; + OodleCompressionLevel CompressionLevel; + uint64_t BlockSize = 0; + if (Compressed.TryGetCompressParameters(/* out */ Compressor, /* out */ CompressionLevel, /* out */ BlockSize)) + { + if (Compressor == OodleCompressor::NotSet) + { + CompositeBuffer Decompressed = Compressed.DecompressToComposite(); + CompressedBuffer NewCompressed = CompressedBuffer::Compress(std::move(Decompressed), + OodleCompressor::Mermaid, + OodleCompressionLevel::Fast, + BlockSize); + + ChunkData = NewCompressed.GetCompressed().Flatten().AsIoBuffer(); + } + } + } + + const uint64_t ChunkSize = ChunkData.GetSize(); + + m_CidStore.AddChunk(ChunkData, AttachData, CidStore::InsertMode::kCopyOnly); + ++m_ChunkCounter; + m_ChunkBytesCounter.fetch_add(ChunkSize); + } + else + { + AllGood = false; + + ZEN_WARN("could not resolve chunk {}", AttachData); + } + }); + + if (AllGood) + { + m_ActionsFile.AppendObject(Action->ActionObj, Action->ActionId); + ++m_ActionsCounter; + + return true; + } + else + { + return false; + } +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/recording/actionrecorder.h b/src/zencompute/recording/actionrecorder.h new file mode 100644 index 000000000..2827b6ac7 --- /dev/null +++ b/src/zencompute/recording/actionrecorder.h @@ -0,0 +1,91 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace zen { +class CbObject; +class CbPackage; +struct IoHash; +} // namespace zen + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen::compute { + +////////////////////////////////////////////////////////////////////////// + +struct RecordingFileWriter +{ + RecordingFileWriter(RecordingFileWriter&&) = delete; + RecordingFileWriter& operator=(RecordingFileWriter&&) = delete; + + RwLock m_FileLock; + BasicFile m_File; + uint64_t m_FileOffset = 0; + CbObjectWriter m_TocWriter; + BasicFile m_TocFile; + + RecordingFileWriter(); + ~RecordingFileWriter(); + + void Open(std::filesystem::path FilePath); + void Close(); + void AppendObject(const CbObject& Object, const IoHash& ObjectHash); +}; + +////////////////////////////////////////////////////////////////////////// + +/** + * Recording "runner" implementation + * + * This class writes out all actions and their attachments to a recording directory + * in a format that can be read back by the RecordingReader. + * + * The contents of the recording directory will be self-contained, with all referenced + * attachments stored in the recording directory itself, so that the recording can be + * moved or shared without needing to maintain references to the main CID store. + * + */ + +class ActionRecorder +{ +public: + ActionRecorder(ChunkResolver& InChunkResolver, const std::filesystem::path& RecordingLogPath); + ~ActionRecorder(); + + ActionRecorder(const ActionRecorder&) = delete; + ActionRecorder& operator=(const ActionRecorder&) = delete; + + void Shutdown(); + void RegisterWorker(const CbPackage& WorkerPackage); + bool RecordAction(Ref Action); + +private: + ChunkResolver& m_ChunkResolver; + std::filesystem::path m_RecordingLogDir; + + RecordingFileWriter m_WorkersFile; + RecordingFileWriter m_ActionsFile; + GcManager m_Gc; + CidStore m_CidStore{m_Gc}; + std::atomic m_ChunkCounter{0}; + std::atomic m_ChunkBytesCounter{0}; + std::atomic m_ActionsCounter{0}; +}; + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/recording/recordingreader.cpp b/src/zencompute/recording/recordingreader.cpp new file mode 100644 index 000000000..1c1a119cf --- /dev/null +++ b/src/zencompute/recording/recordingreader.cpp @@ -0,0 +1,335 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "zencompute/recordingreader.h" + +#include +#include +#include +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +# define ZEN_CONCRT_AVAILABLE 1 +#else +# define ZEN_CONCRT_AVAILABLE 0 +#endif + +#if ZEN_WITH_COMPUTE_SERVICES + +namespace zen::compute { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +# if ZEN_PLATFORM_WINDOWS +# define ZEN_BUILD_ACTION L"Build.action" +# define ZEN_WORKER_UCB L"worker.ucb" +# else +# define ZEN_BUILD_ACTION "Build.action" +# define ZEN_WORKER_UCB "worker.ucb" +# endif + +////////////////////////////////////////////////////////////////////////// + +struct RecordingTreeVisitor : public FileSystemTraversal::TreeVisitor +{ + virtual void VisitFile(const std::filesystem::path& Parent, + const path_view& File, + uint64_t FileSize, + uint32_t NativeModeOrAttributes, + uint64_t NativeModificationTick) + { + ZEN_UNUSED(Parent, File, FileSize, NativeModeOrAttributes, NativeModificationTick); + + if (File.compare(path_view(ZEN_BUILD_ACTION)) == 0) + { + WorkDirs.push_back(Parent); + } + else if (File.compare(path_view(ZEN_WORKER_UCB)) == 0) + { + WorkerDirs.push_back(Parent); + } + } + + virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName, uint32_t NativeModeOrAttributes) + { + ZEN_UNUSED(Parent, DirectoryName, NativeModeOrAttributes); + + return true; + } + + std::vector WorkerDirs; + std::vector WorkDirs; +}; + +////////////////////////////////////////////////////////////////////////// + +void +IterateOverArray(auto Array, auto Func, int TargetParallelism) +{ +# if ZEN_CONCRT_AVAILABLE + if (TargetParallelism > 1) + { + concurrency::simple_partitioner Chunker(Array.size() / TargetParallelism); + concurrency::parallel_for_each(begin(Array), end(Array), [&](const auto& Item) { Func(Item); }); + + return; + } +# else + ZEN_UNUSED(TargetParallelism); +# endif + + for (const auto& Item : Array) + { + Func(Item); + } +} + +////////////////////////////////////////////////////////////////////////// + +RecordingReaderBase::~RecordingReaderBase() = default; + +////////////////////////////////////////////////////////////////////////// + +RecordingReader::RecordingReader(const std::filesystem::path& RecordingPath) : m_RecordingLogDir(RecordingPath) +{ + CidStoreConfiguration CidConfig; + CidConfig.RootDirectory = m_RecordingLogDir / "cid"; + CidConfig.HugeValueThreshold = 128 * 1024 * 1024; + + m_CidStore.Initialize(CidConfig); +} + +RecordingReader::~RecordingReader() +{ + m_CidStore.Flush(); +} + +size_t +RecordingReader::GetActionCount() const +{ + return m_Actions.size(); +} + +IoBuffer +RecordingReader::FindChunkByCid(const IoHash& DecompressedId) +{ + if (IoBuffer Chunk = m_CidStore.FindChunkByCid(DecompressedId)) + { + return Chunk; + } + + ZEN_ERROR("failed lookup of chunk with CID '{}'", DecompressedId); + + return {}; +} + +std::unordered_map +RecordingReader::ReadWorkers() +{ + std::unordered_map WorkerMap; + + { + CbObjectFromFile TocFile = LoadCompactBinaryObject(m_RecordingLogDir / "workers.ztoc"); + CbObject Toc = TocFile.Object; + + m_WorkerDataFile.Open(m_RecordingLogDir / "workers.zdat", BasicFile::Mode::kRead); + + ZEN_ASSERT(Toc["version"sv].AsInt32() == 1); + + for (auto& It : Toc["toc"]) + { + CbArrayView Entry = It.AsArrayView(); + CbFieldViewIterator Vit = Entry.CreateViewIterator(); + + const IoHash WorkerId = Vit++->AsHash(); + const uint64_t Offset = Vit++->AsInt64(0); + const uint64_t Size = Vit++->AsInt64(0); + + IoBuffer WorkerRange = m_WorkerDataFile.ReadRange(Offset, Size); + CbObject WorkerDesc = LoadCompactBinaryObject(WorkerRange); + CbPackage& WorkerPkg = WorkerMap[WorkerId]; + WorkerPkg.SetObject(WorkerDesc); + + WorkerDesc.IterateAttachments([&](const zen::CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + IoBuffer AttachmentData = m_CidStore.FindChunkByCid(AttachmentCid); + + if (AttachmentData) + { + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); + WorkerPkg.AddAttachment(CbAttachment(CompressedData, RawHash)); + } + }); + } + } + + // Scan actions as well (this should be called separately, ideally) + + ScanActions(); + + return WorkerMap; +} + +void +RecordingReader::ScanActions() +{ + CbObjectFromFile TocFile = LoadCompactBinaryObject(m_RecordingLogDir / "actions.ztoc"); + CbObject Toc = TocFile.Object; + + m_ActionDataFile.Open(m_RecordingLogDir / "actions.zdat", BasicFile::Mode::kRead); + + ZEN_ASSERT(Toc["version"sv].AsInt32() == 1); + + for (auto& It : Toc["toc"]) + { + CbArrayView ArrayEntry = It.AsArrayView(); + CbFieldViewIterator Vit = ArrayEntry.CreateViewIterator(); + + ActionEntry Entry; + Entry.ActionId = Vit++->AsHash(); + Entry.Offset = Vit++->AsInt64(0); + Entry.Size = Vit++->AsInt64(0); + + m_Actions.push_back(Entry); + } +} + +void +RecordingReader::IterateActions(std::function&& Callback, int TargetParallelism) +{ + IterateOverArray( + m_Actions, + [&](const ActionEntry& Entry) { + CbObject ActionDesc = LoadCompactBinaryObject(m_ActionDataFile.ReadRange(Entry.Offset, Entry.Size)); + + Callback(ActionDesc, Entry.ActionId); + }, + TargetParallelism); +} + +////////////////////////////////////////////////////////////////////////// + +IoBuffer +LocalResolver::FindChunkByCid(const IoHash& DecompressedId) +{ + RwLock::SharedLockScope _(MapLock); + if (auto It = Attachments.find(DecompressedId); It != Attachments.end()) + { + return It->second; + } + + return {}; +} + +void +LocalResolver::Add(const IoHash& Cid, IoBuffer Data) +{ + RwLock::ExclusiveLockScope _(MapLock); + Data.SetContentType(ZenContentType::kCompressedBinary); + Attachments[Cid] = Data; +} + +/// + +UeRecordingReader::UeRecordingReader(const std::filesystem::path& RecordingPath) : m_RecordingDir(RecordingPath) +{ +} + +UeRecordingReader::~UeRecordingReader() +{ +} + +size_t +UeRecordingReader::GetActionCount() const +{ + return m_WorkDirs.size(); +} + +IoBuffer +UeRecordingReader::FindChunkByCid(const IoHash& DecompressedId) +{ + return m_LocalResolver.FindChunkByCid(DecompressedId); +} + +std::unordered_map +UeRecordingReader::ReadWorkers() +{ + std::unordered_map WorkerMap; + + FileSystemTraversal Traversal; + RecordingTreeVisitor Visitor; + Traversal.TraverseFileSystem(m_RecordingDir, Visitor); + + m_WorkDirs = std::move(Visitor.WorkDirs); + + for (const std::filesystem::path& WorkerDir : Visitor.WorkerDirs) + { + CbObjectFromFile WorkerFile = LoadCompactBinaryObject(WorkerDir / "worker.ucb"); + CbObject WorkerDesc = WorkerFile.Object; + const IoHash& WorkerId = WorkerFile.Hash; + CbPackage& WorkerPkg = WorkerMap[WorkerId]; + WorkerPkg.SetObject(WorkerDesc); + + WorkerDesc.IterateAttachments([&](const zen::CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + IoBuffer AttachmentData = ReadFile(WorkerDir / "chunks" / AttachmentCid.ToHexString()).Flatten(); + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); + WorkerPkg.AddAttachment(CbAttachment(CompressedData, RawHash)); + }); + } + + return WorkerMap; +} + +void +UeRecordingReader::IterateActions(std::function&& Callback, int ParallelismTarget) +{ + IterateOverArray( + m_WorkDirs, + [&](const std::filesystem::path& WorkDir) { + CbPackage WorkPackage = ReadAction(WorkDir); + CbObject ActionObject = WorkPackage.GetObject(); + const IoHash& ActionId = WorkPackage.GetObjectHash(); + + Callback(ActionObject, ActionId); + }, + ParallelismTarget); +} + +CbPackage +UeRecordingReader::ReadAction(std::filesystem::path WorkDir) +{ + CbPackage WorkPackage; + std::filesystem::path WorkDescPath = WorkDir / "Build.action"; + CbObjectFromFile ActionFile = LoadCompactBinaryObject(WorkDescPath); + CbObject& ActionObject = ActionFile.Object; + + WorkPackage.SetObject(ActionObject); + + ActionObject.IterateAttachments([&](const zen::CbFieldView AttachmentField) { + const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); + IoBuffer AttachmentData = ReadFile(WorkDir / "inputs" / AttachmentCid.ToHexString()).Flatten(); + + m_LocalResolver.Add(AttachmentCid, AttachmentData); + + IoHash RawHash; + uint64_t RawSize = 0; + CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); + ZEN_ASSERT(AttachmentCid == RawHash); + WorkPackage.AddAttachment(CbAttachment(CompressedData, RawHash)); + }); + + return WorkPackage; +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/recordingreader.cpp b/src/zencompute/recordingreader.cpp deleted file mode 100644 index 1c1a119cf..000000000 --- a/src/zencompute/recordingreader.cpp +++ /dev/null @@ -1,335 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "zencompute/recordingreader.h" - -#include -#include -#include -#include -#include -#include - -#if ZEN_PLATFORM_WINDOWS -# include -# define ZEN_CONCRT_AVAILABLE 1 -#else -# define ZEN_CONCRT_AVAILABLE 0 -#endif - -#if ZEN_WITH_COMPUTE_SERVICES - -namespace zen::compute { - -using namespace std::literals; - -////////////////////////////////////////////////////////////////////////// - -# if ZEN_PLATFORM_WINDOWS -# define ZEN_BUILD_ACTION L"Build.action" -# define ZEN_WORKER_UCB L"worker.ucb" -# else -# define ZEN_BUILD_ACTION "Build.action" -# define ZEN_WORKER_UCB "worker.ucb" -# endif - -////////////////////////////////////////////////////////////////////////// - -struct RecordingTreeVisitor : public FileSystemTraversal::TreeVisitor -{ - virtual void VisitFile(const std::filesystem::path& Parent, - const path_view& File, - uint64_t FileSize, - uint32_t NativeModeOrAttributes, - uint64_t NativeModificationTick) - { - ZEN_UNUSED(Parent, File, FileSize, NativeModeOrAttributes, NativeModificationTick); - - if (File.compare(path_view(ZEN_BUILD_ACTION)) == 0) - { - WorkDirs.push_back(Parent); - } - else if (File.compare(path_view(ZEN_WORKER_UCB)) == 0) - { - WorkerDirs.push_back(Parent); - } - } - - virtual bool VisitDirectory(const std::filesystem::path& Parent, const path_view& DirectoryName, uint32_t NativeModeOrAttributes) - { - ZEN_UNUSED(Parent, DirectoryName, NativeModeOrAttributes); - - return true; - } - - std::vector WorkerDirs; - std::vector WorkDirs; -}; - -////////////////////////////////////////////////////////////////////////// - -void -IterateOverArray(auto Array, auto Func, int TargetParallelism) -{ -# if ZEN_CONCRT_AVAILABLE - if (TargetParallelism > 1) - { - concurrency::simple_partitioner Chunker(Array.size() / TargetParallelism); - concurrency::parallel_for_each(begin(Array), end(Array), [&](const auto& Item) { Func(Item); }); - - return; - } -# else - ZEN_UNUSED(TargetParallelism); -# endif - - for (const auto& Item : Array) - { - Func(Item); - } -} - -////////////////////////////////////////////////////////////////////////// - -RecordingReaderBase::~RecordingReaderBase() = default; - -////////////////////////////////////////////////////////////////////////// - -RecordingReader::RecordingReader(const std::filesystem::path& RecordingPath) : m_RecordingLogDir(RecordingPath) -{ - CidStoreConfiguration CidConfig; - CidConfig.RootDirectory = m_RecordingLogDir / "cid"; - CidConfig.HugeValueThreshold = 128 * 1024 * 1024; - - m_CidStore.Initialize(CidConfig); -} - -RecordingReader::~RecordingReader() -{ - m_CidStore.Flush(); -} - -size_t -RecordingReader::GetActionCount() const -{ - return m_Actions.size(); -} - -IoBuffer -RecordingReader::FindChunkByCid(const IoHash& DecompressedId) -{ - if (IoBuffer Chunk = m_CidStore.FindChunkByCid(DecompressedId)) - { - return Chunk; - } - - ZEN_ERROR("failed lookup of chunk with CID '{}'", DecompressedId); - - return {}; -} - -std::unordered_map -RecordingReader::ReadWorkers() -{ - std::unordered_map WorkerMap; - - { - CbObjectFromFile TocFile = LoadCompactBinaryObject(m_RecordingLogDir / "workers.ztoc"); - CbObject Toc = TocFile.Object; - - m_WorkerDataFile.Open(m_RecordingLogDir / "workers.zdat", BasicFile::Mode::kRead); - - ZEN_ASSERT(Toc["version"sv].AsInt32() == 1); - - for (auto& It : Toc["toc"]) - { - CbArrayView Entry = It.AsArrayView(); - CbFieldViewIterator Vit = Entry.CreateViewIterator(); - - const IoHash WorkerId = Vit++->AsHash(); - const uint64_t Offset = Vit++->AsInt64(0); - const uint64_t Size = Vit++->AsInt64(0); - - IoBuffer WorkerRange = m_WorkerDataFile.ReadRange(Offset, Size); - CbObject WorkerDesc = LoadCompactBinaryObject(WorkerRange); - CbPackage& WorkerPkg = WorkerMap[WorkerId]; - WorkerPkg.SetObject(WorkerDesc); - - WorkerDesc.IterateAttachments([&](const zen::CbFieldView AttachmentField) { - const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); - IoBuffer AttachmentData = m_CidStore.FindChunkByCid(AttachmentCid); - - if (AttachmentData) - { - IoHash RawHash; - uint64_t RawSize = 0; - CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); - WorkerPkg.AddAttachment(CbAttachment(CompressedData, RawHash)); - } - }); - } - } - - // Scan actions as well (this should be called separately, ideally) - - ScanActions(); - - return WorkerMap; -} - -void -RecordingReader::ScanActions() -{ - CbObjectFromFile TocFile = LoadCompactBinaryObject(m_RecordingLogDir / "actions.ztoc"); - CbObject Toc = TocFile.Object; - - m_ActionDataFile.Open(m_RecordingLogDir / "actions.zdat", BasicFile::Mode::kRead); - - ZEN_ASSERT(Toc["version"sv].AsInt32() == 1); - - for (auto& It : Toc["toc"]) - { - CbArrayView ArrayEntry = It.AsArrayView(); - CbFieldViewIterator Vit = ArrayEntry.CreateViewIterator(); - - ActionEntry Entry; - Entry.ActionId = Vit++->AsHash(); - Entry.Offset = Vit++->AsInt64(0); - Entry.Size = Vit++->AsInt64(0); - - m_Actions.push_back(Entry); - } -} - -void -RecordingReader::IterateActions(std::function&& Callback, int TargetParallelism) -{ - IterateOverArray( - m_Actions, - [&](const ActionEntry& Entry) { - CbObject ActionDesc = LoadCompactBinaryObject(m_ActionDataFile.ReadRange(Entry.Offset, Entry.Size)); - - Callback(ActionDesc, Entry.ActionId); - }, - TargetParallelism); -} - -////////////////////////////////////////////////////////////////////////// - -IoBuffer -LocalResolver::FindChunkByCid(const IoHash& DecompressedId) -{ - RwLock::SharedLockScope _(MapLock); - if (auto It = Attachments.find(DecompressedId); It != Attachments.end()) - { - return It->second; - } - - return {}; -} - -void -LocalResolver::Add(const IoHash& Cid, IoBuffer Data) -{ - RwLock::ExclusiveLockScope _(MapLock); - Data.SetContentType(ZenContentType::kCompressedBinary); - Attachments[Cid] = Data; -} - -/// - -UeRecordingReader::UeRecordingReader(const std::filesystem::path& RecordingPath) : m_RecordingDir(RecordingPath) -{ -} - -UeRecordingReader::~UeRecordingReader() -{ -} - -size_t -UeRecordingReader::GetActionCount() const -{ - return m_WorkDirs.size(); -} - -IoBuffer -UeRecordingReader::FindChunkByCid(const IoHash& DecompressedId) -{ - return m_LocalResolver.FindChunkByCid(DecompressedId); -} - -std::unordered_map -UeRecordingReader::ReadWorkers() -{ - std::unordered_map WorkerMap; - - FileSystemTraversal Traversal; - RecordingTreeVisitor Visitor; - Traversal.TraverseFileSystem(m_RecordingDir, Visitor); - - m_WorkDirs = std::move(Visitor.WorkDirs); - - for (const std::filesystem::path& WorkerDir : Visitor.WorkerDirs) - { - CbObjectFromFile WorkerFile = LoadCompactBinaryObject(WorkerDir / "worker.ucb"); - CbObject WorkerDesc = WorkerFile.Object; - const IoHash& WorkerId = WorkerFile.Hash; - CbPackage& WorkerPkg = WorkerMap[WorkerId]; - WorkerPkg.SetObject(WorkerDesc); - - WorkerDesc.IterateAttachments([&](const zen::CbFieldView AttachmentField) { - const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); - IoBuffer AttachmentData = ReadFile(WorkerDir / "chunks" / AttachmentCid.ToHexString()).Flatten(); - IoHash RawHash; - uint64_t RawSize = 0; - CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); - WorkerPkg.AddAttachment(CbAttachment(CompressedData, RawHash)); - }); - } - - return WorkerMap; -} - -void -UeRecordingReader::IterateActions(std::function&& Callback, int ParallelismTarget) -{ - IterateOverArray( - m_WorkDirs, - [&](const std::filesystem::path& WorkDir) { - CbPackage WorkPackage = ReadAction(WorkDir); - CbObject ActionObject = WorkPackage.GetObject(); - const IoHash& ActionId = WorkPackage.GetObjectHash(); - - Callback(ActionObject, ActionId); - }, - ParallelismTarget); -} - -CbPackage -UeRecordingReader::ReadAction(std::filesystem::path WorkDir) -{ - CbPackage WorkPackage; - std::filesystem::path WorkDescPath = WorkDir / "Build.action"; - CbObjectFromFile ActionFile = LoadCompactBinaryObject(WorkDescPath); - CbObject& ActionObject = ActionFile.Object; - - WorkPackage.SetObject(ActionObject); - - ActionObject.IterateAttachments([&](const zen::CbFieldView AttachmentField) { - const IoHash AttachmentCid = AttachmentField.GetValue().AsHash(); - IoBuffer AttachmentData = ReadFile(WorkDir / "inputs" / AttachmentCid.ToHexString()).Flatten(); - - m_LocalResolver.Add(AttachmentCid, AttachmentData); - - IoHash RawHash; - uint64_t RawSize = 0; - CompressedBuffer CompressedData = CompressedBuffer::FromCompressed(SharedBuffer(AttachmentData), RawHash, RawSize); - ZEN_ASSERT(AttachmentCid == RawHash); - WorkPackage.AddAttachment(CbAttachment(CompressedData, RawHash)); - }); - - return WorkPackage; -} - -} // namespace zen::compute - -#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/remotehttprunner.cpp b/src/zencompute/remotehttprunner.cpp deleted file mode 100644 index 98ced5fe8..000000000 --- a/src/zencompute/remotehttprunner.cpp +++ /dev/null @@ -1,457 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "remotehttprunner.h" - -#if ZEN_WITH_COMPUTE_SERVICES - -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include -# include - -# include - -////////////////////////////////////////////////////////////////////////// - -namespace zen::compute { - -using namespace std::literals; - -////////////////////////////////////////////////////////////////////////// - -RemoteHttpRunner::RemoteHttpRunner(ChunkResolver& InChunkResolver, const std::filesystem::path& BaseDir, std::string_view HostName) -: FunctionRunner(BaseDir) -, m_Log(logging::Get("http_exec")) -, m_ChunkResolver{InChunkResolver} -, m_BaseUrl{fmt::format("{}/apply", HostName)} -, m_Http(m_BaseUrl) -{ - m_MonitorThread = std::thread{&RemoteHttpRunner::MonitorThreadFunction, this}; -} - -RemoteHttpRunner::~RemoteHttpRunner() -{ - Shutdown(); -} - -void -RemoteHttpRunner::Shutdown() -{ - // TODO: should cleanly drain/cancel pending work - - m_MonitorThreadEnabled = false; - m_MonitorThreadEvent.Set(); - if (m_MonitorThread.joinable()) - { - m_MonitorThread.join(); - } -} - -void -RemoteHttpRunner::RegisterWorker(const CbPackage& WorkerPackage) -{ - const IoHash WorkerId = WorkerPackage.GetObjectHash(); - CbPackage WorkerDesc = WorkerPackage; - - std::string WorkerUrl = fmt::format("/workers/{}", WorkerId); - - HttpClient::Response WorkerResponse = m_Http.Get(WorkerUrl); - - if (WorkerResponse.StatusCode == HttpResponseCode::NotFound) - { - HttpClient::Response DescResponse = m_Http.Post(WorkerUrl, WorkerDesc.GetObject()); - - if (DescResponse.StatusCode == HttpResponseCode::NotFound) - { - CbPackage Pkg = WorkerDesc; - - // Build response package by sending only the attachments - // the other end needs. We start with the full package and - // remove the attachments which are not needed. - - { - std::unordered_set Needed; - - CbObject Response = DescResponse.AsObject(); - - for (auto& Item : Response["need"sv]) - { - const IoHash NeedHash = Item.AsHash(); - - Needed.insert(NeedHash); - } - - std::unordered_set ToRemove; - - for (const CbAttachment& Attachment : Pkg.GetAttachments()) - { - const IoHash& Hash = Attachment.GetHash(); - - if (Needed.find(Hash) == Needed.end()) - { - ToRemove.insert(Hash); - } - } - - for (const IoHash& Hash : ToRemove) - { - int RemovedCount = Pkg.RemoveAttachment(Hash); - - ZEN_ASSERT(RemovedCount == 1); - } - } - - // Post resulting package - - HttpClient::Response PayloadResponse = m_Http.Post(WorkerUrl, Pkg); - - if (!IsHttpSuccessCode(PayloadResponse.StatusCode)) - { - ZEN_ERROR("ERROR: unable to register payloads for worker {} at {}{}", WorkerId, m_Http.GetBaseUri(), WorkerUrl); - - // TODO: propagate error - } - } - else if (!IsHttpSuccessCode(DescResponse.StatusCode)) - { - ZEN_ERROR("ERROR: unable to register worker {} at {}{}", WorkerId, m_Http.GetBaseUri(), WorkerUrl); - - // TODO: propagate error - } - else - { - ZEN_ASSERT(DescResponse.StatusCode == HttpResponseCode::NoContent); - } - } - else if (WorkerResponse.StatusCode == HttpResponseCode::OK) - { - // Already known from a previous run - } - else if (!IsHttpSuccessCode(WorkerResponse.StatusCode)) - { - ZEN_ERROR("ERROR: unable to look up worker {} at {}{} (error: {} {})", - WorkerId, - m_Http.GetBaseUri(), - WorkerUrl, - (int)WorkerResponse.StatusCode, - ToString(WorkerResponse.StatusCode)); - - // TODO: propagate error - } -} - -size_t -RemoteHttpRunner::QueryCapacity() -{ - // Estimate how much more work we're ready to accept - - RwLock::SharedLockScope _{m_RunningLock}; - - size_t RunningCount = m_RemoteRunningMap.size(); - - if (RunningCount >= size_t(m_MaxRunningActions)) - { - return 0; - } - - return m_MaxRunningActions - RunningCount; -} - -std::vector -RemoteHttpRunner::SubmitActions(const std::vector>& Actions) -{ - std::vector Results; - - for (const Ref& Action : Actions) - { - Results.push_back(SubmitAction(Action)); - } - - return Results; -} - -SubmitResult -RemoteHttpRunner::SubmitAction(Ref Action) -{ - // Verify whether we can accept more work - - { - RwLock::SharedLockScope _{m_RunningLock}; - if (m_RemoteRunningMap.size() >= size_t(m_MaxRunningActions)) - { - return SubmitResult{.IsAccepted = false}; - } - } - - using namespace std::literals; - - // Each enqueued action is assigned an integer index (logical sequence number), - // which we use as a key for tracking data structures and as an opaque id which - // may be used by clients to reference the scheduled action - - const int32_t ActionLsn = Action->ActionLsn; - const CbObject& ActionObj = Action->ActionObj; - const IoHash ActionId = ActionObj.GetHash(); - - MaybeDumpAction(ActionLsn, ActionObj); - - // Enqueue job - - CbObject Result; - - HttpClient::Response WorkResponse = m_Http.Post("/jobs", ActionObj); - HttpResponseCode WorkResponseCode = WorkResponse.StatusCode; - - if (WorkResponseCode == HttpResponseCode::OK) - { - Result = WorkResponse.AsObject(); - } - else if (WorkResponseCode == HttpResponseCode::NotFound) - { - // Not all attachments are present - - // Build response package including all required attachments - - CbPackage Pkg; - Pkg.SetObject(ActionObj); - - CbObject Response = WorkResponse.AsObject(); - - for (auto& Item : Response["need"sv]) - { - const IoHash NeedHash = Item.AsHash(); - - if (IoBuffer Chunk = m_ChunkResolver.FindChunkByCid(NeedHash)) - { - uint64_t DataRawSize = 0; - IoHash DataRawHash; - CompressedBuffer Compressed = - CompressedBuffer::FromCompressed(SharedBuffer{Chunk}, /* out */ DataRawHash, /* out */ DataRawSize); - - ZEN_ASSERT(DataRawHash == NeedHash); - - Pkg.AddAttachment(CbAttachment(Compressed, NeedHash)); - } - else - { - // No such attachment - - return {.IsAccepted = false, .Reason = fmt::format("missing attachment {}", NeedHash)}; - } - } - - // Post resulting package - - HttpClient::Response PayloadResponse = m_Http.Post("/jobs", Pkg); - - if (!PayloadResponse) - { - ZEN_WARN("unable to register payloads for action {} at {}/jobs", ActionId, m_Http.GetBaseUri()); - - // TODO: include more information about the failure in the response - - return {.IsAccepted = false, .Reason = "HTTP request failed"}; - } - else if (PayloadResponse.StatusCode == HttpResponseCode::OK) - { - Result = PayloadResponse.AsObject(); - } - else - { - // Unexpected response - - const int ResponseStatusCode = (int)PayloadResponse.StatusCode; - - ZEN_WARN("unable to register payloads for action {} at {}/jobs (error: {} {})", - ActionId, - m_Http.GetBaseUri(), - ResponseStatusCode, - ToString(ResponseStatusCode)); - - return {.IsAccepted = false, - .Reason = fmt::format("unexpected response code {} {} from {}/jobs", - ResponseStatusCode, - ToString(ResponseStatusCode), - m_Http.GetBaseUri())}; - } - } - - if (Result) - { - if (const int32_t LsnField = Result["lsn"].AsInt32(0)) - { - HttpRunningAction NewAction; - NewAction.Action = Action; - NewAction.RemoteActionLsn = LsnField; - - { - RwLock::ExclusiveLockScope _(m_RunningLock); - - m_RemoteRunningMap[LsnField] = std::move(NewAction); - } - - ZEN_DEBUG("scheduled action {} with remote LSN {} (local LSN {})", ActionId, LsnField, ActionLsn); - - Action->SetActionState(RunnerAction::State::Running); - - return SubmitResult{.IsAccepted = true}; - } - } - - return {}; -} - -bool -RemoteHttpRunner::IsHealthy() -{ - if (HttpClient::Response Ready = m_Http.Get("/ready")) - { - return true; - } - else - { - // TODO: use response to propagate context - return false; - } -} - -size_t -RemoteHttpRunner::GetSubmittedActionCount() -{ - RwLock::SharedLockScope _(m_RunningLock); - return m_RemoteRunningMap.size(); -} - -void -RemoteHttpRunner::MonitorThreadFunction() -{ - SetCurrentThreadName("RemoteHttpRunner_Monitor"); - - do - { - const int NormalWaitingTime = 1000; - int WaitTimeMs = NormalWaitingTime; - auto WaitOnce = [&] { return m_MonitorThreadEvent.Wait(WaitTimeMs); }; - auto SweepOnce = [&] { - const size_t RetiredCount = SweepRunningActions(); - - m_RunningLock.WithSharedLock([&] { - if (m_RemoteRunningMap.size() > 16) - { - WaitTimeMs = NormalWaitingTime / 4; - } - else - { - if (RetiredCount) - { - WaitTimeMs = NormalWaitingTime / 2; - } - else - { - WaitTimeMs = NormalWaitingTime; - } - } - }); - }; - - while (!WaitOnce()) - { - SweepOnce(); - } - - // Signal received - this may mean we should quit - - SweepOnce(); - } while (m_MonitorThreadEnabled); -} - -size_t -RemoteHttpRunner::SweepRunningActions() -{ - std::vector CompletedActions; - - // Poll remote for list of completed actions - - HttpClient::Response ResponseCompleted = m_Http.Get("/jobs/completed"sv); - - if (CbObject Completed = ResponseCompleted.AsObject()) - { - for (auto& FieldIt : Completed["completed"sv]) - { - const int32_t CompleteLsn = FieldIt.AsInt32(); - - if (HttpClient::Response ResponseJob = m_Http.Get(fmt::format("/jobs/{}"sv, CompleteLsn))) - { - m_RunningLock.WithExclusiveLock([&] { - if (auto CompleteIt = m_RemoteRunningMap.find(CompleteLsn); CompleteIt != m_RemoteRunningMap.end()) - { - HttpRunningAction CompletedAction = std::move(CompleteIt->second); - CompletedAction.ActionResults = ResponseJob.AsPackage(); - CompletedAction.Success = true; - - CompletedActions.push_back(std::move(CompletedAction)); - m_RemoteRunningMap.erase(CompleteIt); - } - else - { - // we received a completion notice for an action we don't know about, - // this can happen if the runner is used by multiple upstream schedulers, - // or if this compute node was recently restarted and lost track of - // previously scheduled actions - } - }); - } - } - - if (CbObjectView Metrics = Completed["metrics"sv].AsObjectView()) - { - // if (const size_t CpuCount = Metrics["core_count"].AsInt32(0)) - if (const int32_t CpuCount = Metrics["lp_count"].AsInt32(0)) - { - const int32_t NewCap = zen::Max(4, CpuCount); - - if (m_MaxRunningActions > NewCap) - { - ZEN_DEBUG("capping {} to {} actions (was {})", m_BaseUrl, NewCap, m_MaxRunningActions); - - m_MaxRunningActions = NewCap; - } - } - } - } - - // Notify outer. Note that this has to be done without holding any local locks - // otherwise we may end up with deadlocks. - - for (HttpRunningAction& HttpAction : CompletedActions) - { - const int ActionLsn = HttpAction.Action->ActionLsn; - - if (HttpAction.Success) - { - ZEN_DEBUG("completed: {} LSN {} (remote LSN {})", HttpAction.Action->ActionId, ActionLsn, HttpAction.RemoteActionLsn); - - HttpAction.Action->SetActionState(RunnerAction::State::Completed); - - HttpAction.Action->SetResult(std::move(HttpAction.ActionResults)); - } - else - { - HttpAction.Action->SetActionState(RunnerAction::State::Failed); - } - } - - return CompletedActions.size(); -} - -} // namespace zen::compute - -#endif diff --git a/src/zencompute/remotehttprunner.h b/src/zencompute/remotehttprunner.h deleted file mode 100644 index 1e885da3d..000000000 --- a/src/zencompute/remotehttprunner.h +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include "zencompute/functionservice.h" - -#if ZEN_WITH_COMPUTE_SERVICES - -# include "functionrunner.h" - -# include -# include -# include -# include - -# include -# include -# include - -namespace zen { -class CidStore; -} - -namespace zen::compute { - -/** HTTP-based runner - - This implements a DDC remote compute execution strategy via REST API - - */ - -class RemoteHttpRunner : public FunctionRunner -{ - RemoteHttpRunner(RemoteHttpRunner&&) = delete; - RemoteHttpRunner& operator=(RemoteHttpRunner&&) = delete; - -public: - RemoteHttpRunner(ChunkResolver& InChunkResolver, const std::filesystem::path& BaseDir, std::string_view HostName); - ~RemoteHttpRunner(); - - virtual void Shutdown() override; - virtual void RegisterWorker(const CbPackage& WorkerPackage) override; - [[nodiscard]] virtual SubmitResult SubmitAction(Ref Action) override; - [[nodiscard]] virtual bool IsHealthy() override; - [[nodiscard]] virtual size_t GetSubmittedActionCount() override; - [[nodiscard]] virtual size_t QueryCapacity() override; - [[nodiscard]] virtual std::vector SubmitActions(const std::vector>& Actions) override; - -protected: - LoggerRef Log() { return m_Log; } - -private: - LoggerRef m_Log; - ChunkResolver& m_ChunkResolver; - std::string m_BaseUrl; - HttpClient m_Http; - - int32_t m_MaxRunningActions = 256; // arbitrary limit for testing - - struct HttpRunningAction - { - Ref Action; - int RemoteActionLsn = 0; // Remote LSN - bool Success = false; - CbPackage ActionResults; - }; - - RwLock m_RunningLock; - std::unordered_map m_RemoteRunningMap; // Note that this is keyed on the *REMOTE* lsn - - std::thread m_MonitorThread; - std::atomic m_MonitorThreadEnabled{true}; - Event m_MonitorThreadEvent; - void MonitorThreadFunction(); - size_t SweepRunningActions(); -}; - -} // namespace zen::compute - -#endif diff --git a/src/zencompute/runners/deferreddeleter.cpp b/src/zencompute/runners/deferreddeleter.cpp new file mode 100644 index 000000000..00977d9fa --- /dev/null +++ b/src/zencompute/runners/deferreddeleter.cpp @@ -0,0 +1,336 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "deferreddeleter.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include + +# include +# include + +namespace zen::compute { + +using namespace std::chrono_literals; + +using Clock = std::chrono::steady_clock; + +// Default deferral: how long to wait before attempting deletion. +// This gives memory-mapped file handles time to close naturally. +static constexpr auto DeferralPeriod = 60s; + +// Shortened deferral after MarkReady(): the client has collected results +// so handles should be released soon, but we still wait briefly. +static constexpr auto ReadyGracePeriod = 5s; + +// Interval between retry attempts for directories that failed deletion. +static constexpr auto RetryInterval = 5s; + +static constexpr int MaxRetries = 10; + +DeferredDirectoryDeleter::DeferredDirectoryDeleter() : m_Thread(&DeferredDirectoryDeleter::ThreadFunction, this) +{ +} + +DeferredDirectoryDeleter::~DeferredDirectoryDeleter() +{ + Shutdown(); +} + +void +DeferredDirectoryDeleter::Enqueue(int ActionLsn, std::filesystem::path Path) +{ + { + std::lock_guard Lock(m_Mutex); + m_Queue.push_back({ActionLsn, std::move(Path)}); + } + m_Cv.notify_one(); +} + +void +DeferredDirectoryDeleter::MarkReady(int ActionLsn) +{ + { + std::lock_guard Lock(m_Mutex); + m_ReadyLsns.push_back(ActionLsn); + } + m_Cv.notify_one(); +} + +void +DeferredDirectoryDeleter::Shutdown() +{ + { + std::lock_guard Lock(m_Mutex); + m_Done = true; + } + m_Cv.notify_one(); + + if (m_Thread.joinable()) + { + m_Thread.join(); + } +} + +void +DeferredDirectoryDeleter::ThreadFunction() +{ + SetCurrentThreadName("ZenDirCleanup"); + + struct PendingEntry + { + int ActionLsn; + std::filesystem::path Path; + Clock::time_point ReadyTime; + int Attempts = 0; + }; + + std::vector PendingList; + + auto TryDelete = [](PendingEntry& Entry) -> bool { + std::error_code Ec; + std::filesystem::remove_all(Entry.Path, Ec); + return !Ec; + }; + + for (;;) + { + bool Shutting = false; + + // Drain the incoming queue and process MarkReady signals + + { + std::unique_lock Lock(m_Mutex); + + if (m_Queue.empty() && m_ReadyLsns.empty() && !m_Done) + { + if (PendingList.empty()) + { + m_Cv.wait(Lock, [this] { return !m_Queue.empty() || !m_ReadyLsns.empty() || m_Done; }); + } + else + { + auto NextReady = PendingList.front().ReadyTime; + for (const auto& Entry : PendingList) + { + if (Entry.ReadyTime < NextReady) + { + NextReady = Entry.ReadyTime; + } + } + + m_Cv.wait_until(Lock, NextReady, [this] { return !m_Queue.empty() || !m_ReadyLsns.empty() || m_Done; }); + } + } + + // Move new items into PendingList with the full deferral deadline + auto Now = Clock::now(); + for (auto& Entry : m_Queue) + { + PendingList.push_back({Entry.ActionLsn, std::move(Entry.Path), Now + DeferralPeriod, 0}); + } + m_Queue.clear(); + + // Apply MarkReady: shorten ReadyTime for matching entries + for (int Lsn : m_ReadyLsns) + { + for (auto& Entry : PendingList) + { + if (Entry.ActionLsn == Lsn) + { + auto NewReady = Now + ReadyGracePeriod; + if (NewReady < Entry.ReadyTime) + { + Entry.ReadyTime = NewReady; + } + } + } + } + m_ReadyLsns.clear(); + + Shutting = m_Done; + } + + // Process items whose deferral period has elapsed (or all items on shutdown) + + auto Now = Clock::now(); + + for (size_t i = 0; i < PendingList.size();) + { + auto& Entry = PendingList[i]; + + if (!Shutting && Now < Entry.ReadyTime) + { + ++i; + continue; + } + + if (TryDelete(Entry)) + { + if (Entry.Attempts > 0) + { + ZEN_INFO("Retry succeeded for directory '{}'", Entry.Path); + } + + PendingList[i] = std::move(PendingList.back()); + PendingList.pop_back(); + } + else + { + ++Entry.Attempts; + + if (Entry.Attempts >= MaxRetries) + { + ZEN_WARN("Giving up on deleting '{}' after {} attempts", Entry.Path, Entry.Attempts); + PendingList[i] = std::move(PendingList.back()); + PendingList.pop_back(); + } + else + { + ZEN_WARN("Unable to delete directory '{}' (attempt {}), will retry", Entry.Path, Entry.Attempts); + Entry.ReadyTime = Now + RetryInterval; + ++i; + } + } + } + + // Exit once shutdown is requested and nothing remains + + if (Shutting && PendingList.empty()) + { + return; + } + } +} + +} // namespace zen::compute + +#endif + +#if ZEN_WITH_TESTS + +# include + +namespace zen::compute { + +void +deferreddeleter_forcelink() +{ +} + +} // namespace zen::compute + +#endif + +#if ZEN_WITH_TESTS && ZEN_WITH_COMPUTE_SERVICES + +# include + +namespace zen::compute { + +TEST_CASE("DeferredDirectoryDeleter.DeletesSingleDirectory") +{ + ScopedTemporaryDirectory TempDir; + std::filesystem::path DirToDelete = TempDir.Path() / "subdir"; + CreateDirectories(DirToDelete / "nested"); + + CHECK(std::filesystem::exists(DirToDelete)); + + { + DeferredDirectoryDeleter Deleter; + Deleter.Enqueue(1, DirToDelete); + } + + CHECK(!std::filesystem::exists(DirToDelete)); +} + +TEST_CASE("DeferredDirectoryDeleter.DeletesMultipleDirectories") +{ + ScopedTemporaryDirectory TempDir; + + constexpr int NumDirs = 10; + std::vector Dirs; + + for (int i = 0; i < NumDirs; ++i) + { + auto Dir = TempDir.Path() / std::to_string(i); + CreateDirectories(Dir / "child"); + Dirs.push_back(std::move(Dir)); + } + + { + DeferredDirectoryDeleter Deleter; + for (int i = 0; i < NumDirs; ++i) + { + CHECK(std::filesystem::exists(Dirs[i])); + Deleter.Enqueue(100 + i, Dirs[i]); + } + } + + for (const auto& Dir : Dirs) + { + CHECK(!std::filesystem::exists(Dir)); + } +} + +TEST_CASE("DeferredDirectoryDeleter.ShutdownIsIdempotent") +{ + ScopedTemporaryDirectory TempDir; + std::filesystem::path Dir = TempDir.Path() / "idempotent"; + CreateDirectories(Dir); + + DeferredDirectoryDeleter Deleter; + Deleter.Enqueue(42, Dir); + Deleter.Shutdown(); + Deleter.Shutdown(); + + CHECK(!std::filesystem::exists(Dir)); +} + +TEST_CASE("DeferredDirectoryDeleter.HandlesNonExistentPath") +{ + ScopedTemporaryDirectory TempDir; + std::filesystem::path NoSuchDir = TempDir.Path() / "does_not_exist"; + + { + DeferredDirectoryDeleter Deleter; + Deleter.Enqueue(99, NoSuchDir); + } +} + +TEST_CASE("DeferredDirectoryDeleter.ExplicitShutdownBeforeDestruction") +{ + ScopedTemporaryDirectory TempDir; + std::filesystem::path Dir = TempDir.Path() / "explicit"; + CreateDirectories(Dir / "inner"); + + DeferredDirectoryDeleter Deleter; + Deleter.Enqueue(7, Dir); + Deleter.Shutdown(); + + CHECK(!std::filesystem::exists(Dir)); +} + +TEST_CASE("DeferredDirectoryDeleter.MarkReadyShortensDeferral") +{ + ScopedTemporaryDirectory TempDir; + std::filesystem::path Dir = TempDir.Path() / "markready"; + CreateDirectories(Dir / "child"); + + DeferredDirectoryDeleter Deleter; + Deleter.Enqueue(50, Dir); + + // Without MarkReady the full deferral (60s) would apply. + // MarkReady shortens it to 5s, and shutdown bypasses even that. + Deleter.MarkReady(50); + Deleter.Shutdown(); + + CHECK(!std::filesystem::exists(Dir)); +} + +} // namespace zen::compute + +#endif // ZEN_WITH_TESTS && ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/runners/deferreddeleter.h b/src/zencompute/runners/deferreddeleter.h new file mode 100644 index 000000000..9b010aa0f --- /dev/null +++ b/src/zencompute/runners/deferreddeleter.h @@ -0,0 +1,68 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencompute/computeservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include +# include + +namespace zen::compute { + +/// Deletes directories on a background thread to avoid blocking callers. +/// Useful when DeleteDirectories may stall (e.g. Wine's deferred-unlink semantics). +/// +/// Enqueued directories wait for a deferral period before deletion, giving +/// file handles time to close. Call MarkReady() with the ActionLsn to shorten +/// the wait to a brief grace period (e.g. once a client has collected results). +/// On shutdown, all pending directories are deleted immediately. +class DeferredDirectoryDeleter +{ + DeferredDirectoryDeleter(const DeferredDirectoryDeleter&) = delete; + DeferredDirectoryDeleter& operator=(const DeferredDirectoryDeleter&) = delete; + +public: + DeferredDirectoryDeleter(); + ~DeferredDirectoryDeleter(); + + /// Enqueue a directory for deferred deletion, associated with an action LSN. + void Enqueue(int ActionLsn, std::filesystem::path Path); + + /// Signal that the action result has been consumed and the directory + /// can be deleted after a short grace period instead of the full deferral. + void MarkReady(int ActionLsn); + + /// Drain the queue and join the background thread. Idempotent. + void Shutdown(); + +private: + struct QueueEntry + { + int ActionLsn; + std::filesystem::path Path; + }; + + std::mutex m_Mutex; + std::condition_variable m_Cv; + std::deque m_Queue; + std::vector m_ReadyLsns; + bool m_Done = false; + std::thread m_Thread; + void ThreadFunction(); +}; + +} // namespace zen::compute + +#endif + +#if ZEN_WITH_TESTS +namespace zen::compute { +void deferreddeleter_forcelink(); // internal +} // namespace zen::compute +#endif diff --git a/src/zencompute/runners/functionrunner.cpp b/src/zencompute/runners/functionrunner.cpp new file mode 100644 index 000000000..768cdf1e1 --- /dev/null +++ b/src/zencompute/runners/functionrunner.cpp @@ -0,0 +1,365 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "functionrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include + +# include +# include + +namespace zen::compute { + +FunctionRunner::FunctionRunner(std::filesystem::path BasePath) : m_ActionsPath(BasePath / "actions") +{ +} + +FunctionRunner::~FunctionRunner() = default; + +size_t +FunctionRunner::QueryCapacity() +{ + return 1; +} + +std::vector +FunctionRunner::SubmitActions(const std::vector>& Actions) +{ + std::vector Results; + Results.reserve(Actions.size()); + + for (const Ref& Action : Actions) + { + Results.push_back(SubmitAction(Action)); + } + + return Results; +} + +void +FunctionRunner::MaybeDumpAction(int ActionLsn, const CbObject& ActionObject) +{ + if (m_DumpActions) + { + std::string UniqueId = fmt::format("{}.ddb", ActionLsn); + std::filesystem::path Path = m_ActionsPath / UniqueId; + + zen::WriteFile(Path, IoBuffer(ActionObject.GetBuffer().AsIoBuffer())); + } +} + +////////////////////////////////////////////////////////////////////////// + +void +BaseRunnerGroup::AddRunnerInternal(FunctionRunner* Runner) +{ + m_RunnersLock.WithExclusiveLock([&] { m_Runners.emplace_back(Runner); }); +} + +size_t +BaseRunnerGroup::QueryCapacity() +{ + size_t TotalCapacity = 0; + m_RunnersLock.WithSharedLock([&] { + for (const auto& Runner : m_Runners) + { + TotalCapacity += Runner->QueryCapacity(); + } + }); + return TotalCapacity; +} + +SubmitResult +BaseRunnerGroup::SubmitAction(Ref Action) +{ + ZEN_TRACE_CPU("BaseRunnerGroup::SubmitAction"); + RwLock::SharedLockScope _(m_RunnersLock); + + const int InitialIndex = m_NextSubmitIndex.load(std::memory_order_acquire); + int Index = InitialIndex; + const int RunnerCount = gsl::narrow(m_Runners.size()); + + if (RunnerCount == 0) + { + return {.IsAccepted = false, .Reason = "No runners available"}; + } + + do + { + while (Index >= RunnerCount) + { + Index -= RunnerCount; + } + + auto& Runner = m_Runners[Index++]; + + SubmitResult Result = Runner->SubmitAction(Action); + + if (Result.IsAccepted == true) + { + m_NextSubmitIndex = Index % RunnerCount; + + return Result; + } + + while (Index >= RunnerCount) + { + Index -= RunnerCount; + } + } while (Index != InitialIndex); + + return {.IsAccepted = false}; +} + +std::vector +BaseRunnerGroup::SubmitActions(const std::vector>& Actions) +{ + ZEN_TRACE_CPU("BaseRunnerGroup::SubmitActions"); + RwLock::SharedLockScope _(m_RunnersLock); + + const int RunnerCount = gsl::narrow(m_Runners.size()); + + if (RunnerCount == 0) + { + return std::vector(Actions.size(), SubmitResult{.IsAccepted = false, .Reason = "No runners available"}); + } + + // Query capacity per runner and compute total + std::vector Capacities(RunnerCount); + size_t TotalCapacity = 0; + + for (int i = 0; i < RunnerCount; ++i) + { + Capacities[i] = m_Runners[i]->QueryCapacity(); + TotalCapacity += Capacities[i]; + } + + if (TotalCapacity == 0) + { + return std::vector(Actions.size(), SubmitResult{.IsAccepted = false, .Reason = "No capacity"}); + } + + // Distribute actions across runners proportionally to their available capacity + std::vector>> PerRunnerActions(RunnerCount); + std::vector ActionRunnerIndex(Actions.size()); + size_t ActionIdx = 0; + + for (int i = 0; i < RunnerCount; ++i) + { + if (Capacities[i] == 0) + { + continue; + } + + size_t Share = (Actions.size() * Capacities[i] + TotalCapacity - 1) / TotalCapacity; + Share = std::min(Share, Capacities[i]); + + for (size_t j = 0; j < Share && ActionIdx < Actions.size(); ++j, ++ActionIdx) + { + PerRunnerActions[i].push_back(Actions[ActionIdx]); + ActionRunnerIndex[ActionIdx] = i; + } + } + + // Assign any remaining actions to runners with capacity (round-robin) + for (int i = 0; ActionIdx < Actions.size(); i = (i + 1) % RunnerCount) + { + if (Capacities[i] > PerRunnerActions[i].size()) + { + PerRunnerActions[i].push_back(Actions[ActionIdx]); + ActionRunnerIndex[ActionIdx] = i; + ++ActionIdx; + } + } + + // Submit batches per runner + std::vector> PerRunnerResults(RunnerCount); + + for (int i = 0; i < RunnerCount; ++i) + { + if (!PerRunnerActions[i].empty()) + { + PerRunnerResults[i] = m_Runners[i]->SubmitActions(PerRunnerActions[i]); + } + } + + // Reassemble results in original action order + std::vector Results(Actions.size()); + std::vector PerRunnerIdx(RunnerCount, 0); + + for (size_t i = 0; i < Actions.size(); ++i) + { + size_t RunnerIdx = ActionRunnerIndex[i]; + size_t Idx = PerRunnerIdx[RunnerIdx]++; + Results[i] = std::move(PerRunnerResults[RunnerIdx][Idx]); + } + + return Results; +} + +size_t +BaseRunnerGroup::GetSubmittedActionCount() +{ + RwLock::SharedLockScope _(m_RunnersLock); + + size_t TotalCount = 0; + + for (const auto& Runner : m_Runners) + { + TotalCount += Runner->GetSubmittedActionCount(); + } + + return TotalCount; +} + +void +BaseRunnerGroup::RegisterWorker(CbPackage Worker) +{ + RwLock::SharedLockScope _(m_RunnersLock); + + for (auto& Runner : m_Runners) + { + Runner->RegisterWorker(Worker); + } +} + +void +BaseRunnerGroup::Shutdown() +{ + RwLock::SharedLockScope _(m_RunnersLock); + + for (auto& Runner : m_Runners) + { + Runner->Shutdown(); + } +} + +bool +BaseRunnerGroup::CancelAction(int ActionLsn) +{ + RwLock::SharedLockScope _(m_RunnersLock); + + for (auto& Runner : m_Runners) + { + if (Runner->CancelAction(ActionLsn)) + { + return true; + } + } + + return false; +} + +void +BaseRunnerGroup::CancelRemoteQueue(int QueueId) +{ + RwLock::SharedLockScope _(m_RunnersLock); + + for (auto& Runner : m_Runners) + { + Runner->CancelRemoteQueue(QueueId); + } +} + +////////////////////////////////////////////////////////////////////////// + +RunnerAction::RunnerAction(ComputeServiceSession* OwnerSession) : m_OwnerSession(OwnerSession) +{ + this->Timestamps[static_cast(State::New)] = DateTime::Now().GetTicks(); +} + +RunnerAction::~RunnerAction() +{ +} + +bool +RunnerAction::ResetActionStateToPending() +{ + // Only allow reset from Failed or Abandoned states + State CurrentState = m_ActionState.load(); + + if (CurrentState != State::Failed && CurrentState != State::Abandoned) + { + return false; + } + + if (!m_ActionState.compare_exchange_strong(CurrentState, State::Pending)) + { + return false; + } + + // Clear timestamps from Submitting through _Count + for (int i = static_cast(State::Submitting); i < static_cast(State::_Count); ++i) + { + this->Timestamps[i] = 0; + } + + // Record new Pending timestamp + this->Timestamps[static_cast(State::Pending)] = DateTime::Now().GetTicks(); + + // Clear execution fields + ExecutionLocation.clear(); + CpuUsagePercent.store(-1.0f, std::memory_order_relaxed); + CpuSeconds.store(0.0f, std::memory_order_relaxed); + + // Increment retry count + RetryCount.fetch_add(1, std::memory_order_relaxed); + + // Re-enter the scheduler pipeline + m_OwnerSession->PostUpdate(this); + + return true; +} + +void +RunnerAction::SetActionState(State NewState) +{ + ZEN_ASSERT(NewState < State::_Count); + this->Timestamps[static_cast(NewState)] = DateTime::Now().GetTicks(); + + do + { + if (State CurrentState = m_ActionState.load(); CurrentState == NewState) + { + // No state change + return; + } + else + { + if (NewState <= CurrentState) + { + // Cannot transition to an earlier or same state + return; + } + + if (m_ActionState.compare_exchange_strong(CurrentState, NewState)) + { + // Successful state change + + m_OwnerSession->PostUpdate(this); + + return; + } + } + } while (true); +} + +void +RunnerAction::SetResult(CbPackage&& Result) +{ + m_Result = std::move(Result); +} + +CbPackage& +RunnerAction::GetResult() +{ + ZEN_ASSERT(IsCompleted()); + return m_Result; +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES \ No newline at end of file diff --git a/src/zencompute/runners/functionrunner.h b/src/zencompute/runners/functionrunner.h new file mode 100644 index 000000000..f67414dbb --- /dev/null +++ b/src/zencompute/runners/functionrunner.h @@ -0,0 +1,214 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include + +namespace zen::compute { + +struct SubmitResult +{ + bool IsAccepted = false; + std::string Reason; +}; + +/** Base interface for classes implementing a remote execution "runner" + */ +class FunctionRunner : public RefCounted +{ + FunctionRunner(FunctionRunner&&) = delete; + FunctionRunner& operator=(FunctionRunner&&) = delete; + +public: + FunctionRunner(std::filesystem::path BasePath); + virtual ~FunctionRunner() = 0; + + virtual void Shutdown() = 0; + virtual void RegisterWorker(const CbPackage& WorkerPackage) = 0; + + [[nodiscard]] virtual SubmitResult SubmitAction(Ref Action) = 0; + [[nodiscard]] virtual size_t GetSubmittedActionCount() = 0; + [[nodiscard]] virtual bool IsHealthy() = 0; + [[nodiscard]] virtual size_t QueryCapacity(); + [[nodiscard]] virtual std::vector SubmitActions(const std::vector>& Actions); + + // Best-effort cancellation of a specific in-flight action. Returns true if the + // cancellation signal was successfully sent. The action will transition to Cancelled + // asynchronously once the platform-level termination completes. + virtual bool CancelAction(int /*ActionLsn*/) { return false; } + + // Cancel the remote queue corresponding to the given local QueueId. + // Only meaningful for remote runners; local runners ignore this. + virtual void CancelRemoteQueue(int /*QueueId*/) {} + +protected: + std::filesystem::path m_ActionsPath; + bool m_DumpActions = false; + void MaybeDumpAction(int ActionLsn, const CbObject& ActionObject); +}; + +/** Base class for RunnerGroup that operates on generic FunctionRunner references. + * All scheduling, capacity, and lifecycle logic lives here. + */ +class BaseRunnerGroup +{ +public: + size_t QueryCapacity(); + SubmitResult SubmitAction(Ref Action); + std::vector SubmitActions(const std::vector>& Actions); + size_t GetSubmittedActionCount(); + void RegisterWorker(CbPackage Worker); + void Shutdown(); + bool CancelAction(int ActionLsn); + void CancelRemoteQueue(int QueueId); + + size_t GetRunnerCount() + { + return m_RunnersLock.WithSharedLock([this] { return m_Runners.size(); }); + } + +protected: + void AddRunnerInternal(FunctionRunner* Runner); + + RwLock m_RunnersLock; + std::vector> m_Runners; + std::atomic m_NextSubmitIndex{0}; +}; + +/** Typed RunnerGroup that adds type-safe runner addition and predicate-based removal. + */ +template +struct RunnerGroup : public BaseRunnerGroup +{ + void AddRunner(RunnerType* Runner) { AddRunnerInternal(Runner); } + + template + size_t RemoveRunnerIf(Predicate&& Pred) + { + size_t RemovedCount = 0; + m_RunnersLock.WithExclusiveLock([&] { + auto It = m_Runners.begin(); + while (It != m_Runners.end()) + { + if (Pred(static_cast(**It))) + { + (*It)->Shutdown(); + It = m_Runners.erase(It); + ++RemovedCount; + } + else + { + ++It; + } + } + }); + return RemovedCount; + } +}; + +/** + * This represents an action going through different stages of scheduling and execution. + */ +struct RunnerAction : public RefCounted +{ + explicit RunnerAction(ComputeServiceSession* OwnerSession); + ~RunnerAction(); + + int ActionLsn = 0; + int QueueId = 0; + WorkerDesc Worker; + IoHash ActionId; + CbObject ActionObj; + int Priority = 0; + std::string ExecutionLocation; // "local" or remote hostname + + // CPU usage and total CPU time of the running process, sampled periodically by the local runner. + // CpuUsagePercent: -1.0 means not yet sampled; >=0.0 is the most recent reading as a percentage. + // CpuSeconds: total CPU time (user+system) consumed since process start, in seconds. 0.0 if not yet sampled. + std::atomic CpuUsagePercent{-1.0f}; + std::atomic CpuSeconds{0.0f}; + std::atomic RetryCount{0}; + + enum class State + { + New, + Pending, + Submitting, + Running, + Completed, + Failed, + Abandoned, + Cancelled, + _Count + }; + + static const char* ToString(State _) + { + switch (_) + { + case State::New: + return "New"; + case State::Pending: + return "Pending"; + case State::Submitting: + return "Submitting"; + case State::Running: + return "Running"; + case State::Completed: + return "Completed"; + case State::Failed: + return "Failed"; + case State::Abandoned: + return "Abandoned"; + case State::Cancelled: + return "Cancelled"; + default: + return "Unknown"; + } + } + + static State FromString(std::string_view Name, State Default = State::Failed) + { + for (int i = 0; i < static_cast(State::_Count); ++i) + { + if (Name == ToString(static_cast(i))) + { + return static_cast(i); + } + } + return Default; + } + + uint64_t Timestamps[static_cast(State::_Count)] = {}; + + State ActionState() const { return m_ActionState; } + void SetActionState(State NewState); + + bool IsSuccess() const { return ActionState() == State::Completed; } + bool ResetActionStateToPending(); + bool IsCompleted() const + { + return ActionState() == State::Completed || ActionState() == State::Failed || ActionState() == State::Abandoned || + ActionState() == State::Cancelled; + } + + void SetResult(CbPackage&& Result); + CbPackage& GetResult(); + + ComputeServiceSession* GetOwnerSession() const { return m_OwnerSession; } + +private: + std::atomic m_ActionState = State::New; + ComputeServiceSession* m_OwnerSession = nullptr; + CbPackage m_Result; +}; + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES \ No newline at end of file diff --git a/src/zencompute/runners/linuxrunner.cpp b/src/zencompute/runners/linuxrunner.cpp new file mode 100644 index 000000000..e79a6c90f --- /dev/null +++ b/src/zencompute/runners/linuxrunner.cpp @@ -0,0 +1,734 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "linuxrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES && ZEN_PLATFORM_LINUX + +# include +# include +# include +# include +# include +# include +# include +# include + +# include +# include +# include +# include +# include +# include +# include +# include + +namespace zen::compute { + +using namespace std::literals; + +namespace { + + // All helper functions in this namespace are async-signal-safe (safe to call + // between fork() and execve()). They use only raw syscalls and avoid any + // heap allocation, stdio, or other non-AS-safe operations. + + void WriteToFd(int Fd, const char* Buf, size_t Len) + { + while (Len > 0) + { + ssize_t Written = write(Fd, Buf, Len); + if (Written <= 0) + { + break; + } + Buf += Written; + Len -= static_cast(Written); + } + } + + [[noreturn]] void WriteErrorAndExit(int ErrorPipeFd, const char* Msg, int Errno) + { + // Write the message prefix + size_t MsgLen = 0; + for (const char* P = Msg; *P; ++P) + { + ++MsgLen; + } + WriteToFd(ErrorPipeFd, Msg, MsgLen); + + // Append ": " and the errno string if non-zero + if (Errno != 0) + { + WriteToFd(ErrorPipeFd, ": ", 2); + const char* ErrStr = strerror(Errno); + size_t ErrLen = 0; + for (const char* P = ErrStr; *P; ++P) + { + ++ErrLen; + } + WriteToFd(ErrorPipeFd, ErrStr, ErrLen); + } + + _exit(127); + } + + int MkdirIfNeeded(const char* Path, mode_t Mode) + { + if (mkdir(Path, Mode) != 0 && errno != EEXIST) + { + return -1; + } + return 0; + } + + int BindMountReadOnly(const char* Src, const char* Dst) + { + if (mount(Src, Dst, nullptr, MS_BIND | MS_REC, nullptr) != 0) + { + return -1; + } + + // Remount read-only + if (mount(nullptr, Dst, nullptr, MS_REMOUNT | MS_BIND | MS_RDONLY | MS_REC, nullptr) != 0) + { + return -1; + } + + return 0; + } + + // Set up namespace-based sandbox isolation in the child process. + // This is called after fork(), before execve(). All operations must be + // async-signal-safe. + // + // The sandbox layout after pivot_root: + // / -> the sandbox directory (tmpfs-like, was SandboxPath) + // /usr -> bind-mount of host /usr (read-only) + // /lib -> bind-mount of host /lib (read-only) + // /lib64 -> bind-mount of host /lib64 (read-only, optional) + // /etc -> bind-mount of host /etc (read-only) + // /worker -> bind-mount of worker directory (read-only) + // /proc -> proc filesystem + // /dev -> tmpfs with null, zero, urandom + void SetupNamespaceSandbox(const char* SandboxPath, uid_t Uid, gid_t Gid, const char* WorkerPath, int ErrorPipeFd) + { + // 1. Unshare user, mount, and network namespaces + if (unshare(CLONE_NEWUSER | CLONE_NEWNS | CLONE_NEWNET) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "unshare() failed", errno); + } + + // 2. Write UID/GID mappings + // Must deny setgroups first (required by kernel for unprivileged user namespaces) + { + int Fd = open("/proc/self/setgroups", O_WRONLY); + if (Fd >= 0) + { + WriteToFd(Fd, "deny", 4); + close(Fd); + } + // setgroups file may not exist on older kernels; not fatal + } + + { + // uid_map: map our UID to 0 inside the namespace + char Buf[64]; + int Len = snprintf(Buf, sizeof(Buf), "0 %u 1\n", static_cast(Uid)); + + int Fd = open("/proc/self/uid_map", O_WRONLY); + if (Fd < 0) + { + WriteErrorAndExit(ErrorPipeFd, "open uid_map failed", errno); + } + WriteToFd(Fd, Buf, static_cast(Len)); + close(Fd); + } + + { + // gid_map: map our GID to 0 inside the namespace + char Buf[64]; + int Len = snprintf(Buf, sizeof(Buf), "0 %u 1\n", static_cast(Gid)); + + int Fd = open("/proc/self/gid_map", O_WRONLY); + if (Fd < 0) + { + WriteErrorAndExit(ErrorPipeFd, "open gid_map failed", errno); + } + WriteToFd(Fd, Buf, static_cast(Len)); + close(Fd); + } + + // 3. Privatize the entire mount tree so our mounts don't propagate + if (mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE, nullptr) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mount MS_PRIVATE failed", errno); + } + + // 4. Create mount points inside the sandbox and bind-mount system directories + + // Helper macro-like pattern for building paths inside sandbox + // We use stack buffers since we can't allocate heap memory safely + char MountPoint[4096]; + + auto BuildPath = [&](const char* Suffix) -> const char* { + snprintf(MountPoint, sizeof(MountPoint), "%s/%s", SandboxPath, Suffix); + return MountPoint; + }; + + // /usr (required) + if (MkdirIfNeeded(BuildPath("usr"), 0755) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mkdir sandbox/usr failed", errno); + } + if (BindMountReadOnly("/usr", BuildPath("usr")) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "bind mount /usr failed", errno); + } + + // /lib (required) + if (MkdirIfNeeded(BuildPath("lib"), 0755) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mkdir sandbox/lib failed", errno); + } + if (BindMountReadOnly("/lib", BuildPath("lib")) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "bind mount /lib failed", errno); + } + + // /lib64 (optional — not all distros have it) + { + struct stat St; + if (stat("/lib64", &St) == 0 && S_ISDIR(St.st_mode)) + { + if (MkdirIfNeeded(BuildPath("lib64"), 0755) == 0) + { + BindMountReadOnly("/lib64", BuildPath("lib64")); + // Failure is non-fatal for lib64 + } + } + } + + // /etc (required — for resolv.conf, ld.so.cache, etc.) + if (MkdirIfNeeded(BuildPath("etc"), 0755) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mkdir sandbox/etc failed", errno); + } + if (BindMountReadOnly("/etc", BuildPath("etc")) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "bind mount /etc failed", errno); + } + + // /worker — bind-mount worker directory (contains the executable) + if (MkdirIfNeeded(BuildPath("worker"), 0755) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mkdir sandbox/worker failed", errno); + } + if (BindMountReadOnly(WorkerPath, BuildPath("worker")) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "bind mount worker dir failed", errno); + } + + // 5. Mount /proc inside sandbox + if (MkdirIfNeeded(BuildPath("proc"), 0755) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mkdir sandbox/proc failed", errno); + } + if (mount("proc", BuildPath("proc"), "proc", MS_NOSUID | MS_NOEXEC | MS_NODEV, nullptr) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mount /proc failed", errno); + } + + // 6. Mount tmpfs /dev and bind-mount essential device nodes + if (MkdirIfNeeded(BuildPath("dev"), 0755) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mkdir sandbox/dev failed", errno); + } + if (mount("tmpfs", BuildPath("dev"), "tmpfs", MS_NOSUID | MS_NOEXEC, "size=64k,mode=0755") != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mount tmpfs /dev failed", errno); + } + + // Bind-mount /dev/null, /dev/zero, /dev/urandom + { + char DevSrc[64]; + char DevDst[4096]; + + auto BindDev = [&](const char* Name) { + snprintf(DevSrc, sizeof(DevSrc), "/dev/%s", Name); + snprintf(DevDst, sizeof(DevDst), "%s/dev/%s", SandboxPath, Name); + + // Create the file to mount over + int Fd = open(DevDst, O_WRONLY | O_CREAT, 0666); + if (Fd >= 0) + { + close(Fd); + } + mount(DevSrc, DevDst, nullptr, MS_BIND, nullptr); + // Non-fatal if individual devices fail + }; + + BindDev("null"); + BindDev("zero"); + BindDev("urandom"); + } + + // 7. pivot_root to sandbox + // pivot_root requires the new root and put_old to be mount points. + // Bind-mount sandbox onto itself to make it a mount point. + if (mount(SandboxPath, SandboxPath, nullptr, MS_BIND | MS_REC, nullptr) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "bind mount sandbox onto itself failed", errno); + } + + // Create .pivot_old inside sandbox + char PivotOld[4096]; + snprintf(PivotOld, sizeof(PivotOld), "%s/.pivot_old", SandboxPath); + if (MkdirIfNeeded(PivotOld, 0755) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "mkdir .pivot_old failed", errno); + } + + if (syscall(SYS_pivot_root, SandboxPath, PivotOld) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "pivot_root failed", errno); + } + + // 8. Now inside new root. Clean up old root. + if (chdir("/") != 0) + { + WriteErrorAndExit(ErrorPipeFd, "chdir / failed", errno); + } + + if (umount2("/.pivot_old", MNT_DETACH) != 0) + { + WriteErrorAndExit(ErrorPipeFd, "umount2 .pivot_old failed", errno); + } + + rmdir("/.pivot_old"); + } + +} // anonymous namespace + +LinuxProcessRunner::LinuxProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool, + bool Sandboxed, + int32_t MaxConcurrentActions) +: LocalProcessRunner(Resolver, BaseDir, Deleter, WorkerPool, MaxConcurrentActions) +, m_Sandboxed(Sandboxed) +{ + // Restore SIGCHLD to default behavior so waitpid() can properly collect + // child exit status. zenserver/main.cpp sets SIGCHLD to SIG_IGN which + // causes the kernel to auto-reap children, making waitpid() return + // -1/ECHILD instead of the exit status we need. + struct sigaction Action = {}; + sigemptyset(&Action.sa_mask); + Action.sa_handler = SIG_DFL; + sigaction(SIGCHLD, &Action, nullptr); + + if (m_Sandboxed) + { + ZEN_INFO("namespace sandboxing enabled for child processes"); + } +} + +SubmitResult +LinuxProcessRunner::SubmitAction(Ref Action) +{ + ZEN_TRACE_CPU("LinuxProcessRunner::SubmitAction"); + std::optional Prepared = PrepareActionSubmission(Action); + + if (!Prepared) + { + return SubmitResult{.IsAccepted = false}; + } + + // Build environment array from worker descriptor + + CbObject WorkerDescription = Prepared->WorkerPackage.GetObject(); + + std::vector EnvStrings; + for (auto& It : WorkerDescription["environment"sv]) + { + EnvStrings.emplace_back(It.AsString()); + } + + std::vector Envp; + Envp.reserve(EnvStrings.size() + 1); + for (auto& Str : EnvStrings) + { + Envp.push_back(Str.data()); + } + Envp.push_back(nullptr); + + // Build argv: -Build=build.action + // Pre-compute all path strings before fork() for async-signal-safety. + + std::string_view ExecPath = WorkerDescription["path"sv].AsString(); + std::string ExePathStr; + std::string SandboxedExePathStr; + + if (m_Sandboxed) + { + // After pivot_root, the worker dir is at /worker inside the new root + std::filesystem::path SandboxedExePath = std::filesystem::path("/worker") / std::filesystem::path(ExecPath); + SandboxedExePathStr = SandboxedExePath.string(); + // We still need the real path for logging + ExePathStr = (Prepared->WorkerPath / std::filesystem::path(ExecPath)).string(); + } + else + { + ExePathStr = (Prepared->WorkerPath / std::filesystem::path(ExecPath)).string(); + } + + std::string BuildArg = "-Build=build.action"; + + // argv[0] should be the path the child will see + const std::string& ChildExePath = m_Sandboxed ? SandboxedExePathStr : ExePathStr; + + std::vector ArgV; + ArgV.push_back(const_cast(ChildExePath.data())); + ArgV.push_back(BuildArg.data()); + ArgV.push_back(nullptr); + + ZEN_DEBUG("Executing: {} {} (sandboxed={})", ExePathStr, BuildArg, m_Sandboxed); + + std::string SandboxPathStr = Prepared->SandboxPath.string(); + std::string WorkerPathStr = Prepared->WorkerPath.string(); + + // Pre-fork: get uid/gid for namespace mapping, create error pipe + uid_t CurrentUid = 0; + gid_t CurrentGid = 0; + int ErrorPipe[2] = {-1, -1}; + + if (m_Sandboxed) + { + CurrentUid = getuid(); + CurrentGid = getgid(); + + if (pipe2(ErrorPipe, O_CLOEXEC) != 0) + { + throw zen::runtime_error("pipe2() for sandbox error pipe failed: {}", strerror(errno)); + } + } + + pid_t ChildPid = fork(); + + if (ChildPid < 0) + { + int SavedErrno = errno; + if (m_Sandboxed) + { + close(ErrorPipe[0]); + close(ErrorPipe[1]); + } + throw zen::runtime_error("fork() failed: {}", strerror(SavedErrno)); + } + + if (ChildPid == 0) + { + // Child process + + if (m_Sandboxed) + { + // Close read end of error pipe — child only writes + close(ErrorPipe[0]); + + SetupNamespaceSandbox(SandboxPathStr.c_str(), CurrentUid, CurrentGid, WorkerPathStr.c_str(), ErrorPipe[1]); + + // After pivot_root, CWD is "/" which is the sandbox root. + // execve with the sandboxed path. + execve(SandboxedExePathStr.c_str(), ArgV.data(), Envp.data()); + + WriteErrorAndExit(ErrorPipe[1], "execve failed", errno); + } + else + { + if (chdir(SandboxPathStr.c_str()) != 0) + { + _exit(127); + } + + execve(ExePathStr.c_str(), ArgV.data(), Envp.data()); + _exit(127); + } + } + + // Parent process + + if (m_Sandboxed) + { + // Close write end of error pipe — parent only reads + close(ErrorPipe[1]); + + // Read from error pipe. If execve succeeded, pipe was closed by O_CLOEXEC + // and read returns 0. If setup failed, child wrote an error message. + char ErrBuf[512]; + ssize_t BytesRead = read(ErrorPipe[0], ErrBuf, sizeof(ErrBuf) - 1); + close(ErrorPipe[0]); + + if (BytesRead > 0) + { + // Sandbox setup or execve failed + ErrBuf[BytesRead] = '\0'; + + // Reap the child (it called _exit(127)) + waitpid(ChildPid, nullptr, 0); + + // Clean up the sandbox in the background + m_DeferredDeleter.Enqueue(Action->ActionLsn, std::move(Prepared->SandboxPath)); + + ZEN_ERROR("Sandbox setup failed for action {}: {}", Action->ActionLsn, ErrBuf); + + Action->SetActionState(RunnerAction::State::Failed); + return SubmitResult{.IsAccepted = false}; + } + } + + // Store child pid as void* (same convention as zencore/process.cpp) + + Ref NewAction{new RunningAction()}; + NewAction->Action = Action; + NewAction->ProcessHandle = reinterpret_cast(static_cast(ChildPid)); + NewAction->SandboxPath = std::move(Prepared->SandboxPath); + + { + RwLock::ExclusiveLockScope _(m_RunningLock); + m_RunningMap[Prepared->ActionLsn] = std::move(NewAction); + } + + Action->SetActionState(RunnerAction::State::Running); + + return SubmitResult{.IsAccepted = true}; +} + +void +LinuxProcessRunner::SweepRunningActions() +{ + ZEN_TRACE_CPU("LinuxProcessRunner::SweepRunningActions"); + std::vector> CompletedActions; + + m_RunningLock.WithExclusiveLock([&] { + for (auto It = begin(m_RunningMap), ItEnd = end(m_RunningMap); It != ItEnd;) + { + Ref Running = It->second; + + pid_t Pid = static_cast(reinterpret_cast(Running->ProcessHandle)); + int Status = 0; + + pid_t Result = waitpid(Pid, &Status, WNOHANG); + + if (Result == Pid) + { + if (WIFEXITED(Status)) + { + Running->ExitCode = WEXITSTATUS(Status); + } + else if (WIFSIGNALED(Status)) + { + Running->ExitCode = 128 + WTERMSIG(Status); + } + else + { + Running->ExitCode = 1; + } + + Running->ProcessHandle = nullptr; + + CompletedActions.push_back(std::move(Running)); + It = m_RunningMap.erase(It); + } + else + { + ++It; + } + } + }); + + ProcessCompletedActions(CompletedActions); +} + +void +LinuxProcessRunner::CancelRunningActions() +{ + ZEN_TRACE_CPU("LinuxProcessRunner::CancelRunningActions"); + Stopwatch Timer; + std::unordered_map> RunningMap; + + m_RunningLock.WithExclusiveLock([&] { std::swap(RunningMap, m_RunningMap); }); + + if (RunningMap.empty()) + { + return; + } + + ZEN_INFO("cancelling all running actions"); + + // Send SIGTERM to all running processes first + + for (const auto& [Lsn, Running] : RunningMap) + { + pid_t Pid = static_cast(reinterpret_cast(Running->ProcessHandle)); + + if (kill(Pid, SIGTERM) != 0) + { + ZEN_WARN("kill(SIGTERM) for LSN {} (pid {}) failed: {}", Running->Action->ActionLsn, Pid, strerror(errno)); + } + } + + // Wait for all processes, regardless of whether SIGTERM succeeded, then clean up. + + for (auto& [Lsn, Running] : RunningMap) + { + pid_t Pid = static_cast(reinterpret_cast(Running->ProcessHandle)); + + // Poll for up to 2 seconds + bool Exited = false; + for (int i = 0; i < 20; ++i) + { + int Status = 0; + pid_t WaitResult = waitpid(Pid, &Status, WNOHANG); + if (WaitResult == Pid) + { + Exited = true; + ZEN_DEBUG("LSN {}: process exit OK", Running->Action->ActionLsn); + break; + } + usleep(100000); // 100ms + } + + if (!Exited) + { + ZEN_WARN("LSN {}: process did not exit after SIGTERM, sending SIGKILL", Running->Action->ActionLsn); + kill(Pid, SIGKILL); + waitpid(Pid, nullptr, 0); + } + + m_DeferredDeleter.Enqueue(Running->Action->ActionLsn, std::move(Running->SandboxPath)); + Running->Action->SetActionState(RunnerAction::State::Failed); + } + + ZEN_INFO("DONE - cancelled {} running processes (took {})", RunningMap.size(), NiceTimeSpanMs(Timer.GetElapsedTimeMs())); +} + +bool +LinuxProcessRunner::CancelAction(int ActionLsn) +{ + ZEN_TRACE_CPU("LinuxProcessRunner::CancelAction"); + + // Hold the shared lock while sending the signal to prevent the sweep thread + // from reaping the PID (via waitpid) between our lookup and kill(). Without + // the lock held, the PID could be recycled by the kernel and we'd signal an + // unrelated process. + bool Sent = false; + + m_RunningLock.WithSharedLock([&] { + auto It = m_RunningMap.find(ActionLsn); + if (It == m_RunningMap.end()) + { + return; + } + + Ref Target = It->second; + if (!Target->ProcessHandle) + { + return; + } + + pid_t Pid = static_cast(reinterpret_cast(Target->ProcessHandle)); + + if (kill(Pid, SIGTERM) != 0) + { + ZEN_WARN("CancelAction: kill(SIGTERM) for LSN {} (pid {}) failed: {}", ActionLsn, Pid, strerror(errno)); + return; + } + + ZEN_DEBUG("CancelAction: sent SIGTERM to LSN {} (pid {})", ActionLsn, Pid); + Sent = true; + }); + + // The monitor thread will pick up the process exit and mark the action as Failed. + return Sent; +} + +static uint64_t +ReadProcStatCpuTicks(pid_t Pid) +{ + char Path[64]; + snprintf(Path, sizeof(Path), "/proc/%d/stat", static_cast(Pid)); + + char Buf[256]; + int Fd = open(Path, O_RDONLY); + if (Fd < 0) + { + return 0; + } + + ssize_t Len = read(Fd, Buf, sizeof(Buf) - 1); + close(Fd); + + if (Len <= 0) + { + return 0; + } + + Buf[Len] = '\0'; + + // Skip past "pid (name) " — find last ')' to handle names containing spaces or parens + const char* P = strrchr(Buf, ')'); + if (!P) + { + return 0; + } + + P += 2; // skip ') ' + + // Remaining fields (space-separated, 0-indexed from here): + // 0:state 1:ppid 2:pgrp 3:session 4:tty_nr 5:tty_pgrp 6:flags + // 7:minflt 8:cminflt 9:majflt 10:cmajflt 11:utime 12:stime + unsigned long UTime = 0; + unsigned long STime = 0; + sscanf(P, "%*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %lu %lu", &UTime, &STime); + return UTime + STime; +} + +void +LinuxProcessRunner::SampleProcessCpu(RunningAction& Running) +{ + static const long ClkTck = sysconf(_SC_CLK_TCK); + + const pid_t Pid = static_cast(reinterpret_cast(Running.ProcessHandle)); + + const uint64_t NowTicks = GetHifreqTimerValue(); + const uint64_t CurrentOsTicks = ReadProcStatCpuTicks(Pid); + + if (CurrentOsTicks == 0) + { + // Process gone or /proc entry unreadable — record timestamp without updating usage + Running.LastCpuSampleTicks = NowTicks; + Running.LastCpuOsTicks = 0; + return; + } + + // Cumulative CPU seconds (absolute, available from first sample) + Running.Action->CpuSeconds.store(static_cast(static_cast(CurrentOsTicks) / ClkTck), std::memory_order_relaxed); + + if (Running.LastCpuSampleTicks != 0 && Running.LastCpuOsTicks != 0) + { + const uint64_t ElapsedMs = Stopwatch::GetElapsedTimeMs(NowTicks - Running.LastCpuSampleTicks); + if (ElapsedMs > 0) + { + const uint64_t DeltaOsTicks = CurrentOsTicks - Running.LastCpuOsTicks; + const float CpuPct = static_cast(static_cast(DeltaOsTicks) * 1000.0 / ClkTck / ElapsedMs * 100.0); + Running.Action->CpuUsagePercent.store(CpuPct, std::memory_order_relaxed); + } + } + + Running.LastCpuSampleTicks = NowTicks; + Running.LastCpuOsTicks = CurrentOsTicks; +} + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/linuxrunner.h b/src/zencompute/runners/linuxrunner.h new file mode 100644 index 000000000..266de366b --- /dev/null +++ b/src/zencompute/runners/linuxrunner.h @@ -0,0 +1,44 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "localrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES && ZEN_PLATFORM_LINUX + +namespace zen::compute { + +/** Native Linux process runner for executing Linux worker executables directly. + + Subclasses LocalProcessRunner, reusing sandbox management, worker manifesting, + input/output handling, and monitor thread infrastructure. Overrides only the + platform-specific methods: process spawning, sweep, and cancellation. + + When Sandboxed is true, child processes are isolated using Linux namespaces: + user, mount, and network namespaces are unshared so the child has no network + access and can only see the sandbox directory (with system libraries bind-mounted + read-only). This requires no special privileges thanks to user namespaces. + */ +class LinuxProcessRunner : public LocalProcessRunner +{ +public: + LinuxProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool, + bool Sandboxed = false, + int32_t MaxConcurrentActions = 0); + + [[nodiscard]] SubmitResult SubmitAction(Ref Action) override; + void SweepRunningActions() override; + void CancelRunningActions() override; + bool CancelAction(int ActionLsn) override; + void SampleProcessCpu(RunningAction& Running) override; + +private: + bool m_Sandboxed = false; +}; + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/localrunner.cpp b/src/zencompute/runners/localrunner.cpp new file mode 100644 index 000000000..7aaefb06e --- /dev/null +++ b/src/zencompute/runners/localrunner.cpp @@ -0,0 +1,674 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "localrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include + +namespace zen::compute { + +using namespace std::literals; + +LocalProcessRunner::LocalProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool, + int32_t MaxConcurrentActions) +: FunctionRunner(BaseDir) +, m_Log(logging::Get("local_exec")) +, m_ChunkResolver(Resolver) +, m_WorkerPath(std::filesystem::weakly_canonical(BaseDir / "workers")) +, m_SandboxPath(std::filesystem::weakly_canonical(BaseDir / "scratch")) +, m_DeferredDeleter(Deleter) +, m_WorkerPool(WorkerPool) +{ + SystemMetrics Sm = GetSystemMetricsForReporting(); + + m_MaxRunningActions = Sm.LogicalProcessorCount * 2; + + if (MaxConcurrentActions > 0) + { + m_MaxRunningActions = MaxConcurrentActions; + } + + ZEN_INFO("Max concurrent action count: {}", m_MaxRunningActions); + + bool DidCleanup = false; + + if (std::filesystem::is_directory(m_ActionsPath)) + { + ZEN_INFO("Cleaning '{}'", m_ActionsPath); + + std::error_code Ec; + CleanDirectory(m_ActionsPath, /* ForceRemoveReadOnlyFiles */ true, Ec); + + if (Ec) + { + ZEN_WARN("Unable to clean '{}': {}", m_ActionsPath, Ec.message()); + } + + DidCleanup = true; + } + + if (std::filesystem::is_directory(m_SandboxPath)) + { + ZEN_INFO("Cleaning '{}'", m_SandboxPath); + std::error_code Ec; + CleanDirectory(m_SandboxPath, /* ForceRemoveReadOnlyFiles */ true, Ec); + + if (Ec) + { + ZEN_WARN("Unable to clean '{}': {}", m_SandboxPath, Ec.message()); + } + + DidCleanup = true; + } + + // We clean out all workers on startup since we can't know they are good. They could be bad + // due to tampering, malware (which I also mean to include AV and antimalware software) or + // other processes we have no control over + if (std::filesystem::is_directory(m_WorkerPath)) + { + ZEN_INFO("Cleaning '{}'", m_WorkerPath); + std::error_code Ec; + CleanDirectory(m_WorkerPath, /* ForceRemoveReadOnlyFiles */ true, Ec); + + if (Ec) + { + ZEN_WARN("Unable to clean '{}': {}", m_WorkerPath, Ec.message()); + } + + DidCleanup = true; + } + + if (DidCleanup) + { + ZEN_INFO("Cleanup complete"); + } + + m_MonitorThread = std::thread{&LocalProcessRunner::MonitorThreadFunction, this}; + +# if ZEN_PLATFORM_WINDOWS + // Suppress any error dialogs caused by missing dependencies + UINT OldMode = ::SetErrorMode(0); + ::SetErrorMode(OldMode | SEM_FAILCRITICALERRORS); +# endif + + m_AcceptNewActions = true; +} + +LocalProcessRunner::~LocalProcessRunner() +{ + try + { + Shutdown(); + } + catch (std::exception& Ex) + { + ZEN_WARN("exception during local process runner shutdown: {}", Ex.what()); + } +} + +void +LocalProcessRunner::Shutdown() +{ + ZEN_TRACE_CPU("LocalProcessRunner::Shutdown"); + m_AcceptNewActions = false; + + m_MonitorThreadEnabled = false; + m_MonitorThreadEvent.Set(); + if (m_MonitorThread.joinable()) + { + m_MonitorThread.join(); + } + + CancelRunningActions(); +} + +std::filesystem::path +LocalProcessRunner::CreateNewSandbox() +{ + ZEN_TRACE_CPU("LocalProcessRunner::CreateNewSandbox"); + std::string UniqueId = std::to_string(++m_SandboxCounter); + std::filesystem::path Path = m_SandboxPath / UniqueId; + zen::CreateDirectories(Path); + + return Path; +} + +void +LocalProcessRunner::RegisterWorker(const CbPackage& WorkerPackage) +{ + ZEN_TRACE_CPU("LocalProcessRunner::RegisterWorker"); + if (m_DumpActions) + { + CbObject WorkerDescriptor = WorkerPackage.GetObject(); + const IoHash& WorkerId = WorkerPackage.GetObjectHash(); + + std::string UniqueId = fmt::format("worker_{}"sv, WorkerId); + std::filesystem::path Path = m_ActionsPath / UniqueId; + + zen::WriteFile(Path / "worker.ucb", WorkerDescriptor.GetBuffer().AsIoBuffer()); + + ManifestWorker(WorkerPackage, Path / "tree", [&](const IoHash& Cid, CompressedBuffer& ChunkBuffer) { + std::filesystem::path ChunkPath = Path / "chunks" / Cid.ToHexString(); + zen::WriteFile(ChunkPath, ChunkBuffer.GetCompressed()); + }); + + ZEN_INFO("dumped worker '{}' to 'file://{}'", WorkerId, Path); + } +} + +size_t +LocalProcessRunner::QueryCapacity() +{ + // Estimate how much more work we're ready to accept + + RwLock::SharedLockScope _{m_RunningLock}; + + if (!m_AcceptNewActions) + { + return 0; + } + + const size_t InFlightCount = m_RunningMap.size() + m_SubmittingCount.load(std::memory_order_relaxed); + + if (const size_t MaxRunningActions = m_MaxRunningActions; InFlightCount >= MaxRunningActions) + { + return 0; + } + else + { + return MaxRunningActions - InFlightCount; + } +} + +std::vector +LocalProcessRunner::SubmitActions(const std::vector>& Actions) +{ + if (Actions.size() <= 1) + { + std::vector Results; + + for (const Ref& Action : Actions) + { + Results.push_back(SubmitAction(Action)); + } + + return Results; + } + + // For nontrivial batches, check capacity upfront and accept what fits. + // Accepted actions are transitioned to Submitting and dispatched to the + // worker pool as fire-and-forget, so SubmitActions returns immediately + // and the scheduler thread is free to handle completions and updates. + + size_t Available = QueryCapacity(); + + std::vector Results(Actions.size()); + + size_t AcceptCount = std::min(Available, Actions.size()); + + for (size_t i = 0; i < AcceptCount; ++i) + { + const Ref& Action = Actions[i]; + + Action->SetActionState(RunnerAction::State::Submitting); + m_SubmittingCount.fetch_add(1, std::memory_order_relaxed); + + Results[i] = SubmitResult{.IsAccepted = true}; + + m_WorkerPool.ScheduleWork( + [this, Action]() { + auto CountGuard = MakeGuard([this] { m_SubmittingCount.fetch_sub(1, std::memory_order_relaxed); }); + + SubmitResult Result = SubmitAction(Action); + + if (!Result.IsAccepted) + { + // This might require another state? We should + // distinguish between outright rejections (e.g. invalid action) + // and transient failures (e.g. failed to launch process) which might + // be retried by the scheduler, but for now just fail the action + Action->SetActionState(RunnerAction::State::Failed); + } + }, + WorkerThreadPool::EMode::EnableBacklog); + } + + for (size_t i = AcceptCount; i < Actions.size(); ++i) + { + Results[i] = SubmitResult{.IsAccepted = false}; + } + + return Results; +} + +std::optional +LocalProcessRunner::PrepareActionSubmission(Ref Action) +{ + ZEN_TRACE_CPU("LocalProcessRunner::PrepareActionSubmission"); + + // Verify whether we can accept more work + + { + RwLock::SharedLockScope _{m_RunningLock}; + + if (!m_AcceptNewActions) + { + return std::nullopt; + } + + if (m_RunningMap.size() >= size_t(m_MaxRunningActions)) + { + return std::nullopt; + } + } + + // Each enqueued action is assigned an integer index (logical sequence number), + // which we use as a key for tracking data structures and as an opaque id which + // may be used by clients to reference the scheduled action + + const int32_t ActionLsn = Action->ActionLsn; + const CbObject& ActionObj = Action->ActionObj; + + MaybeDumpAction(ActionLsn, ActionObj); + + std::filesystem::path SandboxPath = CreateNewSandbox(); + + // Ensure the sandbox directory is cleaned up if any subsequent step throws + auto SandboxGuard = MakeGuard([&] { m_DeferredDeleter.Enqueue(Action->ActionLsn, std::move(SandboxPath)); }); + + CbPackage WorkerPackage = Action->Worker.Descriptor; + + std::filesystem::path WorkerPath = ManifestWorker(Action->Worker); + + // Write out action + + zen::WriteFile(SandboxPath / "build.action", ActionObj.GetBuffer().AsIoBuffer()); + + // Manifest inputs in sandbox + + ActionObj.IterateAttachments([&](CbFieldView Field) { + const IoHash Cid = Field.AsHash(); + std::filesystem::path FilePath{SandboxPath / "Inputs"sv / Cid.ToHexString()}; + IoBuffer DataBuffer = m_ChunkResolver.FindChunkByCid(Cid); + + if (!DataBuffer) + { + throw std::runtime_error(fmt::format("input CID chunk '{}' missing", Cid)); + } + + zen::WriteFile(FilePath, DataBuffer); + }); + + Action->ExecutionLocation = "local"; + + SandboxGuard.Dismiss(); + + return PreparedAction{ + .ActionLsn = ActionLsn, + .SandboxPath = std::move(SandboxPath), + .WorkerPath = std::move(WorkerPath), + .WorkerPackage = std::move(WorkerPackage), + }; +} + +SubmitResult +LocalProcessRunner::SubmitAction(Ref Action) +{ + // Base class is not directly usable — platform subclasses override this + ZEN_UNUSED(Action); + return SubmitResult{.IsAccepted = false}; +} + +size_t +LocalProcessRunner::GetSubmittedActionCount() +{ + RwLock::SharedLockScope _(m_RunningLock); + return m_RunningMap.size(); +} + +std::filesystem::path +LocalProcessRunner::ManifestWorker(const WorkerDesc& Worker) +{ + ZEN_TRACE_CPU("LocalProcessRunner::ManifestWorker"); + RwLock::SharedLockScope _(m_WorkerLock); + + std::filesystem::path WorkerDir = m_WorkerPath / fmt::format("runner_{}", Worker.WorkerId); + + if (!std::filesystem::exists(WorkerDir)) + { + _.ReleaseNow(); + + RwLock::ExclusiveLockScope $(m_WorkerLock); + + if (!std::filesystem::exists(WorkerDir)) + { + ManifestWorker(Worker.Descriptor, WorkerDir, [](const IoHash&, CompressedBuffer&) {}); + } + } + + return WorkerDir; +} + +void +LocalProcessRunner::DecompressAttachmentToFile(const CbPackage& FromPackage, + CbObjectView FileEntry, + const std::filesystem::path& SandboxRootPath, + std::function& ChunkReferenceCallback) +{ + std::string_view Name = FileEntry["name"sv].AsString(); + const IoHash ChunkHash = FileEntry["hash"sv].AsHash(); + const uint64_t Size = FileEntry["size"sv].AsUInt64(); + + CompressedBuffer Compressed; + + if (const CbAttachment* Attachment = FromPackage.FindAttachment(ChunkHash)) + { + Compressed = Attachment->AsCompressedBinary(); + } + else + { + IoBuffer DataBuffer = m_ChunkResolver.FindChunkByCid(ChunkHash); + + if (!DataBuffer) + { + throw std::runtime_error(fmt::format("worker chunk '{}' missing", ChunkHash)); + } + + uint64_t DataRawSize = 0; + IoHash DataRawHash; + Compressed = CompressedBuffer::FromCompressed(SharedBuffer{DataBuffer}, DataRawHash, DataRawSize); + + if (DataRawSize != Size) + { + throw std::runtime_error( + fmt::format("worker chunk '{}' size: {}, action spec expected {}", ChunkHash, DataBuffer.Size(), Size)); + } + } + + ChunkReferenceCallback(ChunkHash, Compressed); + + std::filesystem::path FilePath{SandboxRootPath / std::filesystem::path(Name).make_preferred()}; + + // Validate the resolved path stays within the sandbox to prevent directory traversal + // via malicious names like "../../etc/evil" + // + // This might be worth revisiting to frontload the validation and eliminate some memory + // allocations in the future. + { + std::filesystem::path CanonicalRoot = std::filesystem::weakly_canonical(SandboxRootPath); + std::filesystem::path CanonicalFile = std::filesystem::weakly_canonical(FilePath); + std::string RootStr = CanonicalRoot.string(); + std::string FileStr = CanonicalFile.string(); + + if (FileStr.size() < RootStr.size() || FileStr.compare(0, RootStr.size(), RootStr) != 0) + { + throw zen::runtime_error("path traversal detected: '{}' escapes sandbox root '{}'", Name, SandboxRootPath); + } + } + + SharedBuffer Decompressed = Compressed.Decompress(); + zen::WriteFile(FilePath, Decompressed.AsIoBuffer()); +} + +void +LocalProcessRunner::ManifestWorker(const CbPackage& WorkerPackage, + const std::filesystem::path& SandboxPath, + std::function&& ChunkReferenceCallback) +{ + CbObject WorkerDescription = WorkerPackage.GetObject(); + + // Manifest worker in Sandbox + + for (auto& It : WorkerDescription["executables"sv]) + { + DecompressAttachmentToFile(WorkerPackage, It.AsObjectView(), SandboxPath, ChunkReferenceCallback); +# if !ZEN_PLATFORM_WINDOWS + std::string_view ExeName = It.AsObjectView()["name"sv].AsString(); + std::filesystem::path ExePath{SandboxPath / std::filesystem::path(ExeName).make_preferred()}; + std::filesystem::permissions( + ExePath, + std::filesystem::perms::owner_exec | std::filesystem::perms::group_exec | std::filesystem::perms::others_exec, + std::filesystem::perm_options::add); +# endif + } + + for (auto& It : WorkerDescription["dirs"sv]) + { + std::string_view Name = It.AsString(); + std::filesystem::path DirPath{SandboxPath / std::filesystem::path(Name).make_preferred()}; + + // Validate dir path stays within sandbox + { + std::filesystem::path CanonicalRoot = std::filesystem::weakly_canonical(SandboxPath); + std::filesystem::path CanonicalDir = std::filesystem::weakly_canonical(DirPath); + std::string RootStr = CanonicalRoot.string(); + std::string DirStr = CanonicalDir.string(); + + if (DirStr.size() < RootStr.size() || DirStr.compare(0, RootStr.size(), RootStr) != 0) + { + throw zen::runtime_error("path traversal detected: dir '{}' escapes sandbox root '{}'", Name, SandboxPath); + } + } + + zen::CreateDirectories(DirPath); + } + + for (auto& It : WorkerDescription["files"sv]) + { + DecompressAttachmentToFile(WorkerPackage, It.AsObjectView(), SandboxPath, ChunkReferenceCallback); + } + + WriteFile(SandboxPath / "worker.zcb", WorkerDescription.GetBuffer().AsIoBuffer()); +} + +CbPackage +LocalProcessRunner::GatherActionOutputs(std::filesystem::path SandboxPath) +{ + ZEN_TRACE_CPU("LocalProcessRunner::GatherActionOutputs"); + std::filesystem::path OutputFile = SandboxPath / "build.output"; + FileContents OutputData = zen::ReadFile(OutputFile); + + if (OutputData.ErrorCode) + { + throw std::system_error(OutputData.ErrorCode, fmt::format("Failed to read build output file '{}'", OutputFile)); + } + + CbPackage OutputPackage; + CbObject Output = zen::LoadCompactBinaryObject(OutputData.Flatten()); + + uint64_t TotalAttachmentBytes = 0; + uint64_t TotalRawAttachmentBytes = 0; + + Output.IterateAttachments([&](CbFieldView Field) { + IoHash Hash = Field.AsHash(); + std::filesystem::path OutputPath{SandboxPath / "Outputs" / Hash.ToHexString()}; + FileContents ChunkData = zen::ReadFile(OutputPath); + + if (ChunkData.ErrorCode) + { + throw std::system_error(ChunkData.ErrorCode, fmt::format("Failed to read build output file '{}'", OutputPath)); + } + + uint64_t ChunkDataRawSize = 0; + IoHash ChunkDataHash; + CompressedBuffer AttachmentBuffer = + CompressedBuffer::FromCompressed(SharedBuffer(ChunkData.Flatten()), ChunkDataHash, ChunkDataRawSize); + + if (!AttachmentBuffer) + { + throw std::runtime_error("Invalid output encountered (not valid CompressedBuffer format)"); + } + + TotalAttachmentBytes += AttachmentBuffer.GetCompressedSize(); + TotalRawAttachmentBytes += ChunkDataRawSize; + + CbAttachment Attachment(std::move(AttachmentBuffer), ChunkDataHash); + OutputPackage.AddAttachment(Attachment); + }); + + OutputPackage.SetObject(Output); + + ZEN_DEBUG("Action completed with {} attachments ({} compressed, {} uncompressed)", + OutputPackage.GetAttachments().size(), + NiceBytes(TotalAttachmentBytes), + NiceBytes(TotalRawAttachmentBytes)); + + return OutputPackage; +} + +void +LocalProcessRunner::MonitorThreadFunction() +{ + SetCurrentThreadName("LocalProcessRunner_Monitor"); + + auto _ = MakeGuard([&] { ZEN_INFO("monitor thread exiting"); }); + + do + { + // On Windows it's possible to wait on process handles, so we wait for either a process to exit + // or for the monitor event to be signaled (which indicates we should check for cancellation + // or shutdown). This could be further improved by using a completion port and registering process + // handles with it, but this is a reasonable first implementation given that we shouldn't be dealing + // with an enormous number of concurrent processes. + // + // On other platforms we just wait on the monitor event and poll for process exits at intervals. +# if ZEN_PLATFORM_WINDOWS + auto WaitOnce = [&] { + HANDLE WaitHandles[MAXIMUM_WAIT_OBJECTS]; + + uint32_t NumHandles = 0; + + WaitHandles[NumHandles++] = m_MonitorThreadEvent.GetWindowsHandle(); + + m_RunningLock.WithSharedLock([&] { + for (auto It = begin(m_RunningMap), ItEnd = end(m_RunningMap); It != ItEnd && NumHandles < MAXIMUM_WAIT_OBJECTS; ++It) + { + Ref Action = It->second; + + WaitHandles[NumHandles++] = Action->ProcessHandle; + } + }); + + DWORD WaitResult = WaitForMultipleObjects(NumHandles, WaitHandles, FALSE, 1000); + + // return true if a handle was signaled + return (WaitResult <= NumHandles); + }; +# else + auto WaitOnce = [&] { return m_MonitorThreadEvent.Wait(1000); }; +# endif + + while (!WaitOnce()) + { + if (m_MonitorThreadEnabled == false) + { + return; + } + + SweepRunningActions(); + SampleRunningProcessCpu(); + } + + // Signal received + + SweepRunningActions(); + SampleRunningProcessCpu(); + } while (m_MonitorThreadEnabled); +} + +void +LocalProcessRunner::CancelRunningActions() +{ + // Base class is not directly usable — platform subclasses override this +} + +void +LocalProcessRunner::SampleRunningProcessCpu() +{ + static constexpr uint64_t kSampleIntervalMs = 5'000; + + m_RunningLock.WithSharedLock([&] { + const uint64_t Now = GetHifreqTimerValue(); + for (auto& [Lsn, Running] : m_RunningMap) + { + const bool NeverSampled = Running->LastCpuSampleTicks == 0; + const bool IntervalElapsed = Stopwatch::GetElapsedTimeMs(Now - Running->LastCpuSampleTicks) >= kSampleIntervalMs; + if (NeverSampled || IntervalElapsed) + { + SampleProcessCpu(*Running); + } + } + }); +} + +void +LocalProcessRunner::SweepRunningActions() +{ + ZEN_TRACE_CPU("LocalProcessRunner::SweepRunningActions"); +} + +void +LocalProcessRunner::ProcessCompletedActions(std::vector>& CompletedActions) +{ + ZEN_TRACE_CPU("LocalProcessRunner::ProcessCompletedActions"); + // Shared post-processing: gather outputs, set state, clean sandbox. + // Note that this must be called without holding any local locks + // otherwise we may end up with deadlocks. + + for (Ref Running : CompletedActions) + { + const int ActionLsn = Running->Action->ActionLsn; + + if (Running->ExitCode == 0) + { + try + { + // Gather outputs + + CbPackage OutputPackage = GatherActionOutputs(Running->SandboxPath); + + Running->Action->SetResult(std::move(OutputPackage)); + Running->Action->SetActionState(RunnerAction::State::Completed); + + // Enqueue sandbox for deferred background deletion, giving + // file handles time to close before we attempt removal. + m_DeferredDeleter.Enqueue(ActionLsn, std::move(Running->SandboxPath)); + + // Success -- continue with next iteration of the loop + continue; + } + catch (std::exception& Ex) + { + ZEN_ERROR("Encountered failure while gathering outputs for action lsn {}, '{}'", ActionLsn, Ex.what()); + } + } + + // Failed - clean up the sandbox in the background. + + m_DeferredDeleter.Enqueue(ActionLsn, std::move(Running->SandboxPath)); + Running->Action->SetActionState(RunnerAction::State::Failed); + } +} + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/localrunner.h b/src/zencompute/runners/localrunner.h new file mode 100644 index 000000000..7493e980b --- /dev/null +++ b/src/zencompute/runners/localrunner.h @@ -0,0 +1,138 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencompute/computeservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "functionrunner.h" + +# include +# include +# include +# include +# include + +# include "deferreddeleter.h" + +# include + +# include +# include +# include +# include + +namespace zen { +class CbPackage; +} + +namespace zen::compute { + +/** Direct process spawner + + This runner simply sets up a directory structure for each job and + creates a process to perform the computation in it. It is not very + efficient and is intended mostly for testing. + + */ + +class LocalProcessRunner : public FunctionRunner +{ + LocalProcessRunner(LocalProcessRunner&&) = delete; + LocalProcessRunner& operator=(LocalProcessRunner&&) = delete; + +public: + LocalProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool, + int32_t MaxConcurrentActions = 0); + ~LocalProcessRunner(); + + virtual void Shutdown() override; + virtual void RegisterWorker(const CbPackage& WorkerPackage) override; + [[nodiscard]] virtual SubmitResult SubmitAction(Ref Action) override; + [[nodiscard]] virtual bool IsHealthy() override { return true; } + [[nodiscard]] virtual size_t GetSubmittedActionCount() override; + [[nodiscard]] virtual size_t QueryCapacity() override; + [[nodiscard]] virtual std::vector SubmitActions(const std::vector>& Actions) override; + +protected: + LoggerRef Log() { return m_Log; } + + LoggerRef m_Log; + + struct RunningAction : public RefCounted + { + Ref Action; + void* ProcessHandle = nullptr; + int ExitCode = 0; + std::filesystem::path SandboxPath; + + // State for periodic CPU usage sampling + uint64_t LastCpuSampleTicks = 0; // hifreq timer value at last sample + uint64_t LastCpuOsTicks = 0; // OS CPU ticks (platform-specific units) at last sample + }; + + std::atomic_bool m_AcceptNewActions; + ChunkResolver& m_ChunkResolver; + RwLock m_WorkerLock; + std::filesystem::path m_WorkerPath; + std::atomic m_SandboxCounter = 0; + std::filesystem::path m_SandboxPath; + int32_t m_MaxRunningActions = 64; // arbitrary limit for testing + + // if used in conjuction with m_ResultsLock, this lock must be taken *after* + // m_ResultsLock to avoid deadlocks + RwLock m_RunningLock; + std::unordered_map> m_RunningMap; + + std::atomic m_SubmittingCount = 0; + DeferredDirectoryDeleter& m_DeferredDeleter; + WorkerThreadPool& m_WorkerPool; + + std::thread m_MonitorThread; + std::atomic m_MonitorThreadEnabled{true}; + Event m_MonitorThreadEvent; + void MonitorThreadFunction(); + virtual void SweepRunningActions(); + virtual void CancelRunningActions(); + + // Sample CPU usage for all currently running processes (throttled per-action). + void SampleRunningProcessCpu(); + + // Override in platform runners to sample one process. Called under a shared RunningLock. + virtual void SampleProcessCpu(RunningAction& /*Running*/) {} + + // Shared preamble for SubmitAction: capacity check, sandbox creation, + // worker manifesting, action writing, input manifesting. + struct PreparedAction + { + int32_t ActionLsn; + std::filesystem::path SandboxPath; + std::filesystem::path WorkerPath; + CbPackage WorkerPackage; + }; + std::optional PrepareActionSubmission(Ref Action); + + // Shared post-processing for SweepRunningActions: gather outputs, + // set state, clean sandbox. + void ProcessCompletedActions(std::vector>& CompletedActions); + + std::filesystem::path CreateNewSandbox(); + void ManifestWorker(const CbPackage& WorkerPackage, + const std::filesystem::path& SandboxPath, + std::function&& ChunkReferenceCallback); + std::filesystem::path ManifestWorker(const WorkerDesc& Worker); + CbPackage GatherActionOutputs(std::filesystem::path SandboxPath); + + void DecompressAttachmentToFile(const CbPackage& FromPackage, + CbObjectView FileEntry, + const std::filesystem::path& SandboxRootPath, + std::function& ChunkReferenceCallback); +}; + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/macrunner.cpp b/src/zencompute/runners/macrunner.cpp new file mode 100644 index 000000000..5cec90699 --- /dev/null +++ b/src/zencompute/runners/macrunner.cpp @@ -0,0 +1,491 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "macrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES && ZEN_PLATFORM_MAC + +# include +# include +# include +# include +# include +# include +# include +# include + +# include +# include +# include +# include +# include +# include + +namespace zen::compute { + +using namespace std::literals; + +namespace { + + // All helper functions in this namespace are async-signal-safe (safe to call + // between fork() and execve()). They use only raw syscalls and avoid any + // heap allocation, stdio, or other non-AS-safe operations. + + void WriteToFd(int Fd, const char* Buf, size_t Len) + { + while (Len > 0) + { + ssize_t Written = write(Fd, Buf, Len); + if (Written <= 0) + { + break; + } + Buf += Written; + Len -= static_cast(Written); + } + } + + [[noreturn]] void WriteErrorAndExit(int ErrorPipeFd, const char* Msg, int Errno) + { + // Write the message prefix + size_t MsgLen = 0; + for (const char* P = Msg; *P; ++P) + { + ++MsgLen; + } + WriteToFd(ErrorPipeFd, Msg, MsgLen); + + // Append ": " and the errno string if non-zero + if (Errno != 0) + { + WriteToFd(ErrorPipeFd, ": ", 2); + const char* ErrStr = strerror(Errno); + size_t ErrLen = 0; + for (const char* P = ErrStr; *P; ++P) + { + ++ErrLen; + } + WriteToFd(ErrorPipeFd, ErrStr, ErrLen); + } + + _exit(127); + } + + // Build a Seatbelt profile string that denies everything by default and + // allows only the minimum needed for the worker to execute: process ops, + // system library reads, worker directory (read-only), and sandbox directory + // (read-write). Network access is denied implicitly by the deny-default policy. + std::string BuildSandboxProfile(const std::string& SandboxPath, const std::string& WorkerPath) + { + std::string Profile; + Profile.reserve(1024); + + Profile += "(version 1)\n"; + Profile += "(deny default)\n"; + Profile += "(allow process*)\n"; + Profile += "(allow sysctl-read)\n"; + Profile += "(allow file-read-metadata)\n"; + + // System library paths needed for dynamic linker and runtime + Profile += "(allow file-read* (subpath \"/usr\"))\n"; + Profile += "(allow file-read* (subpath \"/System\"))\n"; + Profile += "(allow file-read* (subpath \"/Library\"))\n"; + Profile += "(allow file-read* (subpath \"/dev\"))\n"; + Profile += "(allow file-read* (subpath \"/private/var/db/dyld\"))\n"; + Profile += "(allow file-read* (subpath \"/etc\"))\n"; + + // Worker directory: read-only + Profile += "(allow file-read* (subpath \""; + Profile += WorkerPath; + Profile += "\"))\n"; + + // Sandbox directory: read+write + Profile += "(allow file-read* file-write* (subpath \""; + Profile += SandboxPath; + Profile += "\"))\n"; + + return Profile; + } + +} // anonymous namespace + +MacProcessRunner::MacProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool, + bool Sandboxed, + int32_t MaxConcurrentActions) +: LocalProcessRunner(Resolver, BaseDir, Deleter, WorkerPool, MaxConcurrentActions) +, m_Sandboxed(Sandboxed) +{ + // Restore SIGCHLD to default behavior so waitpid() can properly collect + // child exit status. zenserver/main.cpp sets SIGCHLD to SIG_IGN which + // causes the kernel to auto-reap children, making waitpid() return + // -1/ECHILD instead of the exit status we need. + struct sigaction Action = {}; + sigemptyset(&Action.sa_mask); + Action.sa_handler = SIG_DFL; + sigaction(SIGCHLD, &Action, nullptr); + + if (m_Sandboxed) + { + ZEN_INFO("Seatbelt sandboxing enabled for child processes"); + } +} + +SubmitResult +MacProcessRunner::SubmitAction(Ref Action) +{ + ZEN_TRACE_CPU("MacProcessRunner::SubmitAction"); + std::optional Prepared = PrepareActionSubmission(Action); + + if (!Prepared) + { + return SubmitResult{.IsAccepted = false}; + } + + // Build environment array from worker descriptor + + CbObject WorkerDescription = Prepared->WorkerPackage.GetObject(); + + std::vector EnvStrings; + for (auto& It : WorkerDescription["environment"sv]) + { + EnvStrings.emplace_back(It.AsString()); + } + + std::vector Envp; + Envp.reserve(EnvStrings.size() + 1); + for (auto& Str : EnvStrings) + { + Envp.push_back(Str.data()); + } + Envp.push_back(nullptr); + + // Build argv: -Build=build.action + + std::string_view ExecPath = WorkerDescription["path"sv].AsString(); + std::filesystem::path ExePath = Prepared->WorkerPath / std::filesystem::path(ExecPath); + std::string ExePathStr = ExePath.string(); + std::string BuildArg = "-Build=build.action"; + + std::vector ArgV; + ArgV.push_back(ExePathStr.data()); + ArgV.push_back(BuildArg.data()); + ArgV.push_back(nullptr); + + ZEN_DEBUG("Executing: {} {} (sandboxed={})", ExePathStr, BuildArg, m_Sandboxed); + + std::string SandboxPathStr = Prepared->SandboxPath.string(); + std::string WorkerPathStr = Prepared->WorkerPath.string(); + + // Pre-fork: build sandbox profile and create error pipe + std::string SandboxProfile; + int ErrorPipe[2] = {-1, -1}; + + if (m_Sandboxed) + { + SandboxProfile = BuildSandboxProfile(SandboxPathStr, WorkerPathStr); + + if (pipe(ErrorPipe) != 0) + { + throw zen::runtime_error("pipe() for sandbox error pipe failed: {}", strerror(errno)); + } + fcntl(ErrorPipe[0], F_SETFD, FD_CLOEXEC); + fcntl(ErrorPipe[1], F_SETFD, FD_CLOEXEC); + } + + pid_t ChildPid = fork(); + + if (ChildPid < 0) + { + int SavedErrno = errno; + if (m_Sandboxed) + { + close(ErrorPipe[0]); + close(ErrorPipe[1]); + } + throw zen::runtime_error("fork() failed: {}", strerror(SavedErrno)); + } + + if (ChildPid == 0) + { + // Child process + + if (m_Sandboxed) + { + // Close read end of error pipe — child only writes + close(ErrorPipe[0]); + + // Apply Seatbelt sandbox profile + char* ErrorBuf = nullptr; + if (sandbox_init(SandboxProfile.c_str(), 0, &ErrorBuf) != 0) + { + // sandbox_init failed — write error to pipe and exit + if (ErrorBuf) + { + WriteErrorAndExit(ErrorPipe[1], ErrorBuf, 0); + // WriteErrorAndExit does not return, but sandbox_free_error + // is not needed since we _exit + } + WriteErrorAndExit(ErrorPipe[1], "sandbox_init failed", errno); + } + if (ErrorBuf) + { + sandbox_free_error(ErrorBuf); + } + + if (chdir(SandboxPathStr.c_str()) != 0) + { + WriteErrorAndExit(ErrorPipe[1], "chdir to sandbox failed", errno); + } + + execve(ExePathStr.c_str(), ArgV.data(), Envp.data()); + + WriteErrorAndExit(ErrorPipe[1], "execve failed", errno); + } + else + { + if (chdir(SandboxPathStr.c_str()) != 0) + { + _exit(127); + } + + execve(ExePathStr.c_str(), ArgV.data(), Envp.data()); + _exit(127); + } + } + + // Parent process + + if (m_Sandboxed) + { + // Close write end of error pipe — parent only reads + close(ErrorPipe[1]); + + // Read from error pipe. If execve succeeded, pipe was closed by O_CLOEXEC + // and read returns 0. If setup failed, child wrote an error message. + char ErrBuf[512]; + ssize_t BytesRead = read(ErrorPipe[0], ErrBuf, sizeof(ErrBuf) - 1); + close(ErrorPipe[0]); + + if (BytesRead > 0) + { + // Sandbox setup or execve failed + ErrBuf[BytesRead] = '\0'; + + // Reap the child (it called _exit(127)) + waitpid(ChildPid, nullptr, 0); + + // Clean up the sandbox in the background + m_DeferredDeleter.Enqueue(Action->ActionLsn, std::move(Prepared->SandboxPath)); + + ZEN_ERROR("Sandbox setup failed for action {}: {}", Action->ActionLsn, ErrBuf); + + Action->SetActionState(RunnerAction::State::Failed); + return SubmitResult{.IsAccepted = false}; + } + } + + // Store child pid as void* (same convention as zencore/process.cpp) + + Ref NewAction{new RunningAction()}; + NewAction->Action = Action; + NewAction->ProcessHandle = reinterpret_cast(static_cast(ChildPid)); + NewAction->SandboxPath = std::move(Prepared->SandboxPath); + + { + RwLock::ExclusiveLockScope _(m_RunningLock); + m_RunningMap[Prepared->ActionLsn] = std::move(NewAction); + } + + Action->SetActionState(RunnerAction::State::Running); + + return SubmitResult{.IsAccepted = true}; +} + +void +MacProcessRunner::SweepRunningActions() +{ + ZEN_TRACE_CPU("MacProcessRunner::SweepRunningActions"); + std::vector> CompletedActions; + + m_RunningLock.WithExclusiveLock([&] { + for (auto It = begin(m_RunningMap), ItEnd = end(m_RunningMap); It != ItEnd;) + { + Ref Running = It->second; + + pid_t Pid = static_cast(reinterpret_cast(Running->ProcessHandle)); + int Status = 0; + + pid_t Result = waitpid(Pid, &Status, WNOHANG); + + if (Result == Pid) + { + if (WIFEXITED(Status)) + { + Running->ExitCode = WEXITSTATUS(Status); + } + else if (WIFSIGNALED(Status)) + { + Running->ExitCode = 128 + WTERMSIG(Status); + } + else + { + Running->ExitCode = 1; + } + + Running->ProcessHandle = nullptr; + + CompletedActions.push_back(std::move(Running)); + It = m_RunningMap.erase(It); + } + else + { + ++It; + } + } + }); + + ProcessCompletedActions(CompletedActions); +} + +void +MacProcessRunner::CancelRunningActions() +{ + ZEN_TRACE_CPU("MacProcessRunner::CancelRunningActions"); + Stopwatch Timer; + std::unordered_map> RunningMap; + + m_RunningLock.WithExclusiveLock([&] { std::swap(RunningMap, m_RunningMap); }); + + if (RunningMap.empty()) + { + return; + } + + ZEN_INFO("cancelling all running actions"); + + // Send SIGTERM to all running processes first + + for (const auto& [Lsn, Running] : RunningMap) + { + pid_t Pid = static_cast(reinterpret_cast(Running->ProcessHandle)); + + if (kill(Pid, SIGTERM) != 0) + { + ZEN_WARN("kill(SIGTERM) for LSN {} (pid {}) failed: {}", Running->Action->ActionLsn, Pid, strerror(errno)); + } + } + + // Wait for all processes, regardless of whether SIGTERM succeeded, then clean up. + + for (auto& [Lsn, Running] : RunningMap) + { + pid_t Pid = static_cast(reinterpret_cast(Running->ProcessHandle)); + + // Poll for up to 2 seconds + bool Exited = false; + for (int i = 0; i < 20; ++i) + { + int Status = 0; + pid_t WaitResult = waitpid(Pid, &Status, WNOHANG); + if (WaitResult == Pid) + { + Exited = true; + ZEN_DEBUG("LSN {}: process exit OK", Running->Action->ActionLsn); + break; + } + usleep(100000); // 100ms + } + + if (!Exited) + { + ZEN_WARN("LSN {}: process did not exit after SIGTERM, sending SIGKILL", Running->Action->ActionLsn); + kill(Pid, SIGKILL); + waitpid(Pid, nullptr, 0); + } + + m_DeferredDeleter.Enqueue(Running->Action->ActionLsn, std::move(Running->SandboxPath)); + Running->Action->SetActionState(RunnerAction::State::Failed); + } + + ZEN_INFO("DONE - cancelled {} running processes (took {})", RunningMap.size(), NiceTimeSpanMs(Timer.GetElapsedTimeMs())); +} + +bool +MacProcessRunner::CancelAction(int ActionLsn) +{ + ZEN_TRACE_CPU("MacProcessRunner::CancelAction"); + + // Hold the shared lock while sending the signal to prevent the sweep thread + // from reaping the PID (via waitpid) between our lookup and kill(). Without + // the lock held, the PID could be recycled by the kernel and we'd signal an + // unrelated process. + bool Sent = false; + + m_RunningLock.WithSharedLock([&] { + auto It = m_RunningMap.find(ActionLsn); + if (It == m_RunningMap.end()) + { + return; + } + + Ref Target = It->second; + if (!Target->ProcessHandle) + { + return; + } + + pid_t Pid = static_cast(reinterpret_cast(Target->ProcessHandle)); + + if (kill(Pid, SIGTERM) != 0) + { + ZEN_WARN("CancelAction: kill(SIGTERM) for LSN {} (pid {}) failed: {}", ActionLsn, Pid, strerror(errno)); + return; + } + + ZEN_DEBUG("CancelAction: sent SIGTERM to LSN {} (pid {})", ActionLsn, Pid); + Sent = true; + }); + + // The monitor thread will pick up the process exit and mark the action as Failed. + return Sent; +} + +void +MacProcessRunner::SampleProcessCpu(RunningAction& Running) +{ + const pid_t Pid = static_cast(reinterpret_cast(Running.ProcessHandle)); + + struct proc_taskinfo Info; + if (proc_pidinfo(Pid, PROC_PIDTASKINFO, 0, &Info, sizeof(Info)) <= 0) + { + return; + } + + // pti_total_user and pti_total_system are in nanoseconds + const uint64_t CurrentOsTicks = Info.pti_total_user + Info.pti_total_system; + const uint64_t NowTicks = GetHifreqTimerValue(); + + // Cumulative CPU seconds (absolute, available from first sample): ns → seconds + Running.Action->CpuSeconds.store(static_cast(static_cast(CurrentOsTicks) / 1'000'000'000.0), std::memory_order_relaxed); + + if (Running.LastCpuSampleTicks != 0 && Running.LastCpuOsTicks != 0) + { + const uint64_t ElapsedMs = Stopwatch::GetElapsedTimeMs(NowTicks - Running.LastCpuSampleTicks); + if (ElapsedMs > 0) + { + const uint64_t DeltaOsTicks = CurrentOsTicks - Running.LastCpuOsTicks; + // ns → ms: divide by 1,000,000; then as percent of elapsed ms + const float CpuPct = static_cast(static_cast(DeltaOsTicks) / 1'000'000.0 / ElapsedMs * 100.0); + Running.Action->CpuUsagePercent.store(CpuPct, std::memory_order_relaxed); + } + } + + Running.LastCpuSampleTicks = NowTicks; + Running.LastCpuOsTicks = CurrentOsTicks; +} + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/macrunner.h b/src/zencompute/runners/macrunner.h new file mode 100644 index 000000000..d653b923a --- /dev/null +++ b/src/zencompute/runners/macrunner.h @@ -0,0 +1,43 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "localrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES && ZEN_PLATFORM_MAC + +namespace zen::compute { + +/** Native macOS process runner for executing Mac worker executables directly. + + Subclasses LocalProcessRunner, reusing sandbox management, worker manifesting, + input/output handling, and monitor thread infrastructure. Overrides only the + platform-specific methods: process spawning, sweep, and cancellation. + + When Sandboxed is true, child processes are isolated using macOS Seatbelt + (sandbox_init): no network access and no filesystem access outside the + explicitly allowed sandbox and worker directories. This requires no elevation. + */ +class MacProcessRunner : public LocalProcessRunner +{ +public: + MacProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool, + bool Sandboxed = false, + int32_t MaxConcurrentActions = 0); + + [[nodiscard]] SubmitResult SubmitAction(Ref Action) override; + void SweepRunningActions() override; + void CancelRunningActions() override; + bool CancelAction(int ActionLsn) override; + void SampleProcessCpu(RunningAction& Running) override; + +private: + bool m_Sandboxed = false; +}; + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/remotehttprunner.cpp b/src/zencompute/runners/remotehttprunner.cpp new file mode 100644 index 000000000..672636d06 --- /dev/null +++ b/src/zencompute/runners/remotehttprunner.cpp @@ -0,0 +1,618 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "remotehttprunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include + +////////////////////////////////////////////////////////////////////////// + +namespace zen::compute { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +RemoteHttpRunner::RemoteHttpRunner(ChunkResolver& InChunkResolver, + const std::filesystem::path& BaseDir, + std::string_view HostName, + WorkerThreadPool& InWorkerPool) +: FunctionRunner(BaseDir) +, m_Log(logging::Get("http_exec")) +, m_ChunkResolver{InChunkResolver} +, m_WorkerPool{InWorkerPool} +, m_HostName{HostName} +, m_BaseUrl{fmt::format("{}/compute", HostName)} +, m_Http(m_BaseUrl) +, m_InstanceId(Oid::NewOid()) +{ + m_MonitorThread = std::thread{&RemoteHttpRunner::MonitorThreadFunction, this}; +} + +RemoteHttpRunner::~RemoteHttpRunner() +{ + Shutdown(); +} + +void +RemoteHttpRunner::Shutdown() +{ + // TODO: should cleanly drain/cancel pending work + + m_MonitorThreadEnabled = false; + m_MonitorThreadEvent.Set(); + if (m_MonitorThread.joinable()) + { + m_MonitorThread.join(); + } +} + +void +RemoteHttpRunner::RegisterWorker(const CbPackage& WorkerPackage) +{ + ZEN_TRACE_CPU("RemoteHttpRunner::RegisterWorker"); + const IoHash WorkerId = WorkerPackage.GetObjectHash(); + CbPackage WorkerDesc = WorkerPackage; + + std::string WorkerUrl = fmt::format("/workers/{}", WorkerId); + + HttpClient::Response WorkerResponse = m_Http.Get(WorkerUrl); + + if (WorkerResponse.StatusCode == HttpResponseCode::NotFound) + { + HttpClient::Response DescResponse = m_Http.Post(WorkerUrl, WorkerDesc.GetObject()); + + if (DescResponse.StatusCode == HttpResponseCode::NotFound) + { + CbPackage Pkg = WorkerDesc; + + // Build response package by sending only the attachments + // the other end needs. We start with the full package and + // remove the attachments which are not needed. + + { + std::unordered_set Needed; + + CbObject Response = DescResponse.AsObject(); + + for (auto& Item : Response["need"sv]) + { + const IoHash NeedHash = Item.AsHash(); + + Needed.insert(NeedHash); + } + + std::unordered_set ToRemove; + + for (const CbAttachment& Attachment : Pkg.GetAttachments()) + { + const IoHash& Hash = Attachment.GetHash(); + + if (Needed.find(Hash) == Needed.end()) + { + ToRemove.insert(Hash); + } + } + + for (const IoHash& Hash : ToRemove) + { + int RemovedCount = Pkg.RemoveAttachment(Hash); + + ZEN_ASSERT(RemovedCount == 1); + } + } + + // Post resulting package + + HttpClient::Response PayloadResponse = m_Http.Post(WorkerUrl, Pkg); + + if (!IsHttpSuccessCode(PayloadResponse.StatusCode)) + { + ZEN_ERROR("ERROR: unable to register payloads for worker {} at {}{}", WorkerId, m_Http.GetBaseUri(), WorkerUrl); + + // TODO: propagate error + } + } + else if (!IsHttpSuccessCode(DescResponse.StatusCode)) + { + ZEN_ERROR("ERROR: unable to register worker {} at {}{}", WorkerId, m_Http.GetBaseUri(), WorkerUrl); + + // TODO: propagate error + } + else + { + ZEN_ASSERT(DescResponse.StatusCode == HttpResponseCode::NoContent); + } + } + else if (WorkerResponse.StatusCode == HttpResponseCode::OK) + { + // Already known from a previous run + } + else if (!IsHttpSuccessCode(WorkerResponse.StatusCode)) + { + ZEN_ERROR("ERROR: unable to look up worker {} at {}{} (error: {} {})", + WorkerId, + m_Http.GetBaseUri(), + WorkerUrl, + (int)WorkerResponse.StatusCode, + ToString(WorkerResponse.StatusCode)); + + // TODO: propagate error + } +} + +size_t +RemoteHttpRunner::QueryCapacity() +{ + // Estimate how much more work we're ready to accept + + RwLock::SharedLockScope _{m_RunningLock}; + + size_t RunningCount = m_RemoteRunningMap.size(); + + if (RunningCount >= size_t(m_MaxRunningActions)) + { + return 0; + } + + return m_MaxRunningActions - RunningCount; +} + +std::vector +RemoteHttpRunner::SubmitActions(const std::vector>& Actions) +{ + ZEN_TRACE_CPU("RemoteHttpRunner::SubmitActions"); + + if (Actions.size() <= 1) + { + std::vector Results; + + for (const Ref& Action : Actions) + { + Results.push_back(SubmitAction(Action)); + } + + return Results; + } + + // For larger batches, submit HTTP requests in parallel via the shared worker pool + + std::vector> Futures; + Futures.reserve(Actions.size()); + + for (const Ref& Action : Actions) + { + std::packaged_task Task([this, Action]() { return SubmitAction(Action); }); + + Futures.push_back(m_WorkerPool.EnqueueTask(std::move(Task), WorkerThreadPool::EMode::EnableBacklog)); + } + + std::vector Results; + Results.reserve(Futures.size()); + + for (auto& Future : Futures) + { + Results.push_back(Future.get()); + } + + return Results; +} + +SubmitResult +RemoteHttpRunner::SubmitAction(Ref Action) +{ + ZEN_TRACE_CPU("RemoteHttpRunner::SubmitAction"); + + // Verify whether we can accept more work + + { + RwLock::SharedLockScope _{m_RunningLock}; + if (m_RemoteRunningMap.size() >= size_t(m_MaxRunningActions)) + { + return SubmitResult{.IsAccepted = false}; + } + } + + using namespace std::literals; + + // Each enqueued action is assigned an integer index (logical sequence number), + // which we use as a key for tracking data structures and as an opaque id which + // may be used by clients to reference the scheduled action + + Action->ExecutionLocation = m_HostName; + + const int32_t ActionLsn = Action->ActionLsn; + const CbObject& ActionObj = Action->ActionObj; + const IoHash ActionId = ActionObj.GetHash(); + + MaybeDumpAction(ActionLsn, ActionObj); + + // Determine the submission URL. If the action belongs to a queue, ensure a + // corresponding remote queue exists on the target node and submit via it. + + std::string SubmitUrl = "/jobs"; + if (const int QueueId = Action->QueueId; QueueId != 0) + { + CbObject QueueMeta = Action->GetOwnerSession()->GetQueueMetadata(QueueId); + CbObject QueueConfig = Action->GetOwnerSession()->GetQueueConfig(QueueId); + if (Oid Token = EnsureRemoteQueue(QueueId, QueueMeta, QueueConfig); Token != Oid::Zero) + { + SubmitUrl = fmt::format("/queues/{}/jobs", Token); + } + } + + // Enqueue job. If the remote returns FailedDependency (424), it means it + // cannot resolve the worker/function — re-register the worker and retry once. + + CbObject Result; + HttpClient::Response WorkResponse; + HttpResponseCode WorkResponseCode{}; + + for (int Attempt = 0; Attempt < 2; ++Attempt) + { + WorkResponse = m_Http.Post(SubmitUrl, ActionObj); + WorkResponseCode = WorkResponse.StatusCode; + + if (WorkResponseCode == HttpResponseCode::FailedDependency && Attempt == 0) + { + ZEN_WARN("remote {} returned FailedDependency for action {} — re-registering worker and retrying", + m_Http.GetBaseUri(), + ActionId); + + RegisterWorker(Action->Worker.Descriptor); + } + else + { + break; + } + } + + if (WorkResponseCode == HttpResponseCode::OK) + { + Result = WorkResponse.AsObject(); + } + else if (WorkResponseCode == HttpResponseCode::NotFound) + { + // Not all attachments are present + + // Build response package including all required attachments + + CbPackage Pkg; + Pkg.SetObject(ActionObj); + + CbObject Response = WorkResponse.AsObject(); + + for (auto& Item : Response["need"sv]) + { + const IoHash NeedHash = Item.AsHash(); + + if (IoBuffer Chunk = m_ChunkResolver.FindChunkByCid(NeedHash)) + { + uint64_t DataRawSize = 0; + IoHash DataRawHash; + CompressedBuffer Compressed = + CompressedBuffer::FromCompressed(SharedBuffer{Chunk}, /* out */ DataRawHash, /* out */ DataRawSize); + + ZEN_ASSERT(DataRawHash == NeedHash); + + Pkg.AddAttachment(CbAttachment(Compressed, NeedHash)); + } + else + { + // No such attachment + + return {.IsAccepted = false, .Reason = fmt::format("missing attachment {}", NeedHash)}; + } + } + + // Post resulting package + + HttpClient::Response PayloadResponse = m_Http.Post(SubmitUrl, Pkg); + + if (!PayloadResponse) + { + ZEN_WARN("unable to register payloads for action {} at {}{}", ActionId, m_Http.GetBaseUri(), SubmitUrl); + + // TODO: include more information about the failure in the response + + return {.IsAccepted = false, .Reason = "HTTP request failed"}; + } + else if (PayloadResponse.StatusCode == HttpResponseCode::OK) + { + Result = PayloadResponse.AsObject(); + } + else + { + // Unexpected response + + const int ResponseStatusCode = (int)PayloadResponse.StatusCode; + + ZEN_WARN("unable to register payloads for action {} at {}{} (error: {} {})", + ActionId, + m_Http.GetBaseUri(), + SubmitUrl, + ResponseStatusCode, + ToString(ResponseStatusCode)); + + return {.IsAccepted = false, + .Reason = fmt::format("unexpected response code {} {} from {}{}", + ResponseStatusCode, + ToString(ResponseStatusCode), + m_Http.GetBaseUri(), + SubmitUrl)}; + } + } + + if (Result) + { + if (const int32_t LsnField = Result["lsn"].AsInt32(0)) + { + HttpRunningAction NewAction; + NewAction.Action = Action; + NewAction.RemoteActionLsn = LsnField; + + { + RwLock::ExclusiveLockScope _(m_RunningLock); + + m_RemoteRunningMap[LsnField] = std::move(NewAction); + } + + ZEN_DEBUG("scheduled action {} with remote LSN {} (local LSN {})", ActionId, LsnField, ActionLsn); + + Action->SetActionState(RunnerAction::State::Running); + + return SubmitResult{.IsAccepted = true}; + } + } + + return {}; +} + +Oid +RemoteHttpRunner::EnsureRemoteQueue(int QueueId, const CbObject& Metadata, const CbObject& Config) +{ + { + RwLock::SharedLockScope _(m_QueueTokenLock); + if (auto It = m_RemoteQueueTokens.find(QueueId); It != m_RemoteQueueTokens.end()) + { + return It->second; + } + } + + // Build a stable idempotency key that uniquely identifies this (runner instance, local queue) + // pair. The server uses this to return the same remote queue token for concurrent or redundant + // requests, preventing orphaned remote queues when multiple threads race through here. + // Also send hostname so the server can associate the queue with its origin for diagnostics. + CbObjectWriter Body; + Body << "idempotency_key"sv << fmt::format("{}/{}", m_InstanceId, QueueId); + Body << "hostname"sv << GetMachineName(); + if (Metadata) + { + Body << "metadata"sv << Metadata; + } + if (Config) + { + Body << "config"sv << Config; + } + + HttpClient::Response Resp = m_Http.Post("/queues/remote", Body.Save()); + if (!Resp) + { + ZEN_WARN("failed to create remote queue for local queue {} on {}", QueueId, m_HostName); + return Oid::Zero; + } + + Oid Token = Oid::TryFromHexString(Resp.AsObject()["queue_token"sv].AsString()); + if (Token == Oid::Zero) + { + return Oid::Zero; + } + + ZEN_DEBUG("created remote queue '{}' for local queue {} on {}", Token, QueueId, m_HostName); + + RwLock::ExclusiveLockScope _(m_QueueTokenLock); + auto [It, Inserted] = m_RemoteQueueTokens.try_emplace(QueueId, Token); + return It->second; +} + +void +RemoteHttpRunner::CancelRemoteQueue(int QueueId) +{ + Oid Token; + { + RwLock::SharedLockScope _(m_QueueTokenLock); + if (auto It = m_RemoteQueueTokens.find(QueueId); It != m_RemoteQueueTokens.end()) + { + Token = It->second; + } + } + + if (Token == Oid::Zero) + { + return; + } + + HttpClient::Response Resp = m_Http.Delete(fmt::format("/queues/{}", Token)); + + if (Resp.StatusCode == HttpResponseCode::NoContent) + { + ZEN_DEBUG("cancelled remote queue '{}' (local queue {}) on {}", Token, QueueId, m_HostName); + } + else + { + ZEN_WARN("failed to cancel remote queue '{}' on {}: {}", Token, m_HostName, int(Resp.StatusCode)); + } +} + +bool +RemoteHttpRunner::IsHealthy() +{ + if (HttpClient::Response Ready = m_Http.Get("/ready")) + { + return true; + } + else + { + // TODO: use response to propagate context + return false; + } +} + +size_t +RemoteHttpRunner::GetSubmittedActionCount() +{ + RwLock::SharedLockScope _(m_RunningLock); + return m_RemoteRunningMap.size(); +} + +void +RemoteHttpRunner::MonitorThreadFunction() +{ + SetCurrentThreadName("RemoteHttpRunner_Monitor"); + + do + { + const int NormalWaitingTime = 200; + int WaitTimeMs = NormalWaitingTime; + auto WaitOnce = [&] { return m_MonitorThreadEvent.Wait(WaitTimeMs); }; + auto SweepOnce = [&] { + const size_t RetiredCount = SweepRunningActions(); + + m_RunningLock.WithSharedLock([&] { + if (m_RemoteRunningMap.size() > 16) + { + WaitTimeMs = NormalWaitingTime / 4; + } + else + { + if (RetiredCount) + { + WaitTimeMs = NormalWaitingTime / 2; + } + else + { + WaitTimeMs = NormalWaitingTime; + } + } + }); + }; + + while (!WaitOnce()) + { + SweepOnce(); + } + + // Signal received - this may mean we should quit + + SweepOnce(); + } while (m_MonitorThreadEnabled); +} + +size_t +RemoteHttpRunner::SweepRunningActions() +{ + ZEN_TRACE_CPU("RemoteHttpRunner::SweepRunningActions"); + std::vector CompletedActions; + + // Poll remote for list of completed actions + + HttpClient::Response ResponseCompleted = m_Http.Get("/jobs/completed"sv); + + if (CbObject Completed = ResponseCompleted.AsObject()) + { + for (auto& FieldIt : Completed["completed"sv]) + { + CbObjectView EntryObj = FieldIt.AsObjectView(); + const int32_t CompleteLsn = EntryObj["lsn"sv].AsInt32(); + std::string_view StateName = EntryObj["state"sv].AsString(); + + RunnerAction::State RemoteState = RunnerAction::FromString(StateName); + + // Always fetch to drain the result from the remote's results map, + // but only keep the result package for successfully completed actions. + HttpClient::Response ResponseJob = m_Http.Get(fmt::format("/jobs/{}"sv, CompleteLsn)); + + m_RunningLock.WithExclusiveLock([&] { + if (auto CompleteIt = m_RemoteRunningMap.find(CompleteLsn); CompleteIt != m_RemoteRunningMap.end()) + { + HttpRunningAction CompletedAction = std::move(CompleteIt->second); + CompletedAction.RemoteState = RemoteState; + + if (RemoteState == RunnerAction::State::Completed && ResponseJob) + { + CompletedAction.ActionResults = ResponseJob.AsPackage(); + } + + CompletedActions.push_back(std::move(CompletedAction)); + m_RemoteRunningMap.erase(CompleteIt); + } + else + { + // we received a completion notice for an action we don't know about, + // this can happen if the runner is used by multiple upstream schedulers, + // or if this compute node was recently restarted and lost track of + // previously scheduled actions + } + }); + } + + if (CbObjectView Metrics = Completed["metrics"sv].AsObjectView()) + { + // if (const size_t CpuCount = Metrics["core_count"].AsInt32(0)) + if (const int32_t CpuCount = Metrics["lp_count"].AsInt32(0)) + { + const int32_t NewCap = zen::Max(4, CpuCount); + + if (m_MaxRunningActions > NewCap) + { + ZEN_DEBUG("capping {} to {} actions (was {})", m_BaseUrl, NewCap, m_MaxRunningActions); + + m_MaxRunningActions = NewCap; + } + } + } + } + + // Notify outer. Note that this has to be done without holding any local locks + // otherwise we may end up with deadlocks. + + for (HttpRunningAction& HttpAction : CompletedActions) + { + const int ActionLsn = HttpAction.Action->ActionLsn; + + ZEN_DEBUG("action {} LSN {} (remote LSN {}) -> {}", + HttpAction.Action->ActionId, + ActionLsn, + HttpAction.RemoteActionLsn, + RunnerAction::ToString(HttpAction.RemoteState)); + + if (HttpAction.RemoteState == RunnerAction::State::Completed) + { + HttpAction.Action->SetResult(std::move(HttpAction.ActionResults)); + } + + HttpAction.Action->SetActionState(HttpAction.RemoteState); + } + + return CompletedActions.size(); +} + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/remotehttprunner.h b/src/zencompute/runners/remotehttprunner.h new file mode 100644 index 000000000..9119992a9 --- /dev/null +++ b/src/zencompute/runners/remotehttprunner.h @@ -0,0 +1,100 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "zencompute/computeservice.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include "functionrunner.h" + +# include +# include +# include +# include +# include +# include + +# include +# include +# include +# include + +namespace zen { +class CidStore; +} + +namespace zen::compute { + +/** HTTP-based runner + + This implements a DDC remote compute execution strategy via REST API + + */ + +class RemoteHttpRunner : public FunctionRunner +{ + RemoteHttpRunner(RemoteHttpRunner&&) = delete; + RemoteHttpRunner& operator=(RemoteHttpRunner&&) = delete; + +public: + RemoteHttpRunner(ChunkResolver& InChunkResolver, + const std::filesystem::path& BaseDir, + std::string_view HostName, + WorkerThreadPool& InWorkerPool); + ~RemoteHttpRunner(); + + virtual void Shutdown() override; + virtual void RegisterWorker(const CbPackage& WorkerPackage) override; + [[nodiscard]] virtual SubmitResult SubmitAction(Ref Action) override; + [[nodiscard]] virtual bool IsHealthy() override; + [[nodiscard]] virtual size_t GetSubmittedActionCount() override; + [[nodiscard]] virtual size_t QueryCapacity() override; + [[nodiscard]] virtual std::vector SubmitActions(const std::vector>& Actions) override; + virtual void CancelRemoteQueue(int QueueId) override; + + std::string_view GetHostName() const { return m_HostName; } + +protected: + LoggerRef Log() { return m_Log; } + +private: + LoggerRef m_Log; + ChunkResolver& m_ChunkResolver; + WorkerThreadPool& m_WorkerPool; + std::string m_HostName; + std::string m_BaseUrl; + HttpClient m_Http; + + int32_t m_MaxRunningActions = 256; // arbitrary limit for testing + + struct HttpRunningAction + { + Ref Action; + int RemoteActionLsn = 0; // Remote LSN + RunnerAction::State RemoteState = RunnerAction::State::Failed; + CbPackage ActionResults; + }; + + RwLock m_RunningLock; + std::unordered_map m_RemoteRunningMap; // Note that this is keyed on the *REMOTE* lsn + + std::thread m_MonitorThread; + std::atomic m_MonitorThreadEnabled{true}; + Event m_MonitorThreadEvent; + void MonitorThreadFunction(); + size_t SweepRunningActions(); + + RwLock m_QueueTokenLock; + std::unordered_map m_RemoteQueueTokens; // local QueueId → remote queue token + + // Stable identity for this runner instance, used as part of the idempotency key when + // creating remote queues. Generated once at construction and never changes. + Oid m_InstanceId; + + Oid EnsureRemoteQueue(int QueueId, const CbObject& Metadata, const CbObject& Config); +}; + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/windowsrunner.cpp b/src/zencompute/runners/windowsrunner.cpp new file mode 100644 index 000000000..e9a1ae8b6 --- /dev/null +++ b/src/zencompute/runners/windowsrunner.cpp @@ -0,0 +1,460 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "windowsrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES && ZEN_PLATFORM_WINDOWS + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +# include +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::compute { + +using namespace std::literals; + +WindowsProcessRunner::WindowsProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool, + bool Sandboxed, + int32_t MaxConcurrentActions) +: LocalProcessRunner(Resolver, BaseDir, Deleter, WorkerPool, MaxConcurrentActions) +, m_Sandboxed(Sandboxed) +{ + if (!m_Sandboxed) + { + return; + } + + // Build a unique profile name per process to avoid collisions + m_AppContainerName = L"zenserver-sandbox-" + std::to_wstring(GetCurrentProcessId()); + + // Clean up any stale profile from a previous crash + DeleteAppContainerProfile(m_AppContainerName.c_str()); + + PSID Sid = nullptr; + + HRESULT Hr = CreateAppContainerProfile(m_AppContainerName.c_str(), + m_AppContainerName.c_str(), // display name + m_AppContainerName.c_str(), // description + nullptr, // no capabilities + 0, // capability count + &Sid); + + if (FAILED(Hr)) + { + throw zen::runtime_error("CreateAppContainerProfile failed: HRESULT 0x{:08X}", static_cast(Hr)); + } + + m_AppContainerSid = Sid; + + ZEN_INFO("AppContainer sandboxing enabled for child processes (profile={})", WideToUtf8(m_AppContainerName)); +} + +WindowsProcessRunner::~WindowsProcessRunner() +{ + if (m_AppContainerSid) + { + FreeSid(m_AppContainerSid); + m_AppContainerSid = nullptr; + } + + if (!m_AppContainerName.empty()) + { + DeleteAppContainerProfile(m_AppContainerName.c_str()); + } +} + +void +WindowsProcessRunner::GrantAppContainerAccess(const std::filesystem::path& Path, DWORD AccessMask) +{ + PACL ExistingDacl = nullptr; + PSECURITY_DESCRIPTOR SecurityDescriptor = nullptr; + + DWORD Result = GetNamedSecurityInfoW(Path.c_str(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + nullptr, + nullptr, + &ExistingDacl, + nullptr, + &SecurityDescriptor); + + if (Result != ERROR_SUCCESS) + { + throw zen::runtime_error("GetNamedSecurityInfoW failed for '{}': {}", Path.string(), GetSystemErrorAsString(Result)); + } + + auto $0 = MakeGuard([&] { LocalFree(SecurityDescriptor); }); + + EXPLICIT_ACCESSW Access{}; + Access.grfAccessPermissions = AccessMask; + Access.grfAccessMode = SET_ACCESS; + Access.grfInheritance = OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE; + Access.Trustee.TrusteeForm = TRUSTEE_IS_SID; + Access.Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP; + Access.Trustee.ptstrName = static_cast(m_AppContainerSid); + + PACL NewDacl = nullptr; + + Result = SetEntriesInAclW(1, &Access, ExistingDacl, &NewDacl); + if (Result != ERROR_SUCCESS) + { + throw zen::runtime_error("SetEntriesInAclW failed for '{}': {}", Path.string(), GetSystemErrorAsString(Result)); + } + + auto $1 = MakeGuard([&] { LocalFree(NewDacl); }); + + Result = SetNamedSecurityInfoW(const_cast(Path.c_str()), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + nullptr, + nullptr, + NewDacl, + nullptr); + + if (Result != ERROR_SUCCESS) + { + throw zen::runtime_error("SetNamedSecurityInfoW failed for '{}': {}", Path.string(), GetSystemErrorAsString(Result)); + } +} + +SubmitResult +WindowsProcessRunner::SubmitAction(Ref Action) +{ + ZEN_TRACE_CPU("WindowsProcessRunner::SubmitAction"); + std::optional Prepared = PrepareActionSubmission(Action); + + if (!Prepared) + { + return SubmitResult{.IsAccepted = false}; + } + + // Set up environment variables + + CbObject WorkerDescription = Prepared->WorkerPackage.GetObject(); + + StringBuilder<1024> EnvironmentBlock; + + for (auto& It : WorkerDescription["environment"sv]) + { + EnvironmentBlock.Append(It.AsString()); + EnvironmentBlock.Append('\0'); + } + EnvironmentBlock.Append('\0'); + EnvironmentBlock.Append('\0'); + + // Execute process - this spawns the child process immediately without waiting + // for completion + + std::string_view ExecPath = WorkerDescription["path"sv].AsString(); + std::filesystem::path ExePath = Prepared->WorkerPath / std::filesystem::path(ExecPath).make_preferred(); + + ExtendableWideStringBuilder<512> CommandLine; + CommandLine.Append(L'"'); + CommandLine.Append(ExePath.c_str()); + CommandLine.Append(L'"'); + CommandLine.Append(L" -Build=build.action"); + + LPSECURITY_ATTRIBUTES lpProcessAttributes = nullptr; + LPSECURITY_ATTRIBUTES lpThreadAttributes = nullptr; + BOOL bInheritHandles = FALSE; + DWORD dwCreationFlags = 0; + + ZEN_DEBUG("Executing: {} (sandboxed={})", WideToUtf8(CommandLine.c_str()), m_Sandboxed); + + CommandLine.EnsureNulTerminated(); + + PROCESS_INFORMATION ProcessInformation{}; + + if (m_Sandboxed) + { + // Grant AppContainer access to sandbox and worker directories + GrantAppContainerAccess(Prepared->SandboxPath, FILE_ALL_ACCESS); + GrantAppContainerAccess(Prepared->WorkerPath, FILE_GENERIC_READ | FILE_GENERIC_EXECUTE); + + // Set up extended startup info with AppContainer security capabilities + SECURITY_CAPABILITIES SecurityCapabilities{}; + SecurityCapabilities.AppContainerSid = m_AppContainerSid; + SecurityCapabilities.Capabilities = nullptr; + SecurityCapabilities.CapabilityCount = 0; + + SIZE_T AttrListSize = 0; + InitializeProcThreadAttributeList(nullptr, 1, 0, &AttrListSize); + + auto AttrList = static_cast(malloc(AttrListSize)); + auto $0 = MakeGuard([&] { free(AttrList); }); + + if (!InitializeProcThreadAttributeList(AttrList, 1, 0, &AttrListSize)) + { + zen::ThrowLastError("InitializeProcThreadAttributeList failed"); + } + + auto $1 = MakeGuard([&] { DeleteProcThreadAttributeList(AttrList); }); + + if (!UpdateProcThreadAttribute(AttrList, + 0, + PROC_THREAD_ATTRIBUTE_SECURITY_CAPABILITIES, + &SecurityCapabilities, + sizeof(SecurityCapabilities), + nullptr, + nullptr)) + { + zen::ThrowLastError("UpdateProcThreadAttribute (SECURITY_CAPABILITIES) failed"); + } + + STARTUPINFOEXW StartupInfoEx{}; + StartupInfoEx.StartupInfo.cb = sizeof(STARTUPINFOEXW); + StartupInfoEx.lpAttributeList = AttrList; + + dwCreationFlags |= EXTENDED_STARTUPINFO_PRESENT; + + BOOL Success = CreateProcessW(nullptr, + CommandLine.Data(), + lpProcessAttributes, + lpThreadAttributes, + bInheritHandles, + dwCreationFlags, + (LPVOID)EnvironmentBlock.Data(), + Prepared->SandboxPath.c_str(), + &StartupInfoEx.StartupInfo, + /* out */ &ProcessInformation); + + if (!Success) + { + zen::ThrowLastError("Unable to launch sandboxed process"); + } + } + else + { + STARTUPINFO StartupInfo{}; + StartupInfo.cb = sizeof StartupInfo; + + BOOL Success = CreateProcessW(nullptr, + CommandLine.Data(), + lpProcessAttributes, + lpThreadAttributes, + bInheritHandles, + dwCreationFlags, + (LPVOID)EnvironmentBlock.Data(), + Prepared->SandboxPath.c_str(), + &StartupInfo, + /* out */ &ProcessInformation); + + if (!Success) + { + zen::ThrowLastError("Unable to launch process"); + } + } + + CloseHandle(ProcessInformation.hThread); + + Ref NewAction{new RunningAction()}; + NewAction->Action = Action; + NewAction->ProcessHandle = ProcessInformation.hProcess; + NewAction->SandboxPath = std::move(Prepared->SandboxPath); + + { + RwLock::ExclusiveLockScope _(m_RunningLock); + + m_RunningMap[Prepared->ActionLsn] = std::move(NewAction); + } + + Action->SetActionState(RunnerAction::State::Running); + + return SubmitResult{.IsAccepted = true}; +} + +void +WindowsProcessRunner::SweepRunningActions() +{ + ZEN_TRACE_CPU("WindowsProcessRunner::SweepRunningActions"); + std::vector> CompletedActions; + + m_RunningLock.WithExclusiveLock([&] { + for (auto It = begin(m_RunningMap), ItEnd = end(m_RunningMap); It != ItEnd;) + { + Ref Running = It->second; + + DWORD ExitCode = 0; + BOOL IsSuccess = GetExitCodeProcess(Running->ProcessHandle, &ExitCode); + + if (IsSuccess && ExitCode != STILL_ACTIVE) + { + CloseHandle(Running->ProcessHandle); + Running->ProcessHandle = INVALID_HANDLE_VALUE; + Running->ExitCode = ExitCode; + + CompletedActions.push_back(std::move(Running)); + It = m_RunningMap.erase(It); + } + else + { + ++It; + } + } + }); + + ProcessCompletedActions(CompletedActions); +} + +void +WindowsProcessRunner::CancelRunningActions() +{ + ZEN_TRACE_CPU("WindowsProcessRunner::CancelRunningActions"); + Stopwatch Timer; + std::unordered_map> RunningMap; + + m_RunningLock.WithExclusiveLock([&] { std::swap(RunningMap, m_RunningMap); }); + + if (RunningMap.empty()) + { + return; + } + + ZEN_INFO("cancelling all running actions"); + + // For expedience we initiate the process termination for all known + // processes before attempting to wait for them to exit. + + // Initiate termination for all known processes before waiting for them to exit. + + for (const auto& Kv : RunningMap) + { + Ref Running = Kv.second; + + BOOL TermSuccess = TerminateProcess(Running->ProcessHandle, 222); + + if (!TermSuccess) + { + DWORD LastError = GetLastError(); + + if (LastError != ERROR_ACCESS_DENIED) + { + ZEN_WARN("TerminateProcess for LSN {} not successful: {}", Running->Action->ActionLsn, GetSystemErrorAsString(LastError)); + } + } + } + + // Wait for all processes and clean up, regardless of whether TerminateProcess succeeded. + + for (auto& [Lsn, Running] : RunningMap) + { + if (Running->ProcessHandle != INVALID_HANDLE_VALUE) + { + DWORD WaitResult = WaitForSingleObject(Running->ProcessHandle, 2000); + + if (WaitResult != WAIT_OBJECT_0) + { + ZEN_WARN("wait for LSN {}: process exit did not succeed, result = {}", Running->Action->ActionLsn, WaitResult); + } + else + { + ZEN_DEBUG("LSN {}: process exit OK", Running->Action->ActionLsn); + } + + CloseHandle(Running->ProcessHandle); + Running->ProcessHandle = INVALID_HANDLE_VALUE; + } + + m_DeferredDeleter.Enqueue(Running->Action->ActionLsn, std::move(Running->SandboxPath)); + Running->Action->SetActionState(RunnerAction::State::Failed); + } + + ZEN_INFO("DONE - cancelled {} running processes (took {})", RunningMap.size(), NiceTimeSpanMs(Timer.GetElapsedTimeMs())); +} + +bool +WindowsProcessRunner::CancelAction(int ActionLsn) +{ + ZEN_TRACE_CPU("WindowsProcessRunner::CancelAction"); + + // Hold the shared lock while terminating to prevent the sweep thread from + // closing the handle between our lookup and TerminateProcess call. + bool Sent = false; + + m_RunningLock.WithSharedLock([&] { + auto It = m_RunningMap.find(ActionLsn); + if (It == m_RunningMap.end()) + { + return; + } + + Ref Target = It->second; + if (Target->ProcessHandle == INVALID_HANDLE_VALUE) + { + return; + } + + BOOL TermSuccess = TerminateProcess(Target->ProcessHandle, 222); + + if (!TermSuccess) + { + DWORD LastError = GetLastError(); + + if (LastError != ERROR_ACCESS_DENIED) + { + ZEN_WARN("CancelAction: TerminateProcess for LSN {} not successful: {}", ActionLsn, GetSystemErrorAsString(LastError)); + } + + return; + } + + ZEN_DEBUG("CancelAction: initiated cancellation of LSN {}", ActionLsn); + Sent = true; + }); + + // The monitor thread will pick up the process exit and mark the action as Failed. + return Sent; +} + +void +WindowsProcessRunner::SampleProcessCpu(RunningAction& Running) +{ + FILETIME CreationTime, ExitTime, KernelTime, UserTime; + if (!GetProcessTimes(Running.ProcessHandle, &CreationTime, &ExitTime, &KernelTime, &UserTime)) + { + return; + } + + auto FtToU64 = [](FILETIME Ft) -> uint64_t { return (static_cast(Ft.dwHighDateTime) << 32) | Ft.dwLowDateTime; }; + + // FILETIME values are in 100-nanosecond intervals + const uint64_t CurrentOsTicks = FtToU64(KernelTime) + FtToU64(UserTime); + const uint64_t NowTicks = GetHifreqTimerValue(); + + // Cumulative CPU seconds (absolute, available from first sample): 100ns → seconds + Running.Action->CpuSeconds.store(static_cast(static_cast(CurrentOsTicks) / 10'000'000.0), std::memory_order_relaxed); + + if (Running.LastCpuSampleTicks != 0 && Running.LastCpuOsTicks != 0) + { + const uint64_t ElapsedMs = Stopwatch::GetElapsedTimeMs(NowTicks - Running.LastCpuSampleTicks); + if (ElapsedMs > 0) + { + const uint64_t DeltaOsTicks = CurrentOsTicks - Running.LastCpuOsTicks; + // 100ns → ms: divide by 10000; then as percent of elapsed ms + const float CpuPct = static_cast(static_cast(DeltaOsTicks) / 10000.0 / ElapsedMs * 100.0); + Running.Action->CpuUsagePercent.store(CpuPct, std::memory_order_relaxed); + } + } + + Running.LastCpuSampleTicks = NowTicks; + Running.LastCpuOsTicks = CurrentOsTicks; +} + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/windowsrunner.h b/src/zencompute/runners/windowsrunner.h new file mode 100644 index 000000000..9f2385cc4 --- /dev/null +++ b/src/zencompute/runners/windowsrunner.h @@ -0,0 +1,53 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "localrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES && ZEN_PLATFORM_WINDOWS + +# include + +# include + +namespace zen::compute { + +/** Windows process runner using CreateProcessW for executing worker executables. + + Subclasses LocalProcessRunner, reusing sandbox management, worker manifesting, + input/output handling, and monitor thread infrastructure. Overrides only the + platform-specific methods: process spawning, sweep, and cancellation. + + When Sandboxed is true, child processes are isolated using a Windows AppContainer: + no network access (AppContainer blocks network by default when no capabilities are + granted) and no filesystem access outside explicitly granted sandbox and worker + directories. This requires no elevation. + */ +class WindowsProcessRunner : public LocalProcessRunner +{ +public: + WindowsProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool, + bool Sandboxed = false, + int32_t MaxConcurrentActions = 0); + ~WindowsProcessRunner(); + + [[nodiscard]] SubmitResult SubmitAction(Ref Action) override; + void SweepRunningActions() override; + void CancelRunningActions() override; + bool CancelAction(int ActionLsn) override; + void SampleProcessCpu(RunningAction& Running) override; + +private: + void GrantAppContainerAccess(const std::filesystem::path& Path, DWORD AccessMask); + + bool m_Sandboxed = false; + PSID m_AppContainerSid = nullptr; + std::wstring m_AppContainerName; +}; + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/winerunner.cpp b/src/zencompute/runners/winerunner.cpp new file mode 100644 index 000000000..506bec73b --- /dev/null +++ b/src/zencompute/runners/winerunner.cpp @@ -0,0 +1,237 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "winerunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES && ZEN_PLATFORM_LINUX + +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include +# include +# include + +namespace zen::compute { + +using namespace std::literals; + +WineProcessRunner::WineProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool) +: LocalProcessRunner(Resolver, BaseDir, Deleter, WorkerPool) +{ + // Restore SIGCHLD to default behavior so waitpid() can properly collect + // child exit status. zenserver/main.cpp sets SIGCHLD to SIG_IGN which + // causes the kernel to auto-reap children, making waitpid() return + // -1/ECHILD instead of the exit status we need. + struct sigaction Action = {}; + sigemptyset(&Action.sa_mask); + Action.sa_handler = SIG_DFL; + sigaction(SIGCHLD, &Action, nullptr); +} + +SubmitResult +WineProcessRunner::SubmitAction(Ref Action) +{ + ZEN_TRACE_CPU("WineProcessRunner::SubmitAction"); + std::optional Prepared = PrepareActionSubmission(Action); + + if (!Prepared) + { + return SubmitResult{.IsAccepted = false}; + } + + // Build environment array from worker descriptor + + CbObject WorkerDescription = Prepared->WorkerPackage.GetObject(); + + std::vector EnvStrings; + for (auto& It : WorkerDescription["environment"sv]) + { + EnvStrings.emplace_back(It.AsString()); + } + + std::vector Envp; + Envp.reserve(EnvStrings.size() + 1); + for (auto& Str : EnvStrings) + { + Envp.push_back(Str.data()); + } + Envp.push_back(nullptr); + + // Build argv: wine -Build=build.action + + std::string_view ExecPath = WorkerDescription["path"sv].AsString(); + std::filesystem::path ExePath = Prepared->WorkerPath / std::filesystem::path(ExecPath); + std::string ExePathStr = ExePath.string(); + std::string WinePathStr = m_WinePath; + std::string BuildArg = "-Build=build.action"; + + std::vector ArgV; + ArgV.push_back(WinePathStr.data()); + ArgV.push_back(ExePathStr.data()); + ArgV.push_back(BuildArg.data()); + ArgV.push_back(nullptr); + + ZEN_DEBUG("Executing via Wine: {} {} {}", WinePathStr, ExePathStr, BuildArg); + + std::string SandboxPathStr = Prepared->SandboxPath.string(); + + pid_t ChildPid = fork(); + + if (ChildPid < 0) + { + throw std::runtime_error(fmt::format("fork() failed: {}", strerror(errno))); + } + + if (ChildPid == 0) + { + // Child process + if (chdir(SandboxPathStr.c_str()) != 0) + { + _exit(127); + } + + execve(WinePathStr.c_str(), ArgV.data(), Envp.data()); + + // execve only returns on failure + _exit(127); + } + + // Parent: store child pid as void* (same convention as zencore/process.cpp) + + Ref NewAction{new RunningAction()}; + NewAction->Action = Action; + NewAction->ProcessHandle = reinterpret_cast(static_cast(ChildPid)); + NewAction->SandboxPath = std::move(Prepared->SandboxPath); + + { + RwLock::ExclusiveLockScope _(m_RunningLock); + m_RunningMap[Prepared->ActionLsn] = std::move(NewAction); + } + + Action->SetActionState(RunnerAction::State::Running); + + return SubmitResult{.IsAccepted = true}; +} + +void +WineProcessRunner::SweepRunningActions() +{ + ZEN_TRACE_CPU("WineProcessRunner::SweepRunningActions"); + std::vector> CompletedActions; + + m_RunningLock.WithExclusiveLock([&] { + for (auto It = begin(m_RunningMap), ItEnd = end(m_RunningMap); It != ItEnd;) + { + Ref Running = It->second; + + pid_t Pid = static_cast(reinterpret_cast(Running->ProcessHandle)); + int Status = 0; + + pid_t Result = waitpid(Pid, &Status, WNOHANG); + + if (Result == Pid) + { + if (WIFEXITED(Status)) + { + Running->ExitCode = WEXITSTATUS(Status); + } + else if (WIFSIGNALED(Status)) + { + Running->ExitCode = 128 + WTERMSIG(Status); + } + else + { + Running->ExitCode = 1; + } + + Running->ProcessHandle = nullptr; + + CompletedActions.push_back(std::move(Running)); + It = m_RunningMap.erase(It); + } + else + { + ++It; + } + } + }); + + ProcessCompletedActions(CompletedActions); +} + +void +WineProcessRunner::CancelRunningActions() +{ + ZEN_TRACE_CPU("WineProcessRunner::CancelRunningActions"); + Stopwatch Timer; + std::unordered_map> RunningMap; + + m_RunningLock.WithExclusiveLock([&] { std::swap(RunningMap, m_RunningMap); }); + + if (RunningMap.empty()) + { + return; + } + + ZEN_INFO("cancelling all running actions"); + + // Send SIGTERM to all running processes first + + for (const auto& [Lsn, Running] : RunningMap) + { + pid_t Pid = static_cast(reinterpret_cast(Running->ProcessHandle)); + + if (kill(Pid, SIGTERM) != 0) + { + ZEN_WARN("kill(SIGTERM) for LSN {} (pid {}) failed: {}", Running->Action->ActionLsn, Pid, strerror(errno)); + } + } + + // Wait for all processes, regardless of whether SIGTERM succeeded, then clean up. + + for (auto& [Lsn, Running] : RunningMap) + { + pid_t Pid = static_cast(reinterpret_cast(Running->ProcessHandle)); + + // Poll for up to 2 seconds + bool Exited = false; + for (int i = 0; i < 20; ++i) + { + int Status = 0; + pid_t WaitResult = waitpid(Pid, &Status, WNOHANG); + if (WaitResult == Pid) + { + Exited = true; + ZEN_DEBUG("LSN {}: process exit OK", Running->Action->ActionLsn); + break; + } + usleep(100000); // 100ms + } + + if (!Exited) + { + ZEN_WARN("LSN {}: process did not exit after SIGTERM, sending SIGKILL", Running->Action->ActionLsn); + kill(Pid, SIGKILL); + waitpid(Pid, nullptr, 0); + } + + m_DeferredDeleter.Enqueue(Running->Action->ActionLsn, std::move(Running->SandboxPath)); + Running->Action->SetActionState(RunnerAction::State::Failed); + } + + ZEN_INFO("DONE - cancelled {} running processes (took {})", RunningMap.size(), NiceTimeSpanMs(Timer.GetElapsedTimeMs())); +} + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/runners/winerunner.h b/src/zencompute/runners/winerunner.h new file mode 100644 index 000000000..7df62e7c0 --- /dev/null +++ b/src/zencompute/runners/winerunner.h @@ -0,0 +1,37 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "localrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES && ZEN_PLATFORM_LINUX + +# include + +namespace zen::compute { + +/** Wine-based process runner for executing Windows worker executables on Linux. + + Subclasses LocalProcessRunner, reusing sandbox management, worker manifesting, + input/output handling, and monitor thread infrastructure. Overrides only the + platform-specific methods: process spawning, sweep, and cancellation. + */ +class WineProcessRunner : public LocalProcessRunner +{ +public: + WineProcessRunner(ChunkResolver& Resolver, + const std::filesystem::path& BaseDir, + DeferredDirectoryDeleter& Deleter, + WorkerThreadPool& WorkerPool); + + [[nodiscard]] SubmitResult SubmitAction(Ref Action) override; + void SweepRunningActions() override; + void CancelRunningActions() override; + +private: + std::string m_WinePath = "wine"; +}; + +} // namespace zen::compute + +#endif diff --git a/src/zencompute/testing/mockimds.cpp b/src/zencompute/testing/mockimds.cpp new file mode 100644 index 000000000..dd09312df --- /dev/null +++ b/src/zencompute/testing/mockimds.cpp @@ -0,0 +1,205 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include + +#if ZEN_WITH_TESTS + +namespace zen::compute { + +const char* +MockImdsService::BaseUri() const +{ + return "/"; +} + +void +MockImdsService::HandleRequest(HttpServerRequest& Request) +{ + std::string_view Uri = Request.RelativeUri(); + + // AWS endpoints live under /latest/ + if (Uri.starts_with("latest/")) + { + if (ActiveProvider == CloudProvider::AWS) + { + HandleAwsRequest(Request); + return; + } + Request.WriteResponse(HttpResponseCode::NotFound); + return; + } + + // Azure endpoints live under /metadata/ + if (Uri.starts_with("metadata/")) + { + if (ActiveProvider == CloudProvider::Azure) + { + HandleAzureRequest(Request); + return; + } + Request.WriteResponse(HttpResponseCode::NotFound); + return; + } + + // GCP endpoints live under /computeMetadata/ + if (Uri.starts_with("computeMetadata/")) + { + if (ActiveProvider == CloudProvider::GCP) + { + HandleGcpRequest(Request); + return; + } + Request.WriteResponse(HttpResponseCode::NotFound); + return; + } + + Request.WriteResponse(HttpResponseCode::NotFound); +} + +// --------------------------------------------------------------------------- +// AWS +// --------------------------------------------------------------------------- + +void +MockImdsService::HandleAwsRequest(HttpServerRequest& Request) +{ + std::string_view Uri = Request.RelativeUri(); + + // IMDSv2 token acquisition (PUT only) + if (Uri == "latest/api/token" && Request.RequestVerb() == HttpVerb::kPut) + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.Token); + return; + } + + // Instance identity + if (Uri == "latest/meta-data/instance-id") + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.InstanceId); + return; + } + + if (Uri == "latest/meta-data/placement/availability-zone") + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.AvailabilityZone); + return; + } + + if (Uri == "latest/meta-data/instance-life-cycle") + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.LifeCycle); + return; + } + + // Autoscaling lifecycle state — 404 when not in an ASG + if (Uri == "latest/meta-data/autoscaling/target-lifecycle-state") + { + if (Aws.AutoscalingState.empty()) + { + Request.WriteResponse(HttpResponseCode::NotFound); + return; + } + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.AutoscalingState); + return; + } + + // Spot interruption notice — 404 when no interruption pending + if (Uri == "latest/meta-data/spot/instance-action") + { + if (Aws.SpotAction.empty()) + { + Request.WriteResponse(HttpResponseCode::NotFound); + return; + } + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Aws.SpotAction); + return; + } + + Request.WriteResponse(HttpResponseCode::NotFound); +} + +// --------------------------------------------------------------------------- +// Azure +// --------------------------------------------------------------------------- + +void +MockImdsService::HandleAzureRequest(HttpServerRequest& Request) +{ + std::string_view Uri = Request.RelativeUri(); + + // Instance metadata (single JSON document) + if (Uri == "metadata/instance") + { + std::string Json = fmt::format(R"({{"compute":{{"vmId":"{}","location":"{}","priority":"{}","vmScaleSetName":"{}"}}}})", + Azure.VmId, + Azure.Location, + Azure.Priority, + Azure.VmScaleSetName); + + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Json); + return; + } + + // Scheduled events for termination monitoring + if (Uri == "metadata/scheduledevents") + { + std::string Json; + if (Azure.ScheduledEventType.empty()) + { + Json = R"({"Events":[]})"; + } + else + { + Json = fmt::format(R"({{"Events":[{{"EventType":"{}","EventStatus":"{}"}}]}})", + Azure.ScheduledEventType, + Azure.ScheduledEventStatus); + } + + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Json); + return; + } + + Request.WriteResponse(HttpResponseCode::NotFound); +} + +// --------------------------------------------------------------------------- +// GCP +// --------------------------------------------------------------------------- + +void +MockImdsService::HandleGcpRequest(HttpServerRequest& Request) +{ + std::string_view Uri = Request.RelativeUri(); + + if (Uri == "computeMetadata/v1/instance/id") + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.InstanceId); + return; + } + + if (Uri == "computeMetadata/v1/instance/zone") + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.Zone); + return; + } + + if (Uri == "computeMetadata/v1/instance/scheduling/preemptible") + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.Preemptible); + return; + } + + if (Uri == "computeMetadata/v1/instance/maintenance-event") + { + Request.WriteResponse(HttpResponseCode::OK, HttpContentType::kText, Gcp.MaintenanceEvent); + return; + } + + Request.WriteResponse(HttpResponseCode::NotFound); +} + +} // namespace zen::compute + +#endif // ZEN_WITH_TESTS diff --git a/src/zencompute/timeline/workertimeline.cpp b/src/zencompute/timeline/workertimeline.cpp new file mode 100644 index 000000000..88ef5b62d --- /dev/null +++ b/src/zencompute/timeline/workertimeline.cpp @@ -0,0 +1,430 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "workertimeline.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include + +# include + +namespace zen::compute { + +WorkerTimeline::WorkerTimeline(std::string_view WorkerId) : m_WorkerId(WorkerId) +{ +} + +WorkerTimeline::~WorkerTimeline() +{ +} + +void +WorkerTimeline::RecordProvisioned() +{ + AppendEvent({ + .Type = EventType::WorkerProvisioned, + .Timestamp = DateTime::Now(), + }); +} + +void +WorkerTimeline::RecordDeprovisioned() +{ + AppendEvent({ + .Type = EventType::WorkerDeprovisioned, + .Timestamp = DateTime::Now(), + }); +} + +void +WorkerTimeline::RecordActionAccepted(int ActionLsn, const IoHash& ActionId) +{ + AppendEvent({ + .Type = EventType::ActionAccepted, + .Timestamp = DateTime::Now(), + .ActionLsn = ActionLsn, + .ActionId = ActionId, + }); +} + +void +WorkerTimeline::RecordActionRejected(int ActionLsn, const IoHash& ActionId, std::string_view Reason) +{ + AppendEvent({ + .Type = EventType::ActionRejected, + .Timestamp = DateTime::Now(), + .ActionLsn = ActionLsn, + .ActionId = ActionId, + .Reason = std::string(Reason), + }); +} + +void +WorkerTimeline::RecordActionStateChanged(int ActionLsn, + const IoHash& ActionId, + RunnerAction::State PreviousState, + RunnerAction::State NewState) +{ + AppendEvent({ + .Type = EventType::ActionStateChanged, + .Timestamp = DateTime::Now(), + .ActionLsn = ActionLsn, + .ActionId = ActionId, + .ActionState = NewState, + .PreviousState = PreviousState, + }); +} + +std::vector +WorkerTimeline::QueryTimeline(DateTime StartTime, DateTime EndTime) const +{ + std::vector Result; + + m_EventsLock.WithSharedLock([&] { + for (const auto& Evt : m_Events) + { + if (Evt.Timestamp >= StartTime && Evt.Timestamp <= EndTime) + { + Result.push_back(Evt); + } + } + }); + + return Result; +} + +std::vector +WorkerTimeline::QueryRecent(int Limit) const +{ + std::vector Result; + + m_EventsLock.WithSharedLock([&] { + const int Count = std::min(Limit, gsl::narrow(m_Events.size())); + auto It = m_Events.end() - Count; + Result.assign(It, m_Events.end()); + }); + + return Result; +} + +size_t +WorkerTimeline::GetEventCount() const +{ + size_t Count = 0; + m_EventsLock.WithSharedLock([&] { Count = m_Events.size(); }); + return Count; +} + +WorkerTimeline::TimeRange +WorkerTimeline::GetTimeRange() const +{ + TimeRange Range; + m_EventsLock.WithSharedLock([&] { + if (!m_Events.empty()) + { + Range.First = m_Events.front().Timestamp; + Range.Last = m_Events.back().Timestamp; + } + }); + return Range; +} + +void +WorkerTimeline::AppendEvent(Event&& Evt) +{ + m_EventsLock.WithExclusiveLock([&] { + while (m_Events.size() >= m_MaxEvents) + { + m_Events.pop_front(); + } + + m_Events.push_back(std::move(Evt)); + }); +} + +const char* +WorkerTimeline::ToString(EventType Type) +{ + switch (Type) + { + case EventType::WorkerProvisioned: + return "provisioned"; + case EventType::WorkerDeprovisioned: + return "deprovisioned"; + case EventType::ActionAccepted: + return "accepted"; + case EventType::ActionRejected: + return "rejected"; + case EventType::ActionStateChanged: + return "state_changed"; + default: + return "unknown"; + } +} + +static WorkerTimeline::EventType +EventTypeFromString(std::string_view Str) +{ + if (Str == "provisioned") + return WorkerTimeline::EventType::WorkerProvisioned; + if (Str == "deprovisioned") + return WorkerTimeline::EventType::WorkerDeprovisioned; + if (Str == "accepted") + return WorkerTimeline::EventType::ActionAccepted; + if (Str == "rejected") + return WorkerTimeline::EventType::ActionRejected; + if (Str == "state_changed") + return WorkerTimeline::EventType::ActionStateChanged; + return WorkerTimeline::EventType::WorkerProvisioned; +} + +void +WorkerTimeline::WriteTo(const std::filesystem::path& Path) const +{ + CbObjectWriter Cbo; + Cbo << "worker_id" << m_WorkerId; + + m_EventsLock.WithSharedLock([&] { + if (!m_Events.empty()) + { + Cbo.AddDateTime("time_first", m_Events.front().Timestamp); + Cbo.AddDateTime("time_last", m_Events.back().Timestamp); + } + + Cbo.BeginArray("events"); + for (const auto& Evt : m_Events) + { + Cbo.BeginObject(); + Cbo << "type" << ToString(Evt.Type); + Cbo.AddDateTime("ts", Evt.Timestamp); + + if (Evt.ActionLsn != 0) + { + Cbo << "lsn" << Evt.ActionLsn; + Cbo << "action_id" << Evt.ActionId; + } + + if (Evt.Type == EventType::ActionStateChanged) + { + Cbo << "prev_state" << static_cast(Evt.PreviousState); + Cbo << "state" << static_cast(Evt.ActionState); + } + + if (!Evt.Reason.empty()) + { + Cbo << "reason" << std::string_view(Evt.Reason); + } + + Cbo.EndObject(); + } + Cbo.EndArray(); + }); + + CbObject Obj = Cbo.Save(); + + BasicFile File(Path, BasicFile::Mode::kTruncate); + File.Write(Obj.GetBuffer().GetView(), 0); +} + +void +WorkerTimeline::ReadFrom(const std::filesystem::path& Path) +{ + CbObjectFromFile Loaded = LoadCompactBinaryObject(Path); + CbObject Root = std::move(Loaded.Object); + + if (!Root) + { + return; + } + + std::deque LoadedEvents; + + for (CbFieldView Field : Root["events"].AsArrayView()) + { + CbObjectView EventObj = Field.AsObjectView(); + + Event Evt; + Evt.Type = EventTypeFromString(EventObj["type"].AsString()); + Evt.Timestamp = EventObj["ts"].AsDateTime(); + + Evt.ActionLsn = EventObj["lsn"].AsInt32(); + Evt.ActionId = EventObj["action_id"].AsHash(); + + if (Evt.Type == EventType::ActionStateChanged) + { + Evt.PreviousState = static_cast(EventObj["prev_state"].AsInt32()); + Evt.ActionState = static_cast(EventObj["state"].AsInt32()); + } + + std::string_view Reason = EventObj["reason"].AsString(); + if (!Reason.empty()) + { + Evt.Reason = std::string(Reason); + } + + LoadedEvents.push_back(std::move(Evt)); + } + + m_EventsLock.WithExclusiveLock([&] { m_Events = std::move(LoadedEvents); }); +} + +WorkerTimeline::TimeRange +WorkerTimeline::ReadTimeRange(const std::filesystem::path& Path) +{ + CbObjectFromFile Loaded = LoadCompactBinaryObject(Path); + + if (!Loaded.Object) + { + return {}; + } + + return { + .First = Loaded.Object["time_first"].AsDateTime(), + .Last = Loaded.Object["time_last"].AsDateTime(), + }; +} + +// WorkerTimelineStore + +static constexpr std::string_view kTimelineExtension = ".ztimeline"; + +WorkerTimelineStore::WorkerTimelineStore(std::filesystem::path PersistenceDir) : m_PersistenceDir(std::move(PersistenceDir)) +{ + std::error_code Ec; + std::filesystem::create_directories(m_PersistenceDir, Ec); +} + +Ref +WorkerTimelineStore::GetOrCreate(std::string_view WorkerId) +{ + // Fast path: check if it already exists in memory + { + RwLock::SharedLockScope _(m_Lock); + auto It = m_Timelines.find(std::string(WorkerId)); + if (It != m_Timelines.end()) + { + return It->second; + } + } + + // Slow path: create under exclusive lock, loading from disk if available + RwLock::ExclusiveLockScope _(m_Lock); + + auto& Entry = m_Timelines[std::string(WorkerId)]; + if (!Entry) + { + Entry = Ref(new WorkerTimeline(WorkerId)); + + std::filesystem::path Path = TimelinePath(WorkerId); + std::error_code Ec; + if (std::filesystem::is_regular_file(Path, Ec)) + { + Entry->ReadFrom(Path); + } + } + return Entry; +} + +Ref +WorkerTimelineStore::Find(std::string_view WorkerId) +{ + RwLock::SharedLockScope _(m_Lock); + auto It = m_Timelines.find(std::string(WorkerId)); + if (It != m_Timelines.end()) + { + return It->second; + } + return {}; +} + +std::vector +WorkerTimelineStore::GetActiveWorkerIds() const +{ + std::vector Result; + + RwLock::SharedLockScope $(m_Lock); + Result.reserve(m_Timelines.size()); + for (const auto& [Id, _] : m_Timelines) + { + Result.push_back(Id); + } + + return Result; +} + +std::vector +WorkerTimelineStore::GetAllWorkerInfo() const +{ + std::unordered_map InfoMap; + + { + RwLock::SharedLockScope _(m_Lock); + for (const auto& [Id, Timeline] : m_Timelines) + { + InfoMap[Id] = Timeline->GetTimeRange(); + } + } + + std::error_code Ec; + for (const auto& Entry : std::filesystem::directory_iterator(m_PersistenceDir, Ec)) + { + if (!Entry.is_regular_file()) + { + continue; + } + + const auto& Path = Entry.path(); + if (Path.extension().string() != kTimelineExtension) + { + continue; + } + + std::string Id = Path.stem().string(); + if (InfoMap.find(Id) == InfoMap.end()) + { + InfoMap[Id] = WorkerTimeline::ReadTimeRange(Path); + } + } + + std::vector Result; + Result.reserve(InfoMap.size()); + for (auto& [Id, Range] : InfoMap) + { + Result.push_back({.WorkerId = std::move(Id), .Range = Range}); + } + return Result; +} + +void +WorkerTimelineStore::Save(std::string_view WorkerId) +{ + RwLock::SharedLockScope _(m_Lock); + auto It = m_Timelines.find(std::string(WorkerId)); + if (It != m_Timelines.end()) + { + It->second->WriteTo(TimelinePath(WorkerId)); + } +} + +void +WorkerTimelineStore::SaveAll() +{ + RwLock::SharedLockScope _(m_Lock); + for (const auto& [Id, Timeline] : m_Timelines) + { + Timeline->WriteTo(TimelinePath(Id)); + } +} + +std::filesystem::path +WorkerTimelineStore::TimelinePath(std::string_view WorkerId) const +{ + return m_PersistenceDir / (std::string(WorkerId) + std::string(kTimelineExtension)); +} + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/timeline/workertimeline.h b/src/zencompute/timeline/workertimeline.h new file mode 100644 index 000000000..87e19bc28 --- /dev/null +++ b/src/zencompute/timeline/workertimeline.h @@ -0,0 +1,169 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "../runners/functionrunner.h" + +#if ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include + +# include +# include +# include +# include +# include +# include + +namespace zen::compute { + +struct RunnerAction; + +/** Worker activity timeline for tracking and visualizing worker activity over time. + * + * Records worker lifecycle events (provisioning/deprovisioning) and action lifecycle + * events (accept, reject, state changes) with timestamps, enabling time-range queries + * for dashboard visualization. + */ +class WorkerTimeline : public RefCounted +{ +public: + explicit WorkerTimeline(std::string_view WorkerId); + ~WorkerTimeline() override; + + struct TimeRange + { + DateTime First = DateTime(0); + DateTime Last = DateTime(0); + + explicit operator bool() const { return First.GetTicks() != 0; } + }; + + enum class EventType + { + WorkerProvisioned, + WorkerDeprovisioned, + ActionAccepted, + ActionRejected, + ActionStateChanged + }; + + static const char* ToString(EventType Type); + + struct Event + { + EventType Type; + DateTime Timestamp = DateTime(0); + + // Action context (only set for action events) + int ActionLsn = 0; + IoHash ActionId; + RunnerAction::State ActionState = RunnerAction::State::New; + RunnerAction::State PreviousState = RunnerAction::State::New; + + // Optional reason (e.g. rejection reason) + std::string Reason; + }; + + /** Record that this worker has been provisioned and is available for work. */ + void RecordProvisioned(); + + /** Record that this worker has been deprovisioned and is no longer available. */ + void RecordDeprovisioned(); + + /** Record that an action was accepted by this worker. */ + void RecordActionAccepted(int ActionLsn, const IoHash& ActionId); + + /** Record that an action was rejected by this worker. */ + void RecordActionRejected(int ActionLsn, const IoHash& ActionId, std::string_view Reason); + + /** Record an action state transition on this worker. */ + void RecordActionStateChanged(int ActionLsn, const IoHash& ActionId, RunnerAction::State PreviousState, RunnerAction::State NewState); + + /** Query events within a time range (inclusive). Returns events ordered by timestamp. */ + [[nodiscard]] std::vector QueryTimeline(DateTime StartTime, DateTime EndTime) const; + + /** Query the most recent N events. */ + [[nodiscard]] std::vector QueryRecent(int Limit = 100) const; + + /** Return the total number of recorded events. */ + [[nodiscard]] size_t GetEventCount() const; + + /** Return the time range covered by the events in this timeline. */ + [[nodiscard]] TimeRange GetTimeRange() const; + + [[nodiscard]] const std::string& GetWorkerId() const { return m_WorkerId; } + + /** Write the timeline to a file at the given path. */ + void WriteTo(const std::filesystem::path& Path) const; + + /** Read the timeline from a file at the given path. Replaces current in-memory events. */ + void ReadFrom(const std::filesystem::path& Path); + + /** Read only the time range from a persisted timeline file, without loading events. */ + [[nodiscard]] static TimeRange ReadTimeRange(const std::filesystem::path& Path); + +private: + void AppendEvent(Event&& Evt); + + std::string m_WorkerId; + mutable RwLock m_EventsLock; + std::deque m_Events; + size_t m_MaxEvents = 10'000; +}; + +/** Manages a set of WorkerTimeline instances, keyed by worker ID. + * + * Provides thread-safe lookup and on-demand creation of timelines, backed by + * a persistence directory. Each timeline is stored as a separate file named + * {WorkerId}.ztimeline within the directory. + */ +class WorkerTimelineStore +{ +public: + explicit WorkerTimelineStore(std::filesystem::path PersistenceDir); + ~WorkerTimelineStore() = default; + + WorkerTimelineStore(const WorkerTimelineStore&) = delete; + WorkerTimelineStore& operator=(const WorkerTimelineStore&) = delete; + + /** Get the timeline for a worker, creating one if it does not exist. + * If a persisted file exists on disk it will be loaded on first access. */ + Ref GetOrCreate(std::string_view WorkerId); + + /** Get the timeline for a worker, or null ref if it does not exist in memory. */ + [[nodiscard]] Ref Find(std::string_view WorkerId); + + /** Return the worker IDs of currently loaded (in-memory) timelines. */ + [[nodiscard]] std::vector GetActiveWorkerIds() const; + + struct WorkerTimelineInfo + { + std::string WorkerId; + WorkerTimeline::TimeRange Range; + }; + + /** Return info for all known timelines (in-memory and on-disk), including time range. */ + [[nodiscard]] std::vector GetAllWorkerInfo() const; + + /** Persist a single worker's timeline to disk. */ + void Save(std::string_view WorkerId); + + /** Persist all in-memory timelines to disk. */ + void SaveAll(); + +private: + [[nodiscard]] std::filesystem::path TimelinePath(std::string_view WorkerId) const; + + std::filesystem::path m_PersistenceDir; + mutable RwLock m_Lock; + std::unordered_map> m_Timelines; +}; + +} // namespace zen::compute + +#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencompute/xmake.lua b/src/zencompute/xmake.lua index 50877508c..ed0af66a5 100644 --- a/src/zencompute/xmake.lua +++ b/src/zencompute/xmake.lua @@ -6,4 +6,14 @@ target('zencompute') add_headerfiles("**.h") add_files("**.cpp") add_includedirs("include", {public=true}) + add_includedirs(".", {private=true}) add_deps("zencore", "zenstore", "zenutil", "zennet", "zenhttp") + add_packages("json11") + + if is_os("macosx") then + add_cxxflags("-Wno-deprecated-declarations") + end + + if is_plat("windows") then + add_syslinks("Userenv") + end diff --git a/src/zencompute/zencompute.cpp b/src/zencompute/zencompute.cpp index 633250f4e..1f3f6d3f9 100644 --- a/src/zencompute/zencompute.cpp +++ b/src/zencompute/zencompute.cpp @@ -2,11 +2,20 @@ #include "zencompute/zencompute.h" +#if ZEN_WITH_TESTS +# include "runners/deferreddeleter.h" +# include +#endif + namespace zen { void zencompute_forcelinktests() { +#if ZEN_WITH_TESTS + compute::cloudmetadata_forcelink(); + compute::deferreddeleter_forcelink(); +#endif } } // namespace zen diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h index bf3c15d3d..fecbe2dbe 100644 --- a/src/zencore/include/zencore/system.h +++ b/src/zencore/include/zencore/system.h @@ -4,6 +4,8 @@ #include +#include +#include #include namespace zen { @@ -12,6 +14,7 @@ class CbWriter; std::string GetMachineName(); std::string_view GetOperatingSystemName(); +std::string_view GetRuntimePlatformName(); // "windows", "wine", "linux", or "macos" std::string_view GetCpuName(); struct SystemMetrics @@ -25,7 +28,13 @@ struct SystemMetrics uint64_t AvailVirtualMemoryMiB = 0; uint64_t PageFileMiB = 0; uint64_t AvailPageFileMiB = 0; - float CpuUsagePercent = 0.0f; +}; + +/// Extended metrics that include CPU usage percentage, which requires +/// stateful delta tracking via SystemMetricsTracker. +struct ExtendedSystemMetrics : SystemMetrics +{ + float CpuUsagePercent = 0.0f; }; SystemMetrics GetSystemMetrics(); @@ -33,6 +42,31 @@ SystemMetrics GetSystemMetrics(); void SetCpuCountForReporting(int FakeCpuCount); SystemMetrics GetSystemMetricsForReporting(); +ExtendedSystemMetrics ApplyReportingOverrides(ExtendedSystemMetrics Metrics); + void Describe(const SystemMetrics& Metrics, CbWriter& Writer); +void Describe(const ExtendedSystemMetrics& Metrics, CbWriter& Writer); + +/// Stateful tracker that computes CPU usage as a delta between consecutive +/// Query() calls. The first call returns CpuUsagePercent = 0 (no previous +/// sample). Thread-safe: concurrent calls are serialised internally. +/// CPU sampling is rate-limited to MinInterval (default 1 s); calls that +/// arrive sooner return the previously cached value. +class SystemMetricsTracker +{ +public: + explicit SystemMetricsTracker(std::chrono::milliseconds MinInterval = std::chrono::seconds(1)); + ~SystemMetricsTracker(); + + SystemMetricsTracker(const SystemMetricsTracker&) = delete; + SystemMetricsTracker& operator=(const SystemMetricsTracker&) = delete; + + /// Collect current metrics. CPU usage is computed as delta since last Query(). + ExtendedSystemMetrics Query(); + +private: + struct Impl; + std::unique_ptr m_Impl; +}; } // namespace zen diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index 267c87e12..833d3c04b 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -7,6 +7,8 @@ #include #include +#include + #if ZEN_PLATFORM_WINDOWS # include @@ -133,33 +135,6 @@ GetSystemMetrics() Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; } - // Query CPU usage using PDH - // - // TODO: This should be changed to not require a Sleep, perhaps by using some - // background metrics gathering mechanism. - - { - PDH_HQUERY QueryHandle = nullptr; - PDH_HCOUNTER CounterHandle = nullptr; - - if (PdhOpenQueryW(nullptr, 0, &QueryHandle) == ERROR_SUCCESS) - { - if (PdhAddEnglishCounterW(QueryHandle, L"\\Processor(_Total)\\% Processor Time", 0, &CounterHandle) == ERROR_SUCCESS) - { - PdhCollectQueryData(QueryHandle); - Sleep(100); - PdhCollectQueryData(QueryHandle); - - PDH_FMT_COUNTERVALUE CounterValue; - if (PdhGetFormattedCounterValue(CounterHandle, PDH_FMT_DOUBLE, nullptr, &CounterValue) == ERROR_SUCCESS) - { - Metrics.CpuUsagePercent = static_cast(CounterValue.doubleValue); - } - } - PdhCloseQuery(QueryHandle); - } - } - return Metrics; } #elif ZEN_PLATFORM_LINUX @@ -235,39 +210,6 @@ GetSystemMetrics() } } - // Query CPU usage - Metrics.CpuUsagePercent = 0.0f; - if (FILE* Stat = fopen("/proc/stat", "r")) - { - char Line[256]; - unsigned long User, Nice, System, Idle, IoWait, Irq, SoftIrq; - static unsigned long PrevUser = 0, PrevNice = 0, PrevSystem = 0, PrevIdle = 0, PrevIoWait = 0, PrevIrq = 0, PrevSoftIrq = 0; - - if (fgets(Line, sizeof(Line), Stat)) - { - if (sscanf(Line, "cpu %lu %lu %lu %lu %lu %lu %lu", &User, &Nice, &System, &Idle, &IoWait, &Irq, &SoftIrq) == 7) - { - unsigned long TotalDelta = (User + Nice + System + Idle + IoWait + Irq + SoftIrq) - - (PrevUser + PrevNice + PrevSystem + PrevIdle + PrevIoWait + PrevIrq + PrevSoftIrq); - unsigned long IdleDelta = Idle - PrevIdle; - - if (TotalDelta > 0) - { - Metrics.CpuUsagePercent = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; - } - - PrevUser = User; - PrevNice = Nice; - PrevSystem = System; - PrevIdle = Idle; - PrevIoWait = IoWait; - PrevIrq = Irq; - PrevSoftIrq = SoftIrq; - } - } - fclose(Stat); - } - // Get memory information long Pages = sysconf(_SC_PHYS_PAGES); long PageSize = sysconf(_SC_PAGE_SIZE); @@ -348,25 +290,6 @@ GetSystemMetrics() sysctlbyname("hw.packages", &Packages, &Size, nullptr, 0); Metrics.CpuCount = Packages > 0 ? Packages : 1; - // Query CPU usage using host_statistics64 - Metrics.CpuUsagePercent = 0.0f; - host_cpu_load_info_data_t CpuLoad; - mach_msg_type_number_t CpuCount = sizeof(CpuLoad) / sizeof(natural_t); - if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (host_info_t)&CpuLoad, &CpuCount) == KERN_SUCCESS) - { - unsigned long TotalTicks = 0; - for (int i = 0; i < CPU_STATE_MAX; ++i) - { - TotalTicks += CpuLoad.cpu_ticks[i]; - } - - if (TotalTicks > 0) - { - unsigned long IdleTicks = CpuLoad.cpu_ticks[CPU_STATE_IDLE]; - Metrics.CpuUsagePercent = 100.0f * (TotalTicks - IdleTicks) / TotalTicks; - } - } - // Get memory information uint64_t MemSize = 0; Size = sizeof(MemSize); @@ -401,6 +324,17 @@ GetSystemMetrics() # error "Unknown platform" #endif +ExtendedSystemMetrics +ApplyReportingOverrides(ExtendedSystemMetrics Metrics) +{ + if (g_FakeCpuCount) + { + Metrics.CoreCount = g_FakeCpuCount; + Metrics.LogicalProcessorCount = g_FakeCpuCount; + } + return Metrics; +} + SystemMetrics GetSystemMetricsForReporting() { @@ -415,12 +349,249 @@ GetSystemMetricsForReporting() return Sm; } +/////////////////////////////////////////////////////////////////////////// +// SystemMetricsTracker +/////////////////////////////////////////////////////////////////////////// + +// Per-platform CPU sampling helper. Called with m_Mutex held. + +#if ZEN_PLATFORM_WINDOWS || ZEN_PLATFORM_LINUX + +// Samples CPU usage by reading /proc/stat. Used natively on Linux and as a +// Wine fallback on Windows (where /proc/stat is accessible via the Z: drive). +struct ProcStatCpuSampler +{ + const char* Path = "/proc/stat"; + unsigned long PrevUser = 0; + unsigned long PrevNice = 0; + unsigned long PrevSystem = 0; + unsigned long PrevIdle = 0; + unsigned long PrevIoWait = 0; + unsigned long PrevIrq = 0; + unsigned long PrevSoftIrq = 0; + + explicit ProcStatCpuSampler(const char* InPath = "/proc/stat") : Path(InPath) {} + + float Sample() + { + float CpuUsage = 0.0f; + + if (FILE* Stat = fopen(Path, "r")) + { + char Line[256]; + unsigned long User, Nice, System, Idle, IoWait, Irq, SoftIrq; + + if (fgets(Line, sizeof(Line), Stat)) + { + if (sscanf(Line, "cpu %lu %lu %lu %lu %lu %lu %lu", &User, &Nice, &System, &Idle, &IoWait, &Irq, &SoftIrq) == 7) + { + unsigned long TotalDelta = (User + Nice + System + Idle + IoWait + Irq + SoftIrq) - + (PrevUser + PrevNice + PrevSystem + PrevIdle + PrevIoWait + PrevIrq + PrevSoftIrq); + unsigned long IdleDelta = Idle - PrevIdle; + + if (TotalDelta > 0) + { + CpuUsage = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; + } + + PrevUser = User; + PrevNice = Nice; + PrevSystem = System; + PrevIdle = Idle; + PrevIoWait = IoWait; + PrevIrq = Irq; + PrevSoftIrq = SoftIrq; + } + } + fclose(Stat); + } + + return CpuUsage; + } +}; + +#endif + +#if ZEN_PLATFORM_WINDOWS + +struct CpuSampler +{ + PDH_HQUERY QueryHandle = nullptr; + PDH_HCOUNTER CounterHandle = nullptr; + bool HasPreviousSample = false; + bool IsWine = false; + ProcStatCpuSampler ProcStat{"Z:\\proc\\stat"}; + + CpuSampler() + { + IsWine = zen::windows::IsRunningOnWine(); + + if (!IsWine) + { + if (PdhOpenQueryW(nullptr, 0, &QueryHandle) == ERROR_SUCCESS) + { + if (PdhAddEnglishCounterW(QueryHandle, L"\\Processor(_Total)\\% Processor Time", 0, &CounterHandle) != ERROR_SUCCESS) + { + CounterHandle = nullptr; + } + } + } + } + + ~CpuSampler() + { + if (QueryHandle) + { + PdhCloseQuery(QueryHandle); + } + } + + float Sample() + { + if (IsWine) + { + return ProcStat.Sample(); + } + + if (!QueryHandle || !CounterHandle) + { + return 0.0f; + } + + PdhCollectQueryData(QueryHandle); + + if (!HasPreviousSample) + { + HasPreviousSample = true; + return 0.0f; + } + + PDH_FMT_COUNTERVALUE CounterValue; + if (PdhGetFormattedCounterValue(CounterHandle, PDH_FMT_DOUBLE, nullptr, &CounterValue) == ERROR_SUCCESS) + { + return static_cast(CounterValue.doubleValue); + } + + return 0.0f; + } +}; + +#elif ZEN_PLATFORM_LINUX + +struct CpuSampler +{ + ProcStatCpuSampler ProcStat; + + float Sample() { return ProcStat.Sample(); } +}; + +#elif ZEN_PLATFORM_MAC + +struct CpuSampler +{ + unsigned long PrevTotalTicks = 0; + unsigned long PrevIdleTicks = 0; + + float Sample() + { + float CpuUsage = 0.0f; + + host_cpu_load_info_data_t CpuLoad; + mach_msg_type_number_t Count = sizeof(CpuLoad) / sizeof(natural_t); + if (host_statistics(mach_host_self(), HOST_CPU_LOAD_INFO, (host_info_t)&CpuLoad, &Count) == KERN_SUCCESS) + { + unsigned long TotalTicks = 0; + for (int i = 0; i < CPU_STATE_MAX; ++i) + { + TotalTicks += CpuLoad.cpu_ticks[i]; + } + unsigned long IdleTicks = CpuLoad.cpu_ticks[CPU_STATE_IDLE]; + + unsigned long TotalDelta = TotalTicks - PrevTotalTicks; + unsigned long IdleDelta = IdleTicks - PrevIdleTicks; + + if (TotalDelta > 0 && PrevTotalTicks > 0) + { + CpuUsage = 100.0f * (TotalDelta - IdleDelta) / TotalDelta; + } + + PrevTotalTicks = TotalTicks; + PrevIdleTicks = IdleTicks; + } + + return CpuUsage; + } +}; + +#endif + +struct SystemMetricsTracker::Impl +{ + using Clock = std::chrono::steady_clock; + + std::mutex Mutex; + CpuSampler Sampler; + float CachedCpuPercent = 0.0f; + Clock::time_point NextSampleTime = Clock::now(); + std::chrono::milliseconds MinInterval; + + explicit Impl(std::chrono::milliseconds InMinInterval) : MinInterval(InMinInterval) {} + + float SampleCpu() + { + const auto Now = Clock::now(); + if (Now >= NextSampleTime) + { + CachedCpuPercent = Sampler.Sample(); + NextSampleTime = Now + MinInterval; + } + return CachedCpuPercent; + } +}; + +SystemMetricsTracker::SystemMetricsTracker(std::chrono::milliseconds MinInterval) : m_Impl(std::make_unique(MinInterval)) +{ +} + +SystemMetricsTracker::~SystemMetricsTracker() = default; + +ExtendedSystemMetrics +SystemMetricsTracker::Query() +{ + ExtendedSystemMetrics Metrics; + static_cast(Metrics) = GetSystemMetrics(); + + std::lock_guard Lock(m_Impl->Mutex); + Metrics.CpuUsagePercent = m_Impl->SampleCpu(); + return Metrics; +} + +/////////////////////////////////////////////////////////////////////////// + std::string_view GetOperatingSystemName() { return ZEN_PLATFORM_NAME; } +std::string_view +GetRuntimePlatformName() +{ +#if ZEN_PLATFORM_WINDOWS + if (zen::windows::IsRunningOnWine()) + { + return "wine"sv; + } + return "windows"sv; +#elif ZEN_PLATFORM_LINUX + return "linux"sv; +#elif ZEN_PLATFORM_MAC + return "macos"sv; +#else + return "unknown"sv; +#endif +} + std::string_view GetCpuName() { @@ -440,4 +611,11 @@ Describe(const SystemMetrics& Metrics, CbWriter& Writer) << "avail_pagefile_mb" << Metrics.AvailPageFileMiB; } +void +Describe(const ExtendedSystemMetrics& Metrics, CbWriter& Writer) +{ + Describe(static_cast(Metrics), Writer); + Writer << "cpu_usage_percent" << Metrics.CpuUsagePercent; +} + } // namespace zen diff --git a/src/zenhorde/hordeagent.cpp b/src/zenhorde/hordeagent.cpp new file mode 100644 index 000000000..819b2d0cb --- /dev/null +++ b/src/zenhorde/hordeagent.cpp @@ -0,0 +1,297 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hordeagent.h" +#include "hordetransportaes.h" + +#include +#include +#include +#include + +#include +#include + +namespace zen::horde { + +HordeAgent::HordeAgent(const MachineInfo& Info) : m_Log(zen::logging::Get("horde.agent")), m_MachineInfo(Info) +{ + ZEN_TRACE_CPU("HordeAgent::Connect"); + + auto Transport = std::make_unique(Info); + if (!Transport->IsValid()) + { + ZEN_WARN("failed to create TCP transport to '{}:{}'", Info.GetConnectionAddress(), Info.GetConnectionPort()); + return; + } + + // The 64-byte nonce is always sent unencrypted as the first thing on the wire. + // The Horde agent uses this to identify which lease this connection belongs to. + Transport->Send(Info.Nonce, sizeof(Info.Nonce)); + + std::unique_ptr FinalTransport = std::move(Transport); + if (Info.EncryptionMode == Encryption::AES) + { + FinalTransport = std::make_unique(Info.Key, std::move(FinalTransport)); + if (!FinalTransport->IsValid()) + { + ZEN_WARN("failed to create AES transport"); + return; + } + } + + // Create multiplexed socket and channels + m_Socket = std::make_unique(std::move(FinalTransport)); + + // Channel 0 is the agent control channel (handles Attach/Fork handshake). + // Channel 100 is the child I/O channel (handles file upload and remote execution). + Ref AgentComputeChannel = m_Socket->CreateChannel(0); + Ref ChildComputeChannel = m_Socket->CreateChannel(100); + + if (!AgentComputeChannel || !ChildComputeChannel) + { + ZEN_WARN("failed to create compute channels"); + return; + } + + m_AgentChannel = std::make_unique(std::move(AgentComputeChannel)); + m_ChildChannel = std::make_unique(std::move(ChildComputeChannel)); + + m_IsValid = true; +} + +HordeAgent::~HordeAgent() +{ + CloseConnection(); +} + +bool +HordeAgent::BeginCommunication() +{ + ZEN_TRACE_CPU("HordeAgent::BeginCommunication"); + + if (!m_IsValid) + { + return false; + } + + // Start the send/recv pump threads + m_Socket->StartCommunication(); + + // Wait for Attach on agent channel + AgentMessageType Type = m_AgentChannel->ReadResponse(5000); + if (Type == AgentMessageType::None) + { + ZEN_WARN("timed out waiting for Attach on agent channel"); + return false; + } + if (Type != AgentMessageType::Attach) + { + ZEN_WARN("expected Attach on agent channel, got 0x{:02x}", static_cast(Type)); + return false; + } + + // Fork tells the remote agent to create child channel 100 with a 4MB buffer. + // After this, the agent will send an Attach on the child channel. + m_AgentChannel->Fork(100, 4 * 1024 * 1024); + + // Wait for Attach on child channel + Type = m_ChildChannel->ReadResponse(5000); + if (Type == AgentMessageType::None) + { + ZEN_WARN("timed out waiting for Attach on child channel"); + return false; + } + if (Type != AgentMessageType::Attach) + { + ZEN_WARN("expected Attach on child channel, got 0x{:02x}", static_cast(Type)); + return false; + } + + return true; +} + +bool +HordeAgent::UploadBinaries(const std::filesystem::path& BundleDir, const std::string& BundleLocator) +{ + ZEN_TRACE_CPU("HordeAgent::UploadBinaries"); + + m_ChildChannel->UploadFiles("", BundleLocator.c_str()); + + std::unordered_map> BlobFiles; + + auto FindOrOpenBlob = [&](std::string_view Locator) -> BasicFile* { + std::string Key(Locator); + + if (auto It = BlobFiles.find(Key); It != BlobFiles.end()) + { + return It->second.get(); + } + + const std::filesystem::path Path = BundleDir / (Key + ".blob"); + std::error_code Ec; + auto File = std::make_unique(); + File->Open(Path, BasicFile::Mode::kRead, Ec); + + if (Ec) + { + ZEN_ERROR("cannot read blob file: '{}'", Path); + return nullptr; + } + + BasicFile* Ptr = File.get(); + BlobFiles.emplace(std::move(Key), std::move(File)); + return Ptr; + }; + + // The upload protocol is request-driven: we send WriteFiles, then the remote agent + // sends ReadBlob requests for each blob it needs. We respond with Blob data until + // the agent sends WriteFilesResponse indicating the upload is complete. + constexpr int32_t ReadResponseTimeoutMs = 1000; + + for (;;) + { + bool TimedOut = false; + + if (AgentMessageType Type = m_ChildChannel->ReadResponse(ReadResponseTimeoutMs, &TimedOut); Type != AgentMessageType::ReadBlob) + { + if (TimedOut) + { + continue; + } + // End of stream - check if it was a successful upload + if (Type == AgentMessageType::WriteFilesResponse) + { + return true; + } + else if (Type == AgentMessageType::Exception) + { + ExceptionInfo Ex; + m_ChildChannel->ReadException(Ex); + ZEN_ERROR("upload exception: {} - {}", Ex.Message, Ex.Description); + } + else + { + ZEN_ERROR("unexpected message type 0x{:02x} during upload", static_cast(Type)); + } + return false; + } + + BlobRequest Req; + m_ChildChannel->ReadBlobRequest(Req); + + BasicFile* File = FindOrOpenBlob(Req.Locator); + if (!File) + { + return false; + } + + // Read from offset to end of file + const uint64_t TotalSize = File->FileSize(); + const uint64_t Offset = static_cast(Req.Offset); + if (Offset >= TotalSize) + { + ZEN_ERROR("upload got request for data beyond end of file: offset={}, length={}, total_size={}", Offset, Req.Length, TotalSize); + m_ChildChannel->Blob(nullptr, 0); + continue; + } + + const IoBuffer Data = File->ReadRange(Offset, Min(Req.Length, TotalSize - Offset)); + m_ChildChannel->Blob(static_cast(Data.GetData()), Data.GetSize()); + } +} + +void +HordeAgent::Execute(const char* Exe, + const char* const* Args, + size_t NumArgs, + const char* WorkingDir, + const char* const* EnvVars, + size_t NumEnvVars, + bool UseWine) +{ + ZEN_TRACE_CPU("HordeAgent::Execute"); + m_ChildChannel + ->Execute(Exe, Args, NumArgs, WorkingDir, EnvVars, NumEnvVars, UseWine ? ExecuteProcessFlags::UseWine : ExecuteProcessFlags::None); +} + +bool +HordeAgent::Poll(bool LogOutput) +{ + constexpr int32_t ReadResponseTimeoutMs = 100; + AgentMessageType Type; + + while ((Type = m_ChildChannel->ReadResponse(ReadResponseTimeoutMs)) != AgentMessageType::None) + { + switch (Type) + { + case AgentMessageType::ExecuteOutput: + { + if (LogOutput && m_ChildChannel->GetResponseSize() > 0) + { + const char* ResponseData = static_cast(m_ChildChannel->GetResponseData()); + size_t ResponseSize = m_ChildChannel->GetResponseSize(); + + // Trim trailing newlines + while (ResponseSize > 0 && (ResponseData[ResponseSize - 1] == '\n' || ResponseData[ResponseSize - 1] == '\r')) + { + --ResponseSize; + } + + if (ResponseSize > 0) + { + const std::string_view Output(ResponseData, ResponseSize); + ZEN_INFO("[remote] {}", Output); + } + } + break; + } + + case AgentMessageType::ExecuteResult: + { + if (m_ChildChannel->GetResponseSize() == sizeof(int32_t)) + { + int32_t ExitCode; + memcpy(&ExitCode, m_ChildChannel->GetResponseData(), sizeof(int32_t)); + ZEN_INFO("remote process exited with code {}", ExitCode); + } + m_IsValid = false; + return false; + } + + case AgentMessageType::Exception: + { + ExceptionInfo Ex; + m_ChildChannel->ReadException(Ex); + ZEN_ERROR("exception: {} - {}", Ex.Message, Ex.Description); + m_HasErrors = true; + break; + } + + default: + break; + } + } + + return m_IsValid && !m_HasErrors; +} + +void +HordeAgent::CloseConnection() +{ + if (m_ChildChannel) + { + m_ChildChannel->Close(); + } + if (m_AgentChannel) + { + m_AgentChannel->Close(); + } +} + +bool +HordeAgent::IsValid() const +{ + return m_IsValid && !m_HasErrors; +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordeagent.h b/src/zenhorde/hordeagent.h new file mode 100644 index 000000000..e0ae89ead --- /dev/null +++ b/src/zenhorde/hordeagent.h @@ -0,0 +1,77 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "hordeagentmessage.h" +#include "hordecomputesocket.h" + +#include + +#include + +#include +#include +#include + +namespace zen::horde { + +/** Manages the lifecycle of a single Horde compute agent. + * + * Handles the full connection sequence for one provisioned machine: + * 1. Connect via TCP transport (with optional AES encryption wrapping) + * 2. Create a multiplexed ComputeSocket with agent (channel 0) and child (channel 100) + * 3. Perform the Attach/Fork handshake to establish the child channel + * 4. Upload zenserver binary via the WriteFiles/ReadBlob protocol + * 5. Execute zenserver remotely via ExecuteV2 + * 6. Poll for ExecuteOutput (stdout) and ExecuteResult (exit code) + */ +class HordeAgent +{ +public: + explicit HordeAgent(const MachineInfo& Info); + ~HordeAgent(); + + HordeAgent(const HordeAgent&) = delete; + HordeAgent& operator=(const HordeAgent&) = delete; + + /** Perform the channel setup handshake (Attach on agent channel, Fork, Attach on child channel). + * Returns false if the handshake times out or receives an unexpected message. */ + bool BeginCommunication(); + + /** Upload binary files to the remote agent. + * @param BundleDir Directory containing .blob files. + * @param BundleLocator Locator string identifying the bundle (from CreateBundle). */ + bool UploadBinaries(const std::filesystem::path& BundleDir, const std::string& BundleLocator); + + /** Execute a command on the remote machine. */ + void Execute(const char* Exe, + const char* const* Args, + size_t NumArgs, + const char* WorkingDir = nullptr, + const char* const* EnvVars = nullptr, + size_t NumEnvVars = 0, + bool UseWine = false); + + /** Poll for output and results. Returns true if the agent is still running. + * When LogOutput is true, remote stdout is logged via ZEN_INFO. */ + bool Poll(bool LogOutput = true); + + void CloseConnection(); + bool IsValid() const; + + const MachineInfo& GetMachineInfo() const { return m_MachineInfo; } + +private: + LoggerRef Log() { return m_Log; } + + std::unique_ptr m_Socket; + std::unique_ptr m_AgentChannel; ///< Channel 0: agent control + std::unique_ptr m_ChildChannel; ///< Channel 100: child I/O + + LoggerRef m_Log; + bool m_IsValid = false; + bool m_HasErrors = false; + MachineInfo m_MachineInfo; +}; + +} // namespace zen::horde diff --git a/src/zenhorde/hordeagentmessage.cpp b/src/zenhorde/hordeagentmessage.cpp new file mode 100644 index 000000000..998134a96 --- /dev/null +++ b/src/zenhorde/hordeagentmessage.cpp @@ -0,0 +1,340 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hordeagentmessage.h" + +#include + +#include +#include + +namespace zen::horde { + +AgentMessageChannel::AgentMessageChannel(Ref Channel) : m_Channel(std::move(Channel)) +{ +} + +AgentMessageChannel::~AgentMessageChannel() = default; + +void +AgentMessageChannel::Close() +{ + CreateMessage(AgentMessageType::None, 0); + FlushMessage(); +} + +void +AgentMessageChannel::Ping() +{ + CreateMessage(AgentMessageType::Ping, 0); + FlushMessage(); +} + +void +AgentMessageChannel::Fork(int ChannelId, int BufferSize) +{ + CreateMessage(AgentMessageType::Fork, sizeof(int) + sizeof(int)); + WriteInt32(ChannelId); + WriteInt32(BufferSize); + FlushMessage(); +} + +void +AgentMessageChannel::Attach() +{ + CreateMessage(AgentMessageType::Attach, 0); + FlushMessage(); +} + +void +AgentMessageChannel::UploadFiles(const char* Path, const char* Locator) +{ + CreateMessage(AgentMessageType::WriteFiles, strlen(Path) + strlen(Locator) + 20); + WriteString(Path); + WriteString(Locator); + FlushMessage(); +} + +void +AgentMessageChannel::Execute(const char* Exe, + const char* const* Args, + size_t NumArgs, + const char* WorkingDir, + const char* const* EnvVars, + size_t NumEnvVars, + ExecuteProcessFlags Flags) +{ + size_t RequiredSize = 50 + strlen(Exe); + for (size_t i = 0; i < NumArgs; ++i) + { + RequiredSize += strlen(Args[i]) + 10; + } + if (WorkingDir) + { + RequiredSize += strlen(WorkingDir) + 10; + } + for (size_t i = 0; i < NumEnvVars; ++i) + { + RequiredSize += strlen(EnvVars[i]) + 20; + } + + CreateMessage(AgentMessageType::ExecuteV2, RequiredSize); + WriteString(Exe); + + WriteUnsignedVarInt(NumArgs); + for (size_t i = 0; i < NumArgs; ++i) + { + WriteString(Args[i]); + } + + WriteOptionalString(WorkingDir); + + // ExecuteV2 protocol requires env vars as separate key/value pairs. + // Callers pass "KEY=VALUE" strings; we split on the first '=' here. + WriteUnsignedVarInt(NumEnvVars); + for (size_t i = 0; i < NumEnvVars; ++i) + { + const char* Eq = strchr(EnvVars[i], '='); + assert(Eq != nullptr); + + WriteString(std::string_view(EnvVars[i], Eq - EnvVars[i])); + if (*(Eq + 1) == '\0') + { + WriteOptionalString(nullptr); + } + else + { + WriteOptionalString(Eq + 1); + } + } + + WriteInt32(static_cast(Flags)); + FlushMessage(); +} + +void +AgentMessageChannel::Blob(const uint8_t* Data, size_t Length) +{ + // Blob responses are chunked to fit within the compute buffer's chunk size. + // The 128-byte margin accounts for the ReadBlobResponse header (offset + total length fields). + const size_t MaxChunkSize = m_Channel->Writer.GetChunkMaxLength() - 128 - MessageHeaderLength; + for (size_t ChunkOffset = 0; ChunkOffset < Length;) + { + const size_t ChunkLength = std::min(Length - ChunkOffset, MaxChunkSize); + + CreateMessage(AgentMessageType::ReadBlobResponse, ChunkLength + 128); + WriteInt32(static_cast(ChunkOffset)); + WriteInt32(static_cast(Length)); + WriteFixedLengthBytes(Data + ChunkOffset, ChunkLength); + FlushMessage(); + + ChunkOffset += ChunkLength; + } +} + +AgentMessageType +AgentMessageChannel::ReadResponse(int32_t TimeoutMs, bool* OutTimedOut) +{ + // Deferred advance: the previous response's buffer is only released when the next + // ReadResponse is called. This allows callers to read response data between calls + // without copying, since the pointer comes directly from the ring buffer. + if (m_ResponseData) + { + m_Channel->Reader.AdvanceReadPosition(m_ResponseLength + MessageHeaderLength); + m_ResponseData = nullptr; + m_ResponseLength = 0; + } + + const uint8_t* Header = m_Channel->Reader.WaitToRead(MessageHeaderLength, TimeoutMs, OutTimedOut); + if (!Header) + { + return AgentMessageType::None; + } + + uint32_t Length; + memcpy(&Length, Header + 1, sizeof(uint32_t)); + + Header = m_Channel->Reader.WaitToRead(MessageHeaderLength + Length, TimeoutMs, OutTimedOut); + if (!Header) + { + return AgentMessageType::None; + } + + m_ResponseType = static_cast(Header[0]); + m_ResponseData = Header + MessageHeaderLength; + m_ResponseLength = Length; + + return m_ResponseType; +} + +void +AgentMessageChannel::ReadException(ExceptionInfo& Ex) +{ + assert(m_ResponseType == AgentMessageType::Exception); + const uint8_t* Pos = m_ResponseData; + Ex.Message = ReadString(&Pos); + Ex.Description = ReadString(&Pos); +} + +int +AgentMessageChannel::ReadExecuteResult() +{ + assert(m_ResponseType == AgentMessageType::ExecuteResult); + const uint8_t* Pos = m_ResponseData; + return ReadInt32(&Pos); +} + +void +AgentMessageChannel::ReadBlobRequest(BlobRequest& Req) +{ + assert(m_ResponseType == AgentMessageType::ReadBlob); + const uint8_t* Pos = m_ResponseData; + Req.Locator = ReadString(&Pos); + Req.Offset = ReadUnsignedVarInt(&Pos); + Req.Length = ReadUnsignedVarInt(&Pos); +} + +void +AgentMessageChannel::CreateMessage(AgentMessageType Type, size_t MaxLength) +{ + m_RequestData = m_Channel->Writer.WaitToWrite(MessageHeaderLength + MaxLength); + m_RequestData[0] = static_cast(Type); + m_MaxRequestSize = MaxLength; + m_RequestSize = 0; +} + +void +AgentMessageChannel::FlushMessage() +{ + const uint32_t Size = static_cast(m_RequestSize); + memcpy(&m_RequestData[1], &Size, sizeof(uint32_t)); + m_Channel->Writer.AdvanceWritePosition(MessageHeaderLength + m_RequestSize); + m_RequestSize = 0; + m_MaxRequestSize = 0; + m_RequestData = nullptr; +} + +void +AgentMessageChannel::WriteInt32(int Value) +{ + WriteFixedLengthBytes(reinterpret_cast(&Value), sizeof(int)); +} + +int +AgentMessageChannel::ReadInt32(const uint8_t** Pos) +{ + int Value; + memcpy(&Value, *Pos, sizeof(int)); + *Pos += sizeof(int); + return Value; +} + +void +AgentMessageChannel::WriteFixedLengthBytes(const uint8_t* Data, size_t Length) +{ + assert(m_RequestSize + Length <= m_MaxRequestSize); + memcpy(&m_RequestData[MessageHeaderLength + m_RequestSize], Data, Length); + m_RequestSize += Length; +} + +const uint8_t* +AgentMessageChannel::ReadFixedLengthBytes(const uint8_t** Pos, size_t Length) +{ + const uint8_t* Data = *Pos; + *Pos += Length; + return Data; +} + +size_t +AgentMessageChannel::MeasureUnsignedVarInt(size_t Value) +{ + if (Value == 0) + { + return 1; + } + return (FloorLog2_64(static_cast(Value)) / 7) + 1; +} + +void +AgentMessageChannel::WriteUnsignedVarInt(size_t Value) +{ + const size_t ByteCount = MeasureUnsignedVarInt(Value); + assert(m_RequestSize + ByteCount <= m_MaxRequestSize); + + uint8_t* Output = m_RequestData + MessageHeaderLength + m_RequestSize; + for (size_t i = 1; i < ByteCount; ++i) + { + Output[ByteCount - i] = static_cast(Value); + Value >>= 8; + } + Output[0] = static_cast((0xFF << (9 - static_cast(ByteCount))) | static_cast(Value)); + + m_RequestSize += ByteCount; +} + +size_t +AgentMessageChannel::ReadUnsignedVarInt(const uint8_t** Pos) +{ + const uint8_t* Data = *Pos; + const uint8_t FirstByte = Data[0]; + const size_t NumBytes = CountLeadingZeros(0xFF & (~static_cast(FirstByte))) + 1 - 24; + + size_t Value = static_cast(FirstByte & (0xFF >> NumBytes)); + for (size_t i = 1; i < NumBytes; ++i) + { + Value <<= 8; + Value |= Data[i]; + } + + *Pos += NumBytes; + return Value; +} + +size_t +AgentMessageChannel::MeasureString(const char* Text) const +{ + const size_t Length = strlen(Text); + return MeasureUnsignedVarInt(Length) + Length; +} + +void +AgentMessageChannel::WriteString(const char* Text) +{ + const size_t Length = strlen(Text); + WriteUnsignedVarInt(Length); + WriteFixedLengthBytes(reinterpret_cast(Text), Length); +} + +void +AgentMessageChannel::WriteString(std::string_view Text) +{ + WriteUnsignedVarInt(Text.size()); + WriteFixedLengthBytes(reinterpret_cast(Text.data()), Text.size()); +} + +std::string_view +AgentMessageChannel::ReadString(const uint8_t** Pos) +{ + const size_t Length = ReadUnsignedVarInt(Pos); + const char* Start = reinterpret_cast(ReadFixedLengthBytes(Pos, Length)); + return std::string_view(Start, Length); +} + +void +AgentMessageChannel::WriteOptionalString(const char* Text) +{ + // Optional strings use length+1 encoding: 0 means null/absent, + // N>0 means a string of length N-1 follows. This matches the UE + // FAgentMessageChannel serialization convention. + if (!Text) + { + WriteUnsignedVarInt(0); + } + else + { + const size_t Length = strlen(Text); + WriteUnsignedVarInt(Length + 1); + WriteFixedLengthBytes(reinterpret_cast(Text), Length); + } +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordeagentmessage.h b/src/zenhorde/hordeagentmessage.h new file mode 100644 index 000000000..38c4375fd --- /dev/null +++ b/src/zenhorde/hordeagentmessage.h @@ -0,0 +1,161 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include "hordecomputechannel.h" + +#include +#include +#include +#include +#include + +namespace zen::horde { + +/** Agent message types matching the UE EAgentMessageType byte values. + * These are the message opcodes exchanged over the agent/child channels. */ +enum class AgentMessageType : uint8_t +{ + None = 0x00, + Ping = 0x01, + Exception = 0x02, + Fork = 0x03, + Attach = 0x04, + WriteFiles = 0x10, + WriteFilesResponse = 0x11, + DeleteFiles = 0x12, + ExecuteV2 = 0x22, + ExecuteOutput = 0x17, + ExecuteResult = 0x18, + ReadBlob = 0x20, + ReadBlobResponse = 0x21, +}; + +/** Flags for the ExecuteV2 message. */ +enum class ExecuteProcessFlags : uint8_t +{ + None = 0, + UseWine = 1, ///< Run the executable under Wine on Linux agents +}; + +/** Parsed exception information from an Exception message. */ +struct ExceptionInfo +{ + std::string_view Message; + std::string_view Description; +}; + +/** Parsed blob read request from a ReadBlob message. */ +struct BlobRequest +{ + std::string_view Locator; + size_t Offset = 0; + size_t Length = 0; +}; + +/** Channel for sending and receiving agent messages over a ComputeChannel. + * + * Implements the Horde agent message protocol, matching the UE + * FAgentMessageChannel serialization format exactly. Messages are framed as + * [type (1B)][payload length (4B)][payload]. Strings use length-prefixed UTF-8; + * integers use variable-length encoding. + * + * The protocol has two directions: + * - Requests (initiator -> remote): Close, Ping, Fork, Attach, UploadFiles, Execute, Blob + * - Responses (remote -> initiator): ReadResponse returns the type, then call the + * appropriate Read* method to parse the payload. + */ +class AgentMessageChannel +{ +public: + explicit AgentMessageChannel(Ref Channel); + ~AgentMessageChannel(); + + AgentMessageChannel(const AgentMessageChannel&) = delete; + AgentMessageChannel& operator=(const AgentMessageChannel&) = delete; + + // --- Requests (Initiator -> Remote) --- + + /** Close the channel. */ + void Close(); + + /** Send a keepalive ping. */ + void Ping(); + + /** Fork communication to a new channel with the given ID and buffer size. */ + void Fork(int ChannelId, int BufferSize); + + /** Send an attach request (used during channel setup handshake). */ + void Attach(); + + /** Request the remote agent to write files from the given bundle locator. */ + void UploadFiles(const char* Path, const char* Locator); + + /** Execute a process on the remote machine. */ + void Execute(const char* Exe, + const char* const* Args, + size_t NumArgs, + const char* WorkingDir, + const char* const* EnvVars, + size_t NumEnvVars, + ExecuteProcessFlags Flags = ExecuteProcessFlags::None); + + /** Send blob data in response to a ReadBlob request. */ + void Blob(const uint8_t* Data, size_t Length); + + // --- Responses (Remote -> Initiator) --- + + /** Read the next response message. Returns the message type, or None on timeout. + * After this returns, use GetResponseData()/GetResponseSize() or the typed + * Read* methods to access the payload. */ + AgentMessageType ReadResponse(int32_t TimeoutMs = -1, bool* OutTimedOut = nullptr); + + const void* GetResponseData() const { return m_ResponseData; } + size_t GetResponseSize() const { return m_ResponseLength; } + + /** Parse an Exception response payload. */ + void ReadException(ExceptionInfo& Ex); + + /** Parse an ExecuteResult response payload. Returns the exit code. */ + int ReadExecuteResult(); + + /** Parse a ReadBlob response payload into a BlobRequest. */ + void ReadBlobRequest(BlobRequest& Req); + +private: + static constexpr size_t MessageHeaderLength = 5; ///< [type(1B)][length(4B)] + + Ref m_Channel; + + uint8_t* m_RequestData = nullptr; + size_t m_RequestSize = 0; + size_t m_MaxRequestSize = 0; + + AgentMessageType m_ResponseType = AgentMessageType::None; + const uint8_t* m_ResponseData = nullptr; + size_t m_ResponseLength = 0; + + void CreateMessage(AgentMessageType Type, size_t MaxLength); + void FlushMessage(); + + void WriteInt32(int Value); + static int ReadInt32(const uint8_t** Pos); + + void WriteFixedLengthBytes(const uint8_t* Data, size_t Length); + static const uint8_t* ReadFixedLengthBytes(const uint8_t** Pos, size_t Length); + + static size_t MeasureUnsignedVarInt(size_t Value); + void WriteUnsignedVarInt(size_t Value); + static size_t ReadUnsignedVarInt(const uint8_t** Pos); + + size_t MeasureString(const char* Text) const; + void WriteString(const char* Text); + void WriteString(std::string_view Text); + static std::string_view ReadString(const uint8_t** Pos); + + void WriteOptionalString(const char* Text); +}; + +} // namespace zen::horde diff --git a/src/zenhorde/hordebundle.cpp b/src/zenhorde/hordebundle.cpp new file mode 100644 index 000000000..d3974bc28 --- /dev/null +++ b/src/zenhorde/hordebundle.cpp @@ -0,0 +1,619 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hordebundle.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace zen::horde { + +static LoggerRef +Log() +{ + static auto s_Logger = zen::logging::Get("horde.bundle"); + return s_Logger; +} + +static constexpr uint8_t PacketSignature[3] = {'U', 'B', 'N'}; +static constexpr uint8_t PacketVersion = 5; +static constexpr int32_t CurrentPacketBaseIdx = -2; +static constexpr int ImportBias = 3; +static constexpr uint32_t ChunkSize = 64 * 1024; // 64KB fixed chunks +static constexpr uint32_t LargeFileThreshold = 128 * 1024; // 128KB + +// BlobType: 20 bytes each = FGuid (16 bytes, 4x uint32 LE) + Version (int32 LE) +// Values from UE SDK: GUIDs stored as 4 uint32 LE values. + +// ChunkLeaf v1: {0xB27AFB68, 0x4A4B9E20, 0x8A78D8A4, 0x39D49840} +static constexpr uint8_t BlobType_ChunkLeafV1[20] = {0x68, 0xFB, 0x7A, 0xB2, 0x20, 0x9E, 0x4B, 0x4A, 0xA4, 0xD8, + 0x78, 0x8A, 0x40, 0x98, 0xD4, 0x39, 0x01, 0x00, 0x00, 0x00}; // version 1 + +// ChunkInterior v2: {0xF4DEDDBC, 0x4C7A70CB, 0x11F04783, 0xB9CDCCAF} +static constexpr uint8_t BlobType_ChunkInteriorV2[20] = {0xBC, 0xDD, 0xDE, 0xF4, 0xCB, 0x70, 0x7A, 0x4C, 0x83, 0x47, + 0xF0, 0x11, 0xAF, 0xCC, 0xCD, 0xB9, 0x02, 0x00, 0x00, 0x00}; // version 2 + +// Directory v1: {0x0714EC11, 0x4D07291A, 0x8AE77F86, 0x799980D6} +static constexpr uint8_t BlobType_DirectoryV1[20] = {0x11, 0xEC, 0x14, 0x07, 0x1A, 0x29, 0x07, 0x4D, 0x86, 0x7F, + 0xE7, 0x8A, 0xD6, 0x80, 0x99, 0x79, 0x01, 0x00, 0x00, 0x00}; // version 1 + +static constexpr size_t BlobTypeSize = 20; + +// ─── VarInt helpers (UE format) ───────────────────────────────────────────── + +static size_t +MeasureVarInt(size_t Value) +{ + if (Value == 0) + { + return 1; + } + return (FloorLog2(static_cast(Value)) / 7) + 1; +} + +static void +WriteVarInt(std::vector& Buffer, size_t Value) +{ + const size_t ByteCount = MeasureVarInt(Value); + const size_t Offset = Buffer.size(); + Buffer.resize(Offset + ByteCount); + + uint8_t* Output = Buffer.data() + Offset; + for (size_t i = 1; i < ByteCount; ++i) + { + Output[ByteCount - i] = static_cast(Value); + Value >>= 8; + } + Output[0] = static_cast((0xFF << (9 - static_cast(ByteCount))) | static_cast(Value)); +} + +// ─── Binary helpers ───────────────────────────────────────────────────────── + +static void +WriteLE32(std::vector& Buffer, int32_t Value) +{ + uint8_t Bytes[4]; + memcpy(Bytes, &Value, 4); + Buffer.insert(Buffer.end(), Bytes, Bytes + 4); +} + +static void +WriteByte(std::vector& Buffer, uint8_t Value) +{ + Buffer.push_back(Value); +} + +static void +WriteBytes(std::vector& Buffer, const void* Data, size_t Size) +{ + auto* Ptr = static_cast(Data); + Buffer.insert(Buffer.end(), Ptr, Ptr + Size); +} + +static void +WriteString(std::vector& Buffer, std::string_view Str) +{ + WriteVarInt(Buffer, Str.size()); + WriteBytes(Buffer, Str.data(), Str.size()); +} + +static void +AlignTo4(std::vector& Buffer) +{ + while (Buffer.size() % 4 != 0) + { + Buffer.push_back(0); + } +} + +static void +PatchLE32(std::vector& Buffer, size_t Offset, int32_t Value) +{ + memcpy(Buffer.data() + Offset, &Value, 4); +} + +// ─── Packet builder ───────────────────────────────────────────────────────── + +// Builds a single uncompressed Horde V2 packet. Layout: +// [Signature(3) + Version(1) + PacketLength(4)] 8 bytes (header) +// [TypeTableOffset(4) + ImportTableOffset(4) + ExportTableOffset(4)] 12 bytes +// [Export data...] +// [Type table: count(4) + count * 20 bytes] +// [Import table: count(4) + (count+1) offset entries(4 each) + import data] +// [Export table: count(4) + (count+1) offset entries(4 each)] +// +// ALL offsets are absolute from byte 0 of the full packet (including the 8-byte header). +// PacketLength in the header = total packet size including the 8-byte header. + +struct PacketBuilder +{ + std::vector Data; + std::vector ExportOffsets; // Absolute byte offset of each export from byte 0 + + // Type table: unique 20-byte BlobType entries + std::vector Types; + + // Import table entries: (baseIdx, fragment) + struct ImportEntry + { + int32_t BaseIdx; + std::string Fragment; + }; + std::vector Imports; + + // Current export's start offset (absolute from byte 0) + size_t CurrentExportStart = 0; + + PacketBuilder() + { + // Reserve packet header (8 bytes) + table offsets (12 bytes) = 20 bytes + Data.resize(20, 0); + + // Write signature + Data[0] = PacketSignature[0]; + Data[1] = PacketSignature[1]; + Data[2] = PacketSignature[2]; + Data[3] = PacketVersion; + // PacketLength, TypeTableOffset, ImportTableOffset, ExportTableOffset + // will be patched in Finish() + } + + int AddType(const uint8_t* BlobType) + { + for (size_t i = 0; i < Types.size(); ++i) + { + if (memcmp(Types[i], BlobType, BlobTypeSize) == 0) + { + return static_cast(i); + } + } + Types.push_back(BlobType); + return static_cast(Types.size() - 1); + } + + int AddImport(int32_t BaseIdx, std::string Fragment) + { + Imports.push_back({BaseIdx, std::move(Fragment)}); + return static_cast(Imports.size() - 1); + } + + void BeginExport() + { + AlignTo4(Data); + CurrentExportStart = Data.size(); + // Reserve space for payload length + WriteLE32(Data, 0); + } + + // Write raw payload data into the current export + void WritePayload(const void* Payload, size_t Size) { WriteBytes(Data, Payload, Size); } + + // Complete the current export: patches payload length, writes type+imports metadata + int CompleteExport(const uint8_t* BlobType, const std::vector& ImportIndices) + { + const int ExportIndex = static_cast(ExportOffsets.size()); + + // Patch payload length (does not include the 4-byte length field itself) + const size_t PayloadStart = CurrentExportStart + 4; + const int32_t PayloadLen = static_cast(Data.size() - PayloadStart); + PatchLE32(Data, CurrentExportStart, PayloadLen); + + // Write type index (varint) + const int TypeIdx = AddType(BlobType); + WriteVarInt(Data, static_cast(TypeIdx)); + + // Write import count + indices + WriteVarInt(Data, ImportIndices.size()); + for (int Idx : ImportIndices) + { + WriteVarInt(Data, static_cast(Idx)); + } + + // Record export offset (absolute from byte 0) + ExportOffsets.push_back(static_cast(CurrentExportStart)); + + return ExportIndex; + } + + // Finalize the packet: write type/import/export tables, patch header. + std::vector Finish() + { + AlignTo4(Data); + + // ── Type table: count(int32) + count * BlobTypeSize bytes ── + const int32_t TypeTableOffset = static_cast(Data.size()); + WriteLE32(Data, static_cast(Types.size())); + for (const uint8_t* TypeEntry : Types) + { + WriteBytes(Data, TypeEntry, BlobTypeSize); + } + + // ── Import table: count(int32) + (count+1) offsets(int32 each) + import data ── + const int32_t ImportTableOffset = static_cast(Data.size()); + const int32_t ImportCount = static_cast(Imports.size()); + WriteLE32(Data, ImportCount); + + // Reserve space for (count+1) offset entries — will be patched below + const size_t ImportOffsetsStart = Data.size(); + for (int32_t i = 0; i <= ImportCount; ++i) + { + WriteLE32(Data, 0); // placeholder + } + + // Write import data and record offsets + for (int32_t i = 0; i < ImportCount; ++i) + { + // Record absolute offset of this import's data + PatchLE32(Data, ImportOffsetsStart + static_cast(i) * 4, static_cast(Data.size())); + + ImportEntry& Imp = Imports[static_cast(i)]; + // BaseIdx encoded as unsigned VarInt with bias: VarInt(BaseIdx + ImportBias) + const size_t EncodedBaseIdx = static_cast(static_cast(Imp.BaseIdx) + ImportBias); + WriteVarInt(Data, EncodedBaseIdx); + // Fragment: raw UTF-8 bytes, NO length prefix (length determined by offset table) + WriteBytes(Data, Imp.Fragment.data(), Imp.Fragment.size()); + } + + // Sentinel offset (points past the last import's data) + PatchLE32(Data, ImportOffsetsStart + static_cast(ImportCount) * 4, static_cast(Data.size())); + + // ── Export table: count(int32) + (count+1) offsets(int32 each) ── + const int32_t ExportTableOffset = static_cast(Data.size()); + const int32_t ExportCount = static_cast(ExportOffsets.size()); + WriteLE32(Data, ExportCount); + + for (int32_t Off : ExportOffsets) + { + WriteLE32(Data, Off); + } + // Sentinel: points to the start of the type table (end of export data region) + WriteLE32(Data, TypeTableOffset); + + // ── Patch header ── + // PacketLength = total packet size including the 8-byte header + const int32_t PacketLength = static_cast(Data.size()); + PatchLE32(Data, 4, PacketLength); + PatchLE32(Data, 8, TypeTableOffset); + PatchLE32(Data, 12, ImportTableOffset); + PatchLE32(Data, 16, ExportTableOffset); + + return std::move(Data); + } +}; + +// ─── Encoded packet wrapper ───────────────────────────────────────────────── + +// Wraps an uncompressed packet with the encoded header: +// [Signature(3) + Version(1) + HeaderLength(4)] 8 bytes +// [DecompressedLength(4)] 4 bytes +// [CompressionFormat(1): 0=None] 1 byte +// [PacketData...] +// +// HeaderLength = total encoded packet size INCLUDING the 8-byte outer header. + +static std::vector +EncodePacket(std::vector UncompressedPacket) +{ + const int32_t DecompressedLen = static_cast(UncompressedPacket.size()); + // HeaderLength includes the 8-byte outer signature header itself + const int32_t HeaderLength = 8 + 4 + 1 + DecompressedLen; + + std::vector Encoded; + Encoded.reserve(static_cast(HeaderLength)); + + // Outer signature: 'U','B','N', version=5, HeaderLength (LE int32) + WriteByte(Encoded, PacketSignature[0]); // 'U' + WriteByte(Encoded, PacketSignature[1]); // 'B' + WriteByte(Encoded, PacketSignature[2]); // 'N' + WriteByte(Encoded, PacketVersion); // 5 + WriteLE32(Encoded, HeaderLength); + + // Decompressed length + compression format + WriteLE32(Encoded, DecompressedLen); + WriteByte(Encoded, 0); // CompressionFormat::None + + // Packet data + WriteBytes(Encoded, UncompressedPacket.data(), UncompressedPacket.size()); + + return Encoded; +} + +// ─── Bundle blob name generation ──────────────────────────────────────────── + +static std::string +GenerateBlobName() +{ + static std::atomic s_Counter{0}; + + const int Pid = GetCurrentProcessId(); + + auto Now = std::chrono::steady_clock::now().time_since_epoch(); + auto Ms = std::chrono::duration_cast(Now).count(); + + ExtendableStringBuilder<64> Name; + Name << Pid << "_" << Ms << "_" << s_Counter.fetch_add(1); + return std::string(Name.ToView()); +} + +// ─── File info for bundling ───────────────────────────────────────────────── + +struct FileInfo +{ + std::filesystem::path Path; + std::string Name; // Filename only (for directory entry) + uint64_t FileSize; + IoHash ContentHash; // IoHash of file content + BLAKE3 StreamHash; // Full BLAKE3 for stream hash + int DirectoryExportImportIndex; // Import index referencing this file's root export + IoHash RootExportHash; // IoHash of the root export for this file +}; + +// ─── CreateBundle implementation ──────────────────────────────────────────── + +bool +BundleCreator::CreateBundle(const std::vector& Files, const std::filesystem::path& OutputDir, BundleResult& OutResult) +{ + ZEN_TRACE_CPU("BundleCreator::CreateBundle"); + + std::error_code Ec; + + // Collect files that exist + std::vector ValidFiles; + for (const BundleFile& F : Files) + { + if (!std::filesystem::exists(F.Path, Ec)) + { + if (F.Optional) + { + continue; + } + ZEN_ERROR("required bundle file does not exist: {}", F.Path.string()); + return false; + } + FileInfo Info; + Info.Path = F.Path; + Info.Name = F.Path.filename().string(); + Info.FileSize = std::filesystem::file_size(F.Path, Ec); + if (Ec) + { + ZEN_ERROR("failed to get file size: {}", F.Path.string()); + return false; + } + ValidFiles.push_back(std::move(Info)); + } + + if (ValidFiles.empty()) + { + ZEN_ERROR("no valid files to bundle"); + return false; + } + + std::filesystem::create_directories(OutputDir, Ec); + if (Ec) + { + ZEN_ERROR("failed to create output directory: {}", OutputDir.string()); + return false; + } + + const std::string BlobName = GenerateBlobName(); + PacketBuilder Packet; + + // Process each file: create chunk exports + for (FileInfo& Info : ValidFiles) + { + BasicFile File; + File.Open(Info.Path, BasicFile::Mode::kRead, Ec); + if (Ec) + { + ZEN_ERROR("failed to open file: {}", Info.Path.string()); + return false; + } + + // Compute stream hash (full BLAKE3) and content hash (IoHash) while reading + BLAKE3Stream StreamHasher; + IoHashStream ContentHasher; + + if (Info.FileSize <= LargeFileThreshold) + { + // Small file: single chunk leaf export + IoBuffer Content = File.ReadAll(); + const auto* Data = static_cast(Content.GetData()); + const size_t Size = Content.GetSize(); + + StreamHasher.Append(Data, Size); + ContentHasher.Append(Data, Size); + + Packet.BeginExport(); + Packet.WritePayload(Data, Size); + + const IoHash ChunkHash = IoHash::HashBuffer(Data, Size); + const int ExportIndex = Packet.CompleteExport(BlobType_ChunkLeafV1, {}); + Info.RootExportHash = ChunkHash; + Info.ContentHash = ContentHasher.GetHash(); + Info.StreamHash = StreamHasher.GetHash(); + + // Add import for this file's root export (references export within same packet) + ExtendableStringBuilder<32> Fragment; + Fragment << "exp=" << ExportIndex; + Info.DirectoryExportImportIndex = Packet.AddImport(CurrentPacketBaseIdx, std::string(Fragment.ToView())); + } + else + { + // Large file: split into fixed 64KB chunks, then create interior node + std::vector ChunkExportIndices; + std::vector ChunkHashes; + + uint64_t Remaining = Info.FileSize; + uint64_t Offset = 0; + + while (Remaining > 0) + { + const uint64_t ReadSize = std::min(static_cast(ChunkSize), Remaining); + IoBuffer Chunk = File.ReadRange(Offset, ReadSize); + const auto* Data = static_cast(Chunk.GetData()); + const size_t Size = Chunk.GetSize(); + + StreamHasher.Append(Data, Size); + ContentHasher.Append(Data, Size); + + Packet.BeginExport(); + Packet.WritePayload(Data, Size); + + const IoHash ChunkHash = IoHash::HashBuffer(Data, Size); + const int ExpIdx = Packet.CompleteExport(BlobType_ChunkLeafV1, {}); + + ChunkExportIndices.push_back(ExpIdx); + ChunkHashes.push_back(ChunkHash); + + Offset += ReadSize; + Remaining -= ReadSize; + } + + Info.ContentHash = ContentHasher.GetHash(); + Info.StreamHash = StreamHasher.GetHash(); + + // Create interior node referencing all chunk leaves + // Interior payload: for each child: [IoHash(20)][node_type=1(1)] + imports + std::vector InteriorImports; + for (size_t i = 0; i < ChunkExportIndices.size(); ++i) + { + ExtendableStringBuilder<32> Fragment; + Fragment << "exp=" << ChunkExportIndices[i]; + const int ImportIdx = Packet.AddImport(CurrentPacketBaseIdx, std::string(Fragment.ToView())); + InteriorImports.push_back(ImportIdx); + } + + Packet.BeginExport(); + + // Write interior payload: [hash(20)][type(1)] per child + for (size_t i = 0; i < ChunkHashes.size(); ++i) + { + Packet.WritePayload(ChunkHashes[i].Hash, sizeof(IoHash)); + const uint8_t NodeType = 1; // ChunkNode type + Packet.WritePayload(&NodeType, 1); + } + + // Hash the interior payload to get the interior node hash + const IoHash InteriorHash = IoHash::HashBuffer(Packet.Data.data() + (Packet.CurrentExportStart + 4), + Packet.Data.size() - (Packet.CurrentExportStart + 4)); + + const int InteriorExportIndex = Packet.CompleteExport(BlobType_ChunkInteriorV2, InteriorImports); + + Info.RootExportHash = InteriorHash; + + // Add import for directory to reference this interior node + ExtendableStringBuilder<32> Fragment; + Fragment << "exp=" << InteriorExportIndex; + Info.DirectoryExportImportIndex = Packet.AddImport(CurrentPacketBaseIdx, std::string(Fragment.ToView())); + } + } + + // Create directory node export + // Payload: [flags(varint=0)] [file_count(varint)] [file_entries...] [dir_count(varint=0)] + // FileEntry: [import(varint)] [IoHash(20)] [name(string)] [flags(varint)] [length(varint)] [IoHash_stream(20)] + + Packet.BeginExport(); + + // Build directory payload into a temporary buffer, then write it + std::vector DirPayload; + WriteVarInt(DirPayload, 0); // flags + WriteVarInt(DirPayload, ValidFiles.size()); // file_count + + std::vector DirImports; + for (size_t i = 0; i < ValidFiles.size(); ++i) + { + FileInfo& Info = ValidFiles[i]; + DirImports.push_back(Info.DirectoryExportImportIndex); + + // IoHash of target (20 bytes) — import is consumed sequentially from the + // export's import list by ReadBlobRef, not encoded in the payload + WriteBytes(DirPayload, Info.RootExportHash.Hash, sizeof(IoHash)); + // name (string) + WriteString(DirPayload, Info.Name); + // flags (varint): 1 = Executable + WriteVarInt(DirPayload, 1); + // length (varint) + WriteVarInt(DirPayload, static_cast(Info.FileSize)); + // stream hash: IoHash from full BLAKE3, truncated to 20 bytes + const IoHash StreamIoHash = IoHash::FromBLAKE3(Info.StreamHash); + WriteBytes(DirPayload, StreamIoHash.Hash, sizeof(IoHash)); + } + + WriteVarInt(DirPayload, 0); // dir_count + + Packet.WritePayload(DirPayload.data(), DirPayload.size()); + const int DirExportIndex = Packet.CompleteExport(BlobType_DirectoryV1, DirImports); + + // Finalize packet and encode + std::vector UncompressedPacket = Packet.Finish(); + std::vector EncodedPacket = EncodePacket(std::move(UncompressedPacket)); + + // Write .blob file + const std::filesystem::path BlobFilePath = OutputDir / (BlobName + ".blob"); + { + BasicFile BlobFile(BlobFilePath, BasicFile::Mode::kTruncate, Ec); + if (Ec) + { + ZEN_ERROR("failed to create blob file: {}", BlobFilePath.string()); + return false; + } + BlobFile.Write(EncodedPacket.data(), EncodedPacket.size(), 0); + } + + // Build locator: #pkt=0,&exp= + ExtendableStringBuilder<256> Locator; + Locator << BlobName << "#pkt=0," << uint64_t(EncodedPacket.size()) << "&exp=" << DirExportIndex; + const std::string LocatorStr(Locator.ToView()); + + // Write .ref file (use first file's name as the ref base) + const std::filesystem::path RefFilePath = OutputDir / (ValidFiles[0].Name + ".Bundle.ref"); + { + BasicFile RefFile(RefFilePath, BasicFile::Mode::kTruncate, Ec); + if (Ec) + { + ZEN_ERROR("failed to create ref file: {}", RefFilePath.string()); + return false; + } + RefFile.Write(LocatorStr.data(), LocatorStr.size(), 0); + } + + OutResult.Locator = LocatorStr; + OutResult.BundleDir = OutputDir; + + ZEN_INFO("created V2 bundle: blob={}.blob locator={} files={}", BlobName, LocatorStr, ValidFiles.size()); + return true; +} + +bool +BundleCreator::ReadLocator(const std::filesystem::path& RefFile, std::string& OutLocator) +{ + BasicFile File; + std::error_code Ec; + File.Open(RefFile, BasicFile::Mode::kRead, Ec); + if (Ec) + { + return false; + } + + IoBuffer Content = File.ReadAll(); + OutLocator.assign(static_cast(Content.GetData()), Content.GetSize()); + + // Strip trailing whitespace/newlines + while (!OutLocator.empty() && (OutLocator.back() == '\n' || OutLocator.back() == '\r' || OutLocator.back() == '\0')) + { + OutLocator.pop_back(); + } + + return !OutLocator.empty(); +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordebundle.h b/src/zenhorde/hordebundle.h new file mode 100644 index 000000000..052f60435 --- /dev/null +++ b/src/zenhorde/hordebundle.h @@ -0,0 +1,49 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +namespace zen::horde { + +/** Describes a file to include in a Horde bundle. */ +struct BundleFile +{ + std::filesystem::path Path; ///< Local file path + bool Optional; ///< If true, skip without error if missing +}; + +/** Result of a successful bundle creation. */ +struct BundleResult +{ + std::string Locator; ///< Root directory locator for WriteFiles + std::filesystem::path BundleDir; ///< Directory containing .blob files +}; + +/** Creates Horde V2 bundles from local files for upload to remote agents. + * + * Produces a proper Horde storage V2 bundle containing: + * - Chunk leaf exports for file data (split into 64KB chunks for large files) + * - Optional interior chunk nodes referencing leaf chunks + * - A directory node listing all bundled files with metadata + * + * The bundle is written as a single .blob file with a corresponding .ref file + * containing the locator string. The locator format is: + * #pkt=0,&exp= + */ +struct BundleCreator +{ + /** Create a V2 bundle from one or more input files. + * @param Files Files to include in the bundle. + * @param OutputDir Directory where .blob and .ref files will be written. + * @param OutResult Receives the locator and output directory on success. + * @return True on success. */ + static bool CreateBundle(const std::vector& Files, const std::filesystem::path& OutputDir, BundleResult& OutResult); + + /** Read a locator string from a .ref file. Strips trailing whitespace/newlines. */ + static bool ReadLocator(const std::filesystem::path& RefFile, std::string& OutLocator); +}; + +} // namespace zen::horde diff --git a/src/zenhorde/hordeclient.cpp b/src/zenhorde/hordeclient.cpp new file mode 100644 index 000000000..fb981f0ba --- /dev/null +++ b/src/zenhorde/hordeclient.cpp @@ -0,0 +1,382 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::horde { + +HordeClient::HordeClient(const HordeConfig& Config) : m_Config(Config), m_Log(zen::logging::Get("horde.client")) +{ +} + +HordeClient::~HordeClient() = default; + +bool +HordeClient::Initialize() +{ + ZEN_TRACE_CPU("HordeClient::Initialize"); + + HttpClientSettings Settings; + Settings.LogCategory = "horde.http"; + Settings.ConnectTimeout = std::chrono::milliseconds{10000}; + Settings.Timeout = std::chrono::milliseconds{60000}; + Settings.RetryCount = 1; + Settings.ExpectedErrorCodes = {HttpResponseCode::ServiceUnavailable, HttpResponseCode::TooManyRequests}; + + if (!m_Config.AuthToken.empty()) + { + Settings.AccessTokenProvider = [token = m_Config.AuthToken]() -> HttpClientAccessToken { + HttpClientAccessToken Token; + Token.Value = token; + Token.ExpireTime = HttpClientAccessToken::Clock::now() + std::chrono::hours{24}; + return Token; + }; + } + + m_Http = std::make_unique(m_Config.ServerUrl, Settings); + + if (!m_Config.AuthToken.empty()) + { + if (!m_Http->Authenticate()) + { + ZEN_WARN("failed to authenticate with Horde server"); + return false; + } + } + + return true; +} + +std::string +HordeClient::BuildRequestBody() const +{ + json11::Json::object Requirements; + + if (m_Config.Mode == ConnectionMode::Direct && !m_Config.Pool.empty()) + { + Requirements["pool"] = m_Config.Pool; + } + + std::string Condition; +#if ZEN_PLATFORM_WINDOWS + ExtendableStringBuilder<256> CondBuf; + CondBuf << "(OSFamily == 'Windows' || WineEnabled == '" << (m_Config.AllowWine ? "true" : "false") << "')"; + Condition = std::string(CondBuf); +#elif ZEN_PLATFORM_MAC + Condition = "OSFamily == 'MacOS'"; +#else + Condition = "OSFamily == 'Linux'"; +#endif + + if (!m_Config.Condition.empty()) + { + Condition += " "; + Condition += m_Config.Condition; + } + + Requirements["condition"] = Condition; + Requirements["exclusive"] = true; + + json11::Json::object Connection; + Connection["modePreference"] = ToString(m_Config.Mode); + + if (m_Config.EncryptionMode != Encryption::None) + { + Connection["encryption"] = ToString(m_Config.EncryptionMode); + } + + // Request configured zen service port to be forwarded. The Horde agent will map this + // to a local port on the provisioned machine and report it back in the response. + json11::Json::object PortsObj; + PortsObj["ZenPort"] = json11::Json(m_Config.ZenServicePort); + Connection["ports"] = PortsObj; + + json11::Json::object Root; + Root["requirements"] = Requirements; + Root["connection"] = Connection; + + return json11::Json(Root).dump(); +} + +bool +HordeClient::ResolveCluster(const std::string& RequestBody, ClusterInfo& OutCluster) +{ + ZEN_TRACE_CPU("HordeClient::ResolveCluster"); + + const IoBuffer Payload = IoBufferBuilder::MakeFromMemory(MemoryView{RequestBody.data(), RequestBody.size()}, ZenContentType::kJSON); + + const HttpClient::Response Response = m_Http->Post("api/v2/compute/_cluster", Payload); + + if (Response.Error) + { + ZEN_WARN("cluster resolution failed: {}", Response.Error->ErrorMessage); + return false; + } + + const int StatusCode = static_cast(Response.StatusCode); + + if (StatusCode == 503 || StatusCode == 429) + { + ZEN_DEBUG("cluster resolution returned HTTP/{}: no resources", StatusCode); + return false; + } + + if (StatusCode == 401) + { + ZEN_WARN("cluster resolution returned HTTP/401: token expired"); + return false; + } + + if (!Response.IsSuccess()) + { + ZEN_WARN("cluster resolution failed with HTTP/{}", StatusCode); + return false; + } + + const std::string Body(Response.AsText()); + std::string Err; + const json11::Json Json = json11::Json::parse(Body, Err); + + if (!Err.empty()) + { + ZEN_WARN("invalid JSON response for cluster resolution: {}", Err); + return false; + } + + const json11::Json ClusterIdVal = Json["clusterId"]; + if (!ClusterIdVal.is_string() || ClusterIdVal.string_value().empty()) + { + ZEN_WARN("missing 'clusterId' in cluster resolution response"); + return false; + } + + OutCluster.ClusterId = ClusterIdVal.string_value(); + return true; +} + +bool +HordeClient::ParseHexBytes(std::string_view Hex, uint8_t* Out, size_t OutSize) +{ + if (Hex.size() != OutSize * 2) + { + return false; + } + + for (size_t i = 0; i < OutSize; ++i) + { + auto HexToByte = [](char c) -> int { + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + return -1; + }; + + const int Hi = HexToByte(Hex[i * 2]); + const int Lo = HexToByte(Hex[i * 2 + 1]); + if (Hi < 0 || Lo < 0) + { + return false; + } + Out[i] = static_cast((Hi << 4) | Lo); + } + + return true; +} + +bool +HordeClient::RequestMachine(const std::string& RequestBody, const std::string& ClusterId, MachineInfo& OutMachine) +{ + ZEN_TRACE_CPU("HordeClient::RequestMachine"); + + ZEN_INFO("requesting machine from Horde with cluster '{}'", ClusterId.empty() ? "default" : ClusterId.c_str()); + + ExtendableStringBuilder<128> ResourcePath; + ResourcePath << "api/v2/compute/" << (ClusterId.empty() ? "default" : ClusterId.c_str()); + + const IoBuffer Payload = IoBufferBuilder::MakeFromMemory(MemoryView{RequestBody.data(), RequestBody.size()}, ZenContentType::kJSON); + const HttpClient::Response Response = m_Http->Post(ResourcePath.ToView(), Payload); + + // Reset output to invalid state + OutMachine = {}; + OutMachine.Port = 0xFFFF; + + if (Response.Error) + { + ZEN_WARN("machine request failed: {}", Response.Error->ErrorMessage); + return false; + } + + const int StatusCode = static_cast(Response.StatusCode); + + if (StatusCode == 404 || StatusCode == 503 || StatusCode == 429) + { + ZEN_DEBUG("machine request returned HTTP/{}: no resources", StatusCode); + return false; + } + + if (StatusCode == 401) + { + ZEN_WARN("machine request returned HTTP/401: token expired"); + return false; + } + + if (!Response.IsSuccess()) + { + ZEN_WARN("machine request failed with HTTP/{}", StatusCode); + return false; + } + + const std::string Body(Response.AsText()); + std::string Err; + const json11::Json Json = json11::Json::parse(Body, Err); + + if (!Err.empty()) + { + ZEN_WARN("invalid JSON response for machine request: {}", Err); + return false; + } + + // Required fields + const json11::Json NonceVal = Json["nonce"]; + const json11::Json IpVal = Json["ip"]; + const json11::Json PortVal = Json["port"]; + + if (!NonceVal.is_string() || !IpVal.is_string() || !PortVal.is_number()) + { + ZEN_WARN("missing 'nonce', 'ip', or 'port' in machine response"); + return false; + } + + OutMachine.Ip = IpVal.string_value(); + OutMachine.Port = static_cast(PortVal.int_value()); + + if (!ParseHexBytes(NonceVal.string_value(), OutMachine.Nonce, NonceSize)) + { + ZEN_WARN("invalid nonce hex string in machine response"); + return false; + } + + if (const json11::Json PortsVal = Json["ports"]; PortsVal.is_object()) + { + for (const auto& [Key, Val] : PortsVal.object_items()) + { + PortInfo Info; + if (Val["port"].is_number()) + { + Info.Port = static_cast(Val["port"].int_value()); + } + if (Val["agentPort"].is_number()) + { + Info.AgentPort = static_cast(Val["agentPort"].int_value()); + } + OutMachine.Ports[Key] = Info; + } + } + + if (const json11::Json ConnectionModeVal = Json["connectionMode"]; ConnectionModeVal.is_string()) + { + if (FromString(OutMachine.Mode, ConnectionModeVal.string_value())) + { + if (const json11::Json ConnectionAddressVal = Json["connectionAddress"]; ConnectionAddressVal.is_string()) + { + OutMachine.ConnectionAddress = ConnectionAddressVal.string_value(); + } + } + } + + // Properties are a flat string array of "Key=Value" pairs describing the machine. + // We extract OS family and core counts for sizing decisions. If neither core count + // is available, we fall back to 16 as a conservative default. + uint16_t LogicalCores = 0; + uint16_t PhysicalCores = 0; + + if (const json11::Json PropertiesVal = Json["properties"]; PropertiesVal.is_array()) + { + for (const json11::Json& PropVal : PropertiesVal.array_items()) + { + if (!PropVal.is_string()) + { + continue; + } + + const std::string Prop = PropVal.string_value(); + if (Prop.starts_with("OSFamily=")) + { + if (Prop.substr(9) == "Windows") + { + OutMachine.IsWindows = true; + } + } + else if (Prop.starts_with("LogicalCores=")) + { + LogicalCores = static_cast(std::atoi(Prop.c_str() + 13)); + } + else if (Prop.starts_with("PhysicalCores=")) + { + PhysicalCores = static_cast(std::atoi(Prop.c_str() + 14)); + } + } + } + + if (LogicalCores > 0) + { + OutMachine.LogicalCores = LogicalCores; + } + else if (PhysicalCores > 0) + { + OutMachine.LogicalCores = PhysicalCores * 2; + } + else + { + OutMachine.LogicalCores = 16; + } + + if (const json11::Json EncryptionVal = Json["encryption"]; EncryptionVal.is_string()) + { + if (FromString(OutMachine.EncryptionMode, EncryptionVal.string_value())) + { + if (OutMachine.EncryptionMode == Encryption::AES) + { + const json11::Json KeyVal = Json["key"]; + if (KeyVal.is_string() && !KeyVal.string_value().empty()) + { + if (!ParseHexBytes(KeyVal.string_value(), OutMachine.Key, KeySize)) + { + ZEN_WARN("invalid AES key in machine response"); + } + } + else + { + ZEN_WARN("AES encryption requested but no key provided"); + } + } + } + } + + if (const json11::Json LeaseIdVal = Json["leaseId"]; LeaseIdVal.is_string()) + { + OutMachine.LeaseId = LeaseIdVal.string_value(); + } + + ZEN_INFO("Horde machine assigned [{}:{}] cores={} lease={}", + OutMachine.GetConnectionAddress(), + OutMachine.GetConnectionPort(), + OutMachine.LogicalCores, + OutMachine.LeaseId); + + return true; +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordecomputebuffer.cpp b/src/zenhorde/hordecomputebuffer.cpp new file mode 100644 index 000000000..0d032b5d5 --- /dev/null +++ b/src/zenhorde/hordecomputebuffer.cpp @@ -0,0 +1,454 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hordecomputebuffer.h" + +#include +#include +#include +#include +#include + +namespace zen::horde { + +// Simplified ring buffer implementation for in-process use only. +// Uses a single contiguous buffer with write/read cursors and +// mutex+condvar for synchronization. This is simpler than the UE version +// which uses lock-free atomics and shared memory, but sufficient for our +// use case where we're the initiator side of the compute protocol. + +struct ComputeBuffer::Detail : TRefCounted +{ + std::vector Data; + size_t NumChunks = 0; + size_t ChunkLength = 0; + + // Current write state + size_t WriteChunkIdx = 0; + size_t WriteOffset = 0; + bool WriteComplete = false; + + // Current read state + size_t ReadChunkIdx = 0; + size_t ReadOffset = 0; + bool Detached = false; + + // Per-chunk written length + std::vector ChunkWrittenLength; + std::vector ChunkFinished; // Writer moved to next chunk + + std::mutex Mutex; + std::condition_variable ReadCV; ///< Signaled when new data is written or stream completes + std::condition_variable WriteCV; ///< Signaled when reader advances past a chunk, freeing space + + bool HasWriter = false; + bool HasReader = false; + + uint8_t* ChunkPtr(size_t ChunkIdx) { return Data.data() + ChunkIdx * ChunkLength; } + const uint8_t* ChunkPtr(size_t ChunkIdx) const { return Data.data() + ChunkIdx * ChunkLength; } +}; + +// ComputeBuffer + +ComputeBuffer::ComputeBuffer() +{ +} +ComputeBuffer::~ComputeBuffer() +{ +} + +bool +ComputeBuffer::CreateNew(const Params& InParams) +{ + auto* NewDetail = new Detail(); + NewDetail->NumChunks = InParams.NumChunks; + NewDetail->ChunkLength = InParams.ChunkLength; + NewDetail->Data.resize(InParams.NumChunks * InParams.ChunkLength, 0); + NewDetail->ChunkWrittenLength.resize(InParams.NumChunks, 0); + NewDetail->ChunkFinished.resize(InParams.NumChunks, false); + + m_Detail = NewDetail; + return true; +} + +void +ComputeBuffer::Close() +{ + m_Detail = nullptr; +} + +bool +ComputeBuffer::IsValid() const +{ + return static_cast(m_Detail); +} + +ComputeBufferReader +ComputeBuffer::CreateReader() +{ + assert(m_Detail); + m_Detail->HasReader = true; + return ComputeBufferReader(m_Detail); +} + +ComputeBufferWriter +ComputeBuffer::CreateWriter() +{ + assert(m_Detail); + m_Detail->HasWriter = true; + return ComputeBufferWriter(m_Detail); +} + +// ComputeBufferReader + +ComputeBufferReader::ComputeBufferReader() +{ +} +ComputeBufferReader::~ComputeBufferReader() +{ +} + +ComputeBufferReader::ComputeBufferReader(const ComputeBufferReader& Other) = default; +ComputeBufferReader::ComputeBufferReader(ComputeBufferReader&& Other) noexcept = default; +ComputeBufferReader& ComputeBufferReader::operator=(const ComputeBufferReader& Other) = default; +ComputeBufferReader& ComputeBufferReader::operator=(ComputeBufferReader&& Other) noexcept = default; + +ComputeBufferReader::ComputeBufferReader(Ref InDetail) : m_Detail(std::move(InDetail)) +{ +} + +void +ComputeBufferReader::Close() +{ + m_Detail = nullptr; +} + +void +ComputeBufferReader::Detach() +{ + if (m_Detail) + { + std::lock_guard Lock(m_Detail->Mutex); + m_Detail->Detached = true; + m_Detail->ReadCV.notify_all(); + } +} + +bool +ComputeBufferReader::IsValid() const +{ + return static_cast(m_Detail); +} + +bool +ComputeBufferReader::IsComplete() const +{ + if (!m_Detail) + { + return true; + } + std::lock_guard Lock(m_Detail->Mutex); + if (m_Detail->Detached) + { + return true; + } + return m_Detail->WriteComplete && m_Detail->ReadChunkIdx == m_Detail->WriteChunkIdx && + m_Detail->ReadOffset >= m_Detail->ChunkWrittenLength[m_Detail->ReadChunkIdx]; +} + +void +ComputeBufferReader::AdvanceReadPosition(size_t Size) +{ + if (!m_Detail) + { + return; + } + + std::lock_guard Lock(m_Detail->Mutex); + + m_Detail->ReadOffset += Size; + + // Check if we need to move to next chunk + const size_t ReadChunk = m_Detail->ReadChunkIdx; + if (m_Detail->ChunkFinished[ReadChunk] && m_Detail->ReadOffset >= m_Detail->ChunkWrittenLength[ReadChunk]) + { + const size_t NextChunk = (ReadChunk + 1) % m_Detail->NumChunks; + m_Detail->ReadChunkIdx = NextChunk; + m_Detail->ReadOffset = 0; + m_Detail->WriteCV.notify_all(); + } + + m_Detail->ReadCV.notify_all(); +} + +size_t +ComputeBufferReader::GetMaxReadSize() const +{ + if (!m_Detail) + { + return 0; + } + std::lock_guard Lock(m_Detail->Mutex); + const size_t ReadChunk = m_Detail->ReadChunkIdx; + return m_Detail->ChunkWrittenLength[ReadChunk] - m_Detail->ReadOffset; +} + +const uint8_t* +ComputeBufferReader::WaitToRead(size_t MinSize, int TimeoutMs, bool* OutTimedOut) +{ + if (!m_Detail) + { + return nullptr; + } + + std::unique_lock Lock(m_Detail->Mutex); + + auto Predicate = [&]() -> bool { + if (m_Detail->Detached) + { + return true; + } + + const size_t ReadChunk = m_Detail->ReadChunkIdx; + const size_t Available = m_Detail->ChunkWrittenLength[ReadChunk] - m_Detail->ReadOffset; + + if (Available >= MinSize) + { + return true; + } + + // If chunk is finished and we've read everything, try to move to next + if (m_Detail->ChunkFinished[ReadChunk] && m_Detail->ReadOffset >= m_Detail->ChunkWrittenLength[ReadChunk]) + { + if (m_Detail->WriteComplete) + { + return true; // End of stream + } + // Move to next chunk + const size_t NextChunk = (ReadChunk + 1) % m_Detail->NumChunks; + m_Detail->ReadChunkIdx = NextChunk; + m_Detail->ReadOffset = 0; + m_Detail->WriteCV.notify_all(); + return false; // Re-check with new chunk + } + + if (m_Detail->WriteComplete) + { + return true; // End of stream + } + + return false; + }; + + if (TimeoutMs < 0) + { + m_Detail->ReadCV.wait(Lock, Predicate); + } + else + { + if (!m_Detail->ReadCV.wait_for(Lock, std::chrono::milliseconds(TimeoutMs), Predicate)) + { + if (OutTimedOut) + { + *OutTimedOut = true; + } + return nullptr; + } + } + + if (m_Detail->Detached) + { + return nullptr; + } + + const size_t ReadChunk = m_Detail->ReadChunkIdx; + const size_t Available = m_Detail->ChunkWrittenLength[ReadChunk] - m_Detail->ReadOffset; + + if (Available < MinSize) + { + return nullptr; // End of stream + } + + return m_Detail->ChunkPtr(ReadChunk) + m_Detail->ReadOffset; +} + +size_t +ComputeBufferReader::Read(void* Buffer, size_t MaxSize, int TimeoutMs, bool* OutTimedOut) +{ + const uint8_t* Data = WaitToRead(1, TimeoutMs, OutTimedOut); + if (!Data) + { + return 0; + } + + const size_t Available = GetMaxReadSize(); + const size_t ToCopy = std::min(Available, MaxSize); + memcpy(Buffer, Data, ToCopy); + AdvanceReadPosition(ToCopy); + return ToCopy; +} + +// ComputeBufferWriter + +ComputeBufferWriter::ComputeBufferWriter() = default; +ComputeBufferWriter::ComputeBufferWriter(const ComputeBufferWriter& Other) = default; +ComputeBufferWriter::ComputeBufferWriter(ComputeBufferWriter&& Other) noexcept = default; +ComputeBufferWriter::~ComputeBufferWriter() = default; +ComputeBufferWriter& ComputeBufferWriter::operator=(const ComputeBufferWriter& Other) = default; +ComputeBufferWriter& ComputeBufferWriter::operator=(ComputeBufferWriter&& Other) noexcept = default; + +ComputeBufferWriter::ComputeBufferWriter(Ref InDetail) : m_Detail(std::move(InDetail)) +{ +} + +void +ComputeBufferWriter::Close() +{ + if (m_Detail) + { + { + std::lock_guard Lock(m_Detail->Mutex); + if (!m_Detail->WriteComplete) + { + m_Detail->WriteComplete = true; + m_Detail->ReadCV.notify_all(); + } + } + m_Detail = nullptr; + } +} + +bool +ComputeBufferWriter::IsValid() const +{ + return static_cast(m_Detail); +} + +void +ComputeBufferWriter::MarkComplete() +{ + if (m_Detail) + { + std::lock_guard Lock(m_Detail->Mutex); + m_Detail->WriteComplete = true; + m_Detail->ReadCV.notify_all(); + } +} + +void +ComputeBufferWriter::AdvanceWritePosition(size_t Size) +{ + if (!m_Detail || Size == 0) + { + return; + } + + std::lock_guard Lock(m_Detail->Mutex); + const size_t WriteChunk = m_Detail->WriteChunkIdx; + m_Detail->ChunkWrittenLength[WriteChunk] += Size; + m_Detail->WriteOffset += Size; + m_Detail->ReadCV.notify_all(); +} + +size_t +ComputeBufferWriter::GetMaxWriteSize() const +{ + if (!m_Detail) + { + return 0; + } + std::lock_guard Lock(m_Detail->Mutex); + const size_t WriteChunk = m_Detail->WriteChunkIdx; + return m_Detail->ChunkLength - m_Detail->ChunkWrittenLength[WriteChunk]; +} + +size_t +ComputeBufferWriter::GetChunkMaxLength() const +{ + if (!m_Detail) + { + return 0; + } + return m_Detail->ChunkLength; +} + +size_t +ComputeBufferWriter::Write(const void* Buffer, size_t MaxSize, int TimeoutMs) +{ + uint8_t* Dest = WaitToWrite(1, TimeoutMs); + if (!Dest) + { + return 0; + } + + const size_t Available = GetMaxWriteSize(); + const size_t ToCopy = std::min(Available, MaxSize); + memcpy(Dest, Buffer, ToCopy); + AdvanceWritePosition(ToCopy); + return ToCopy; +} + +uint8_t* +ComputeBufferWriter::WaitToWrite(size_t MinSize, int TimeoutMs) +{ + if (!m_Detail) + { + return nullptr; + } + + std::unique_lock Lock(m_Detail->Mutex); + + if (m_Detail->WriteComplete) + { + return nullptr; + } + + const size_t WriteChunk = m_Detail->WriteChunkIdx; + const size_t Available = m_Detail->ChunkLength - m_Detail->ChunkWrittenLength[WriteChunk]; + + // If current chunk has enough space, return pointer + if (Available >= MinSize) + { + return m_Detail->ChunkPtr(WriteChunk) + m_Detail->ChunkWrittenLength[WriteChunk]; + } + + // Current chunk is full - mark it as finished and move to next. + // The writer cannot advance until the reader has fully consumed the next chunk, + // preventing the writer from overwriting data the reader hasn't processed yet. + m_Detail->ChunkFinished[WriteChunk] = true; + m_Detail->ReadCV.notify_all(); + + const size_t NextChunk = (WriteChunk + 1) % m_Detail->NumChunks; + + // Wait until reader has consumed the next chunk + auto Predicate = [&]() -> bool { + // Check if read has moved past this chunk + return m_Detail->ReadChunkIdx != NextChunk || m_Detail->Detached; + }; + + if (TimeoutMs < 0) + { + m_Detail->WriteCV.wait(Lock, Predicate); + } + else + { + if (!m_Detail->WriteCV.wait_for(Lock, std::chrono::milliseconds(TimeoutMs), Predicate)) + { + return nullptr; + } + } + + if (m_Detail->Detached) + { + return nullptr; + } + + // Reset next chunk + m_Detail->ChunkWrittenLength[NextChunk] = 0; + m_Detail->ChunkFinished[NextChunk] = false; + m_Detail->WriteChunkIdx = NextChunk; + m_Detail->WriteOffset = 0; + + return m_Detail->ChunkPtr(NextChunk); +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordecomputebuffer.h b/src/zenhorde/hordecomputebuffer.h new file mode 100644 index 000000000..64ef91b7a --- /dev/null +++ b/src/zenhorde/hordecomputebuffer.h @@ -0,0 +1,136 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace zen::horde { + +class ComputeBufferReader; +class ComputeBufferWriter; + +/** Simplified in-process ring buffer for the Horde compute protocol. + * + * Unlike the UE FComputeBuffer which supports shared-memory and memory-mapped files, + * this implementation uses plain heap-allocated memory since we only need in-process + * communication between channel and transport threads. The buffer is divided into + * fixed-size chunks; readers and writers block when no space is available. + */ +class ComputeBuffer +{ +public: + struct Params + { + size_t NumChunks = 2; + size_t ChunkLength = 512 * 1024; + }; + + ComputeBuffer(); + ~ComputeBuffer(); + + ComputeBuffer(const ComputeBuffer&) = delete; + ComputeBuffer& operator=(const ComputeBuffer&) = delete; + + bool CreateNew(const Params& InParams); + void Close(); + + bool IsValid() const; + + ComputeBufferReader CreateReader(); + ComputeBufferWriter CreateWriter(); + +private: + struct Detail; + Ref m_Detail; + + friend class ComputeBufferReader; + friend class ComputeBufferWriter; +}; + +/** Read endpoint for a ComputeBuffer. + * + * Provides blocking reads from the ring buffer. WaitToRead() returns a pointer + * directly into the buffer memory (zero-copy); the caller must call + * AdvanceReadPosition() after consuming the data. + */ +class ComputeBufferReader +{ +public: + ComputeBufferReader(); + ComputeBufferReader(const ComputeBufferReader&); + ComputeBufferReader(ComputeBufferReader&&) noexcept; + ~ComputeBufferReader(); + + ComputeBufferReader& operator=(const ComputeBufferReader&); + ComputeBufferReader& operator=(ComputeBufferReader&&) noexcept; + + void Close(); + void Detach(); + bool IsValid() const; + bool IsComplete() const; + + void AdvanceReadPosition(size_t Size); + size_t GetMaxReadSize() const; + + /** Copy up to MaxSize bytes from the buffer into Buffer. Blocks until data is available. */ + size_t Read(void* Buffer, size_t MaxSize, int TimeoutMs = -1, bool* OutTimedOut = nullptr); + + /** Wait until at least MinSize bytes are available and return a direct pointer. + * Returns nullptr on timeout or if the writer has completed. */ + const uint8_t* WaitToRead(size_t MinSize, int TimeoutMs = -1, bool* OutTimedOut = nullptr); + +private: + friend class ComputeBuffer; + explicit ComputeBufferReader(Ref InDetail); + + Ref m_Detail; +}; + +/** Write endpoint for a ComputeBuffer. + * + * Provides blocking writes into the ring buffer. WaitToWrite() returns a pointer + * directly into the buffer memory (zero-copy); the caller must call + * AdvanceWritePosition() after filling the data. Call MarkComplete() to signal + * that no more data will be written. + */ +class ComputeBufferWriter +{ +public: + ComputeBufferWriter(); + ComputeBufferWriter(const ComputeBufferWriter&); + ComputeBufferWriter(ComputeBufferWriter&&) noexcept; + ~ComputeBufferWriter(); + + ComputeBufferWriter& operator=(const ComputeBufferWriter&); + ComputeBufferWriter& operator=(ComputeBufferWriter&&) noexcept; + + void Close(); + bool IsValid() const; + + /** Signal that no more data will be written. Unblocks any waiting readers. */ + void MarkComplete(); + + void AdvanceWritePosition(size_t Size); + size_t GetMaxWriteSize() const; + size_t GetChunkMaxLength() const; + + /** Copy up to MaxSize bytes from Buffer into the ring buffer. Blocks until space is available. */ + size_t Write(const void* Buffer, size_t MaxSize, int TimeoutMs = -1); + + /** Wait until at least MinSize bytes of write space are available and return a direct pointer. + * Returns nullptr on timeout. */ + uint8_t* WaitToWrite(size_t MinSize, int TimeoutMs = -1); + +private: + friend class ComputeBuffer; + explicit ComputeBufferWriter(Ref InDetail); + + Ref m_Detail; +}; + +} // namespace zen::horde diff --git a/src/zenhorde/hordecomputechannel.cpp b/src/zenhorde/hordecomputechannel.cpp new file mode 100644 index 000000000..ee2a6f327 --- /dev/null +++ b/src/zenhorde/hordecomputechannel.cpp @@ -0,0 +1,37 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hordecomputechannel.h" + +namespace zen::horde { + +ComputeChannel::ComputeChannel(ComputeBufferReader InReader, ComputeBufferWriter InWriter) +: Reader(std::move(InReader)) +, Writer(std::move(InWriter)) +{ +} + +bool +ComputeChannel::IsValid() const +{ + return Reader.IsValid() && Writer.IsValid(); +} + +size_t +ComputeChannel::Send(const void* Data, size_t Size, int TimeoutMs) +{ + return Writer.Write(Data, Size, TimeoutMs); +} + +size_t +ComputeChannel::Recv(void* Data, size_t Size, int TimeoutMs) +{ + return Reader.Read(Data, Size, TimeoutMs); +} + +void +ComputeChannel::MarkComplete() +{ + Writer.MarkComplete(); +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordecomputechannel.h b/src/zenhorde/hordecomputechannel.h new file mode 100644 index 000000000..c1dff20e4 --- /dev/null +++ b/src/zenhorde/hordecomputechannel.h @@ -0,0 +1,32 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "hordecomputebuffer.h" + +namespace zen::horde { + +/** Bidirectional communication channel using a pair of compute buffers. + * + * Pairs a ComputeBufferReader (for receiving data) with a ComputeBufferWriter + * (for sending data). Used by ComputeSocket to represent one logical channel + * within a multiplexed connection. + */ +class ComputeChannel : public TRefCounted +{ +public: + ComputeBufferReader Reader; + ComputeBufferWriter Writer; + + ComputeChannel(ComputeBufferReader InReader, ComputeBufferWriter InWriter); + + bool IsValid() const; + + size_t Send(const void* Data, size_t Size, int TimeoutMs = -1); + size_t Recv(void* Data, size_t Size, int TimeoutMs = -1); + + /** Signal that no more data will be sent on this channel. */ + void MarkComplete(); +}; + +} // namespace zen::horde diff --git a/src/zenhorde/hordecomputesocket.cpp b/src/zenhorde/hordecomputesocket.cpp new file mode 100644 index 000000000..6ef67760c --- /dev/null +++ b/src/zenhorde/hordecomputesocket.cpp @@ -0,0 +1,204 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hordecomputesocket.h" + +#include + +namespace zen::horde { + +ComputeSocket::ComputeSocket(std::unique_ptr Transport) +: m_Log(zen::logging::Get("horde.socket")) +, m_Transport(std::move(Transport)) +{ +} + +ComputeSocket::~ComputeSocket() +{ + // Shutdown order matters: first stop the ping thread, then unblock send threads + // by detaching readers, then join send threads, and finally close the transport + // to unblock the recv thread (which is blocked on RecvMessage). + { + std::lock_guard Lock(m_PingMutex); + m_PingShouldStop = true; + m_PingCV.notify_all(); + } + + for (auto& Reader : m_Readers) + { + Reader.Detach(); + } + + for (auto& [Id, Thread] : m_SendThreads) + { + if (Thread.joinable()) + { + Thread.join(); + } + } + + m_Transport->Close(); + + if (m_RecvThread.joinable()) + { + m_RecvThread.join(); + } + if (m_PingThread.joinable()) + { + m_PingThread.join(); + } +} + +Ref +ComputeSocket::CreateChannel(int ChannelId) +{ + ComputeBuffer::Params Params; + + ComputeBuffer RecvBuffer; + if (!RecvBuffer.CreateNew(Params)) + { + return {}; + } + + ComputeBuffer SendBuffer; + if (!SendBuffer.CreateNew(Params)) + { + return {}; + } + + Ref Channel(new ComputeChannel(RecvBuffer.CreateReader(), SendBuffer.CreateWriter())); + + // Attach recv buffer writer (transport recv thread writes into this) + { + std::lock_guard Lock(m_WritersMutex); + m_Writers.emplace(ChannelId, RecvBuffer.CreateWriter()); + } + + // Attach send buffer reader (send thread reads from this) + { + ComputeBufferReader Reader = SendBuffer.CreateReader(); + m_Readers.push_back(Reader); + m_SendThreads.emplace(ChannelId, std::thread(&ComputeSocket::SendThreadProc, this, ChannelId, std::move(Reader))); + } + + return Channel; +} + +void +ComputeSocket::StartCommunication() +{ + m_RecvThread = std::thread(&ComputeSocket::RecvThreadProc, this); + m_PingThread = std::thread(&ComputeSocket::PingThreadProc, this); +} + +void +ComputeSocket::PingThreadProc() +{ + while (true) + { + { + std::unique_lock Lock(m_PingMutex); + if (m_PingCV.wait_for(Lock, std::chrono::milliseconds(2000), [this] { return m_PingShouldStop; })) + { + break; + } + } + + std::lock_guard Lock(m_SendMutex); + FrameHeader Header; + Header.Channel = 0; + Header.Size = ControlPing; + m_Transport->SendMessage(&Header, sizeof(Header)); + } +} + +void +ComputeSocket::RecvThreadProc() +{ + // Writers are cached locally to avoid taking m_WritersMutex on every frame. + // The shared m_Writers map is only accessed when a channel is seen for the first time. + std::unordered_map CachedWriters; + + FrameHeader Header; + while (m_Transport->RecvMessage(&Header, sizeof(Header))) + { + if (Header.Size >= 0) + { + // Data frame + auto It = CachedWriters.find(Header.Channel); + if (It == CachedWriters.end()) + { + std::lock_guard Lock(m_WritersMutex); + auto WIt = m_Writers.find(Header.Channel); + if (WIt == m_Writers.end()) + { + ZEN_WARN("recv frame for unknown channel {}", Header.Channel); + // Skip the data + std::vector Discard(Header.Size); + m_Transport->RecvMessage(Discard.data(), Header.Size); + continue; + } + It = CachedWriters.emplace(Header.Channel, WIt->second).first; + } + + ComputeBufferWriter& Writer = It->second; + uint8_t* Dest = Writer.WaitToWrite(Header.Size); + if (!Dest || !m_Transport->RecvMessage(Dest, Header.Size)) + { + ZEN_WARN("failed to read frame data (channel={}, size={})", Header.Channel, Header.Size); + return; + } + Writer.AdvanceWritePosition(Header.Size); + } + else if (Header.Size == ControlDetach) + { + // Detach the recv buffer for this channel + CachedWriters.erase(Header.Channel); + + std::lock_guard Lock(m_WritersMutex); + auto It = m_Writers.find(Header.Channel); + if (It != m_Writers.end()) + { + It->second.MarkComplete(); + m_Writers.erase(It); + } + } + else if (Header.Size == ControlPing) + { + // Ping response - ignore + } + else + { + ZEN_WARN("invalid frame header size: {}", Header.Size); + return; + } + } +} + +void +ComputeSocket::SendThreadProc(int Channel, ComputeBufferReader Reader) +{ + // Each channel has its own send thread. All send threads share m_SendMutex + // to serialize writes to the transport, since TCP requires atomic frame writes. + FrameHeader Header; + Header.Channel = Channel; + + const uint8_t* Data; + while ((Data = Reader.WaitToRead(1)) != nullptr) + { + std::lock_guard Lock(m_SendMutex); + + Header.Size = static_cast(Reader.GetMaxReadSize()); + m_Transport->SendMessage(&Header, sizeof(Header)); + m_Transport->SendMessage(Data, Header.Size); + Reader.AdvanceReadPosition(Header.Size); + } + + if (Reader.IsComplete()) + { + std::lock_guard Lock(m_SendMutex); + Header.Size = ControlDetach; + m_Transport->SendMessage(&Header, sizeof(Header)); + } +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordecomputesocket.h b/src/zenhorde/hordecomputesocket.h new file mode 100644 index 000000000..0c3cb4195 --- /dev/null +++ b/src/zenhorde/hordecomputesocket.h @@ -0,0 +1,79 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "hordecomputebuffer.h" +#include "hordecomputechannel.h" +#include "hordetransport.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace zen::horde { + +/** Multiplexed socket that routes data between multiple channels over a single transport. + * + * Each channel is identified by an integer ID and backed by a pair of ComputeBuffers. + * A recv thread demultiplexes incoming frames to channel-specific buffers, while + * per-channel send threads multiplex outgoing data onto the shared transport. + * + * Wire format per frame: [channelId (4B)][size (4B)][data] + * Control messages use negative sizes: -2 = detach (channel closed), -3 = ping. + */ +class ComputeSocket +{ +public: + explicit ComputeSocket(std::unique_ptr Transport); + ~ComputeSocket(); + + ComputeSocket(const ComputeSocket&) = delete; + ComputeSocket& operator=(const ComputeSocket&) = delete; + + /** Create a channel with the given ID. + * Allocates anonymous in-process buffers and spawns a send thread for the channel. */ + Ref CreateChannel(int ChannelId); + + /** Start the recv pump and ping threads. Must be called after all channels are created. */ + void StartCommunication(); + +private: + struct FrameHeader + { + int32_t Channel = 0; + int32_t Size = 0; + }; + + static constexpr int32_t ControlDetach = -2; + static constexpr int32_t ControlPing = -3; + + LoggerRef Log() { return m_Log; } + + void RecvThreadProc(); + void SendThreadProc(int Channel, ComputeBufferReader Reader); + void PingThreadProc(); + + LoggerRef m_Log; + std::unique_ptr m_Transport; + std::mutex m_SendMutex; ///< Serializes writes to the transport + + std::mutex m_WritersMutex; + std::unordered_map m_Writers; ///< Recv-side: writers keyed by channel ID + + std::vector m_Readers; ///< Send-side: readers for join on destruction + std::unordered_map m_SendThreads; ///< One send thread per channel + + std::thread m_RecvThread; + std::thread m_PingThread; + + bool m_PingShouldStop = false; + std::mutex m_PingMutex; + std::condition_variable m_PingCV; +}; + +} // namespace zen::horde diff --git a/src/zenhorde/hordeconfig.cpp b/src/zenhorde/hordeconfig.cpp new file mode 100644 index 000000000..2dca228d9 --- /dev/null +++ b/src/zenhorde/hordeconfig.cpp @@ -0,0 +1,89 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +namespace zen::horde { + +bool +HordeConfig::Validate() const +{ + if (ServerUrl.empty()) + { + return false; + } + + // Relay mode implies AES encryption + if (Mode == ConnectionMode::Relay && EncryptionMode != Encryption::AES) + { + return false; + } + + return true; +} + +const char* +ToString(ConnectionMode Mode) +{ + switch (Mode) + { + case ConnectionMode::Direct: + return "direct"; + case ConnectionMode::Tunnel: + return "tunnel"; + case ConnectionMode::Relay: + return "relay"; + } + return "direct"; +} + +const char* +ToString(Encryption Enc) +{ + switch (Enc) + { + case Encryption::None: + return "none"; + case Encryption::AES: + return "aes"; + } + return "none"; +} + +bool +FromString(ConnectionMode& OutMode, std::string_view Str) +{ + if (Str == "direct") + { + OutMode = ConnectionMode::Direct; + return true; + } + if (Str == "tunnel") + { + OutMode = ConnectionMode::Tunnel; + return true; + } + if (Str == "relay") + { + OutMode = ConnectionMode::Relay; + return true; + } + return false; +} + +bool +FromString(Encryption& OutEnc, std::string_view Str) +{ + if (Str == "none") + { + OutEnc = Encryption::None; + return true; + } + if (Str == "aes") + { + OutEnc = Encryption::AES; + return true; + } + return false; +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordeprovisioner.cpp b/src/zenhorde/hordeprovisioner.cpp new file mode 100644 index 000000000..f88c95da2 --- /dev/null +++ b/src/zenhorde/hordeprovisioner.cpp @@ -0,0 +1,367 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include + +#include "hordeagent.h" +#include "hordebundle.h" + +#include +#include +#include +#include +#include + +#include +#include + +namespace zen::horde { + +struct HordeProvisioner::AgentWrapper +{ + std::thread Thread; + std::atomic ShouldExit{false}; +}; + +HordeProvisioner::HordeProvisioner(const HordeConfig& Config, + const std::filesystem::path& BinariesPath, + const std::filesystem::path& WorkingDir, + std::string_view OrchestratorEndpoint) +: m_Config(Config) +, m_BinariesPath(BinariesPath) +, m_WorkingDir(WorkingDir) +, m_OrchestratorEndpoint(OrchestratorEndpoint) +, m_Log(zen::logging::Get("horde.provisioner")) +{ +} + +HordeProvisioner::~HordeProvisioner() +{ + std::lock_guard Lock(m_AgentsLock); + for (auto& Agent : m_Agents) + { + Agent->ShouldExit.store(true); + } + for (auto& Agent : m_Agents) + { + if (Agent->Thread.joinable()) + { + Agent->Thread.join(); + } + } +} + +void +HordeProvisioner::SetTargetCoreCount(uint32_t Count) +{ + ZEN_TRACE_CPU("HordeProvisioner::SetTargetCoreCount"); + + m_TargetCoreCount.store(std::min(Count, static_cast(m_Config.MaxCores))); + + while (m_EstimatedCoreCount.load() < m_TargetCoreCount.load()) + { + if (!m_AskForAgents.load()) + { + return; + } + RequestAgent(); + } + + // Clean up finished agent threads + std::lock_guard Lock(m_AgentsLock); + for (auto It = m_Agents.begin(); It != m_Agents.end();) + { + if ((*It)->ShouldExit.load()) + { + if ((*It)->Thread.joinable()) + { + (*It)->Thread.join(); + } + It = m_Agents.erase(It); + } + else + { + ++It; + } + } +} + +ProvisioningStats +HordeProvisioner::GetStats() const +{ + ProvisioningStats Stats; + Stats.TargetCoreCount = m_TargetCoreCount.load(); + Stats.EstimatedCoreCount = m_EstimatedCoreCount.load(); + Stats.ActiveCoreCount = m_ActiveCoreCount.load(); + Stats.AgentsActive = m_AgentsActive.load(); + Stats.AgentsRequesting = m_AgentsRequesting.load(); + return Stats; +} + +uint32_t +HordeProvisioner::GetAgentCount() const +{ + std::lock_guard Lock(m_AgentsLock); + return static_cast(m_Agents.size()); +} + +void +HordeProvisioner::RequestAgent() +{ + m_EstimatedCoreCount.fetch_add(EstimatedCoresPerAgent); + + std::lock_guard Lock(m_AgentsLock); + + auto Wrapper = std::make_unique(); + AgentWrapper& Ref = *Wrapper; + Wrapper->Thread = std::thread([this, &Ref] { ThreadAgent(Ref); }); + + m_Agents.push_back(std::move(Wrapper)); +} + +void +HordeProvisioner::ThreadAgent(AgentWrapper& Wrapper) +{ + ZEN_TRACE_CPU("HordeProvisioner::ThreadAgent"); + + static std::atomic ThreadIndex{0}; + const uint32_t CurrentIndex = ThreadIndex.fetch_add(1); + + zen::SetCurrentThreadName(fmt::format("horde_agent_{}", CurrentIndex)); + + std::unique_ptr Agent; + uint32_t MachineCoreCount = 0; + + auto _ = MakeGuard([&] { + if (Agent) + { + Agent->CloseConnection(); + } + Wrapper.ShouldExit.store(true); + }); + + { + // EstimatedCoreCount is incremented speculatively when the agent is requested + // (in RequestAgent) so that SetTargetCoreCount doesn't over-provision. + auto $ = MakeGuard([&] { m_EstimatedCoreCount.fetch_sub(EstimatedCoresPerAgent); }); + + { + ZEN_TRACE_CPU("HordeProvisioner::CreateBundles"); + + std::lock_guard BundleLock(m_BundleLock); + + if (!m_BundlesCreated) + { + const std::filesystem::path OutputDir = m_WorkingDir / "horde_bundles"; + + std::vector Files; + +#if ZEN_PLATFORM_WINDOWS + Files.emplace_back(m_BinariesPath / "zenserver.exe", false); +#elif ZEN_PLATFORM_LINUX + Files.emplace_back(m_BinariesPath / "zenserver", false); + Files.emplace_back(m_BinariesPath / "zenserver.debug", true); +#elif ZEN_PLATFORM_MAC + Files.emplace_back(m_BinariesPath / "zenserver", false); +#endif + + BundleResult Result; + if (!BundleCreator::CreateBundle(Files, OutputDir, Result)) + { + ZEN_WARN("failed to create bundle, cannot provision any agents!"); + m_AskForAgents.store(false); + return; + } + + m_Bundles.emplace_back(Result.Locator, Result.BundleDir); + m_BundlesCreated = true; + } + + if (!m_HordeClient) + { + m_HordeClient = std::make_unique(m_Config); + if (!m_HordeClient->Initialize()) + { + ZEN_WARN("failed to initialize Horde HTTP client, cannot provision any agents!"); + m_AskForAgents.store(false); + return; + } + } + } + + if (!m_AskForAgents.load()) + { + return; + } + + m_AgentsRequesting.fetch_add(1); + auto ReqGuard = MakeGuard([this] { m_AgentsRequesting.fetch_sub(1); }); + + // Simple backoff: if the last machine request failed, wait up to 5 seconds + // before trying again. + // + // Note however that it's possible that multiple threads enter this code at + // the same time if multiple agents are requested at once, and they will all + // see the same last failure time and back off accordingly. We might want to + // use a semaphore or similar to limit the number of concurrent requests. + + if (const uint64_t LastFail = m_LastRequestFailTime.load(); LastFail != 0) + { + auto Now = static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); + const uint64_t ElapsedNs = Now - LastFail; + const uint64_t ElapsedMs = ElapsedNs / 1'000'000; + if (ElapsedMs < 5000) + { + const uint64_t WaitMs = 5000 - ElapsedMs; + for (uint64_t Waited = 0; Waited < WaitMs && !Wrapper.ShouldExit.load(); Waited += 100) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + if (Wrapper.ShouldExit.load()) + { + return; + } + } + } + + if (m_ActiveCoreCount.load() >= m_TargetCoreCount.load()) + { + return; + } + + std::string RequestBody = m_HordeClient->BuildRequestBody(); + + // Resolve cluster if needed + std::string ClusterId = m_Config.Cluster; + if (ClusterId == HordeConfig::ClusterAuto) + { + ClusterInfo Cluster; + if (!m_HordeClient->ResolveCluster(RequestBody, Cluster)) + { + ZEN_WARN("failed to resolve cluster"); + m_LastRequestFailTime.store(static_cast(std::chrono::steady_clock::now().time_since_epoch().count())); + return; + } + ClusterId = Cluster.ClusterId; + } + + MachineInfo Machine; + if (!m_HordeClient->RequestMachine(RequestBody, ClusterId, /* out */ Machine) || !Machine.IsValid()) + { + m_LastRequestFailTime.store(static_cast(std::chrono::steady_clock::now().time_since_epoch().count())); + return; + } + + m_LastRequestFailTime.store(0); + + if (Wrapper.ShouldExit.load()) + { + return; + } + + // Connect to agent and perform handshake + Agent = std::make_unique(Machine); + if (!Agent->IsValid()) + { + ZEN_WARN("agent creation failed for {}:{}", Machine.GetConnectionAddress(), Machine.GetConnectionPort()); + return; + } + + if (!Agent->BeginCommunication()) + { + ZEN_WARN("BeginCommunication failed"); + return; + } + + for (auto& [Locator, BundleDir] : m_Bundles) + { + if (Wrapper.ShouldExit.load()) + { + return; + } + + if (!Agent->UploadBinaries(BundleDir, Locator)) + { + ZEN_WARN("UploadBinaries failed"); + return; + } + } + + if (Wrapper.ShouldExit.load()) + { + return; + } + + // Build command line for remote zenserver + std::vector ArgStrings; + ArgStrings.push_back("compute"); + ArgStrings.push_back("--http=asio"); + + // TEMP HACK - these should be made fully dynamic + // these are currently here to allow spawning the compute agent locally + // for debugging purposes (i.e with a local Horde Server+Agent setup) + ArgStrings.push_back(fmt::format("--port={}", m_Config.ZenServicePort)); + ArgStrings.push_back("--data-dir=c:\\temp\\123"); + + if (!m_OrchestratorEndpoint.empty()) + { + ExtendableStringBuilder<256> CoordArg; + CoordArg << "--coordinator-endpoint=" << m_OrchestratorEndpoint; + ArgStrings.emplace_back(CoordArg.ToView()); + } + + { + ExtendableStringBuilder<128> IdArg; + IdArg << "--instance-id=horde-" << Machine.LeaseId; + ArgStrings.emplace_back(IdArg.ToView()); + } + + std::vector Args; + Args.reserve(ArgStrings.size()); + for (const std::string& Arg : ArgStrings) + { + Args.push_back(Arg.c_str()); + } + +#if ZEN_PLATFORM_WINDOWS + const bool UseWine = !Machine.IsWindows; + const char* AppName = "zenserver.exe"; +#else + const bool UseWine = false; + const char* AppName = "zenserver"; +#endif + + Agent->Execute(AppName, Args.data(), Args.size(), nullptr, nullptr, 0, UseWine); + + ZEN_INFO("remote execution started on [{}:{}] lease={}", + Machine.GetConnectionAddress(), + Machine.GetConnectionPort(), + Machine.LeaseId); + + MachineCoreCount = Machine.LogicalCores; + m_EstimatedCoreCount.fetch_add(MachineCoreCount); + m_ActiveCoreCount.fetch_add(MachineCoreCount); + m_AgentsActive.fetch_add(1); + } + + // Agent poll loop + + auto ActiveGuard = MakeGuard([&]() { + m_EstimatedCoreCount.fetch_sub(MachineCoreCount); + m_ActiveCoreCount.fetch_sub(MachineCoreCount); + m_AgentsActive.fetch_sub(1); + }); + + while (Agent->IsValid() && !Wrapper.ShouldExit.load()) + { + const bool LogOutput = false; + if (!Agent->Poll(LogOutput)) + { + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordetransport.cpp b/src/zenhorde/hordetransport.cpp new file mode 100644 index 000000000..69766e73e --- /dev/null +++ b/src/zenhorde/hordetransport.cpp @@ -0,0 +1,169 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hordetransport.h" + +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#if ZEN_PLATFORM_WINDOWS +# undef SendMessage +#endif + +namespace zen::horde { + +// ComputeTransport base + +bool +ComputeTransport::SendMessage(const void* Data, size_t Size) +{ + const uint8_t* Ptr = static_cast(Data); + size_t Remaining = Size; + + while (Remaining > 0) + { + const size_t Sent = Send(Ptr, Remaining); + if (Sent == 0) + { + return false; + } + Ptr += Sent; + Remaining -= Sent; + } + + return true; +} + +bool +ComputeTransport::RecvMessage(void* Data, size_t Size) +{ + uint8_t* Ptr = static_cast(Data); + size_t Remaining = Size; + + while (Remaining > 0) + { + const size_t Received = Recv(Ptr, Remaining); + if (Received == 0) + { + return false; + } + Ptr += Received; + Remaining -= Received; + } + + return true; +} + +// TcpComputeTransport - ASIO pimpl + +struct TcpComputeTransport::Impl +{ + asio::io_context IoContext; + asio::ip::tcp::socket Socket; + + Impl() : Socket(IoContext) {} +}; + +// Uses ASIO in synchronous mode only — no async operations or io_context::run(). +// The io_context is only needed because ASIO sockets require one to be constructed. +TcpComputeTransport::TcpComputeTransport(const MachineInfo& Info) +: m_Impl(std::make_unique()) +, m_Log(zen::logging::Get("horde.transport")) +{ + ZEN_TRACE_CPU("TcpComputeTransport::Connect"); + + asio::error_code Ec; + + const asio::ip::address Address = asio::ip::make_address(Info.GetConnectionAddress(), Ec); + if (Ec) + { + ZEN_WARN("invalid address '{}': {}", Info.GetConnectionAddress(), Ec.message()); + m_HasErrors = true; + return; + } + + const asio::ip::tcp::endpoint Endpoint(Address, Info.GetConnectionPort()); + + m_Impl->Socket.connect(Endpoint, Ec); + if (Ec) + { + ZEN_WARN("failed to connect to Horde compute [{}:{}]: {}", Info.GetConnectionAddress(), Info.GetConnectionPort(), Ec.message()); + m_HasErrors = true; + return; + } + + // Disable Nagle's algorithm for lower latency + m_Impl->Socket.set_option(asio::ip::tcp::no_delay(true), Ec); +} + +TcpComputeTransport::~TcpComputeTransport() +{ + Close(); +} + +bool +TcpComputeTransport::IsValid() const +{ + return m_Impl && m_Impl->Socket.is_open() && !m_HasErrors && !m_IsClosed; +} + +size_t +TcpComputeTransport::Send(const void* Data, size_t Size) +{ + if (!IsValid()) + { + return 0; + } + + asio::error_code Ec; + const size_t Sent = m_Impl->Socket.send(asio::buffer(Data, Size), 0, Ec); + + if (Ec) + { + m_HasErrors = true; + return 0; + } + + return Sent; +} + +size_t +TcpComputeTransport::Recv(void* Data, size_t Size) +{ + if (!IsValid()) + { + return 0; + } + + asio::error_code Ec; + const size_t Received = m_Impl->Socket.receive(asio::buffer(Data, Size), 0, Ec); + + if (Ec) + { + return 0; + } + + return Received; +} + +void +TcpComputeTransport::MarkComplete() +{ +} + +void +TcpComputeTransport::Close() +{ + if (!m_IsClosed && m_Impl && m_Impl->Socket.is_open()) + { + asio::error_code Ec; + m_Impl->Socket.shutdown(asio::ip::tcp::socket::shutdown_both, Ec); + m_Impl->Socket.close(Ec); + } + m_IsClosed = true; +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordetransport.h b/src/zenhorde/hordetransport.h new file mode 100644 index 000000000..1b178dc0f --- /dev/null +++ b/src/zenhorde/hordetransport.h @@ -0,0 +1,71 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# undef SendMessage +#endif + +namespace zen::horde { + +/** Abstract base interface for compute transports. + * + * Matches the UE FComputeTransport pattern. Concrete implementations handle + * the underlying I/O (TCP, AES-wrapped, etc.) while this interface provides + * blocking message helpers on top. + */ +class ComputeTransport +{ +public: + virtual ~ComputeTransport() = default; + + virtual bool IsValid() const = 0; + virtual size_t Send(const void* Data, size_t Size) = 0; + virtual size_t Recv(void* Data, size_t Size) = 0; + virtual void MarkComplete() = 0; + virtual void Close() = 0; + + /** Blocking send that loops until all bytes are transferred. Returns false on error. */ + bool SendMessage(const void* Data, size_t Size); + + /** Blocking receive that loops until all bytes are transferred. Returns false on error. */ + bool RecvMessage(void* Data, size_t Size); +}; + +/** TCP socket transport using ASIO. + * + * Connects to the Horde compute endpoint specified by MachineInfo and provides + * raw TCP send/receive. ASIO internals are hidden behind a pimpl to keep the + * header clean. + */ +class TcpComputeTransport final : public ComputeTransport +{ +public: + explicit TcpComputeTransport(const MachineInfo& Info); + ~TcpComputeTransport() override; + + bool IsValid() const override; + size_t Send(const void* Data, size_t Size) override; + size_t Recv(void* Data, size_t Size) override; + void MarkComplete() override; + void Close() override; + +private: + LoggerRef Log() { return m_Log; } + + struct Impl; + std::unique_ptr m_Impl; + LoggerRef m_Log; + bool m_IsClosed = false; + bool m_HasErrors = false; +}; + +} // namespace zen::horde diff --git a/src/zenhorde/hordetransportaes.cpp b/src/zenhorde/hordetransportaes.cpp new file mode 100644 index 000000000..986dd3705 --- /dev/null +++ b/src/zenhorde/hordetransportaes.cpp @@ -0,0 +1,425 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "hordetransportaes.h" + +#include +#include + +#include +#include +#include + +#if ZEN_PLATFORM_WINDOWS +# include +# include +# pragma comment(lib, "Bcrypt.lib") +#else +ZEN_THIRD_PARTY_INCLUDES_START +# include +# include +ZEN_THIRD_PARTY_INCLUDES_END +#endif + +namespace zen::horde { + +struct AesComputeTransport::CryptoContext +{ + uint8_t Key[KeySize] = {}; + uint8_t EncryptNonce[NonceBytes] = {}; + uint8_t DecryptNonce[NonceBytes] = {}; + bool HasErrors = false; + +#if !ZEN_PLATFORM_WINDOWS + EVP_CIPHER_CTX* EncCtx = nullptr; + EVP_CIPHER_CTX* DecCtx = nullptr; +#endif + + CryptoContext(const uint8_t (&InKey)[KeySize]) + { + memcpy(Key, InKey, KeySize); + + // The encrypt nonce is randomly initialized and then deterministically mutated + // per message via UpdateNonce(). The decrypt nonce is not used — it comes from + // the wire (each received message carries its own nonce in the header). + std::random_device Rd; + std::mt19937 Gen(Rd()); + std::uniform_int_distribution Dist(0, 255); + for (auto& Byte : EncryptNonce) + { + Byte = static_cast(Dist(Gen)); + } + +#if !ZEN_PLATFORM_WINDOWS + // Drain any stale OpenSSL errors + while (ERR_get_error() != 0) + { + } + + EncCtx = EVP_CIPHER_CTX_new(); + EVP_EncryptInit_ex(EncCtx, EVP_aes_256_gcm(), nullptr, nullptr, nullptr); + + DecCtx = EVP_CIPHER_CTX_new(); + EVP_DecryptInit_ex(DecCtx, EVP_aes_256_gcm(), nullptr, nullptr, nullptr); +#endif + } + + ~CryptoContext() + { +#if ZEN_PLATFORM_WINDOWS + SecureZeroMemory(Key, sizeof(Key)); + SecureZeroMemory(EncryptNonce, sizeof(EncryptNonce)); + SecureZeroMemory(DecryptNonce, sizeof(DecryptNonce)); +#else + OPENSSL_cleanse(Key, sizeof(Key)); + OPENSSL_cleanse(EncryptNonce, sizeof(EncryptNonce)); + OPENSSL_cleanse(DecryptNonce, sizeof(DecryptNonce)); + + if (EncCtx) + { + EVP_CIPHER_CTX_free(EncCtx); + } + if (DecCtx) + { + EVP_CIPHER_CTX_free(DecCtx); + } +#endif + } + + void UpdateNonce() + { + uint32_t* N32 = reinterpret_cast(EncryptNonce); + N32[0]++; + N32[1]--; + N32[2] = N32[0] ^ N32[1]; + } + + // Returns total encrypted message size, or 0 on failure + // Output format: [length(4B)][nonce(12B)][ciphertext][tag(16B)] + int32_t EncryptMessage(uint8_t* Out, const void* In, int32_t InLength) + { + UpdateNonce(); + + // On Windows, BCrypt algorithm/key handles are created per call. This is simpler than + // caching but has some overhead. For our use case (relatively large, infrequent messages) + // this is acceptable. +#if ZEN_PLATFORM_WINDOWS + BCRYPT_ALG_HANDLE hAlg = nullptr; + BCRYPT_KEY_HANDLE hKey = nullptr; + + BCryptOpenAlgorithmProvider(&hAlg, BCRYPT_AES_ALGORITHM, nullptr, 0); + BCryptSetProperty(hAlg, BCRYPT_CHAINING_MODE, (PUCHAR)BCRYPT_CHAIN_MODE_GCM, sizeof(BCRYPT_CHAIN_MODE_GCM), 0); + BCryptGenerateSymmetricKey(hAlg, &hKey, nullptr, 0, (PUCHAR)Key, KeySize, 0); + + BCRYPT_AUTHENTICATED_CIPHER_MODE_INFO AuthInfo; + BCRYPT_INIT_AUTH_MODE_INFO(AuthInfo); + AuthInfo.pbNonce = EncryptNonce; + AuthInfo.cbNonce = NonceBytes; + uint8_t Tag[TagBytes] = {}; + AuthInfo.pbTag = Tag; + AuthInfo.cbTag = TagBytes; + + ULONG CipherLen = 0; + NTSTATUS Status = + BCryptEncrypt(hKey, (PUCHAR)In, (ULONG)InLength, &AuthInfo, nullptr, 0, Out + 4 + NonceBytes, (ULONG)InLength, &CipherLen, 0); + + if (!BCRYPT_SUCCESS(Status)) + { + HasErrors = true; + BCryptDestroyKey(hKey); + BCryptCloseAlgorithmProvider(hAlg, 0); + return 0; + } + + // Write header: length + nonce + memcpy(Out, &InLength, 4); + memcpy(Out + 4, EncryptNonce, NonceBytes); + // Write tag after ciphertext + memcpy(Out + 4 + NonceBytes + CipherLen, Tag, TagBytes); + + BCryptDestroyKey(hKey); + BCryptCloseAlgorithmProvider(hAlg, 0); + + return 4 + NonceBytes + static_cast(CipherLen) + TagBytes; +#else + if (EVP_EncryptInit_ex(EncCtx, nullptr, nullptr, Key, EncryptNonce) != 1) + { + HasErrors = true; + return 0; + } + + int32_t Offset = 0; + // Write length + memcpy(Out + Offset, &InLength, 4); + Offset += 4; + // Write nonce + memcpy(Out + Offset, EncryptNonce, NonceBytes); + Offset += NonceBytes; + + // Encrypt + int OutLen = 0; + if (EVP_EncryptUpdate(EncCtx, Out + Offset, &OutLen, static_cast(In), InLength) != 1) + { + HasErrors = true; + return 0; + } + Offset += OutLen; + + // Finalize + int FinalLen = 0; + if (EVP_EncryptFinal_ex(EncCtx, Out + Offset, &FinalLen) != 1) + { + HasErrors = true; + return 0; + } + Offset += FinalLen; + + // Get tag + if (EVP_CIPHER_CTX_ctrl(EncCtx, EVP_CTRL_GCM_GET_TAG, TagBytes, Out + Offset) != 1) + { + HasErrors = true; + return 0; + } + Offset += TagBytes; + + return Offset; +#endif + } + + // Decrypt a message. Returns decrypted data length, or 0 on failure. + // Input must be [ciphertext][tag], with nonce provided separately. + int32_t DecryptMessage(void* Out, const uint8_t* Nonce, const uint8_t* CipherAndTag, int32_t DataLength) + { +#if ZEN_PLATFORM_WINDOWS + BCRYPT_ALG_HANDLE hAlg = nullptr; + BCRYPT_KEY_HANDLE hKey = nullptr; + + BCryptOpenAlgorithmProvider(&hAlg, BCRYPT_AES_ALGORITHM, nullptr, 0); + BCryptSetProperty(hAlg, BCRYPT_CHAINING_MODE, (PUCHAR)BCRYPT_CHAIN_MODE_GCM, sizeof(BCRYPT_CHAIN_MODE_GCM), 0); + BCryptGenerateSymmetricKey(hAlg, &hKey, nullptr, 0, (PUCHAR)Key, KeySize, 0); + + BCRYPT_AUTHENTICATED_CIPHER_MODE_INFO AuthInfo; + BCRYPT_INIT_AUTH_MODE_INFO(AuthInfo); + AuthInfo.pbNonce = const_cast(Nonce); + AuthInfo.cbNonce = NonceBytes; + AuthInfo.pbTag = const_cast(CipherAndTag + DataLength); + AuthInfo.cbTag = TagBytes; + + ULONG PlainLen = 0; + NTSTATUS Status = BCryptDecrypt(hKey, + (PUCHAR)CipherAndTag, + (ULONG)DataLength, + &AuthInfo, + nullptr, + 0, + (PUCHAR)Out, + (ULONG)DataLength, + &PlainLen, + 0); + + BCryptDestroyKey(hKey); + BCryptCloseAlgorithmProvider(hAlg, 0); + + if (!BCRYPT_SUCCESS(Status)) + { + HasErrors = true; + return 0; + } + + return static_cast(PlainLen); +#else + if (EVP_DecryptInit_ex(DecCtx, nullptr, nullptr, Key, Nonce) != 1) + { + HasErrors = true; + return 0; + } + + int OutLen = 0; + if (EVP_DecryptUpdate(DecCtx, static_cast(Out), &OutLen, CipherAndTag, DataLength) != 1) + { + HasErrors = true; + return 0; + } + + // Set the tag for verification + if (EVP_CIPHER_CTX_ctrl(DecCtx, EVP_CTRL_GCM_SET_TAG, TagBytes, const_cast(CipherAndTag + DataLength)) != 1) + { + HasErrors = true; + return 0; + } + + int FinalLen = 0; + if (EVP_DecryptFinal_ex(DecCtx, static_cast(Out) + OutLen, &FinalLen) != 1) + { + HasErrors = true; + return 0; + } + + return OutLen + FinalLen; +#endif + } +}; + +AesComputeTransport::AesComputeTransport(const uint8_t (&Key)[KeySize], std::unique_ptr InnerTransport) +: m_Crypto(std::make_unique(Key)) +, m_Inner(std::move(InnerTransport)) +{ +} + +AesComputeTransport::~AesComputeTransport() +{ + Close(); +} + +bool +AesComputeTransport::IsValid() const +{ + return m_Inner && m_Inner->IsValid() && m_Crypto && !m_Crypto->HasErrors && !m_IsClosed; +} + +size_t +AesComputeTransport::Send(const void* Data, size_t Size) +{ + ZEN_TRACE_CPU("AesComputeTransport::Send"); + + if (!IsValid()) + { + return 0; + } + + std::lock_guard Lock(m_Lock); + + const int32_t DataLength = static_cast(Size); + const size_t MessageLength = 4 + NonceBytes + Size + TagBytes; + + if (m_EncryptBuffer.size() < MessageLength) + { + m_EncryptBuffer.resize(MessageLength); + } + + const int32_t EncryptedLen = m_Crypto->EncryptMessage(m_EncryptBuffer.data(), Data, DataLength); + if (EncryptedLen == 0) + { + return 0; + } + + if (!m_Inner->SendMessage(m_EncryptBuffer.data(), static_cast(EncryptedLen))) + { + return 0; + } + + return Size; +} + +size_t +AesComputeTransport::Recv(void* Data, size_t Size) +{ + if (!IsValid()) + { + return 0; + } + + // AES-GCM decrypts entire messages at once, but the caller may request fewer bytes + // than the decrypted message contains. Excess bytes are buffered in m_RemainingData + // and returned on subsequent Recv calls without another decryption round-trip. + ZEN_TRACE_CPU("AesComputeTransport::Recv"); + + std::lock_guard Lock(m_Lock); + + if (!m_RemainingData.empty()) + { + const size_t Available = m_RemainingData.size() - m_RemainingOffset; + const size_t ToCopy = std::min(Available, Size); + + memcpy(Data, m_RemainingData.data() + m_RemainingOffset, ToCopy); + m_RemainingOffset += ToCopy; + + if (m_RemainingOffset >= m_RemainingData.size()) + { + m_RemainingData.clear(); + m_RemainingOffset = 0; + } + + return ToCopy; + } + + // Receive packet header: [length(4B)][nonce(12B)] + struct PacketHeader + { + int32_t DataLength = 0; + uint8_t Nonce[NonceBytes] = {}; + } Header; + + if (!m_Inner->RecvMessage(&Header, sizeof(Header))) + { + return 0; + } + + // Validate DataLength to prevent OOM from malicious/corrupt peers + static constexpr int32_t MaxDataLength = 64 * 1024 * 1024; // 64 MiB + + if (Header.DataLength <= 0 || Header.DataLength > MaxDataLength) + { + ZEN_WARN("AES recv: invalid DataLength {} from peer", Header.DataLength); + return 0; + } + + // Receive ciphertext + tag + const size_t MessageLength = static_cast(Header.DataLength) + TagBytes; + + if (m_EncryptBuffer.size() < MessageLength) + { + m_EncryptBuffer.resize(MessageLength); + } + + if (!m_Inner->RecvMessage(m_EncryptBuffer.data(), MessageLength)) + { + return 0; + } + + // Decrypt + const size_t BytesToReturn = std::min(static_cast(Header.DataLength), Size); + + // We need a temporary buffer for decryption if we can't decrypt directly into output + std::vector DecryptedBuf(static_cast(Header.DataLength)); + + const int32_t Decrypted = m_Crypto->DecryptMessage(DecryptedBuf.data(), Header.Nonce, m_EncryptBuffer.data(), Header.DataLength); + if (Decrypted == 0) + { + return 0; + } + + memcpy(Data, DecryptedBuf.data(), BytesToReturn); + + // Store remaining data if we couldn't return everything + if (static_cast(Header.DataLength) > BytesToReturn) + { + m_RemainingOffset = 0; + m_RemainingData.assign(DecryptedBuf.begin() + BytesToReturn, DecryptedBuf.begin() + Header.DataLength); + } + + return BytesToReturn; +} + +void +AesComputeTransport::MarkComplete() +{ + if (IsValid()) + { + m_Inner->MarkComplete(); + } +} + +void +AesComputeTransport::Close() +{ + if (!m_IsClosed) + { + if (m_Inner && m_Inner->IsValid()) + { + m_Inner->Close(); + } + m_IsClosed = true; + } +} + +} // namespace zen::horde diff --git a/src/zenhorde/hordetransportaes.h b/src/zenhorde/hordetransportaes.h new file mode 100644 index 000000000..efcad9835 --- /dev/null +++ b/src/zenhorde/hordetransportaes.h @@ -0,0 +1,52 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include "hordetransport.h" + +#include +#include +#include +#include + +namespace zen::horde { + +/** AES-256-GCM encrypted transport wrapper. + * + * Wraps an inner ComputeTransport, encrypting all outgoing data and decrypting + * all incoming data using AES-256-GCM. The nonce is mutated per message using + * the Horde nonce mangling scheme: n32[0]++; n32[1]--; n32[2] = n32[0] ^ n32[1]. + * + * Wire format per encrypted message: + * [plaintext length (4B little-endian)][nonce (12B)][ciphertext][GCM tag (16B)] + * + * Uses BCrypt on Windows and OpenSSL EVP on Linux/macOS (selected at compile time). + */ +class AesComputeTransport final : public ComputeTransport +{ +public: + AesComputeTransport(const uint8_t (&Key)[KeySize], std::unique_ptr InnerTransport); + ~AesComputeTransport() override; + + bool IsValid() const override; + size_t Send(const void* Data, size_t Size) override; + size_t Recv(void* Data, size_t Size) override; + void MarkComplete() override; + void Close() override; + +private: + static constexpr size_t NonceBytes = 12; ///< AES-GCM nonce size + static constexpr size_t TagBytes = 16; ///< AES-GCM authentication tag size + + struct CryptoContext; + + std::unique_ptr m_Crypto; + std::unique_ptr m_Inner; + std::vector m_EncryptBuffer; + std::vector m_RemainingData; ///< Buffered decrypted data from a partially consumed Recv + size_t m_RemainingOffset = 0; + std::mutex m_Lock; + bool m_IsClosed = false; +}; + +} // namespace zen::horde diff --git a/src/zenhorde/include/zenhorde/hordeclient.h b/src/zenhorde/include/zenhorde/hordeclient.h new file mode 100644 index 000000000..201d68b83 --- /dev/null +++ b/src/zenhorde/include/zenhorde/hordeclient.h @@ -0,0 +1,116 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include + +namespace zen { +class HttpClient; +} + +namespace zen::horde { + +static constexpr size_t NonceSize = 64; +static constexpr size_t KeySize = 32; + +/** Port mapping information returned by Horde for a provisioned machine. */ +struct PortInfo +{ + uint16_t Port = 0; + uint16_t AgentPort = 0; +}; + +/** Describes a provisioned compute machine returned by the Horde API. + * + * Contains the network address, encryption credentials, and capabilities + * needed to establish a compute transport connection to the machine. + */ +struct MachineInfo +{ + std::string Ip; + ConnectionMode Mode = ConnectionMode::Direct; + std::string ConnectionAddress; ///< Relay/tunnel address (used when Mode != Direct) + uint16_t Port = 0; + uint16_t LogicalCores = 0; + Encryption EncryptionMode = Encryption::None; + uint8_t Nonce[NonceSize] = {}; ///< 64-byte nonce sent during TCP handshake + uint8_t Key[KeySize] = {}; ///< 32-byte AES key (when EncryptionMode == AES) + bool IsWindows = false; + std::string LeaseId; + + std::map Ports; + + /** Return the address to connect to, accounting for connection mode. */ + const std::string& GetConnectionAddress() const { return Mode == ConnectionMode::Relay ? ConnectionAddress : Ip; } + + /** Return the port to connect to, accounting for connection mode and port mapping. */ + uint16_t GetConnectionPort() const + { + if (Mode == ConnectionMode::Relay) + { + auto It = Ports.find("_horde_compute"); + if (It != Ports.end()) + { + return It->second.Port; + } + } + return Port; + } + + bool IsValid() const { return !Ip.empty() && Port != 0xFFFF; } +}; + +/** Result of cluster auto-resolution via the Horde API. */ +struct ClusterInfo +{ + std::string ClusterId = "default"; +}; + +/** HTTP client for the Horde compute REST API. + * + * Handles cluster resolution and machine provisioning requests. Each call + * is synchronous and returns success/failure. Thread safety: individual + * methods are not thread-safe; callers must synchronize access. + */ +class HordeClient +{ +public: + explicit HordeClient(const HordeConfig& Config); + ~HordeClient(); + + HordeClient(const HordeClient&) = delete; + HordeClient& operator=(const HordeClient&) = delete; + + /** Initialize the underlying HTTP client. Must be called before other methods. */ + bool Initialize(); + + /** Build the JSON request body for cluster resolution and machine requests. + * Encodes pool, condition, connection mode, encryption, and port requirements. */ + std::string BuildRequestBody() const; + + /** Resolve the best cluster for the given request via POST /api/v2/compute/_cluster. */ + bool ResolveCluster(const std::string& RequestBody, ClusterInfo& OutCluster); + + /** Request a compute machine from the given cluster via POST /api/v2/compute/{clusterId}. + * On success, populates OutMachine with connection details and credentials. */ + bool RequestMachine(const std::string& RequestBody, const std::string& ClusterId, MachineInfo& OutMachine); + + LoggerRef Log() { return m_Log; } + +private: + bool ParseHexBytes(std::string_view Hex, uint8_t* Out, size_t OutSize); + + HordeConfig m_Config; + std::unique_ptr m_Http; + LoggerRef m_Log; +}; + +} // namespace zen::horde diff --git a/src/zenhorde/include/zenhorde/hordeconfig.h b/src/zenhorde/include/zenhorde/hordeconfig.h new file mode 100644 index 000000000..dd70f9832 --- /dev/null +++ b/src/zenhorde/include/zenhorde/hordeconfig.h @@ -0,0 +1,62 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +namespace zen::horde { + +/** Transport connection mode for Horde compute agents. */ +enum class ConnectionMode +{ + Direct, ///< Connect directly to the agent IP + Tunnel, ///< Connect through a Horde tunnel relay + Relay, ///< Connect through a Horde relay with port mapping +}; + +/** Transport encryption mode for Horde compute channels. */ +enum class Encryption +{ + None, ///< No encryption + AES, ///< AES-256-GCM encryption (required for Relay mode) +}; + +/** Configuration for connecting to an Epic Horde compute cluster. + * + * Specifies the Horde server URL, authentication token, pool selection, + * connection mode, and resource limits. Used by HordeClient and HordeProvisioner. + */ +struct HordeConfig +{ + static constexpr const char* ClusterDefault = "default"; + static constexpr const char* ClusterAuto = "_auto"; + + bool Enabled = false; ///< Whether Horde provisioning is active + std::string ServerUrl; ///< Horde server base URL (e.g. "https://horde.example.com") + std::string AuthToken; ///< Authentication token for the Horde API + std::string Pool; ///< Pool name to request machines from + std::string Cluster = ClusterDefault; ///< Cluster ID, or "_auto" to auto-resolve + std::string Condition; ///< Agent filter expression for machine selection + std::string HostAddress; ///< Address that provisioned agents use to connect back to us + std::string BinariesPath; ///< Path to directory containing zenserver binary for remote upload + uint16_t ZenServicePort = 8558; ///< Port number that provisioned agents should forward to us for Zen service communication + + int MaxCores = 2048; + bool AllowWine = true; ///< Allow running Windows binaries under Wine on Linux agents + ConnectionMode Mode = ConnectionMode::Direct; + Encryption EncryptionMode = Encryption::None; + + /** Validate the configuration. Returns false if the configuration is invalid + * (e.g. Relay mode without AES encryption). */ + bool Validate() const; +}; + +const char* ToString(ConnectionMode Mode); +const char* ToString(Encryption Enc); + +bool FromString(ConnectionMode& OutMode, std::string_view Str); +bool FromString(Encryption& OutEnc, std::string_view Str); + +} // namespace zen::horde diff --git a/src/zenhorde/include/zenhorde/hordeprovisioner.h b/src/zenhorde/include/zenhorde/hordeprovisioner.h new file mode 100644 index 000000000..4e2e63bbd --- /dev/null +++ b/src/zenhorde/include/zenhorde/hordeprovisioner.h @@ -0,0 +1,110 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace zen::horde { + +class HordeClient; + +/** Snapshot of the current provisioning state, returned by HordeProvisioner::GetStats(). */ +struct ProvisioningStats +{ + uint32_t TargetCoreCount = 0; ///< Requested number of cores (clamped to MaxCores) + uint32_t EstimatedCoreCount = 0; ///< Cores expected once pending requests complete + uint32_t ActiveCoreCount = 0; ///< Cores on machines that are currently running zenserver + uint32_t AgentsActive = 0; ///< Number of agents with a running remote process + uint32_t AgentsRequesting = 0; ///< Number of agents currently requesting a machine from Horde +}; + +/** Multi-agent lifecycle manager for Horde worker provisioning. + * + * Provisions remote compute workers by requesting machines from the Horde API, + * connecting via the Horde compute transport protocol, uploading the zenserver + * binary, and executing it remotely. Each provisioned machine runs zenserver + * in compute mode, which announces itself back to the orchestrator. + * + * Spawns one thread per agent. Each thread handles the full lifecycle: + * HTTP request -> TCP connect -> nonce handshake -> optional AES encryption -> + * channel setup -> binary upload -> remote execution -> poll until exit. + * + * Thread safety: SetTargetCoreCount and GetStats may be called from any thread. + */ +class HordeProvisioner +{ +public: + /** Construct a provisioner. + * @param Config Horde connection and pool configuration. + * @param BinariesPath Directory containing the zenserver binary to upload. + * @param WorkingDir Local directory for bundle staging and working files. + * @param OrchestratorEndpoint URL of the orchestrator that remote workers announce to. */ + HordeProvisioner(const HordeConfig& Config, + const std::filesystem::path& BinariesPath, + const std::filesystem::path& WorkingDir, + std::string_view OrchestratorEndpoint); + + /** Signals all agent threads to exit and joins them. */ + ~HordeProvisioner(); + + HordeProvisioner(const HordeProvisioner&) = delete; + HordeProvisioner& operator=(const HordeProvisioner&) = delete; + + /** Set the target number of cores to provision. + * Clamped to HordeConfig::MaxCores. Spawns new agent threads if the + * estimated core count is below the target. Also joins any finished + * agent threads. */ + void SetTargetCoreCount(uint32_t Count); + + /** Return a snapshot of the current provisioning counters. */ + ProvisioningStats GetStats() const; + + uint32_t GetActiveCoreCount() const { return m_ActiveCoreCount.load(); } + uint32_t GetAgentCount() const; + +private: + LoggerRef Log() { return m_Log; } + + struct AgentWrapper; + + void RequestAgent(); + void ThreadAgent(AgentWrapper& Wrapper); + + HordeConfig m_Config; + std::filesystem::path m_BinariesPath; + std::filesystem::path m_WorkingDir; + std::string m_OrchestratorEndpoint; + + std::unique_ptr m_HordeClient; + + std::mutex m_BundleLock; + std::vector> m_Bundles; ///< (locator, bundleDir) pairs + bool m_BundlesCreated = false; + + mutable std::mutex m_AgentsLock; + std::vector> m_Agents; + + std::atomic m_LastRequestFailTime{0}; + std::atomic m_TargetCoreCount{0}; + std::atomic m_EstimatedCoreCount{0}; + std::atomic m_ActiveCoreCount{0}; + std::atomic m_AgentsActive{0}; + std::atomic m_AgentsRequesting{0}; + std::atomic m_AskForAgents{true}; + + LoggerRef m_Log; + + static constexpr uint32_t EstimatedCoresPerAgent = 32; +}; + +} // namespace zen::horde diff --git a/src/zenhorde/include/zenhorde/zenhorde.h b/src/zenhorde/include/zenhorde/zenhorde.h new file mode 100644 index 000000000..35147ff75 --- /dev/null +++ b/src/zenhorde/include/zenhorde/zenhorde.h @@ -0,0 +1,9 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if !defined(ZEN_WITH_HORDE) +# define ZEN_WITH_HORDE 1 +#endif diff --git a/src/zenhorde/xmake.lua b/src/zenhorde/xmake.lua new file mode 100644 index 000000000..48d028e86 --- /dev/null +++ b/src/zenhorde/xmake.lua @@ -0,0 +1,22 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +target('zenhorde') + set_kind("static") + set_group("libs") + add_headerfiles("**.h") + add_files("**.cpp") + add_includedirs("include", {public=true}) + add_deps("zencore", "zenhttp", "zencompute", "zenutil") + add_packages("asio", "json11") + + if is_plat("windows") then + add_syslinks("Ws2_32", "Bcrypt") + end + + if is_plat("linux") or is_plat("macosx") then + add_packages("openssl") + end + + if is_os("macosx") then + add_cxxflags("-Wno-deprecated-declarations") + end diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 62c080a7b..02cccc540 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -103,6 +103,7 @@ public: virtual bool IsLocalMachineRequest() const = 0; virtual std::string_view GetAuthorizationHeader() const = 0; + virtual std::string_view GetRemoteAddress() const { return {}; } /** Respond with payload diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index c4d9ee777..33f182df9 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -544,7 +544,8 @@ public: HttpService& Service, IoBuffer PayloadBuffer, uint32_t RequestNumber, - bool IsLocalMachineRequest); + bool IsLocalMachineRequest, + std::string RemoteAddress); ~HttpAsioServerRequest(); virtual Oid ParseSessionId() const override; @@ -552,6 +553,7 @@ public: virtual bool IsLocalMachineRequest() const override; virtual std::string_view GetAuthorizationHeader() const override; + virtual std::string_view GetRemoteAddress() const override; virtual IoBuffer ReadPayload() override; virtual void WriteResponse(HttpResponseCode ResponseCode) override; @@ -569,6 +571,7 @@ public: uint32_t m_RequestNumber = 0; // Note: different to request ID which is derived from headers IoBuffer m_PayloadBuffer; bool m_IsLocalMachineRequest; + std::string m_RemoteAddress; std::unique_ptr m_Response; }; @@ -1238,9 +1241,15 @@ HttpServerConnection::HandleRequest() { ZEN_TRACE_CPU("asio::HandleRequest"); - bool IsLocalConnection = m_Socket->local_endpoint().address() == m_Socket->remote_endpoint().address(); + auto RemoteEndpoint = m_Socket->remote_endpoint(); + bool IsLocalConnection = m_Socket->local_endpoint().address() == RemoteEndpoint.address(); - HttpAsioServerRequest Request(m_RequestData, *Service, m_RequestData.Body(), RequestNumber, IsLocalConnection); + HttpAsioServerRequest Request(m_RequestData, + *Service, + m_RequestData.Body(), + RequestNumber, + IsLocalConnection, + RemoteEndpoint.address().to_string()); ZEN_TRACE_VERBOSE("handle request, connection: {}, request: {}'", m_ConnectionId, RequestNumber); @@ -1725,12 +1734,14 @@ HttpAsioServerRequest::HttpAsioServerRequest(HttpRequestParser& Request, HttpService& Service, IoBuffer PayloadBuffer, uint32_t RequestNumber, - bool IsLocalMachineRequest) + bool IsLocalMachineRequest, + std::string RemoteAddress) : HttpServerRequest(Service) , m_Request(Request) , m_RequestNumber(RequestNumber) , m_PayloadBuffer(std::move(PayloadBuffer)) , m_IsLocalMachineRequest(IsLocalMachineRequest) +, m_RemoteAddress(std::move(RemoteAddress)) { const int PrefixLength = Service.UriPrefixLength(); @@ -1808,6 +1819,12 @@ HttpAsioServerRequest::IsLocalMachineRequest() const return m_IsLocalMachineRequest; } +std::string_view +HttpAsioServerRequest::GetRemoteAddress() const +{ + return m_RemoteAddress; +} + std::string_view HttpAsioServerRequest::GetAuthorizationHeader() const { diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index a48f1d316..cf639c114 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -336,8 +336,9 @@ public: virtual Oid ParseSessionId() const override; virtual uint32_t ParseRequestId() const override; - virtual bool IsLocalMachineRequest() const; + virtual bool IsLocalMachineRequest() const override; virtual std::string_view GetAuthorizationHeader() const override; + virtual std::string_view GetRemoteAddress() const override; virtual IoBuffer ReadPayload() override; virtual void WriteResponse(HttpResponseCode ResponseCode) override; @@ -353,11 +354,12 @@ public: HttpSysServerRequest(const HttpSysServerRequest&) = delete; HttpSysServerRequest& operator=(const HttpSysServerRequest&) = delete; - HttpSysTransaction& m_HttpTx; - HttpSysRequestHandler* m_NextCompletionHandler = nullptr; - IoBuffer m_PayloadBuffer; - ExtendableStringBuilder<128> m_UriUtf8; - ExtendableStringBuilder<128> m_QueryStringUtf8; + HttpSysTransaction& m_HttpTx; + HttpSysRequestHandler* m_NextCompletionHandler = nullptr; + IoBuffer m_PayloadBuffer; + ExtendableStringBuilder<128> m_UriUtf8; + ExtendableStringBuilder<128> m_QueryStringUtf8; + mutable ExtendableStringBuilder<64> m_RemoteAddress; }; /** HTTP transaction @@ -1901,6 +1903,17 @@ HttpSysServerRequest::IsLocalMachineRequest() const } } +std::string_view +HttpSysServerRequest::GetRemoteAddress() const +{ + if (m_RemoteAddress.Size() == 0) + { + const SOCKADDR* SockAddr = m_HttpTx.HttpRequest()->Address.pRemoteAddress; + GetAddressString(m_RemoteAddress, SockAddr, /* IncludePort */ false); + } + return m_RemoteAddress.ToView(); +} + std::string_view HttpSysServerRequest::GetAuthorizationHeader() const { diff --git a/src/zennomad/include/zennomad/nomadclient.h b/src/zennomad/include/zennomad/nomadclient.h new file mode 100644 index 000000000..0a3411ace --- /dev/null +++ b/src/zennomad/include/zennomad/nomadclient.h @@ -0,0 +1,77 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +#include +#include +#include + +namespace zen { +class HttpClient; +} + +namespace zen::nomad { + +/** Summary of a Nomad job returned by the API. */ +struct NomadJobInfo +{ + std::string Id; + std::string Status; ///< "pending", "running", "dead" + std::string StatusDescription; +}; + +/** Summary of a Nomad allocation returned by the API. */ +struct NomadAllocInfo +{ + std::string Id; + std::string ClientStatus; ///< "pending", "running", "complete", "failed" + std::string TaskState; ///< State of the task within the allocation +}; + +/** HTTP client for the Nomad REST API (v1). + * + * Handles job submission, status polling, and job termination. + * All calls are synchronous. Thread safety: individual methods are + * not thread-safe; callers must synchronize access. + */ +class NomadClient +{ +public: + explicit NomadClient(const NomadConfig& Config); + ~NomadClient(); + + NomadClient(const NomadClient&) = delete; + NomadClient& operator=(const NomadClient&) = delete; + + /** Initialize the underlying HTTP client. Must be called before other methods. */ + bool Initialize(); + + /** Build the Nomad job registration JSON for the given job ID and orchestrator endpoint. + * The JSON structure varies based on the configured driver and distribution mode. */ + std::string BuildJobJson(const std::string& JobId, const std::string& OrchestratorEndpoint) const; + + /** Submit a job via PUT /v1/jobs. On success, populates OutJob with the job info. */ + bool SubmitJob(const std::string& JobJson, NomadJobInfo& OutJob); + + /** Get the status of a job via GET /v1/job/{jobId}. */ + bool GetJobStatus(const std::string& JobId, NomadJobInfo& OutJob); + + /** Get allocations for a job via GET /v1/job/{jobId}/allocations. */ + bool GetAllocations(const std::string& JobId, std::vector& OutAllocs); + + /** Stop a job via DELETE /v1/job/{jobId}. */ + bool StopJob(const std::string& JobId); + + LoggerRef Log() { return m_Log; } + +private: + NomadConfig m_Config; + std::unique_ptr m_Http; + LoggerRef m_Log; +}; + +} // namespace zen::nomad diff --git a/src/zennomad/include/zennomad/nomadconfig.h b/src/zennomad/include/zennomad/nomadconfig.h new file mode 100644 index 000000000..92d2bbaca --- /dev/null +++ b/src/zennomad/include/zennomad/nomadconfig.h @@ -0,0 +1,65 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +namespace zen::nomad { + +/** Nomad task driver type. */ +enum class Driver +{ + RawExec, ///< Use Nomad raw_exec driver (direct process execution) + Docker, ///< Use Nomad Docker driver +}; + +/** How the zenserver binary is made available on Nomad clients. */ +enum class BinaryDistribution +{ + PreDeployed, ///< Binary is already present on Nomad client nodes + Artifact, ///< Download binary via Nomad artifact stanza +}; + +/** Configuration for Nomad worker provisioning. + * + * Specifies the Nomad server URL, authentication, resource limits, and + * job configuration. Used by NomadClient and NomadProvisioner. + */ +struct NomadConfig +{ + bool Enabled = false; ///< Whether Nomad provisioning is active + std::string ServerUrl; ///< Nomad HTTP API URL (e.g. "http://localhost:4646") + std::string AclToken; ///< Nomad ACL token (sent as X-Nomad-Token header) + std::string Datacenter = "dc1"; ///< Target datacenter + std::string Namespace = "default"; ///< Nomad namespace + std::string Region; ///< Nomad region (empty = server default) + + Driver TaskDriver = Driver::RawExec; ///< Task driver for job execution + BinaryDistribution BinDistribution = BinaryDistribution::PreDeployed; ///< How to distribute the zenserver binary + + std::string BinaryPath; ///< Path to zenserver on Nomad clients (PreDeployed mode) + std::string ArtifactSource; ///< URL to download zenserver binary (Artifact mode) + std::string DockerImage; ///< Docker image name (Docker driver mode) + + int MaxJobs = 64; ///< Maximum concurrent Nomad jobs + int CpuMhz = 1000; ///< CPU MHz allocated per task + int MemoryMb = 2048; ///< Memory MB allocated per task + int CoresPerJob = 32; ///< Estimated cores per job (for scaling calculations) + int MaxCores = 2048; ///< Maximum total cores to provision + + std::string JobPrefix = "zenserver-worker"; ///< Prefix for generated Nomad job IDs + + /** Validate the configuration. Returns false if required fields are missing + * or incompatible options are set. */ + bool Validate() const; +}; + +const char* ToString(Driver D); +const char* ToString(BinaryDistribution Dist); + +bool FromString(Driver& OutDriver, std::string_view Str); +bool FromString(BinaryDistribution& OutDist, std::string_view Str); + +} // namespace zen::nomad diff --git a/src/zennomad/include/zennomad/nomadprocess.h b/src/zennomad/include/zennomad/nomadprocess.h new file mode 100644 index 000000000..a66c2ce41 --- /dev/null +++ b/src/zennomad/include/zennomad/nomadprocess.h @@ -0,0 +1,78 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include +#include + +namespace zen::nomad { + +struct NomadJobInfo; +struct NomadAllocInfo; + +/** Manages a Nomad agent process running in dev mode for testing. + * + * Spawns `nomad agent -dev` and polls the HTTP API until the agent + * is ready. On destruction or via StopNomadAgent(), the agent + * process is killed. + */ +class NomadProcess +{ +public: + NomadProcess(); + ~NomadProcess(); + + NomadProcess(const NomadProcess&) = delete; + NomadProcess& operator=(const NomadProcess&) = delete; + + /** Spawn a Nomad dev agent and block until the leader endpoint responds (10 s timeout). */ + void SpawnNomadAgent(); + + /** Kill the Nomad agent process. */ + void StopNomadAgent(); + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +/** Lightweight HTTP wrapper around the Nomad v1 REST API for use in tests. + * + * Unlike the production NomadClient (which requires a NomadConfig and + * supports all driver/distribution modes), this client exposes a simpler + * interface geared towards test scenarios. + */ +class NomadTestClient +{ +public: + explicit NomadTestClient(std::string_view BaseUri); + ~NomadTestClient(); + + NomadTestClient(const NomadTestClient&) = delete; + NomadTestClient& operator=(const NomadTestClient&) = delete; + + /** Submit a raw_exec batch job. + * Returns the parsed job info on success; Id will be empty on failure. */ + NomadJobInfo SubmitJob(std::string_view JobId, std::string_view Command, const std::vector& Args); + + /** Query the status of an existing job. */ + NomadJobInfo GetJobStatus(std::string_view JobId); + + /** Stop (deregister) a running job. */ + void StopJob(std::string_view JobId); + + /** Get allocations for a job. */ + std::vector GetAllocations(std::string_view JobId); + + /** List all jobs, optionally filtered by prefix. */ + std::vector ListJobs(std::string_view Prefix = ""); + +private: + HttpClient m_HttpClient; +}; + +} // namespace zen::nomad diff --git a/src/zennomad/include/zennomad/nomadprovisioner.h b/src/zennomad/include/zennomad/nomadprovisioner.h new file mode 100644 index 000000000..750693b3f --- /dev/null +++ b/src/zennomad/include/zennomad/nomadprovisioner.h @@ -0,0 +1,107 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace zen::nomad { + +class NomadClient; + +/** Snapshot of the current Nomad provisioning state, returned by NomadProvisioner::GetStats(). */ +struct NomadProvisioningStats +{ + uint32_t TargetCoreCount = 0; ///< Requested number of cores (clamped to MaxCores) + uint32_t EstimatedCoreCount = 0; ///< Cores expected from submitted jobs + uint32_t ActiveJobCount = 0; ///< Number of currently tracked Nomad jobs + uint32_t RunningJobCount = 0; ///< Number of jobs in "running" status +}; + +/** Job lifecycle manager for Nomad worker provisioning. + * + * Provisions remote compute workers by submitting batch jobs to a Nomad + * cluster via the REST API. Each job runs zenserver in compute mode, which + * announces itself back to the orchestrator. + * + * Uses a single management thread that periodically: + * 1. Submits new jobs when estimated cores < target cores + * 2. Polls existing jobs for status changes + * 3. Cleans up dead/failed jobs and adjusts counters + * + * Thread safety: SetTargetCoreCount and GetStats may be called from any thread. + */ +class NomadProvisioner +{ +public: + /** Construct a provisioner. + * @param Config Nomad connection and job configuration. + * @param OrchestratorEndpoint URL of the orchestrator that remote workers announce to. */ + NomadProvisioner(const NomadConfig& Config, std::string_view OrchestratorEndpoint); + + /** Signals the management thread to exit and stops all tracked jobs. */ + ~NomadProvisioner(); + + NomadProvisioner(const NomadProvisioner&) = delete; + NomadProvisioner& operator=(const NomadProvisioner&) = delete; + + /** Set the target number of cores to provision. + * Clamped to NomadConfig::MaxCores. The management thread will + * submit new jobs to approach this target. */ + void SetTargetCoreCount(uint32_t Count); + + /** Return a snapshot of the current provisioning counters. */ + NomadProvisioningStats GetStats() const; + +private: + LoggerRef Log() { return m_Log; } + + struct TrackedJob + { + std::string JobId; + std::string Status; ///< "pending", "running", "dead" + int Cores = 0; + }; + + void ManagementThread(); + void SubmitNewJobs(); + void PollExistingJobs(); + void CleanupDeadJobs(); + void StopAllJobs(); + + std::string GenerateJobId(); + + NomadConfig m_Config; + std::string m_OrchestratorEndpoint; + + std::unique_ptr m_Client; + + mutable std::mutex m_JobsLock; + std::vector m_Jobs; + std::atomic m_JobIndex{0}; + + std::atomic m_TargetCoreCount{0}; + std::atomic m_EstimatedCoreCount{0}; + std::atomic m_RunningJobCount{0}; + + std::thread m_Thread; + std::mutex m_WakeMutex; + std::condition_variable m_WakeCV; + std::atomic m_ShouldExit{false}; + + uint32_t m_ProcessId = 0; + + LoggerRef m_Log; +}; + +} // namespace zen::nomad diff --git a/src/zennomad/include/zennomad/zennomad.h b/src/zennomad/include/zennomad/zennomad.h new file mode 100644 index 000000000..09fb98dfe --- /dev/null +++ b/src/zennomad/include/zennomad/zennomad.h @@ -0,0 +1,9 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if !defined(ZEN_WITH_NOMAD) +# define ZEN_WITH_NOMAD 1 +#endif diff --git a/src/zennomad/nomadclient.cpp b/src/zennomad/nomadclient.cpp new file mode 100644 index 000000000..9edcde125 --- /dev/null +++ b/src/zennomad/nomadclient.cpp @@ -0,0 +1,366 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::nomad { + +namespace { + + HttpClient::KeyValueMap MakeNomadHeaders(const NomadConfig& Config) + { + HttpClient::KeyValueMap Headers; + if (!Config.AclToken.empty()) + { + Headers->emplace("X-Nomad-Token", Config.AclToken); + } + return Headers; + } + +} // namespace + +NomadClient::NomadClient(const NomadConfig& Config) : m_Config(Config), m_Log(zen::logging::Get("nomad.client")) +{ +} + +NomadClient::~NomadClient() = default; + +bool +NomadClient::Initialize() +{ + ZEN_TRACE_CPU("NomadClient::Initialize"); + + HttpClientSettings Settings; + Settings.LogCategory = "nomad.http"; + Settings.ConnectTimeout = std::chrono::milliseconds{10000}; + Settings.Timeout = std::chrono::milliseconds{60000}; + Settings.RetryCount = 1; + + // Ensure the base URL ends with a slash so path concatenation works correctly + std::string BaseUrl = m_Config.ServerUrl; + if (!BaseUrl.empty() && BaseUrl.back() != '/') + { + BaseUrl += '/'; + } + + m_Http = std::make_unique(BaseUrl, Settings); + + return true; +} + +std::string +NomadClient::BuildJobJson(const std::string& JobId, const std::string& OrchestratorEndpoint) const +{ + ZEN_TRACE_CPU("NomadClient::BuildJobJson"); + + // Build the task config based on driver and distribution mode + json11::Json::object TaskConfig; + + if (m_Config.TaskDriver == Driver::RawExec) + { + std::string Command; + if (m_Config.BinDistribution == BinaryDistribution::PreDeployed) + { + Command = m_Config.BinaryPath; + } + else + { + // Artifact mode: binary is downloaded to local/zenserver + Command = "local/zenserver"; + } + + TaskConfig["command"] = Command; + + json11::Json::array Args; + Args.push_back("compute"); + Args.push_back("--http=asio"); + if (!OrchestratorEndpoint.empty()) + { + ExtendableStringBuilder<256> CoordArg; + CoordArg << "--coordinator-endpoint=" << OrchestratorEndpoint; + Args.push_back(std::string(CoordArg.ToView())); + } + { + ExtendableStringBuilder<128> IdArg; + IdArg << "--instance-id=nomad-" << JobId; + Args.push_back(std::string(IdArg.ToView())); + } + TaskConfig["args"] = Args; + } + else + { + // Docker driver + TaskConfig["image"] = m_Config.DockerImage; + + json11::Json::array Args; + Args.push_back("compute"); + Args.push_back("--http=asio"); + if (!OrchestratorEndpoint.empty()) + { + ExtendableStringBuilder<256> CoordArg; + CoordArg << "--coordinator-endpoint=" << OrchestratorEndpoint; + Args.push_back(std::string(CoordArg.ToView())); + } + { + ExtendableStringBuilder<128> IdArg; + IdArg << "--instance-id=nomad-" << JobId; + Args.push_back(std::string(IdArg.ToView())); + } + TaskConfig["args"] = Args; + } + + // Build resource stanza + json11::Json::object Resources; + Resources["CPU"] = m_Config.CpuMhz; + Resources["MemoryMB"] = m_Config.MemoryMb; + + // Build the task + json11::Json::object Task; + Task["Name"] = "zenserver"; + Task["Driver"] = (m_Config.TaskDriver == Driver::RawExec) ? "raw_exec" : "docker"; + Task["Config"] = TaskConfig; + Task["Resources"] = Resources; + + // Add artifact stanza if using artifact distribution + if (m_Config.BinDistribution == BinaryDistribution::Artifact && !m_Config.ArtifactSource.empty()) + { + json11::Json::object Artifact; + Artifact["GetterSource"] = m_Config.ArtifactSource; + + json11::Json::array Artifacts; + Artifacts.push_back(Artifact); + Task["Artifacts"] = Artifacts; + } + + json11::Json::array Tasks; + Tasks.push_back(Task); + + // Build the task group + json11::Json::object Group; + Group["Name"] = "zenserver-group"; + Group["Count"] = 1; + Group["Tasks"] = Tasks; + + json11::Json::array Groups; + Groups.push_back(Group); + + // Build datacenters array + json11::Json::array Datacenters; + Datacenters.push_back(m_Config.Datacenter); + + // Build the job + json11::Json::object Job; + Job["ID"] = JobId; + Job["Name"] = JobId; + Job["Type"] = "batch"; + Job["Datacenters"] = Datacenters; + Job["TaskGroups"] = Groups; + + if (!m_Config.Namespace.empty() && m_Config.Namespace != "default") + { + Job["Namespace"] = m_Config.Namespace; + } + + if (!m_Config.Region.empty()) + { + Job["Region"] = m_Config.Region; + } + + // Wrap in the registration envelope + json11::Json::object Root; + Root["Job"] = Job; + + return json11::Json(Root).dump(); +} + +bool +NomadClient::SubmitJob(const std::string& JobJson, NomadJobInfo& OutJob) +{ + ZEN_TRACE_CPU("NomadClient::SubmitJob"); + + const IoBuffer Payload = IoBufferBuilder::MakeFromMemory(MemoryView{JobJson.data(), JobJson.size()}, ZenContentType::kJSON); + + const HttpClient::Response Response = m_Http->Put("v1/jobs", Payload, MakeNomadHeaders(m_Config)); + + if (Response.Error) + { + ZEN_WARN("Nomad job submit failed: {}", Response.Error->ErrorMessage); + return false; + } + + const int StatusCode = static_cast(Response.StatusCode); + + if (!Response.IsSuccess()) + { + ZEN_WARN("Nomad job submit failed with HTTP/{}", StatusCode); + return false; + } + + const std::string Body(Response.AsText()); + std::string Err; + const json11::Json Json = json11::Json::parse(Body, Err); + + if (!Err.empty()) + { + ZEN_WARN("invalid JSON response from Nomad job submit: {}", Err); + return false; + } + + // The response contains EvalID; the job ID is what we submitted + OutJob.Id = Json["JobModifyIndex"].is_number() ? OutJob.Id : ""; + OutJob.Status = "pending"; + + ZEN_INFO("Nomad job submitted: eval_id={}", Json["EvalID"].string_value()); + + return true; +} + +bool +NomadClient::GetJobStatus(const std::string& JobId, NomadJobInfo& OutJob) +{ + ZEN_TRACE_CPU("NomadClient::GetJobStatus"); + + ExtendableStringBuilder<128> Path; + Path << "v1/job/" << JobId; + + const HttpClient::Response Response = m_Http->Get(Path.ToView(), MakeNomadHeaders(m_Config)); + + if (Response.Error) + { + ZEN_WARN("Nomad job status query failed for '{}': {}", JobId, Response.Error->ErrorMessage); + return false; + } + + const int StatusCode = static_cast(Response.StatusCode); + + if (StatusCode == 404) + { + ZEN_INFO("Nomad job '{}' not found", JobId); + OutJob.Status = "dead"; + return true; + } + + if (!Response.IsSuccess()) + { + ZEN_WARN("Nomad job status query failed with HTTP/{}", StatusCode); + return false; + } + + const std::string Body(Response.AsText()); + std::string Err; + const json11::Json Json = json11::Json::parse(Body, Err); + + if (!Err.empty()) + { + ZEN_WARN("invalid JSON in Nomad job status response: {}", Err); + return false; + } + + OutJob.Id = Json["ID"].string_value(); + OutJob.Status = Json["Status"].string_value(); + if (const json11::Json Desc = Json["StatusDescription"]; Desc.is_string()) + { + OutJob.StatusDescription = Desc.string_value(); + } + + return true; +} + +bool +NomadClient::GetAllocations(const std::string& JobId, std::vector& OutAllocs) +{ + ZEN_TRACE_CPU("NomadClient::GetAllocations"); + + ExtendableStringBuilder<128> Path; + Path << "v1/job/" << JobId << "/allocations"; + + const HttpClient::Response Response = m_Http->Get(Path.ToView(), MakeNomadHeaders(m_Config)); + + if (Response.Error) + { + ZEN_WARN("Nomad allocation query failed for '{}': {}", JobId, Response.Error->ErrorMessage); + return false; + } + + if (!Response.IsSuccess()) + { + ZEN_WARN("Nomad allocation query failed with HTTP/{}", static_cast(Response.StatusCode)); + return false; + } + + const std::string Body(Response.AsText()); + std::string Err; + const json11::Json Json = json11::Json::parse(Body, Err); + + if (!Err.empty()) + { + ZEN_WARN("invalid JSON in Nomad allocation response: {}", Err); + return false; + } + + OutAllocs.clear(); + if (!Json.is_array()) + { + return true; + } + + for (const json11::Json& AllocVal : Json.array_items()) + { + NomadAllocInfo Alloc; + Alloc.Id = AllocVal["ID"].string_value(); + Alloc.ClientStatus = AllocVal["ClientStatus"].string_value(); + + // Extract task state if available + if (const json11::Json TaskStates = AllocVal["TaskStates"]; TaskStates.is_object()) + { + for (const auto& [TaskName, TaskState] : TaskStates.object_items()) + { + if (TaskState["State"].is_string()) + { + Alloc.TaskState = TaskState["State"].string_value(); + } + } + } + + OutAllocs.push_back(std::move(Alloc)); + } + + return true; +} + +bool +NomadClient::StopJob(const std::string& JobId) +{ + ZEN_TRACE_CPU("NomadClient::StopJob"); + + ExtendableStringBuilder<128> Path; + Path << "v1/job/" << JobId; + + const HttpClient::Response Response = m_Http->Delete(Path.ToView(), MakeNomadHeaders(m_Config)); + + if (Response.Error) + { + ZEN_WARN("Nomad job stop failed for '{}': {}", JobId, Response.Error->ErrorMessage); + return false; + } + + if (!Response.IsSuccess()) + { + ZEN_WARN("Nomad job stop failed with HTTP/{}", static_cast(Response.StatusCode)); + return false; + } + + ZEN_INFO("Nomad job '{}' stopped", JobId); + return true; +} + +} // namespace zen::nomad diff --git a/src/zennomad/nomadconfig.cpp b/src/zennomad/nomadconfig.cpp new file mode 100644 index 000000000..d55b3da9a --- /dev/null +++ b/src/zennomad/nomadconfig.cpp @@ -0,0 +1,91 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +namespace zen::nomad { + +bool +NomadConfig::Validate() const +{ + if (ServerUrl.empty()) + { + return false; + } + + if (BinDistribution == BinaryDistribution::PreDeployed && BinaryPath.empty()) + { + return false; + } + + if (BinDistribution == BinaryDistribution::Artifact && ArtifactSource.empty()) + { + return false; + } + + if (TaskDriver == Driver::Docker && DockerImage.empty()) + { + return false; + } + + return true; +} + +const char* +ToString(Driver D) +{ + switch (D) + { + case Driver::RawExec: + return "raw_exec"; + case Driver::Docker: + return "docker"; + } + return "raw_exec"; +} + +const char* +ToString(BinaryDistribution Dist) +{ + switch (Dist) + { + case BinaryDistribution::PreDeployed: + return "predeployed"; + case BinaryDistribution::Artifact: + return "artifact"; + } + return "predeployed"; +} + +bool +FromString(Driver& OutDriver, std::string_view Str) +{ + if (Str == "raw_exec") + { + OutDriver = Driver::RawExec; + return true; + } + if (Str == "docker") + { + OutDriver = Driver::Docker; + return true; + } + return false; +} + +bool +FromString(BinaryDistribution& OutDist, std::string_view Str) +{ + if (Str == "predeployed") + { + OutDist = BinaryDistribution::PreDeployed; + return true; + } + if (Str == "artifact") + { + OutDist = BinaryDistribution::Artifact; + return true; + } + return false; +} + +} // namespace zen::nomad diff --git a/src/zennomad/nomadprocess.cpp b/src/zennomad/nomadprocess.cpp new file mode 100644 index 000000000..1ae968fb7 --- /dev/null +++ b/src/zennomad/nomadprocess.cpp @@ -0,0 +1,354 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +ZEN_THIRD_PARTY_INCLUDES_END + +#include + +namespace zen::nomad { + +////////////////////////////////////////////////////////////////////////// + +struct NomadProcess::Impl +{ + Impl(std::string_view BaseUri) : m_HttpClient(BaseUri) {} + ~Impl() = default; + + void SpawnNomadAgent() + { + ZEN_TRACE_CPU("SpawnNomadAgent"); + + if (m_ProcessHandle.IsValid()) + { + return; + } + + CreateProcOptions Options; + Options.Flags |= CreateProcOptions::Flag_Windows_NewProcessGroup; + + CreateProcResult Result = CreateProc("nomad" ZEN_EXE_SUFFIX_LITERAL, "nomad" ZEN_EXE_SUFFIX_LITERAL " agent -dev", Options); + + if (Result) + { + m_ProcessHandle.Initialize(Result); + + Stopwatch Timer; + + // Poll to check when the agent is ready + + do + { + Sleep(100); + HttpClient::Response Resp = m_HttpClient.Get("v1/status/leader"); + if (Resp) + { + ZEN_INFO("Nomad agent started successfully (waited {})", NiceTimeSpanMs(Timer.GetElapsedTimeMs())); + + return; + } + } while (Timer.GetElapsedTimeMs() < 30000); + } + + // Report failure! + + ZEN_WARN("Nomad agent failed to start within timeout period"); + } + + void StopNomadAgent() + { + if (!m_ProcessHandle.IsValid()) + { + return; + } + + // This waits for the process to exit and also resets the handle + m_ProcessHandle.Kill(); + } + +private: + ProcessHandle m_ProcessHandle; + HttpClient m_HttpClient; +}; + +NomadProcess::NomadProcess() : m_Impl(std::make_unique("http://localhost:4646/")) +{ +} + +NomadProcess::~NomadProcess() +{ +} + +void +NomadProcess::SpawnNomadAgent() +{ + m_Impl->SpawnNomadAgent(); +} + +void +NomadProcess::StopNomadAgent() +{ + m_Impl->StopNomadAgent(); +} + +////////////////////////////////////////////////////////////////////////// + +NomadTestClient::NomadTestClient(std::string_view BaseUri) : m_HttpClient(BaseUri) +{ +} + +NomadTestClient::~NomadTestClient() +{ +} + +NomadJobInfo +NomadTestClient::SubmitJob(std::string_view JobId, std::string_view Command, const std::vector& Args) +{ + ZEN_TRACE_CPU("SubmitNomadJob"); + + NomadJobInfo Result; + + // Build the job JSON for a raw_exec batch job + json11::Json::object TaskConfig; + TaskConfig["command"] = std::string(Command); + + json11::Json::array JsonArgs; + for (const auto& Arg : Args) + { + JsonArgs.push_back(Arg); + } + TaskConfig["args"] = JsonArgs; + + json11::Json::object Resources; + Resources["CPU"] = 100; + Resources["MemoryMB"] = 64; + + json11::Json::object Task; + Task["Name"] = "test-task"; + Task["Driver"] = "raw_exec"; + Task["Config"] = TaskConfig; + Task["Resources"] = Resources; + + json11::Json::array Tasks; + Tasks.push_back(Task); + + json11::Json::object Group; + Group["Name"] = "test-group"; + Group["Count"] = 1; + Group["Tasks"] = Tasks; + + json11::Json::array Groups; + Groups.push_back(Group); + + json11::Json::array Datacenters; + Datacenters.push_back("dc1"); + + json11::Json::object Job; + Job["ID"] = std::string(JobId); + Job["Name"] = std::string(JobId); + Job["Type"] = "batch"; + Job["Datacenters"] = Datacenters; + Job["TaskGroups"] = Groups; + + json11::Json::object Root; + Root["Job"] = Job; + + std::string Body = json11::Json(Root).dump(); + + IoBuffer Payload = IoBufferBuilder::MakeFromMemory(MemoryView{Body.data(), Body.size()}, ZenContentType::kJSON); + + HttpClient::Response Response = + m_HttpClient.Put("v1/jobs", Payload, {{"Content-Type", "application/json"}, {"Accept", "application/json"}}); + + if (!Response || !Response.IsSuccess()) + { + ZEN_WARN("NomadTestClient: SubmitJob failed for '{}'", JobId); + return Result; + } + + std::string ResponseBody(Response.AsText()); + std::string Err; + const json11::Json Json = json11::Json::parse(ResponseBody, Err); + + if (!Err.empty()) + { + ZEN_WARN("NomadTestClient: invalid JSON in SubmitJob response: {}", Err); + return Result; + } + + Result.Id = std::string(JobId); + Result.Status = "pending"; + + ZEN_INFO("NomadTestClient: job '{}' submitted (eval_id={})", JobId, Json["EvalID"].string_value()); + + return Result; +} + +NomadJobInfo +NomadTestClient::GetJobStatus(std::string_view JobId) +{ + ZEN_TRACE_CPU("GetNomadJobStatus"); + + NomadJobInfo Result; + + HttpClient::Response Response = m_HttpClient.Get(fmt::format("v1/job/{}", JobId)); + + if (Response.Error) + { + ZEN_WARN("NomadTestClient: GetJobStatus failed for '{}': {}", JobId, Response.Error->ErrorMessage); + return Result; + } + + if (static_cast(Response.StatusCode) == 404) + { + Result.Status = "dead"; + return Result; + } + + if (!Response.IsSuccess()) + { + ZEN_WARN("NomadTestClient: GetJobStatus failed with HTTP/{}", static_cast(Response.StatusCode)); + return Result; + } + + std::string Body(Response.AsText()); + std::string Err; + const json11::Json Json = json11::Json::parse(Body, Err); + + if (!Err.empty()) + { + ZEN_WARN("NomadTestClient: invalid JSON in GetJobStatus response: {}", Err); + return Result; + } + + Result.Id = Json["ID"].string_value(); + Result.Status = Json["Status"].string_value(); + if (const json11::Json Desc = Json["StatusDescription"]; Desc.is_string()) + { + Result.StatusDescription = Desc.string_value(); + } + + return Result; +} + +void +NomadTestClient::StopJob(std::string_view JobId) +{ + ZEN_TRACE_CPU("StopNomadJob"); + + HttpClient::Response Response = m_HttpClient.Delete(fmt::format("v1/job/{}", JobId)); + + if (!Response || !Response.IsSuccess()) + { + ZEN_WARN("NomadTestClient: StopJob failed for '{}'", JobId); + return; + } + + ZEN_INFO("NomadTestClient: job '{}' stopped", JobId); +} + +std::vector +NomadTestClient::GetAllocations(std::string_view JobId) +{ + ZEN_TRACE_CPU("GetNomadAllocations"); + + std::vector Allocs; + + HttpClient::Response Response = m_HttpClient.Get(fmt::format("v1/job/{}/allocations", JobId)); + + if (!Response || !Response.IsSuccess()) + { + ZEN_WARN("NomadTestClient: GetAllocations failed for '{}'", JobId); + return Allocs; + } + + std::string Body(Response.AsText()); + std::string Err; + const json11::Json Json = json11::Json::parse(Body, Err); + + if (!Err.empty() || !Json.is_array()) + { + return Allocs; + } + + for (const json11::Json& AllocVal : Json.array_items()) + { + NomadAllocInfo Alloc; + Alloc.Id = AllocVal["ID"].string_value(); + Alloc.ClientStatus = AllocVal["ClientStatus"].string_value(); + + if (const json11::Json TaskStates = AllocVal["TaskStates"]; TaskStates.is_object()) + { + for (const auto& [TaskName, TaskState] : TaskStates.object_items()) + { + if (TaskState["State"].is_string()) + { + Alloc.TaskState = TaskState["State"].string_value(); + } + } + } + + Allocs.push_back(std::move(Alloc)); + } + + return Allocs; +} + +std::vector +NomadTestClient::ListJobs(std::string_view Prefix) +{ + ZEN_TRACE_CPU("ListNomadJobs"); + + std::vector Jobs; + + std::string Url = "v1/jobs"; + if (!Prefix.empty()) + { + Url = fmt::format("v1/jobs?prefix={}", Prefix); + } + + HttpClient::Response Response = m_HttpClient.Get(Url); + + if (!Response || !Response.IsSuccess()) + { + ZEN_WARN("NomadTestClient: ListJobs failed"); + return Jobs; + } + + std::string Body(Response.AsText()); + std::string Err; + const json11::Json Json = json11::Json::parse(Body, Err); + + if (!Err.empty() || !Json.is_array()) + { + return Jobs; + } + + for (const json11::Json& JobVal : Json.array_items()) + { + NomadJobInfo Job; + Job.Id = JobVal["ID"].string_value(); + Job.Status = JobVal["Status"].string_value(); + if (const json11::Json Desc = JobVal["StatusDescription"]; Desc.is_string()) + { + Job.StatusDescription = Desc.string_value(); + } + Jobs.push_back(std::move(Job)); + } + + return Jobs; +} + +} // namespace zen::nomad diff --git a/src/zennomad/nomadprovisioner.cpp b/src/zennomad/nomadprovisioner.cpp new file mode 100644 index 000000000..3fe9c0ac3 --- /dev/null +++ b/src/zennomad/nomadprovisioner.cpp @@ -0,0 +1,264 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +namespace zen::nomad { + +NomadProvisioner::NomadProvisioner(const NomadConfig& Config, std::string_view OrchestratorEndpoint) +: m_Config(Config) +, m_OrchestratorEndpoint(OrchestratorEndpoint) +, m_ProcessId(static_cast(zen::GetCurrentProcessId())) +, m_Log(zen::logging::Get("nomad.provisioner")) +{ + ZEN_DEBUG("initializing provisioner (server: {}, driver: {}, max_cores: {}, cores_per_job: {}, max_jobs: {})", + m_Config.ServerUrl, + ToString(m_Config.TaskDriver), + m_Config.MaxCores, + m_Config.CoresPerJob, + m_Config.MaxJobs); + + m_Client = std::make_unique(m_Config); + if (!m_Client->Initialize()) + { + ZEN_ERROR("failed to initialize Nomad HTTP client"); + return; + } + + ZEN_DEBUG("Nomad HTTP client initialized, starting management thread"); + + m_Thread = std::thread([this] { ManagementThread(); }); +} + +NomadProvisioner::~NomadProvisioner() +{ + ZEN_DEBUG("provisioner shutting down"); + + m_ShouldExit.store(true); + m_WakeCV.notify_all(); + + if (m_Thread.joinable()) + { + m_Thread.join(); + } + + StopAllJobs(); + + ZEN_DEBUG("provisioner shutdown complete"); +} + +void +NomadProvisioner::SetTargetCoreCount(uint32_t Count) +{ + const uint32_t Clamped = std::min(Count, static_cast(m_Config.MaxCores)); + const uint32_t Previous = m_TargetCoreCount.exchange(Clamped); + + if (Clamped != Previous) + { + ZEN_DEBUG("target core count changed: {} -> {}", Previous, Clamped); + } + + m_WakeCV.notify_all(); +} + +NomadProvisioningStats +NomadProvisioner::GetStats() const +{ + NomadProvisioningStats Stats; + Stats.TargetCoreCount = m_TargetCoreCount.load(); + Stats.EstimatedCoreCount = m_EstimatedCoreCount.load(); + Stats.RunningJobCount = m_RunningJobCount.load(); + + { + std::lock_guard Lock(m_JobsLock); + Stats.ActiveJobCount = static_cast(m_Jobs.size()); + } + + return Stats; +} + +std::string +NomadProvisioner::GenerateJobId() +{ + const uint32_t Index = m_JobIndex.fetch_add(1); + + ExtendableStringBuilder<128> Builder; + Builder << m_Config.JobPrefix << "-" << m_ProcessId << "-" << Index; + return std::string(Builder.ToView()); +} + +void +NomadProvisioner::ManagementThread() +{ + ZEN_TRACE_CPU("Nomad_Mgmt"); + zen::SetCurrentThreadName("nomad_mgmt"); + + ZEN_INFO("Nomad management thread started"); + + while (!m_ShouldExit.load()) + { + ZEN_DEBUG("management cycle: target={} estimated={} running={} active={}", + m_TargetCoreCount.load(), + m_EstimatedCoreCount.load(), + m_RunningJobCount.load(), + [this] { + std::lock_guard Lock(m_JobsLock); + return m_Jobs.size(); + }()); + + SubmitNewJobs(); + PollExistingJobs(); + CleanupDeadJobs(); + + // Wait up to 5 seconds or until woken + std::unique_lock Lock(m_WakeMutex); + m_WakeCV.wait_for(Lock, std::chrono::seconds(5), [this] { return m_ShouldExit.load(); }); + } + + ZEN_INFO("Nomad management thread exiting"); +} + +void +NomadProvisioner::SubmitNewJobs() +{ + ZEN_TRACE_CPU("NomadProvisioner::SubmitNewJobs"); + + const uint32_t CoresPerJob = static_cast(m_Config.CoresPerJob); + + while (m_EstimatedCoreCount.load() < m_TargetCoreCount.load()) + { + { + std::lock_guard Lock(m_JobsLock); + if (static_cast(m_Jobs.size()) >= m_Config.MaxJobs) + { + ZEN_INFO("Nomad max jobs limit reached ({})", m_Config.MaxJobs); + break; + } + } + + if (m_ShouldExit.load()) + { + break; + } + + const std::string JobId = GenerateJobId(); + + ZEN_DEBUG("submitting job '{}' (estimated: {}, target: {})", JobId, m_EstimatedCoreCount.load(), m_TargetCoreCount.load()); + + const std::string JobJson = m_Client->BuildJobJson(JobId, m_OrchestratorEndpoint); + + NomadJobInfo JobInfo; + JobInfo.Id = JobId; + + if (!m_Client->SubmitJob(JobJson, JobInfo)) + { + ZEN_WARN("failed to submit Nomad job '{}'", JobId); + break; + } + + TrackedJob Tracked; + Tracked.JobId = JobId; + Tracked.Status = "pending"; + Tracked.Cores = static_cast(CoresPerJob); + + { + std::lock_guard Lock(m_JobsLock); + m_Jobs.push_back(std::move(Tracked)); + } + + m_EstimatedCoreCount.fetch_add(CoresPerJob); + + ZEN_INFO("Nomad job '{}' submitted (estimated cores: {})", JobId, m_EstimatedCoreCount.load()); + } +} + +void +NomadProvisioner::PollExistingJobs() +{ + ZEN_TRACE_CPU("NomadProvisioner::PollExistingJobs"); + + std::lock_guard Lock(m_JobsLock); + + for (auto& Job : m_Jobs) + { + if (m_ShouldExit.load()) + { + break; + } + + NomadJobInfo Info; + if (!m_Client->GetJobStatus(Job.JobId, Info)) + { + ZEN_DEBUG("failed to poll status for job '{}'", Job.JobId); + continue; + } + + const std::string PrevStatus = Job.Status; + Job.Status = Info.Status; + + if (PrevStatus != Job.Status) + { + ZEN_INFO("Nomad job '{}' status changed: {} -> {}", Job.JobId, PrevStatus, Job.Status); + + if (Job.Status == "running" && PrevStatus != "running") + { + m_RunningJobCount.fetch_add(1); + } + else if (Job.Status != "running" && PrevStatus == "running") + { + m_RunningJobCount.fetch_sub(1); + } + } + } +} + +void +NomadProvisioner::CleanupDeadJobs() +{ + ZEN_TRACE_CPU("NomadProvisioner::CleanupDeadJobs"); + + std::lock_guard Lock(m_JobsLock); + + for (auto It = m_Jobs.begin(); It != m_Jobs.end();) + { + if (It->Status == "dead") + { + ZEN_INFO("Nomad job '{}' is dead, removing from tracked jobs", It->JobId); + m_EstimatedCoreCount.fetch_sub(static_cast(It->Cores)); + It = m_Jobs.erase(It); + } + else + { + ++It; + } + } +} + +void +NomadProvisioner::StopAllJobs() +{ + ZEN_TRACE_CPU("NomadProvisioner::StopAllJobs"); + + std::lock_guard Lock(m_JobsLock); + + for (const auto& Job : m_Jobs) + { + ZEN_INFO("stopping Nomad job '{}' during shutdown", Job.JobId); + m_Client->StopJob(Job.JobId); + } + + m_Jobs.clear(); + m_EstimatedCoreCount.store(0); + m_RunningJobCount.store(0); +} + +} // namespace zen::nomad diff --git a/src/zennomad/xmake.lua b/src/zennomad/xmake.lua new file mode 100644 index 000000000..ef1a8b201 --- /dev/null +++ b/src/zennomad/xmake.lua @@ -0,0 +1,10 @@ +-- Copyright Epic Games, Inc. All Rights Reserved. + +target('zennomad') + set_kind("static") + set_group("libs") + add_headerfiles("**.h") + add_files("**.cpp") + add_includedirs("include", {public=true}) + add_deps("zencore", "zenhttp", "zenutil") + add_packages("json11") diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index f4b4d592b..43a4937f0 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -8186,7 +8186,7 @@ TEST_CASE("buildstorageoperations.partial.block.download" * doctest::skip(true)) Headers); REQUIRE(GetBlobRangesResponse.IsSuccess()); - MemoryView RangesMemoryView = GetBlobRangesResponse.ResponsePayload.GetView(); + [[maybe_unused]] MemoryView RangesMemoryView = GetBlobRangesResponse.ResponsePayload.GetView(); std::vector> PayloadRanges = GetBlobRangesResponse.GetRanges(Ranges); if (PayloadRanges.empty()) diff --git a/src/zenremotestore/chunking/chunkingcache.cpp b/src/zenremotestore/chunking/chunkingcache.cpp index f4e1c7837..e9b783a00 100644 --- a/src/zenremotestore/chunking/chunkingcache.cpp +++ b/src/zenremotestore/chunking/chunkingcache.cpp @@ -75,13 +75,13 @@ public: { Lock.ReleaseNow(); RwLock::ExclusiveLockScope EditLock(m_Lock); - if (auto RemoveIt = m_PathHashToEntry.find(PathHash); It != m_PathHashToEntry.end()) + if (auto RemoveIt = m_PathHashToEntry.find(PathHash); RemoveIt != m_PathHashToEntry.end()) { - CachedEntry& DeleteEntry = m_Entries[It->second]; + CachedEntry& DeleteEntry = m_Entries[RemoveIt->second]; DeleteEntry.Chunked = {}; DeleteEntry.ModificationTick = 0; - m_FreeEntryIndexes.push_back(It->second); - m_PathHashToEntry.erase(It); + m_FreeEntryIndexes.push_back(RemoveIt->second); + m_PathHashToEntry.erase(RemoveIt); } } } diff --git a/src/zenserver-test/compute-tests.cpp b/src/zenserver-test/compute-tests.cpp new file mode 100644 index 000000000..c90ac5d8b --- /dev/null +++ b/src/zenserver-test/compute-tests.cpp @@ -0,0 +1,1700 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#if ZEN_WITH_TESTS && ZEN_WITH_COMPUTE_SERVICES + +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include + +# include "zenserver-test.h" + +# include + +namespace zen::tests::compute { + +using namespace std::literals; + +// BuildSystemVersion and function version GUIDs matching zentest-appstub +static constexpr std::string_view kBuildSystemVersion = "17fe280d-ccd8-4be8-a9d1-89c944a70969"; +static constexpr std::string_view kRot13Version = "13131313-1313-1313-1313-131313131313"; +static constexpr std::string_view kSleepVersion = "88888888-8888-8888-8888-888888888888"; + +// In-memory implementation of ChunkResolver for test use. +// Stores compressed data keyed by decompressed content hash. +class InMemoryChunkResolver : public ChunkResolver +{ +public: + IoBuffer FindChunkByCid(const IoHash& DecompressedId) override + { + auto It = m_Chunks.find(DecompressedId); + if (It != m_Chunks.end()) + { + return It->second; + } + return {}; + } + + void AddChunk(const IoHash& DecompressedId, IoBuffer Data) { m_Chunks[DecompressedId] = std::move(Data); } + +private: + std::unordered_map m_Chunks; +}; + +// Read, compress, and register zentest-appstub as a worker. +// Returns the WorkerId (hash of the worker package object). +static IoHash +RegisterWorker(HttpClient& Client, ZenServerEnvironment& Env) +{ + std::filesystem::path AppStubPath = Env.ProgramBaseDir() / ("zentest-appstub" ZEN_EXE_SUFFIX_LITERAL); + + FileContents AppStubData = zen::ReadFile(AppStubPath); + REQUIRE_MESSAGE(!AppStubData.ErrorCode, fmt::format("Failed to read '{}': {}", AppStubPath.string(), AppStubData.ErrorCode.message())); + + IoBuffer AppStubBuffer = AppStubData.Flatten(); + + CompressedBuffer AppStubCompressed = CompressedBuffer::Compress(SharedBuffer::MakeView(AppStubBuffer.GetData(), AppStubBuffer.Size()), + OodleCompressor::Selkie, + OodleCompressionLevel::HyperFast4); + + const IoHash AppStubRawHash = AppStubCompressed.DecodeRawHash(); + const uint64_t AppStubRawSize = AppStubBuffer.Size(); + + CbAttachment AppStubAttachment(std::move(AppStubCompressed), AppStubRawHash); + + CbObjectWriter WorkerWriter; + WorkerWriter << "buildsystem_version"sv << Guid::FromString(kBuildSystemVersion); + WorkerWriter << "path"sv + << "zentest-appstub"sv; + + WorkerWriter.BeginArray("executables"sv); + WorkerWriter.BeginObject(); + WorkerWriter << "name"sv + << "zentest-appstub"sv; + WorkerWriter.AddAttachment("hash"sv, AppStubAttachment); + WorkerWriter << "size"sv << AppStubRawSize; + WorkerWriter.EndObject(); + WorkerWriter.EndArray(); + + WorkerWriter.BeginArray("functions"sv); + WorkerWriter.BeginObject(); + WorkerWriter << "name"sv + << "Rot13"sv; + WorkerWriter << "version"sv << Guid::FromString(kRot13Version); + WorkerWriter.EndObject(); + WorkerWriter.BeginObject(); + WorkerWriter << "name"sv + << "Sleep"sv; + WorkerWriter << "version"sv << Guid::FromString(kSleepVersion); + WorkerWriter.EndObject(); + WorkerWriter.EndArray(); + + CbPackage WorkerPackage; + WorkerPackage.SetObject(WorkerWriter.Save()); + WorkerPackage.AddAttachment(AppStubAttachment); + + const IoHash WorkerId = WorkerPackage.GetObjectHash(); + + const std::string WorkerUrl = fmt::format("/workers/{}", WorkerId.ToHexString()); + HttpClient::Response RegisterResp = Client.Post(WorkerUrl, std::move(WorkerPackage)); + REQUIRE_MESSAGE(RegisterResp, + fmt::format("Worker registration failed: status={}, body={}", int(RegisterResp.StatusCode), RegisterResp.ToText())); + + return WorkerId; +} + +// Build a Rot13 action CbPackage for the given input string. +static CbPackage +BuildRot13ActionPackage(std::string_view Input) +{ + CompressedBuffer InputCompressed = CompressedBuffer::Compress(SharedBuffer::MakeView(Input.data(), Input.size()), + OodleCompressor::Selkie, + OodleCompressionLevel::HyperFast4); + + const IoHash InputRawHash = InputCompressed.DecodeRawHash(); + const uint64_t InputRawSize = Input.size(); + + CbAttachment InputAttachment(std::move(InputCompressed), InputRawHash); + + CbObjectWriter ActionWriter; + ActionWriter << "Function"sv + << "Rot13"sv; + ActionWriter << "FunctionVersion"sv << Guid::FromString(kRot13Version); + ActionWriter << "BuildSystemVersion"sv << Guid::FromString(kBuildSystemVersion); + ActionWriter.BeginObject("Inputs"sv); + ActionWriter.BeginObject("Source"sv); + ActionWriter.AddAttachment("RawHash"sv, InputAttachment); + ActionWriter << "RawSize"sv << InputRawSize; + ActionWriter.EndObject(); + ActionWriter.EndObject(); + + CbPackage ActionPackage; + ActionPackage.SetObject(ActionWriter.Save()); + ActionPackage.AddAttachment(InputAttachment); + + return ActionPackage; +} + +// Build a Sleep action CbPackage. The worker sleeps for SleepTimeMs before returning its input. +static CbPackage +BuildSleepActionPackage(std::string_view Input, uint64_t SleepTimeMs) +{ + CompressedBuffer InputCompressed = CompressedBuffer::Compress(SharedBuffer::MakeView(Input.data(), Input.size()), + OodleCompressor::Selkie, + OodleCompressionLevel::HyperFast4); + + const IoHash InputRawHash = InputCompressed.DecodeRawHash(); + const uint64_t InputRawSize = Input.size(); + + CbAttachment InputAttachment(std::move(InputCompressed), InputRawHash); + + CbObjectWriter ActionWriter; + ActionWriter << "Function"sv + << "Sleep"sv; + ActionWriter << "FunctionVersion"sv << Guid::FromString(kSleepVersion); + ActionWriter << "BuildSystemVersion"sv << Guid::FromString(kBuildSystemVersion); + ActionWriter.BeginObject("Inputs"sv); + ActionWriter.BeginObject("Source"sv); + ActionWriter.AddAttachment("RawHash"sv, InputAttachment); + ActionWriter << "RawSize"sv << InputRawSize; + ActionWriter.EndObject(); + ActionWriter.EndObject(); + ActionWriter.BeginObject("Constants"sv); + ActionWriter << "SleepTimeMs"sv << SleepTimeMs; + ActionWriter.EndObject(); + + CbPackage ActionPackage; + ActionPackage.SetObject(ActionWriter.Save()); + ActionPackage.AddAttachment(InputAttachment); + + return ActionPackage; +} + +// Build a Sleep action CbObject and populate the chunk resolver with the input attachment. +static CbObject +BuildSleepActionForSession(std::string_view Input, uint64_t SleepTimeMs, InMemoryChunkResolver& Resolver) +{ + CompressedBuffer InputCompressed = CompressedBuffer::Compress(SharedBuffer::MakeView(Input.data(), Input.size()), + OodleCompressor::Selkie, + OodleCompressionLevel::HyperFast4); + + const IoHash InputRawHash = InputCompressed.DecodeRawHash(); + const uint64_t InputRawSize = Input.size(); + + Resolver.AddChunk(InputRawHash, InputCompressed.GetCompressed().Flatten().AsIoBuffer()); + + CbAttachment InputAttachment(std::move(InputCompressed), InputRawHash); + + CbObjectWriter ActionWriter; + ActionWriter << "Function"sv + << "Sleep"sv; + ActionWriter << "FunctionVersion"sv << Guid::FromString(kSleepVersion); + ActionWriter << "BuildSystemVersion"sv << Guid::FromString(kBuildSystemVersion); + ActionWriter.BeginObject("Inputs"sv); + ActionWriter.BeginObject("Source"sv); + ActionWriter.AddAttachment("RawHash"sv, InputAttachment); + ActionWriter << "RawSize"sv << InputRawSize; + ActionWriter.EndObject(); + ActionWriter.EndObject(); + ActionWriter.BeginObject("Constants"sv); + ActionWriter << "SleepTimeMs"sv << SleepTimeMs; + ActionWriter.EndObject(); + + return ActionWriter.Save(); +} + +static HttpClient::Response +PollForResult(HttpClient& Client, const std::string& ResultUrl, uint64_t TimeoutMs = 30'000) +{ + HttpClient::Response Resp; + Stopwatch Timer; + + while (Timer.GetElapsedTimeMs() < TimeoutMs) + { + Resp = Client.Get(ResultUrl); + + if (Resp.StatusCode == HttpResponseCode::OK) + { + break; + } + + Sleep(100); + } + + return Resp; +} + +static bool +PollForLsnInCompleted(HttpClient& Client, const std::string& CompletedUrl, int Lsn, uint64_t TimeoutMs = 30'000) +{ + Stopwatch Timer; + + while (Timer.GetElapsedTimeMs() < TimeoutMs) + { + HttpClient::Response Resp = Client.Get(CompletedUrl); + + if (Resp) + { + for (auto& Item : Resp.AsObject()["completed"sv]) + { + if (Item.AsInt32() == Lsn) + { + return true; + } + } + } + + Sleep(100); + } + + return false; +} + +static std::string +GetRot13Output(const CbPackage& ResultPackage) +{ + CbObject ResultObj = ResultPackage.GetObject(); + + IoHash OutputHash; + CbFieldView ValuesField = ResultObj["Values"sv]; + + if (CbFieldViewIterator It = begin(ValuesField); It.HasValue()) + { + OutputHash = (*It).AsObjectView()["RawHash"sv].AsHash(); + } + + REQUIRE_MESSAGE(OutputHash != IoHash::Zero, "Expected non-zero output hash in result Values array"); + + const CbAttachment* OutputAttachment = ResultPackage.FindAttachment(OutputHash); + REQUIRE_MESSAGE(OutputAttachment != nullptr, "Output attachment not found in result package"); + + CompressedBuffer OutputCompressed = OutputAttachment->AsCompressedBinary(); + SharedBuffer OutputData = OutputCompressed.Decompress(); + + return std::string(static_cast(OutputData.GetData()), OutputData.GetSize()); +} + +// Mock orchestrator HTTP service that serves GET /orch/agents with a controllable response. +class MockOrchestratorService : public HttpService +{ +public: + MockOrchestratorService() + { + // Initialize with empty worker list + CbObjectWriter Cbo; + Cbo.BeginArray("workers"sv); + Cbo.EndArray(); + m_WorkerList = Cbo.Save(); + } + + const char* BaseUri() const override { return "/orch/"; } + + void HandleRequest(HttpServerRequest& Request) override + { + if (Request.RequestVerb() == HttpVerb::kGet && Request.RelativeUri() == "agents"sv) + { + RwLock::SharedLockScope Lock(m_Lock); + Request.WriteResponse(HttpResponseCode::OK, m_WorkerList); + return; + } + Request.WriteResponse(HttpResponseCode::NotFound); + } + + void SetWorkerList(CbObject WorkerList) + { + RwLock::ExclusiveLockScope Lock(m_Lock); + m_WorkerList = std::move(WorkerList); + } + +private: + RwLock m_Lock; + CbObject m_WorkerList; +}; + +// Manages in-process ASIO HTTP server lifecycle for mock orchestrator. +struct MockOrchestratorFixture +{ + MockOrchestratorService Service; + ScopedTemporaryDirectory TmpDir; + Ref Server; + std::thread ServerThread; + uint16_t Port = 0; + + MockOrchestratorFixture() + { + HttpServerConfig Config; + Config.ServerClass = "asio"; + Config.ForceLoopback = true; + Server = CreateHttpServer(Config); + Server->RegisterService(Service); + Port = static_cast(Server->Initialize(TestEnv.GetNewPortNumber(), TmpDir.Path())); + ZEN_ASSERT(Port != 0); + ServerThread = std::thread([this]() { Server->Run(false); }); + } + + ~MockOrchestratorFixture() + { + Server->RequestExit(); + if (ServerThread.joinable()) + { + ServerThread.join(); + } + Server->Close(); + } + + std::string GetEndpoint() const { return fmt::format("http://localhost:{}", Port); } +}; + +// Build the CbObject response for /orch/agents matching the format UpdateCoordinatorState expects. +static CbObject +BuildAgentListResponse(std::initializer_list> Workers) +{ + CbObjectWriter Cbo; + Cbo.BeginArray("workers"sv); + for (const auto& [Id, Uri] : Workers) + { + Cbo.BeginObject(); + Cbo << "id"sv << Id; + Cbo << "uri"sv << Uri; + Cbo << "hostname"sv + << "localhost"sv; + Cbo << "reachable"sv << true; + Cbo << "dt"sv << uint64_t(0); + Cbo.EndObject(); + } + Cbo.EndArray(); + return Cbo.Save(); +} + +// Build the worker CbPackage for zentest-appstub AND populate the chunk resolver. +// This is the same logic as RegisterWorker() but returns the package instead of POSTing it. +static CbPackage +BuildWorkerPackage(ZenServerEnvironment& Env, InMemoryChunkResolver& Resolver) +{ + std::filesystem::path AppStubPath = Env.ProgramBaseDir() / ("zentest-appstub" ZEN_EXE_SUFFIX_LITERAL); + + FileContents AppStubData = zen::ReadFile(AppStubPath); + REQUIRE_MESSAGE(!AppStubData.ErrorCode, fmt::format("Failed to read '{}': {}", AppStubPath.string(), AppStubData.ErrorCode.message())); + + IoBuffer AppStubBuffer = AppStubData.Flatten(); + + CompressedBuffer AppStubCompressed = CompressedBuffer::Compress(SharedBuffer::MakeView(AppStubBuffer.GetData(), AppStubBuffer.Size()), + OodleCompressor::Selkie, + OodleCompressionLevel::HyperFast4); + + const IoHash AppStubRawHash = AppStubCompressed.DecodeRawHash(); + const uint64_t AppStubRawSize = AppStubBuffer.Size(); + + // Store compressed data in chunk resolver for when the remote runner needs it + Resolver.AddChunk(AppStubRawHash, AppStubCompressed.GetCompressed().Flatten().AsIoBuffer()); + + CbAttachment AppStubAttachment(std::move(AppStubCompressed), AppStubRawHash); + + CbObjectWriter WorkerWriter; + WorkerWriter << "buildsystem_version"sv << Guid::FromString(kBuildSystemVersion); + WorkerWriter << "path"sv + << "zentest-appstub"sv; + + WorkerWriter.BeginArray("executables"sv); + WorkerWriter.BeginObject(); + WorkerWriter << "name"sv + << "zentest-appstub"sv; + WorkerWriter.AddAttachment("hash"sv, AppStubAttachment); + WorkerWriter << "size"sv << AppStubRawSize; + WorkerWriter.EndObject(); + WorkerWriter.EndArray(); + + WorkerWriter.BeginArray("functions"sv); + WorkerWriter.BeginObject(); + WorkerWriter << "name"sv + << "Rot13"sv; + WorkerWriter << "version"sv << Guid::FromString(kRot13Version); + WorkerWriter.EndObject(); + WorkerWriter.BeginObject(); + WorkerWriter << "name"sv + << "Sleep"sv; + WorkerWriter << "version"sv << Guid::FromString(kSleepVersion); + WorkerWriter.EndObject(); + WorkerWriter.EndArray(); + + CbPackage WorkerPackage; + WorkerPackage.SetObject(WorkerWriter.Save()); + WorkerPackage.AddAttachment(AppStubAttachment); + + return WorkerPackage; +} + +// Build a Rot13 action CbObject (not CbPackage) and populate the chunk resolver with the input attachment. +static CbObject +BuildRot13ActionForSession(std::string_view Input, InMemoryChunkResolver& Resolver) +{ + CompressedBuffer InputCompressed = CompressedBuffer::Compress(SharedBuffer::MakeView(Input.data(), Input.size()), + OodleCompressor::Selkie, + OodleCompressionLevel::HyperFast4); + + const IoHash InputRawHash = InputCompressed.DecodeRawHash(); + const uint64_t InputRawSize = Input.size(); + + // Store compressed data in chunk resolver + Resolver.AddChunk(InputRawHash, InputCompressed.GetCompressed().Flatten().AsIoBuffer()); + + CbAttachment InputAttachment(std::move(InputCompressed), InputRawHash); + + CbObjectWriter ActionWriter; + ActionWriter << "Function"sv + << "Rot13"sv; + ActionWriter << "FunctionVersion"sv << Guid::FromString(kRot13Version); + ActionWriter << "BuildSystemVersion"sv << Guid::FromString(kBuildSystemVersion); + ActionWriter.BeginObject("Inputs"sv); + ActionWriter.BeginObject("Source"sv); + ActionWriter.AddAttachment("RawHash"sv, InputAttachment); + ActionWriter << "RawSize"sv << InputRawSize; + ActionWriter.EndObject(); + ActionWriter.EndObject(); + + return ActionWriter.Save(); +} + +TEST_SUITE_BEGIN("server.function"); + +TEST_CASE("function.rot13") +{ + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // Submit action via legacy /jobs/{worker} endpoint + const std::string JobUrl = fmt::format("/jobs/{}", WorkerId.ToHexString()); + HttpClient::Response SubmitResp = Client.Post(JobUrl, BuildRot13ActionPackage("Hello World"sv)); + REQUIRE_MESSAGE(SubmitResp, fmt::format("Job submission failed: status={}, body={}", int(SubmitResp.StatusCode), SubmitResp.ToText())); + + const int Lsn = SubmitResp.AsObject()["lsn"sv].AsInt32(); + REQUIRE_MESSAGE(Lsn != 0, "Expected non-zero LSN from job submission"); + + // Poll for result via legacy /jobs/{lsn} endpoint + const std::string ResultUrl = fmt::format("/jobs/{}", Lsn); + HttpClient::Response ResultResp = PollForResult(Client, ResultUrl); + REQUIRE_MESSAGE( + ResultResp.StatusCode == HttpResponseCode::OK, + fmt::format("Job did not complete in time. Last status: {}\nServer log:\n{}", int(ResultResp.StatusCode), Instance.GetLogOutput())); + + // Verify result: Rot13("Hello World") == "Uryyb Jbeyq" + CbPackage ResultPackage = ResultResp.AsPackage(); + REQUIRE_MESSAGE(bool(ResultPackage), fmt::format("Action failed (empty result package)\nServer log:\n{}", Instance.GetLogOutput())); + + CHECK_EQ(GetRot13Output(ResultPackage), "Uryyb Jbeyq"sv); +} + +TEST_CASE("function.workers") +{ + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + // Before registration, GET /workers should return an empty list + HttpClient::Response EmptyListResp = Client.Get("/workers"sv); + REQUIRE_MESSAGE(EmptyListResp, "Failed to list workers before registration"); + CHECK_EQ(EmptyListResp.AsObject()["workers"sv].AsArrayView().Num(), 0); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // GET /workers — the registered worker should appear in the listing + HttpClient::Response ListResp = Client.Get("/workers"sv); + REQUIRE_MESSAGE(ListResp, "Failed to list workers after registration"); + + bool WorkerFound = false; + for (auto& Item : ListResp.AsObject()["workers"sv]) + { + if (Item.AsHash() == WorkerId) + { + WorkerFound = true; + break; + } + } + + REQUIRE_MESSAGE(WorkerFound, fmt::format("Worker {} not found in worker listing", WorkerId.ToHexString())); + + // GET /workers/{worker} — descriptor should match what was registered + const std::string WorkerUrl = fmt::format("/workers/{}", WorkerId.ToHexString()); + HttpClient::Response DescResp = Client.Get(WorkerUrl); + REQUIRE_MESSAGE(DescResp, fmt::format("Failed to get worker descriptor: status={}", int(DescResp.StatusCode))); + + CbObject Desc = DescResp.AsObject(); + CHECK_EQ(Desc["buildsystem_version"sv].AsUuid(), Guid::FromString(kBuildSystemVersion)); + CHECK_EQ(Desc["path"sv].AsString(), "zentest-appstub"sv); + + bool Rot13Found = false; + bool SleepFound = false; + for (auto& Item : Desc["functions"sv]) + { + std::string_view Name = Item.AsObjectView()["name"sv].AsString(); + if (Name == "Rot13"sv) + { + CHECK_EQ(Item.AsObjectView()["version"sv].AsUuid(), Guid::FromString(kRot13Version)); + Rot13Found = true; + } + else if (Name == "Sleep"sv) + { + CHECK_EQ(Item.AsObjectView()["version"sv].AsUuid(), Guid::FromString(kSleepVersion)); + SleepFound = true; + } + } + + CHECK_MESSAGE(Rot13Found, "Rot13 function not found in worker descriptor"); + CHECK_MESSAGE(SleepFound, "Sleep function not found in worker descriptor"); + + // GET /workers/{unknown} — should return 404 + const std::string UnknownUrl = fmt::format("/workers/{}", IoHash::Zero.ToHexString()); + HttpClient::Response NotFoundResp = Client.Get(UnknownUrl); + CHECK_EQ(NotFoundResp.StatusCode, HttpResponseCode::NotFound); +} + +TEST_CASE("function.queues.lifecycle") +{ + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // Create a queue + HttpClient::Response CreateResp = Client.Post("/queues"sv); + REQUIRE_MESSAGE(CreateResp, fmt::format("Queue creation failed: status={}, body={}", int(CreateResp.StatusCode), CreateResp.ToText())); + + const int QueueId = CreateResp.AsObject()["queue_id"sv].AsInt32(); + REQUIRE_MESSAGE(QueueId != 0, "Expected non-zero queue_id from queue creation"); + + // Verify the queue appears in the listing + HttpClient::Response ListResp = Client.Get("/queues"sv); + REQUIRE_MESSAGE(ListResp, "Failed to list queues"); + + bool QueueFound = false; + for (auto& Item : ListResp.AsObject()["queues"sv]) + { + if (Item.AsObjectView()["queue_id"sv].AsInt32() == QueueId) + { + QueueFound = true; + break; + } + } + + REQUIRE_MESSAGE(QueueFound, fmt::format("Queue {} not found in queue listing", QueueId)); + + // Submit action via queue-scoped endpoint + const std::string JobUrl = fmt::format("/queues/{}/jobs/{}", QueueId, WorkerId.ToHexString()); + HttpClient::Response SubmitResp = Client.Post(JobUrl, BuildRot13ActionPackage("Hello World"sv)); + REQUIRE_MESSAGE(SubmitResp, + fmt::format("Queue job submission failed: status={}, body={}", int(SubmitResp.StatusCode), SubmitResp.ToText())); + + const int Lsn = SubmitResp.AsObject()["lsn"sv].AsInt32(); + REQUIRE_MESSAGE(Lsn != 0, "Expected non-zero LSN from queue job submission"); + + // Poll for completion via queue-scoped /completed endpoint + const std::string CompletedUrl = fmt::format("/queues/{}/completed", QueueId); + REQUIRE_MESSAGE(PollForLsnInCompleted(Client, CompletedUrl, Lsn), + fmt::format("LSN {} did not appear in queue {} completed list within timeout\nServer log:\n{}", + Lsn, + QueueId, + Instance.GetLogOutput())); + + // Retrieve result via queue-scoped /jobs/{lsn} endpoint + const std::string ResultUrl = fmt::format("/queues/{}/jobs/{}", QueueId, Lsn); + HttpClient::Response ResultResp = Client.Get(ResultUrl); + REQUIRE_MESSAGE( + ResultResp.StatusCode == HttpResponseCode::OK, + fmt::format("Failed to retrieve result: status={}\nServer log:\n{}", int(ResultResp.StatusCode), Instance.GetLogOutput())); + + // Verify result: Rot13("Hello World") == "Uryyb Jbeyq" + CbPackage ResultPackage = ResultResp.AsPackage(); + REQUIRE_MESSAGE(bool(ResultPackage), fmt::format("Empty result package\nServer log:\n{}", Instance.GetLogOutput())); + + CHECK_EQ(GetRot13Output(ResultPackage), "Uryyb Jbeyq"sv); + + // Verify queue status reflects completion + const std::string StatusUrl = fmt::format("/queues/{}", QueueId); + HttpClient::Response StatusResp = Client.Get(StatusUrl); + REQUIRE_MESSAGE(StatusResp, "Failed to get queue status"); + + CbObject QueueStatus = StatusResp.AsObject(); + CHECK_EQ(QueueStatus["completed_count"sv].AsInt32(), 1); + CHECK_EQ(QueueStatus["active_count"sv].AsInt32(), 0); + CHECK_EQ(QueueStatus["failed_count"sv].AsInt32(), 0); + CHECK_EQ(std::string(QueueStatus["state"sv].AsString()), "active"); +} + +TEST_CASE("function.queues.cancel") +{ + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // Create a queue + HttpClient::Response CreateResp = Client.Post("/queues"sv); + REQUIRE_MESSAGE(CreateResp, "Queue creation failed"); + + const int QueueId = CreateResp.AsObject()["queue_id"sv].AsInt32(); + REQUIRE_MESSAGE(QueueId != 0, "Expected non-zero queue_id from queue creation"); + + // Submit a job + const std::string JobUrl = fmt::format("/queues/{}/jobs/{}", QueueId, WorkerId.ToHexString()); + HttpClient::Response SubmitResp = Client.Post(JobUrl, BuildRot13ActionPackage("Hello World"sv)); + REQUIRE_MESSAGE(SubmitResp, fmt::format("Job submission failed: status={}, body={}", int(SubmitResp.StatusCode), SubmitResp.ToText())); + + // Cancel the queue + const std::string QueueUrl = fmt::format("/queues/{}", QueueId); + HttpClient::Response CancelResp = Client.Delete(QueueUrl); + REQUIRE_MESSAGE(CancelResp.StatusCode == HttpResponseCode::NoContent, + fmt::format("Queue cancellation failed: status={}, body={}", int(CancelResp.StatusCode), CancelResp.ToText())); + + // Verify queue status shows cancelled + HttpClient::Response StatusResp = Client.Get(QueueUrl); + REQUIRE_MESSAGE(StatusResp, "Failed to get queue status after cancel"); + + CbObject QueueStatus = StatusResp.AsObject(); + CHECK_EQ(std::string(QueueStatus["state"sv].AsString()), "cancelled"); +} + +TEST_CASE("function.queues.remote") +{ + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // Create a remote queue — response includes both an integer queue_id and an OID queue_token + HttpClient::Response CreateResp = Client.Post("/queues/remote"sv); + REQUIRE_MESSAGE(CreateResp, + fmt::format("Remote queue creation failed: status={}, body={}", int(CreateResp.StatusCode), CreateResp.ToText())); + + CbObject CreateObj = CreateResp.AsObject(); + const std::string QueueToken = std::string(CreateObj["queue_token"sv].AsString()); + REQUIRE_MESSAGE(!QueueToken.empty(), "Expected non-empty queue_token from remote queue creation"); + + // All subsequent requests use the opaque token in place of the integer queue id + const std::string JobUrl = fmt::format("/queues/{}/jobs/{}", QueueToken, WorkerId.ToHexString()); + HttpClient::Response SubmitResp = Client.Post(JobUrl, BuildRot13ActionPackage("Hello World"sv)); + REQUIRE_MESSAGE(SubmitResp, + fmt::format("Remote queue job submission failed: status={}, body={}", int(SubmitResp.StatusCode), SubmitResp.ToText())); + + const int Lsn = SubmitResp.AsObject()["lsn"sv].AsInt32(); + REQUIRE_MESSAGE(Lsn != 0, "Expected non-zero LSN from remote queue job submission"); + + // Poll for completion via the token-addressed /completed endpoint + const std::string CompletedUrl = fmt::format("/queues/{}/completed", QueueToken); + REQUIRE_MESSAGE( + PollForLsnInCompleted(Client, CompletedUrl, Lsn), + fmt::format("LSN {} did not appear in remote queue completed list within timeout\nServer log:\n{}", Lsn, Instance.GetLogOutput())); + + // Retrieve result via the token-addressed /jobs/{lsn} endpoint + const std::string ResultUrl = fmt::format("/queues/{}/jobs/{}", QueueToken, Lsn); + HttpClient::Response ResultResp = Client.Get(ResultUrl); + REQUIRE_MESSAGE(ResultResp.StatusCode == HttpResponseCode::OK, + fmt::format("Failed to retrieve result from remote queue: status={}\nServer log:\n{}", + int(ResultResp.StatusCode), + Instance.GetLogOutput())); + + // Verify result: Rot13("Hello World") == "Uryyb Jbeyq" + CbPackage ResultPackage = ResultResp.AsPackage(); + REQUIRE_MESSAGE(bool(ResultPackage), fmt::format("Empty result package\nServer log:\n{}", Instance.GetLogOutput())); + + CHECK_EQ(GetRot13Output(ResultPackage), "Uryyb Jbeyq"sv); +} + +TEST_CASE("function.queues.cancel_running") +{ + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // Create a queue + HttpClient::Response CreateResp = Client.Post("/queues"sv); + REQUIRE_MESSAGE(CreateResp, "Queue creation failed"); + + const int QueueId = CreateResp.AsObject()["queue_id"sv].AsInt32(); + REQUIRE_MESSAGE(QueueId != 0, "Expected non-zero queue_id from queue creation"); + + // Submit a Sleep job long enough that it will still be running when we cancel + const std::string JobUrl = fmt::format("/queues/{}/jobs/{}", QueueId, WorkerId.ToHexString()); + HttpClient::Response SubmitResp = Client.Post(JobUrl, BuildSleepActionPackage("data"sv, 30'000)); + REQUIRE_MESSAGE(SubmitResp, + fmt::format("Sleep job submission failed: status={}, body={}", int(SubmitResp.StatusCode), SubmitResp.ToText())); + + const int Lsn = SubmitResp.AsObject()["lsn"sv].AsInt32(); + REQUIRE_MESSAGE(Lsn != 0, "Expected non-zero LSN from Sleep job submission"); + + // Wait for the worker process to start executing before cancelling + Sleep(1'000); + + // Cancel the queue, which should interrupt the running Sleep job + const std::string QueueUrl = fmt::format("/queues/{}", QueueId); + HttpClient::Response CancelResp = Client.Delete(QueueUrl); + REQUIRE_MESSAGE(CancelResp.StatusCode == HttpResponseCode::NoContent, + fmt::format("Queue cancellation failed: status={}, body={}", int(CancelResp.StatusCode), CancelResp.ToText())); + + // The cancelled job should appear in the /completed endpoint once the process exits + const std::string CompletedUrl = fmt::format("/queues/{}/completed", QueueId); + REQUIRE_MESSAGE(PollForLsnInCompleted(Client, CompletedUrl, Lsn), + fmt::format("LSN {} did not appear in queue {} completed list after cancel\nServer log:\n{}", + Lsn, + QueueId, + Instance.GetLogOutput())); + + // Verify the queue reflects one cancelled action + HttpClient::Response StatusResp = Client.Get(QueueUrl); + REQUIRE_MESSAGE(StatusResp, "Failed to get queue status after cancel"); + + CbObject QueueStatus = StatusResp.AsObject(); + CHECK_EQ(std::string(QueueStatus["state"sv].AsString()), "cancelled"); + CHECK_EQ(QueueStatus["cancelled_count"sv].AsInt32(), 1); + CHECK_EQ(QueueStatus["completed_count"sv].AsInt32(), 0); +} + +TEST_CASE("function.queues.remote_cancel") +{ + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // Create a remote queue to obtain an OID token for token-addressed cancellation + HttpClient::Response CreateResp = Client.Post("/queues/remote"sv); + REQUIRE_MESSAGE(CreateResp, + fmt::format("Remote queue creation failed: status={}, body={}", int(CreateResp.StatusCode), CreateResp.ToText())); + + const std::string QueueToken = std::string(CreateResp.AsObject()["queue_token"sv].AsString()); + REQUIRE_MESSAGE(!QueueToken.empty(), "Expected non-empty queue_token from remote queue creation"); + + // Submit a long-running Sleep job via the token-addressed endpoint + const std::string JobUrl = fmt::format("/queues/{}/jobs/{}", QueueToken, WorkerId.ToHexString()); + HttpClient::Response SubmitResp = Client.Post(JobUrl, BuildSleepActionPackage("data"sv, 30'000)); + REQUIRE_MESSAGE(SubmitResp, + fmt::format("Sleep job submission failed: status={}, body={}", int(SubmitResp.StatusCode), SubmitResp.ToText())); + + const int Lsn = SubmitResp.AsObject()["lsn"sv].AsInt32(); + REQUIRE_MESSAGE(Lsn != 0, "Expected non-zero LSN from Sleep job submission"); + + // Wait for the worker process to start executing before cancelling + Sleep(1'000); + + // Cancel the queue via its OID token + const std::string QueueUrl = fmt::format("/queues/{}", QueueToken); + HttpClient::Response CancelResp = Client.Delete(QueueUrl); + REQUIRE_MESSAGE(CancelResp.StatusCode == HttpResponseCode::NoContent, + fmt::format("Remote queue cancellation failed: status={}, body={}", int(CancelResp.StatusCode), CancelResp.ToText())); + + // The cancelled job should appear in the token-addressed /completed endpoint + const std::string CompletedUrl = fmt::format("/queues/{}/completed", QueueToken); + REQUIRE_MESSAGE( + PollForLsnInCompleted(Client, CompletedUrl, Lsn), + fmt::format("LSN {} did not appear in remote queue completed list after cancel\nServer log:\n{}", Lsn, Instance.GetLogOutput())); + + // Verify the queue status reflects the cancellation + HttpClient::Response StatusResp = Client.Get(QueueUrl); + REQUIRE_MESSAGE(StatusResp, "Failed to get remote queue status after cancel"); + + CbObject QueueStatus = StatusResp.AsObject(); + CHECK_EQ(std::string(QueueStatus["state"sv].AsString()), "cancelled"); + CHECK_EQ(QueueStatus["cancelled_count"sv].AsInt32(), 1); + CHECK_EQ(QueueStatus["completed_count"sv].AsInt32(), 0); +} + +TEST_CASE("function.queues.drain") +{ + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // Create a queue + HttpClient::Response CreateResp = Client.Post("/queues"sv); + REQUIRE_MESSAGE(CreateResp, "Queue creation failed"); + + const int QueueId = CreateResp.AsObject()["queue_id"sv].AsInt32(); + const std::string QueueUrl = fmt::format("/queues/{}", QueueId); + + // Submit a long-running job so we can verify it completes even after drain + const std::string JobUrl = fmt::format("/queues/{}/jobs/{}", QueueId, WorkerId.ToHexString()); + HttpClient::Response Submit1 = Client.Post(JobUrl, BuildSleepActionPackage("data"sv, 2'000)); + REQUIRE_MESSAGE(Submit1, fmt::format("First job submission failed: status={}", int(Submit1.StatusCode))); + const int Lsn1 = Submit1.AsObject()["lsn"sv].AsInt32(); + + // Drain the queue + const std::string DrainUrl = fmt::format("/queues/{}/drain", QueueId); + HttpClient::Response DrainResp = Client.Post(DrainUrl); + REQUIRE_MESSAGE(DrainResp, fmt::format("Drain failed: status={}, body={}", int(DrainResp.StatusCode), DrainResp.ToText())); + CHECK_EQ(std::string(DrainResp.AsObject()["state"sv].AsString()), "draining"); + + // Second submission should be rejected with 424 + HttpClient::Response Submit2 = Client.Post(JobUrl, BuildRot13ActionPackage("Hello"sv)); + CHECK_EQ(Submit2.StatusCode, HttpResponseCode::FailedDependency); + CHECK_EQ(std::string(Submit2.AsObject()["error"sv].AsString()), "queue is draining"); + + // First job should still complete + const std::string CompletedUrl = fmt::format("/queues/{}/completed", QueueId); + REQUIRE_MESSAGE(PollForLsnInCompleted(Client, CompletedUrl, Lsn1), + fmt::format("LSN {} did not complete after drain\nServer log:\n{}", Lsn1, Instance.GetLogOutput())); + + // Queue status should show draining + complete + HttpClient::Response StatusResp = Client.Get(QueueUrl); + REQUIRE_MESSAGE(StatusResp, "Failed to get queue status"); + + CbObject QueueStatus = StatusResp.AsObject(); + CHECK_EQ(std::string(QueueStatus["state"sv].AsString()), "draining"); + CHECK(QueueStatus["is_complete"sv].AsBool()); +} + +TEST_CASE("function.priority") +{ + // Spawn server with max-actions=1 to guarantee serialized action execution, + // which lets us deterministically verify that higher-priority pending jobs + // are scheduled before lower-priority ones. + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady("--max-actions=1"); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // Create a queue for all test jobs + HttpClient::Response CreateResp = Client.Post("/queues"sv); + REQUIRE_MESSAGE(CreateResp, "Queue creation failed"); + + const int QueueId = CreateResp.AsObject()["queue_id"sv].AsInt32(); + REQUIRE_MESSAGE(QueueId != 0, "Expected non-zero queue_id"); + + // Submit a blocker Sleep job to occupy the single execution slot. + // Once the blocker is running, the scheduler must choose among the pending + // jobs by priority when the slot becomes free. + const std::string BlockerJobUrl = fmt::format("/queues/{}/jobs/{}?priority=0", QueueId, WorkerId.ToHexString()); + HttpClient::Response BlockerResp = Client.Post(BlockerJobUrl, BuildSleepActionPackage("data"sv, 1'000)); + REQUIRE_MESSAGE(BlockerResp, fmt::format("Blocker job submission failed: status={}", int(BlockerResp.StatusCode))); + + // Submit 3 low-priority Rot13 jobs + const std::string LowJobUrl = fmt::format("/queues/{}/jobs/{}?priority=0", QueueId, WorkerId.ToHexString()); + + HttpClient::Response LowResp1 = Client.Post(LowJobUrl, BuildRot13ActionPackage("low1"sv)); + REQUIRE_MESSAGE(LowResp1, "Low-priority job 1 submission failed"); + const int LsnLow1 = LowResp1.AsObject()["lsn"sv].AsInt32(); + + HttpClient::Response LowResp2 = Client.Post(LowJobUrl, BuildRot13ActionPackage("low2"sv)); + REQUIRE_MESSAGE(LowResp2, "Low-priority job 2 submission failed"); + const int LsnLow2 = LowResp2.AsObject()["lsn"sv].AsInt32(); + + HttpClient::Response LowResp3 = Client.Post(LowJobUrl, BuildRot13ActionPackage("low3"sv)); + REQUIRE_MESSAGE(LowResp3, "Low-priority job 3 submission failed"); + const int LsnLow3 = LowResp3.AsObject()["lsn"sv].AsInt32(); + + // Submit 1 high-priority Rot13 job — should execute before the low-priority ones + const std::string HighJobUrl = fmt::format("/queues/{}/jobs/{}?priority=10", QueueId, WorkerId.ToHexString()); + HttpClient::Response HighResp = Client.Post(HighJobUrl, BuildRot13ActionPackage("high"sv)); + REQUIRE_MESSAGE(HighResp, "High-priority job submission failed"); + const int LsnHigh = HighResp.AsObject()["lsn"sv].AsInt32(); + + // Wait for all 4 priority-test jobs to appear in the queue's completed list. + // This avoids any snapshot-timing race: by the time we compare timestamps, all + // jobs have already finished and their history entries are stable. + const std::string CompletedUrl = fmt::format("/queues/{}/completed", QueueId); + + { + bool AllCompleted = false; + Stopwatch WaitTimer; + + while (!AllCompleted && WaitTimer.GetElapsedTimeMs() < 30'000) + { + HttpClient::Response Resp = Client.Get(CompletedUrl); + + if (Resp) + { + bool FoundHigh = false; + bool FoundLow1 = false; + bool FoundLow2 = false; + bool FoundLow3 = false; + + CbObject RespObj = Resp.AsObject(); + + for (auto& Item : RespObj["completed"sv]) + { + const int Lsn = Item.AsInt32(); + if (Lsn == LsnHigh) + { + FoundHigh = true; + } + else if (Lsn == LsnLow1) + { + FoundLow1 = true; + } + else if (Lsn == LsnLow2) + { + FoundLow2 = true; + } + else if (Lsn == LsnLow3) + { + FoundLow3 = true; + } + } + + AllCompleted = FoundHigh && FoundLow1 && FoundLow2 && FoundLow3; + } + + if (!AllCompleted) + { + Sleep(100); + } + } + + REQUIRE_MESSAGE( + AllCompleted, + fmt::format( + "Not all priority test jobs completed within timeout (lsnHigh={} lsnLow1={} lsnLow2={} lsnLow3={})\nServer log:\n{}", + LsnHigh, + LsnLow1, + LsnLow2, + LsnLow3, + Instance.GetLogOutput())); + } + + // Query the queue-scoped history to obtain the time_Completed timestamp for each + // job. The history endpoint records when each RunnerAction::State transition + // occurred, so time_Completed is the wall-clock tick at which the action finished. + // Using the queue-scoped endpoint avoids exposing history from other queues. + const std::string HistoryUrl = fmt::format("/queues/{}/history", QueueId); + HttpClient::Response HistoryResp = Client.Get(HistoryUrl); + REQUIRE_MESSAGE(HistoryResp, "Failed to query queue action history"); + + CbObject HistoryObj = HistoryResp.AsObject(); + + auto GetCompletedTimestamp = [&](int Lsn) -> uint64_t { + for (auto& Item : HistoryObj["history"sv]) + { + if (Item.AsObjectView()["lsn"sv].AsInt32() == Lsn) + { + return Item.AsObjectView()["time_Completed"sv].AsUInt64(); + } + } + return 0; + }; + + const uint64_t TimeHigh = GetCompletedTimestamp(LsnHigh); + const uint64_t TimeLow1 = GetCompletedTimestamp(LsnLow1); + const uint64_t TimeLow2 = GetCompletedTimestamp(LsnLow2); + const uint64_t TimeLow3 = GetCompletedTimestamp(LsnLow3); + + REQUIRE_MESSAGE(TimeHigh != 0, fmt::format("lsnHigh={} not found in action history", LsnHigh)); + REQUIRE_MESSAGE(TimeLow1 != 0, fmt::format("lsnLow1={} not found in action history", LsnLow1)); + REQUIRE_MESSAGE(TimeLow2 != 0, fmt::format("lsnLow2={} not found in action history", LsnLow2)); + REQUIRE_MESSAGE(TimeLow3 != 0, fmt::format("lsnLow3={} not found in action history", LsnLow3)); + + // The high-priority job must have completed strictly before every low-priority job + CHECK_MESSAGE(TimeHigh < TimeLow1, + fmt::format("Priority ordering violated: lsnHigh={} completed at t={} but lsnLow1={} completed at t={} (expected later)", + LsnHigh, + TimeHigh, + LsnLow1, + TimeLow1)); + CHECK_MESSAGE(TimeHigh < TimeLow2, + fmt::format("Priority ordering violated: lsnHigh={} completed at t={} but lsnLow2={} completed at t={} (expected later)", + LsnHigh, + TimeHigh, + LsnLow2, + TimeLow2)); + CHECK_MESSAGE(TimeHigh < TimeLow3, + fmt::format("Priority ordering violated: lsnHigh={} completed at t={} but lsnLow3={} completed at t={} (expected later)", + LsnHigh, + TimeHigh, + LsnLow3, + TimeLow3)); +} + +////////////////////////////////////////////////////////////////////////// +// Remote worker synchronization tests +// +// These tests exercise the orchestrator discovery path where new compute +// nodes appear over time and must receive previously registered workers +// via SyncWorkersToRunner(). + +TEST_CASE("function.remote.worker_sync_on_discovery") +{ + // Spawn real zenserver in compute mode + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t ServerPort = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(ServerPort != 0, Instance.GetLogOutput()); + + const std::string ServerUri = fmt::format("http://localhost:{}", ServerPort); + + // Start mock orchestrator with empty worker list + MockOrchestratorFixture MockOrch; + + // Create session infrastructure + InMemoryChunkResolver Resolver; + ScopedTemporaryDirectory SessionBaseDir; + zen::compute::ComputeServiceSession Session(Resolver); + Session.SetOrchestratorEndpoint(MockOrch.GetEndpoint()); + Session.SetOrchestratorBasePath(SessionBaseDir.Path()); + Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Ready); + + // Register worker on session (stored locally, no runners yet) + CbPackage WorkerPackage = BuildWorkerPackage(TestEnv, Resolver); + Session.RegisterWorker(WorkerPackage); + + // Update mock orchestrator to advertise the real server + MockOrch.Service.SetWorkerList(BuildAgentListResponse({{"worker-1", ServerUri}})); + + // Wait for scheduler to discover the runner (~5s throttle + margin) + Sleep(7'000); + + // Submit Rot13 action via session + CbObject ActionObj = BuildRot13ActionForSession("Hello World"sv, Resolver); + + zen::compute::ComputeServiceSession::EnqueueResult EnqueueRes = Session.EnqueueAction(ActionObj, 0); + REQUIRE_MESSAGE(EnqueueRes, "Action enqueue failed"); + + // Poll for result + CbPackage ResultPackage; + HttpResponseCode ResultCode = HttpResponseCode::Accepted; + Stopwatch Timer; + + while (Timer.GetElapsedTimeMs() < 30'000) + { + ResultCode = Session.GetActionResult(EnqueueRes.Lsn, ResultPackage); + if (ResultCode == HttpResponseCode::OK) + { + break; + } + Sleep(200); + } + + REQUIRE_MESSAGE( + ResultCode == HttpResponseCode::OK, + fmt::format("Action did not complete in time. Last status: {}\nServer log:\n{}", int(ResultCode), Instance.GetLogOutput())); + + REQUIRE_MESSAGE(bool(ResultPackage), fmt::format("Empty result package\nServer log:\n{}", Instance.GetLogOutput())); + + CHECK_EQ(GetRot13Output(ResultPackage), "Uryyb Jbeyq"sv); + + Session.Shutdown(); +} + +TEST_CASE("function.remote.late_runner_discovery") +{ + // Spawn first server + ZenServerInstance Instance1(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance1.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port1 = Instance1.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port1 != 0, Instance1.GetLogOutput()); + + const std::string ServerUri1 = fmt::format("http://localhost:{}", Port1); + + // Start mock orchestrator advertising W1 + MockOrchestratorFixture MockOrch; + MockOrch.Service.SetWorkerList(BuildAgentListResponse({{"worker-1", ServerUri1}})); + + // Create session and register worker + InMemoryChunkResolver Resolver; + ScopedTemporaryDirectory SessionBaseDir; + zen::compute::ComputeServiceSession Session(Resolver); + Session.SetOrchestratorEndpoint(MockOrch.GetEndpoint()); + Session.SetOrchestratorBasePath(SessionBaseDir.Path()); + Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Ready); + + CbPackage WorkerPackage = BuildWorkerPackage(TestEnv, Resolver); + Session.RegisterWorker(WorkerPackage); + + // Wait for W1 discovery + Sleep(7'000); + + // Baseline: submit Rot13 action and verify it completes on W1 + { + CbObject ActionObj = BuildRot13ActionForSession("Hello World"sv, Resolver); + + zen::compute::ComputeServiceSession::EnqueueResult EnqueueRes = Session.EnqueueAction(ActionObj, 0); + REQUIRE_MESSAGE(EnqueueRes, "Baseline action enqueue failed"); + + CbPackage ResultPackage; + HttpResponseCode ResultCode = HttpResponseCode::Accepted; + Stopwatch Timer; + + while (Timer.GetElapsedTimeMs() < 30'000) + { + ResultCode = Session.GetActionResult(EnqueueRes.Lsn, ResultPackage); + if (ResultCode == HttpResponseCode::OK) + { + break; + } + Sleep(200); + } + + REQUIRE_MESSAGE(ResultCode == HttpResponseCode::OK, + fmt::format("Baseline action did not complete in time\nServer log:\n{}", Instance1.GetLogOutput())); + + CHECK_EQ(GetRot13Output(ResultPackage), "Uryyb Jbeyq"sv); + } + + // Spawn second server + ZenServerInstance Instance2(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance2.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port2 = Instance2.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port2 != 0, Instance2.GetLogOutput()); + + const std::string ServerUri2 = fmt::format("http://localhost:{}", Port2); + + // Update mock orchestrator to include both W1 and W2 + MockOrch.Service.SetWorkerList(BuildAgentListResponse({{"worker-1", ServerUri1}, {"worker-2", ServerUri2}})); + + // Wait for W2 discovery + Sleep(7'000); + + // Verify W2 received the worker by querying its /compute/workers endpoint directly + { + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port2); + HttpClient Client(ComputeBaseUri); + HttpClient::Response ListResp = Client.Get("/workers"sv); + REQUIRE_MESSAGE(ListResp, "Failed to list workers on W2"); + + bool WorkerFound = false; + for (auto& Item : ListResp.AsObject()["workers"sv]) + { + if (Item.AsHash() == WorkerPackage.GetObjectHash()) + { + WorkerFound = true; + break; + } + } + + REQUIRE_MESSAGE(WorkerFound, + fmt::format("Worker not found on W2 after discovery — SyncWorkersToRunner may have failed\nW2 log:\n{}", + Instance2.GetLogOutput())); + } + + // Submit another action and verify it completes (could run on either W1 or W2) + { + CbObject ActionObj = BuildRot13ActionForSession("Second Test"sv, Resolver); + + zen::compute::ComputeServiceSession::EnqueueResult EnqueueRes = Session.EnqueueAction(ActionObj, 0); + REQUIRE_MESSAGE(EnqueueRes, "Second action enqueue failed"); + + CbPackage ResultPackage; + HttpResponseCode ResultCode = HttpResponseCode::Accepted; + Stopwatch Timer; + + while (Timer.GetElapsedTimeMs() < 30'000) + { + ResultCode = Session.GetActionResult(EnqueueRes.Lsn, ResultPackage); + if (ResultCode == HttpResponseCode::OK) + { + break; + } + Sleep(200); + } + + REQUIRE_MESSAGE(ResultCode == HttpResponseCode::OK, + fmt::format("Second action did not complete in time\nW1 log:\n{}\nW2 log:\n{}", + Instance1.GetLogOutput(), + Instance2.GetLogOutput())); + + // Rot13("Second Test") = "Frpbaq Grfg" + CHECK_EQ(GetRot13Output(ResultPackage), "Frpbaq Grfg"sv); + } + + Session.Shutdown(); +} + +TEST_CASE("function.remote.queue_association") +{ + // Spawn real zenserver as a remote compute node + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + REQUIRE_MESSAGE(Instance.SpawnServerAndWaitUntilReady() != 0, Instance.GetLogOutput()); + + // Start mock orchestrator advertising the server + MockOrchestratorFixture MockOrch; + MockOrch.Service.SetWorkerList(BuildAgentListResponse({{"worker-1", Instance.GetBaseUri()}})); + + // Create session infrastructure + InMemoryChunkResolver Resolver; + ScopedTemporaryDirectory SessionBaseDir; + zen::compute::ComputeServiceSession Session(Resolver); + Session.SetOrchestratorEndpoint(MockOrch.GetEndpoint()); + Session.SetOrchestratorBasePath(SessionBaseDir.Path()); + Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Ready); + + // Register worker on session + CbPackage WorkerPackage = BuildWorkerPackage(TestEnv, Resolver); + Session.RegisterWorker(WorkerPackage); + + // Wait for scheduler to discover the runner + Sleep(7'000); + + // Create a local queue and submit action to it + auto QueueResult = Session.CreateQueue(); + REQUIRE_MESSAGE(QueueResult.QueueId != 0, "Failed to create local queue"); + const int QueueId = QueueResult.QueueId; + + CbObject ActionObj = BuildRot13ActionForSession("Hello World"sv, Resolver); + + zen::compute::ComputeServiceSession::EnqueueResult EnqueueRes = Session.EnqueueActionToQueue(QueueId, ActionObj, 0); + REQUIRE_MESSAGE(EnqueueRes, "Action enqueue to queue failed"); + + // Poll for result + CbPackage ResultPackage; + HttpResponseCode ResultCode = HttpResponseCode::Accepted; + Stopwatch Timer; + + while (Timer.GetElapsedTimeMs() < 30'000) + { + ResultCode = Session.GetActionResult(EnqueueRes.Lsn, ResultPackage); + if (ResultCode == HttpResponseCode::OK) + { + break; + } + Sleep(200); + } + + REQUIRE_MESSAGE( + ResultCode == HttpResponseCode::OK, + fmt::format("Action did not complete in time. Last status: {}\nServer log:\n{}", int(ResultCode), Instance.GetLogOutput())); + + REQUIRE_MESSAGE(bool(ResultPackage), fmt::format("Empty result package\nServer log:\n{}", Instance.GetLogOutput())); + CHECK_EQ(GetRot13Output(ResultPackage), "Uryyb Jbeyq"sv); + + // Verify that a non-implicit remote queue was created on the compute node + HttpClient Client(Instance.GetBaseUri() + "/compute"); + + HttpClient::Response QueuesResp = Client.Get("/queues"sv); + REQUIRE_MESSAGE(QueuesResp, "Failed to list queues on remote server"); + + bool RemoteQueueFound = false; + for (auto& Item : QueuesResp.AsObject()["queues"sv]) + { + if (!Item.AsObjectView()["implicit"sv].AsBool()) + { + RemoteQueueFound = true; + break; + } + } + + CHECK_MESSAGE(RemoteQueueFound, "Expected a non-implicit remote queue on the compute node"); + + Session.Shutdown(); +} + +TEST_CASE("function.remote.queue_cancel_propagation") +{ + // Spawn real zenserver as a remote compute node + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + REQUIRE_MESSAGE(Instance.SpawnServerAndWaitUntilReady() != 0, Instance.GetLogOutput()); + + // Start mock orchestrator advertising the server + MockOrchestratorFixture MockOrch; + MockOrch.Service.SetWorkerList(BuildAgentListResponse({{"worker-1", Instance.GetBaseUri()}})); + + // Create session infrastructure + InMemoryChunkResolver Resolver; + ScopedTemporaryDirectory SessionBaseDir; + zen::compute::ComputeServiceSession Session(Resolver); + Session.SetOrchestratorEndpoint(MockOrch.GetEndpoint()); + Session.SetOrchestratorBasePath(SessionBaseDir.Path()); + Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Ready); + + // Register worker on session + CbPackage WorkerPackage = BuildWorkerPackage(TestEnv, Resolver); + Session.RegisterWorker(WorkerPackage); + + // Wait for scheduler to discover the runner + Sleep(7'000); + + // Create a local queue and submit a long-running Sleep action + auto QueueResult = Session.CreateQueue(); + REQUIRE_MESSAGE(QueueResult.QueueId != 0, "Failed to create local queue"); + const int QueueId = QueueResult.QueueId; + + CbObject ActionObj = BuildSleepActionForSession("data"sv, 30'000, Resolver); + + zen::compute::ComputeServiceSession::EnqueueResult EnqueueRes = Session.EnqueueActionToQueue(QueueId, ActionObj, 0); + REQUIRE_MESSAGE(EnqueueRes, "Sleep action enqueue to queue failed"); + + // Wait for the action to start running on the remote + Sleep(2'000); + + // Cancel the local queue — this should propagate to the remote + Session.CancelQueue(QueueId); + + // Poll for the action to complete (as cancelled) + CbPackage ResultPackage; + HttpResponseCode ResultCode = HttpResponseCode::Accepted; + Stopwatch Timer; + + while (Timer.GetElapsedTimeMs() < 30'000) + { + ResultCode = Session.GetActionResult(EnqueueRes.Lsn, ResultPackage); + if (ResultCode == HttpResponseCode::OK) + { + break; + } + Sleep(200); + } + + // Verify the local queue shows cancelled + auto QueueStatus = Session.GetQueueStatus(QueueId); + CHECK(QueueStatus.State == zen::compute::ComputeServiceSession::QueueState::Cancelled); + + // Verify the remote queue on the compute node is also cancelled + HttpClient Client(Instance.GetBaseUri() + "/compute"); + + HttpClient::Response QueuesResp = Client.Get("/queues"sv); + REQUIRE_MESSAGE(QueuesResp, "Failed to list queues on remote server"); + + bool RemoteQueueCancelled = false; + for (auto& Item : QueuesResp.AsObject()["queues"sv]) + { + if (!Item.AsObjectView()["implicit"sv].AsBool()) + { + RemoteQueueCancelled = std::string(Item.AsObjectView()["state"sv].AsString()) == "cancelled"; + break; + } + } + + CHECK_MESSAGE(RemoteQueueCancelled, "Expected the remote queue to be cancelled"); + + Session.Shutdown(); +} + +TEST_CASE("function.abandon_running_http") +{ + // Spawn a real zenserver to execute a long-running action, then abandon via HTTP endpoint + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + REQUIRE_MESSAGE(Port != 0, Instance.GetLogOutput()); + + const std::string ComputeBaseUri = fmt::format("http://localhost:{}/compute", Port); + HttpClient Client(ComputeBaseUri); + + const IoHash WorkerId = RegisterWorker(Client, TestEnv); + + // Create a queue and submit a long-running Sleep job + HttpClient::Response CreateResp = Client.Post("/queues"sv); + REQUIRE_MESSAGE(CreateResp, "Queue creation failed"); + + const int QueueId = CreateResp.AsObject()["queue_id"sv].AsInt32(); + REQUIRE_MESSAGE(QueueId != 0, "Expected non-zero queue_id"); + + const std::string JobUrl = fmt::format("/queues/{}/jobs/{}", QueueId, WorkerId.ToHexString()); + HttpClient::Response SubmitResp = Client.Post(JobUrl, BuildSleepActionPackage("data"sv, 30'000)); + REQUIRE_MESSAGE(SubmitResp, fmt::format("Sleep job submission failed: status={}", int(SubmitResp.StatusCode))); + + const int Lsn = SubmitResp.AsObject()["lsn"sv].AsInt32(); + REQUIRE_MESSAGE(Lsn != 0, "Expected non-zero LSN"); + + // Wait for the process to start running + Sleep(1'000); + + // Verify the ready endpoint returns OK before abandon + { + HttpClient::Response ReadyResp = Client.Get("/ready"sv); + CHECK(ReadyResp.StatusCode == HttpResponseCode::OK); + } + + // Trigger abandon via the HTTP endpoint + HttpClient::Response AbandonResp = Client.Post("/abandon"sv); + REQUIRE_MESSAGE(AbandonResp.StatusCode == HttpResponseCode::OK, + fmt::format("Abandon request failed: status={}, body={}", int(AbandonResp.StatusCode), AbandonResp.ToText())); + + // Ready endpoint should now return 503 + { + HttpClient::Response ReadyResp = Client.Get("/ready"sv); + CHECK(ReadyResp.StatusCode == HttpResponseCode::ServiceUnavailable); + } + + // The abandoned action should appear in the completed endpoint once the process exits + const std::string CompletedUrl = fmt::format("/queues/{}/completed", QueueId); + REQUIRE_MESSAGE(PollForLsnInCompleted(Client, CompletedUrl, Lsn), + fmt::format("LSN {} did not appear in queue {} completed list after abandon\nServer log:\n{}", + Lsn, + QueueId, + Instance.GetLogOutput())); + + // Verify the queue reflects one abandoned action + const std::string QueueUrl = fmt::format("/queues/{}", QueueId); + HttpClient::Response StatusResp = Client.Get(QueueUrl); + REQUIRE_MESSAGE(StatusResp, "Failed to get queue status after abandon"); + + CbObject QueueStatus = StatusResp.AsObject(); + CHECK_EQ(QueueStatus["abandoned_count"sv].AsInt32(), 1); + CHECK_EQ(QueueStatus["completed_count"sv].AsInt32(), 0); + CHECK_EQ(QueueStatus["active_count"sv].AsInt32(), 0); + + // Submitting new work should be rejected + HttpClient::Response RejectedResp = Client.Post(JobUrl, BuildRot13ActionPackage("rejected"sv)); + CHECK_MESSAGE(RejectedResp.StatusCode != HttpResponseCode::OK, "Expected action submission to be rejected in Abandoned state"); +} + +TEST_CASE("function.session.abandon_pending") +{ + // Create a session with no runners so actions stay pending + InMemoryChunkResolver Resolver; + ScopedTemporaryDirectory SessionBaseDir; + zen::compute::ComputeServiceSession Session(Resolver); + Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Ready); + + CbPackage WorkerPackage = BuildWorkerPackage(TestEnv, Resolver); + Session.RegisterWorker(WorkerPackage); + + // Enqueue several actions — they will stay pending because there are no runners + auto QueueResult = Session.CreateQueue(); + REQUIRE_MESSAGE(QueueResult.QueueId != 0, "Failed to create queue"); + + CbObject ActionObj = BuildRot13ActionForSession("abandon-test"sv, Resolver); + + auto Enqueue1 = Session.EnqueueActionToQueue(QueueResult.QueueId, ActionObj, 0); + auto Enqueue2 = Session.EnqueueActionToQueue(QueueResult.QueueId, ActionObj, 0); + auto Enqueue3 = Session.EnqueueActionToQueue(QueueResult.QueueId, ActionObj, 0); + REQUIRE_MESSAGE(Enqueue1, "Failed to enqueue action 1"); + REQUIRE_MESSAGE(Enqueue2, "Failed to enqueue action 2"); + REQUIRE_MESSAGE(Enqueue3, "Failed to enqueue action 3"); + + // Transition to Abandoned — should mark all pending actions as Abandoned + bool Transitioned = Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Abandoned); + CHECK_MESSAGE(Transitioned, "Failed to transition to Abandoned"); + CHECK(Session.GetSessionState() == zen::compute::ComputeServiceSession::SessionState::Abandoned); + CHECK(!Session.IsHealthy()); + + // Give the scheduler thread time to process the state changes + Sleep(2'000); + + // All three actions should now be in the results map as abandoned + for (int Lsn : {Enqueue1.Lsn, Enqueue2.Lsn, Enqueue3.Lsn}) + { + CbPackage Result; + HttpResponseCode Code = Session.GetActionResult(Lsn, Result); + CHECK_MESSAGE(Code == HttpResponseCode::OK, fmt::format("Expected action LSN {} to be in results (got {})", Lsn, int(Code))); + } + + // Queue should show 0 active, 3 abandoned + auto Status = Session.GetQueueStatus(QueueResult.QueueId); + CHECK_EQ(Status.ActiveCount, 0); + CHECK_EQ(Status.AbandonedCount, 3); + + // New actions should be rejected + auto Rejected = Session.EnqueueActionToQueue(QueueResult.QueueId, ActionObj, 0); + CHECK_MESSAGE(!Rejected, "Expected action submission to be rejected in Abandoned state"); + + // Abandoned → Sunset should be valid + CHECK(Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Sunset)); + + Session.Shutdown(); +} + +TEST_CASE("function.session.abandon_running") +{ + // Spawn a real zenserver as a remote compute node + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + REQUIRE_MESSAGE(Instance.SpawnServerAndWaitUntilReady() != 0, Instance.GetLogOutput()); + + // Start mock orchestrator advertising the server + MockOrchestratorFixture MockOrch; + MockOrch.Service.SetWorkerList(BuildAgentListResponse({{"worker-1", Instance.GetBaseUri()}})); + + // Create session infrastructure + InMemoryChunkResolver Resolver; + ScopedTemporaryDirectory SessionBaseDir; + zen::compute::ComputeServiceSession Session(Resolver); + Session.SetOrchestratorEndpoint(MockOrch.GetEndpoint()); + Session.SetOrchestratorBasePath(SessionBaseDir.Path()); + Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Ready); + + CbPackage WorkerPackage = BuildWorkerPackage(TestEnv, Resolver); + Session.RegisterWorker(WorkerPackage); + + // Wait for scheduler to discover the runner + Sleep(7'000); + + // Create a queue and submit a long-running Sleep action + auto QueueResult = Session.CreateQueue(); + REQUIRE_MESSAGE(QueueResult.QueueId != 0, "Failed to create queue"); + const int QueueId = QueueResult.QueueId; + + CbObject ActionObj = BuildSleepActionForSession("data"sv, 30'000, Resolver); + + auto EnqueueRes = Session.EnqueueActionToQueue(QueueId, ActionObj, 0); + REQUIRE_MESSAGE(EnqueueRes, "Sleep action enqueue to queue failed"); + + // Wait for the action to start running on the remote + Sleep(2'000); + + // Transition to Abandoned — should abandon the running action + bool Transitioned = Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Abandoned); + CHECK_MESSAGE(Transitioned, "Failed to transition to Abandoned"); + CHECK(!Session.IsHealthy()); + + // Poll for the action to complete (as abandoned) + CbPackage ResultPackage; + HttpResponseCode ResultCode = HttpResponseCode::Accepted; + Stopwatch Timer; + + while (Timer.GetElapsedTimeMs() < 30'000) + { + ResultCode = Session.GetActionResult(EnqueueRes.Lsn, ResultPackage); + if (ResultCode == HttpResponseCode::OK) + { + break; + } + Sleep(200); + } + + REQUIRE_MESSAGE(ResultCode == HttpResponseCode::OK, + fmt::format("Action did not complete within timeout\nServer log:\n{}", Instance.GetLogOutput())); + + // Verify the queue shows abandoned, not completed + auto QueueStatus = Session.GetQueueStatus(QueueId); + CHECK_EQ(QueueStatus.ActiveCount, 0); + CHECK_EQ(QueueStatus.AbandonedCount, 1); + CHECK_EQ(QueueStatus.CompletedCount, 0); + + Session.Shutdown(); +} + +TEST_CASE("function.remote.abandon_propagation") +{ + // Spawn real zenserver as a remote compute node + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + REQUIRE_MESSAGE(Instance.SpawnServerAndWaitUntilReady() != 0, Instance.GetLogOutput()); + + // Start mock orchestrator advertising the server + MockOrchestratorFixture MockOrch; + MockOrch.Service.SetWorkerList(BuildAgentListResponse({{"worker-1", Instance.GetBaseUri()}})); + + // Create session infrastructure + InMemoryChunkResolver Resolver; + ScopedTemporaryDirectory SessionBaseDir; + zen::compute::ComputeServiceSession Session(Resolver); + Session.SetOrchestratorEndpoint(MockOrch.GetEndpoint()); + Session.SetOrchestratorBasePath(SessionBaseDir.Path()); + Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Ready); + + // Register worker on session + CbPackage WorkerPackage = BuildWorkerPackage(TestEnv, Resolver); + Session.RegisterWorker(WorkerPackage); + + // Wait for scheduler to discover the runner + Sleep(7'000); + + // Create a local queue and submit a long-running Sleep action + auto QueueResult = Session.CreateQueue(); + REQUIRE_MESSAGE(QueueResult.QueueId != 0, "Failed to create local queue"); + const int QueueId = QueueResult.QueueId; + + CbObject ActionObj = BuildSleepActionForSession("data"sv, 30'000, Resolver); + + auto EnqueueRes = Session.EnqueueActionToQueue(QueueId, ActionObj, 0); + REQUIRE_MESSAGE(EnqueueRes, "Sleep action enqueue to queue failed"); + + // Wait for the action to start running on the remote + Sleep(2'000); + + // Transition to Abandoned — should abandon the running action and propagate + bool Transitioned = Session.RequestStateTransition(zen::compute::ComputeServiceSession::SessionState::Abandoned); + CHECK_MESSAGE(Transitioned, "Failed to transition to Abandoned"); + + // Poll for the action to complete + CbPackage ResultPackage; + HttpResponseCode ResultCode = HttpResponseCode::Accepted; + Stopwatch Timer; + + while (Timer.GetElapsedTimeMs() < 30'000) + { + ResultCode = Session.GetActionResult(EnqueueRes.Lsn, ResultPackage); + if (ResultCode == HttpResponseCode::OK) + { + break; + } + Sleep(200); + } + + REQUIRE_MESSAGE(ResultCode == HttpResponseCode::OK, + fmt::format("Action did not complete within timeout\nServer log:\n{}", Instance.GetLogOutput())); + + // Verify the local queue shows abandoned + auto QueueStatus = Session.GetQueueStatus(QueueId); + CHECK_EQ(QueueStatus.ActiveCount, 0); + CHECK_EQ(QueueStatus.AbandonedCount, 1); + + // Session should not be healthy + CHECK(!Session.IsHealthy()); + + // The remote compute node should still be healthy (only the parent abandoned) + HttpClient RemoteClient(Instance.GetBaseUri() + "/compute"); + HttpClient::Response ReadyResp = RemoteClient.Get("/ready"sv); + CHECK_MESSAGE(ReadyResp.StatusCode == HttpResponseCode::OK, "Remote compute node should still be healthy"); + + Session.Shutdown(); +} + +TEST_SUITE_END(); + +} // namespace zen::tests::compute + +#endif diff --git a/src/zenserver-test/function-tests.cpp b/src/zenserver-test/function-tests.cpp deleted file mode 100644 index 82848c6ad..000000000 --- a/src/zenserver-test/function-tests.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include - -#if ZEN_WITH_TESTS - -# include -# include -# include -# include -# include - -# include "zenserver-test.h" - -namespace zen::tests { - -using namespace std::literals; - -TEST_SUITE_BEGIN("server.function"); - -TEST_CASE("function.run") -{ - std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); - - ZenServerInstance Instance(TestEnv); - Instance.SetDataDir(TestDir); - Instance.SpawnServer(13337); - - ZEN_INFO("Waiting..."); - - Instance.WaitUntilReady(); -} - -TEST_SUITE_END(); - -} // namespace zen::tests - -#endif diff --git a/src/zenserver-test/logging-tests.cpp b/src/zenserver-test/logging-tests.cpp new file mode 100644 index 000000000..fe39e14c0 --- /dev/null +++ b/src/zenserver-test/logging-tests.cpp @@ -0,0 +1,257 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#if ZEN_WITH_TESTS + +# include "zenserver-test.h" + +# include +# include +# include +# include + +namespace zen::tests { + +using namespace std::literals; + +////////////////////////////////////////////////////////////////////////// + +static bool +LogContains(const std::string& Log, std::string_view Needle) +{ + return Log.find(Needle) != std::string::npos; +} + +static std::string +ReadFileToString(const std::filesystem::path& Path) +{ + FileContents Contents = ReadFile(Path); + if (Contents.ErrorCode) + { + return {}; + } + + IoBuffer Content = Contents.Flatten(); + if (!Content) + { + return {}; + } + + return std::string(static_cast(Content.Data()), Content.Size()); +} + +////////////////////////////////////////////////////////////////////////// + +// Verify that a log file is created at the default location (DataDir/logs/zenserver.log) +// even without --abslog. The file must contain "server session id" (logged at INFO +// to all registered loggers during init) and "log starting at" (emitted once a file +// sink is first opened). +TEST_CASE("logging.file.default") +{ + const std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(); + CHECK_MESSAGE(Port != 0, Instance.GetLogOutput()); + + Instance.Shutdown(); + + const std::filesystem::path DefaultLogFile = TestDir / "logs" / "zenserver.log"; + CHECK_MESSAGE(std::filesystem::exists(DefaultLogFile), "Default log file was not created"); + const std::string FileLog = ReadFileToString(DefaultLogFile); + CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog); + CHECK_MESSAGE(LogContains(FileLog, "log starting at"), FileLog); +} + +// --quiet sets the console sink level to WARN. The formatted "[info] ..." +// entry written by the default logger's console sink must therefore not appear +// in captured stdout. (The "console" named logger — used by ZEN_CONSOLE_* +// macros — may still emit plain-text messages without a level marker, so we +// check for the absence of the full_formatter "[info]" prefix rather than the +// message text itself.) +TEST_CASE("logging.console.quiet") +{ + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady("--quiet"); + CHECK_MESSAGE(Port != 0, Instance.GetLogOutput()); + + Instance.Shutdown(); + + const std::string Log = Instance.GetLogOutput(); + CHECK_MESSAGE(!LogContains(Log, "[info] server session id"), Log); +} + +// --noconsole removes the stdout sink entirely, so the captured console output +// must not contain any log entries from the logging system. +TEST_CASE("logging.console.disabled") +{ + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady("--noconsole"); + CHECK_MESSAGE(Port != 0, Instance.GetLogOutput()); + + Instance.Shutdown(); + + const std::string Log = Instance.GetLogOutput(); + CHECK_MESSAGE(!LogContains(Log, "server session id"), Log); +} + +// --abslog creates a rotating log file at the specified path. +// The file must contain "server session id" (logged at INFO to all loggers +// during init) and "log starting at" (emitted once a file sink is active). +TEST_CASE("logging.file.basic") +{ + const std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + const std::filesystem::path LogFile = TestDir / "test.log"; + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + + const std::string LogArg = fmt::format("--abslog {}", LogFile.string()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(LogArg); + CHECK_MESSAGE(Port != 0, Instance.GetLogOutput()); + + Instance.Shutdown(); + + CHECK_MESSAGE(std::filesystem::exists(LogFile), "Log file was not created"); + const std::string FileLog = ReadFileToString(LogFile); + CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog); + CHECK_MESSAGE(LogContains(FileLog, "log starting at"), FileLog); +} + +// --abslog with a .json extension selects the JSON formatter. +// Each log entry must be a JSON object containing at least the "message" +// and "source" fields. +TEST_CASE("logging.file.json") +{ + const std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + const std::filesystem::path LogFile = TestDir / "test.json"; + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + + const std::string LogArg = fmt::format("--abslog {}", LogFile.string()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(LogArg); + CHECK_MESSAGE(Port != 0, Instance.GetLogOutput()); + + Instance.Shutdown(); + + CHECK_MESSAGE(std::filesystem::exists(LogFile), "JSON log file was not created"); + const std::string FileLog = ReadFileToString(LogFile); + CHECK_MESSAGE(LogContains(FileLog, "\"message\""), FileLog); + CHECK_MESSAGE(LogContains(FileLog, "\"source\": \"zenserver\""), FileLog); + CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog); +} + +// --log-id is automatically set to the server instance name in test mode. +// The JSON formatter emits this value as the "id" field, so every entry in a +// .json log file must carry a non-empty "id". +TEST_CASE("logging.log_id") +{ + const std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + const std::filesystem::path LogFile = TestDir / "test.json"; + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + + const std::string LogArg = fmt::format("--abslog {}", LogFile.string()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(LogArg); + CHECK_MESSAGE(Port != 0, Instance.GetLogOutput()); + + Instance.Shutdown(); + + CHECK_MESSAGE(std::filesystem::exists(LogFile), "JSON log file was not created"); + const std::string FileLog = ReadFileToString(LogFile); + // The JSON formatter writes the log-id as: "id": "", + CHECK_MESSAGE(LogContains(FileLog, "\"id\": \""), FileLog); +} + +// --log-warn raises the level threshold above INFO so that INFO messages +// are filtered. "server session id" is broadcast at INFO to all loggers: it must +// appear in the main file sink (default logger unaffected) but must NOT appear in +// http.log where the http_requests logger now has a WARN threshold. +TEST_CASE("logging.level.warn_suppresses_info") +{ + const std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + const std::filesystem::path LogFile = TestDir / "test.log"; + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + + const std::string LogArg = fmt::format("--abslog {} --log-warn http_requests", LogFile.string()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(LogArg); + CHECK_MESSAGE(Port != 0, Instance.GetLogOutput()); + + Instance.Shutdown(); + + CHECK_MESSAGE(std::filesystem::exists(LogFile), "Log file was not created"); + const std::string FileLog = ReadFileToString(LogFile); + CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog); + + const std::filesystem::path HttpLogFile = TestDir / "logs" / "http.log"; + CHECK_MESSAGE(std::filesystem::exists(HttpLogFile), "http.log was not created"); + const std::string HttpLog = ReadFileToString(HttpLogFile); + CHECK_MESSAGE(!LogContains(HttpLog, "server session id"), HttpLog); +} + +// --log-info sets an explicit INFO threshold. The INFO "server session id" +// broadcast must still land in http.log, confirming that INFO messages are not +// filtered when the logger level is exactly INFO. +TEST_CASE("logging.level.info_allows_info") +{ + const std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + const std::filesystem::path LogFile = TestDir / "test.log"; + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + + const std::string LogArg = fmt::format("--abslog {} --log-info http_requests", LogFile.string()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(LogArg); + CHECK_MESSAGE(Port != 0, Instance.GetLogOutput()); + + Instance.Shutdown(); + + const std::filesystem::path HttpLogFile = TestDir / "logs" / "http.log"; + CHECK_MESSAGE(std::filesystem::exists(HttpLogFile), "http.log was not created"); + const std::string HttpLog = ReadFileToString(HttpLogFile); + CHECK_MESSAGE(LogContains(HttpLog, "server session id"), HttpLog); +} + +// --log-off silences a named logger entirely. +// "server session id" is broadcast at INFO to all registered loggers via +// spdlog::apply_all during init. When the "http_requests" logger is set to +// OFF its dedicated http.log file must not contain that message. +// The main file sink (via --abslog) must be unaffected. +TEST_CASE("logging.level.off_specific_logger") +{ + const std::filesystem::path TestDir = TestEnv.CreateNewTestDir(); + const std::filesystem::path LogFile = TestDir / "test.log"; + + ZenServerInstance Instance(TestEnv); + Instance.SetDataDir(TestDir); + + const std::string LogArg = fmt::format("--abslog {} --log-off http_requests", LogFile.string()); + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(LogArg); + CHECK_MESSAGE(Port != 0, Instance.GetLogOutput()); + + Instance.Shutdown(); + + // Main log file must still have the startup message + CHECK_MESSAGE(std::filesystem::exists(LogFile), "Log file was not created"); + const std::string FileLog = ReadFileToString(LogFile); + CHECK_MESSAGE(LogContains(FileLog, "server session id"), FileLog); + + // http.log is created by the RotatingFileSink but the logger is OFF, so + // the broadcast "server session id" message must not have been written to it + const std::filesystem::path HttpLogFile = TestDir / "logs" / "http.log"; + CHECK_MESSAGE(std::filesystem::exists(HttpLogFile), "http.log was not created"); + const std::string HttpLog = ReadFileToString(HttpLogFile); + CHECK_MESSAGE(!LogContains(HttpLog, "server session id"), HttpLog); +} + +} // namespace zen::tests + +#endif diff --git a/src/zenserver-test/nomad-tests.cpp b/src/zenserver-test/nomad-tests.cpp new file mode 100644 index 000000000..6eb99bc3a --- /dev/null +++ b/src/zenserver-test/nomad-tests.cpp @@ -0,0 +1,126 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#if ZEN_WITH_TESTS && ZEN_WITH_NOMAD +# include "zenserver-test.h" +# include +# include +# include +# include +# include +# include +# include +# include + +# include + +namespace zen::tests::nomad_tests { + +using namespace std::literals; + +TEST_CASE("nomad.client.lifecycle" * doctest::skip()) +{ + zen::nomad::NomadProcess NomadProc; + NomadProc.SpawnNomadAgent(); + + zen::nomad::NomadTestClient Client("http://localhost:4646/"); + + // Submit a simple batch job that sleeps briefly +# if ZEN_PLATFORM_WINDOWS + auto Job = Client.SubmitJob("zen-test-job", "cmd.exe", {"/C", "timeout /t 10 /nobreak"}); +# else + auto Job = Client.SubmitJob("zen-test-job", "/bin/sleep", {"10"}); +# endif + REQUIRE(!Job.Id.empty()); + CHECK_EQ(Job.Status, "pending"); + + // Poll until the job is running (or dead) + { + Stopwatch Timer; + bool FoundRunning = false; + while (Timer.GetElapsedTimeMs() < 15000) + { + auto Status = Client.GetJobStatus("zen-test-job"); + if (Status.Status == "running") + { + FoundRunning = true; + break; + } + if (Status.Status == "dead") + { + break; + } + Sleep(500); + } + CHECK(FoundRunning); + } + + // Verify allocations exist + auto Allocs = Client.GetAllocations("zen-test-job"); + CHECK(!Allocs.empty()); + + // Stop the job + Client.StopJob("zen-test-job"); + + // Verify it reaches dead state + { + Stopwatch Timer; + bool FoundDead = false; + while (Timer.GetElapsedTimeMs() < 10000) + { + auto Status = Client.GetJobStatus("zen-test-job"); + if (Status.Status == "dead") + { + FoundDead = true; + break; + } + Sleep(500); + } + CHECK(FoundDead); + } + + NomadProc.StopNomadAgent(); +} + +TEST_CASE("nomad.provisioner.integration" * doctest::skip()) +{ + zen::nomad::NomadProcess NomadProc; + NomadProc.SpawnNomadAgent(); + + // Spawn zenserver in compute mode with Nomad provisioning enabled + ZenServerInstance Instance(TestEnv, ZenServerInstance::ServerMode::kComputeServer); + + Instance.SetDataDir(TestEnv.CreateNewTestDir()); + + std::filesystem::path ZenServerPath = TestEnv.ProgramBaseDir() / "zenserver" ZEN_EXE_SUFFIX_LITERAL; + + std::string NomadArgs = fmt::format( + "--nomad-enabled=true" + " --nomad-server=http://localhost:4646" + " --nomad-driver=raw_exec" + " --nomad-binary-path={}" + " --nomad-max-cores=32" + " --nomad-cores-per-job=32", + ZenServerPath.string()); + + const uint16_t Port = Instance.SpawnServerAndWaitUntilReady(NomadArgs); + REQUIRE(Port != 0); + + // Give the provisioner time to submit jobs. + // The management thread has a 5s wait between cycles, and the HTTP client has + // a 10s connect timeout, so we need to allow enough time for at least one full cycle. + Sleep(15000); + + // Verify jobs were submitted to Nomad + zen::nomad::NomadTestClient NomadClient("http://localhost:4646/"); + + auto Jobs = NomadClient.ListJobs("zenserver-worker"); + + ZEN_INFO("nomad.provisioner.integration: found {} jobs with prefix 'zenserver-worker'", Jobs.size()); + CHECK_MESSAGE(!Jobs.empty(), Instance.GetLogOutput()); + + Instance.Shutdown(); + NomadProc.StopNomadAgent(); +} + +} // namespace zen::tests::nomad_tests +#endif diff --git a/src/zenserver-test/xmake.lua b/src/zenserver-test/xmake.lua index 2a269cea1..7b208bbc7 100644 --- a/src/zenserver-test/xmake.lua +++ b/src/zenserver-test/xmake.lua @@ -6,10 +6,15 @@ target("zenserver-test") add_headerfiles("**.h") add_files("*.cpp") add_files("zenserver-test.cpp", {unity_ignored = true }) - add_deps("zencore", "zenremotestore", "zenhttp") + add_deps("zencore", "zenremotestore", "zenhttp", "zencompute", "zenstore") add_deps("zenserver", {inherit=false}) + add_deps("zentest-appstub", {inherit=false}) add_packages("http_parser") + if has_config("zennomad") then + add_deps("zennomad") + end + if is_plat("macosx") then add_ldflags("-framework CoreFoundation") add_ldflags("-framework Security") diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp index 0f9ef0287..802d06caf 100644 --- a/src/zenserver/compute/computeserver.cpp +++ b/src/zenserver/compute/computeserver.cpp @@ -1,9 +1,9 @@ // Copyright Epic Games, Inc. All Rights Reserved. #include "computeserver.h" -#include -#include "computeservice.h" - +#include +#include +#include #if ZEN_WITH_COMPUTE_SERVICES # include @@ -13,10 +13,20 @@ # include # include # include +# include # include +# include # include # include # include +# if ZEN_WITH_HORDE +# include +# include +# endif +# if ZEN_WITH_NOMAD +# include +# include +# endif ZEN_THIRD_PARTY_INCLUDES_START # include @@ -27,6 +37,13 @@ namespace zen { void ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options) { + Options.add_option("compute", + "", + "max-actions", + "Maximum number of concurrent local actions (0 = auto)", + cxxopts::value(m_ServerOptions.MaxConcurrentActions)->default_value("0"), + ""); + Options.add_option("compute", "", "upstream-notification-endpoint", @@ -40,6 +57,236 @@ ZenComputeServerConfigurator::AddCliOptions(cxxopts::Options& Options) "Instance ID for use in notifications", cxxopts::value(m_ServerOptions.InstanceId)->default_value(""), ""); + + Options.add_option("compute", + "", + "coordinator-endpoint", + "Endpoint URL for coordinator service", + cxxopts::value(m_ServerOptions.CoordinatorEndpoint)->default_value(""), + ""); + + Options.add_option("compute", + "", + "idms", + "Enable IDMS cloud detection; optionally specify a custom probe endpoint", + cxxopts::value(m_ServerOptions.IdmsEndpoint)->default_value("")->implicit_value("auto"), + ""); + + Options.add_option("compute", + "", + "worker-websocket", + "Use WebSocket for worker-orchestrator link (instant reachability detection)", + cxxopts::value(m_ServerOptions.EnableWorkerWebSocket)->default_value("false"), + ""); + +# if ZEN_WITH_HORDE + // Horde provisioning options + Options.add_option("horde", + "", + "horde-enabled", + "Enable Horde worker provisioning", + cxxopts::value(m_ServerOptions.HordeConfig.Enabled)->default_value("false"), + ""); + + Options.add_option("horde", + "", + "horde-server", + "Horde server URL", + cxxopts::value(m_ServerOptions.HordeConfig.ServerUrl)->default_value(""), + ""); + + Options.add_option("horde", + "", + "horde-token", + "Horde authentication token", + cxxopts::value(m_ServerOptions.HordeConfig.AuthToken)->default_value(""), + ""); + + Options.add_option("horde", + "", + "horde-pool", + "Horde pool name", + cxxopts::value(m_ServerOptions.HordeConfig.Pool)->default_value(""), + ""); + + Options.add_option("horde", + "", + "horde-cluster", + "Horde cluster ID ('default' or '_auto' for auto-resolve)", + cxxopts::value(m_ServerOptions.HordeConfig.Cluster)->default_value("default"), + ""); + + Options.add_option("horde", + "", + "horde-mode", + "Horde connection mode (direct, tunnel, relay)", + cxxopts::value(m_HordeModeStr)->default_value("direct"), + ""); + + Options.add_option("horde", + "", + "horde-encryption", + "Horde transport encryption (none, aes)", + cxxopts::value(m_HordeEncryptionStr)->default_value("none"), + ""); + + Options.add_option("horde", + "", + "horde-max-cores", + "Maximum number of Horde cores to provision", + cxxopts::value(m_ServerOptions.HordeConfig.MaxCores)->default_value("2048"), + ""); + + Options.add_option("horde", + "", + "horde-host", + "Host address for Horde agents to connect back to", + cxxopts::value(m_ServerOptions.HordeConfig.HostAddress)->default_value(""), + ""); + + Options.add_option("horde", + "", + "horde-condition", + "Additional Horde agent filter condition", + cxxopts::value(m_ServerOptions.HordeConfig.Condition)->default_value(""), + ""); + + Options.add_option("horde", + "", + "horde-binaries", + "Path to directory containing zenserver binary for remote upload", + cxxopts::value(m_ServerOptions.HordeConfig.BinariesPath)->default_value(""), + ""); + + Options.add_option("horde", + "", + "horde-zen-service-port", + "Port number for Zen service communication", + cxxopts::value(m_ServerOptions.HordeConfig.ZenServicePort)->default_value("8558"), + ""); +# endif + +# if ZEN_WITH_NOMAD + // Nomad provisioning options + Options.add_option("nomad", + "", + "nomad-enabled", + "Enable Nomad worker provisioning", + cxxopts::value(m_ServerOptions.NomadConfig.Enabled)->default_value("false"), + ""); + + Options.add_option("nomad", + "", + "nomad-server", + "Nomad HTTP API URL", + cxxopts::value(m_ServerOptions.NomadConfig.ServerUrl)->default_value(""), + ""); + + Options.add_option("nomad", + "", + "nomad-token", + "Nomad ACL token", + cxxopts::value(m_ServerOptions.NomadConfig.AclToken)->default_value(""), + ""); + + Options.add_option("nomad", + "", + "nomad-datacenter", + "Nomad target datacenter", + cxxopts::value(m_ServerOptions.NomadConfig.Datacenter)->default_value("dc1"), + ""); + + Options.add_option("nomad", + "", + "nomad-namespace", + "Nomad namespace", + cxxopts::value(m_ServerOptions.NomadConfig.Namespace)->default_value("default"), + ""); + + Options.add_option("nomad", + "", + "nomad-region", + "Nomad region (empty for server default)", + cxxopts::value(m_ServerOptions.NomadConfig.Region)->default_value(""), + ""); + + Options.add_option("nomad", + "", + "nomad-driver", + "Nomad task driver (raw_exec, docker)", + cxxopts::value(m_NomadDriverStr)->default_value("raw_exec"), + ""); + + Options.add_option("nomad", + "", + "nomad-distribution", + "Binary distribution mode (predeployed, artifact)", + cxxopts::value(m_NomadDistributionStr)->default_value("predeployed"), + ""); + + Options.add_option("nomad", + "", + "nomad-binary-path", + "Path to zenserver on Nomad clients (predeployed mode)", + cxxopts::value(m_ServerOptions.NomadConfig.BinaryPath)->default_value(""), + ""); + + Options.add_option("nomad", + "", + "nomad-artifact-source", + "URL to download zenserver binary (artifact mode)", + cxxopts::value(m_ServerOptions.NomadConfig.ArtifactSource)->default_value(""), + ""); + + Options.add_option("nomad", + "", + "nomad-docker-image", + "Docker image for zenserver (docker driver)", + cxxopts::value(m_ServerOptions.NomadConfig.DockerImage)->default_value(""), + ""); + + Options.add_option("nomad", + "", + "nomad-max-jobs", + "Maximum concurrent Nomad jobs", + cxxopts::value(m_ServerOptions.NomadConfig.MaxJobs)->default_value("64"), + ""); + + Options.add_option("nomad", + "", + "nomad-cpu-mhz", + "CPU MHz allocated per Nomad task", + cxxopts::value(m_ServerOptions.NomadConfig.CpuMhz)->default_value("1000"), + ""); + + Options.add_option("nomad", + "", + "nomad-memory-mb", + "Memory MB allocated per Nomad task", + cxxopts::value(m_ServerOptions.NomadConfig.MemoryMb)->default_value("2048"), + ""); + + Options.add_option("nomad", + "", + "nomad-cores-per-job", + "Estimated cores per Nomad job (for scaling)", + cxxopts::value(m_ServerOptions.NomadConfig.CoresPerJob)->default_value("32"), + ""); + + Options.add_option("nomad", + "", + "nomad-max-cores", + "Maximum total cores to provision via Nomad", + cxxopts::value(m_ServerOptions.NomadConfig.MaxCores)->default_value("2048"), + ""); + + Options.add_option("nomad", + "", + "nomad-job-prefix", + "Prefix for generated Nomad job IDs", + cxxopts::value(m_ServerOptions.NomadConfig.JobPrefix)->default_value("zenserver-worker"), + ""); +# endif } void @@ -63,6 +310,15 @@ ZenComputeServerConfigurator::OnConfigFileParsed(LuaConfig::Options& LuaOptions) void ZenComputeServerConfigurator::ValidateOptions() { +# if ZEN_WITH_HORDE + horde::FromString(m_ServerOptions.HordeConfig.Mode, m_HordeModeStr); + horde::FromString(m_ServerOptions.HordeConfig.EncryptionMode, m_HordeEncryptionStr); +# endif + +# if ZEN_WITH_NOMAD + nomad::FromString(m_ServerOptions.NomadConfig.TaskDriver, m_NomadDriverStr); + nomad::FromString(m_ServerOptions.NomadConfig.BinDistribution, m_NomadDistributionStr); +# endif } /////////////////////////////////////////////////////////////////////////// @@ -90,10 +346,14 @@ ZenComputeServer::Initialize(const ZenComputeServerConfig& ServerConfig, ZenServ return EffectiveBasePort; } + m_CoordinatorEndpoint = ServerConfig.CoordinatorEndpoint; + m_InstanceId = ServerConfig.InstanceId; + m_EnableWorkerWebSocket = ServerConfig.EnableWorkerWebSocket; + // This is a workaround to make sure we can have automated tests. Without // this the ranges for different child zen compute processes could overlap with // the main test range. - ZenServerEnvironment::SetBaseChildId(1000); + ZenServerEnvironment::SetBaseChildId(2000); m_DebugOptionForcedCrash = ServerConfig.ShouldCrash; @@ -113,6 +373,46 @@ ZenComputeServer::Cleanup() ZEN_INFO(ZEN_APP_NAME " cleaning up"); try { + // Cancel the maintenance timer so it stops re-enqueuing before we + // tear down the provisioners it references. + m_ProvisionerMaintenanceTimer.cancel(); + m_AnnounceTimer.cancel(); + +# if ZEN_WITH_HORDE + // Shut down Horde provisioner first — this signals all agent threads + // to exit and joins them before we tear down HTTP services. + m_HordeProvisioner.reset(); +# endif + +# if ZEN_WITH_NOMAD + // Shut down Nomad provisioner — stops the management thread and + // sends stop requests for all tracked jobs. + m_NomadProvisioner.reset(); +# endif + + // Close the orchestrator WebSocket client before stopping the io_context + m_WsReconnectTimer.cancel(); + if (m_OrchestratorWsClient) + { + m_OrchestratorWsClient->Close(); + m_OrchestratorWsClient.reset(); + } + m_OrchestratorWsHandler.reset(); + + ResolveCloudMetadata(); + m_CloudMetadata.reset(); + + // Shut down services that own threads or use the io_context before we + // stop the io_context and close the HTTP server. + if (m_OrchestratorService) + { + m_OrchestratorService->Shutdown(); + } + if (m_ComputeService) + { + m_ComputeService->Shutdown(); + } + m_IoContext.stop(); if (m_IoRunner.joinable()) { @@ -139,7 +439,8 @@ ZenComputeServer::InitializeState(const ZenComputeServerConfig& ServerConfig) void ZenComputeServer::InitializeServices(const ZenComputeServerConfig& ServerConfig) { - ZEN_INFO("initializing storage"); + ZEN_TRACE_CPU("ZenComputeServer::InitializeServices"); + ZEN_INFO("initializing compute services"); CidStoreConfiguration Config; Config.RootDirectory = m_DataRoot / "cas"; @@ -147,46 +448,405 @@ ZenComputeServer::InitializeServices(const ZenComputeServerConfig& ServerConfig) m_CidStore = std::make_unique(m_GcManager); m_CidStore->Initialize(Config); + if (!ServerConfig.IdmsEndpoint.empty()) + { + ZEN_INFO("detecting cloud environment (async)"); + if (ServerConfig.IdmsEndpoint == "auto") + { + m_CloudMetadataFuture = std::async(std::launch::async, [DataDir = ServerConfig.DataDir] { + return std::make_unique(DataDir / "cloud"); + }); + } + else + { + ZEN_INFO("using custom IDMS endpoint: {}", ServerConfig.IdmsEndpoint); + m_CloudMetadataFuture = std::async(std::launch::async, [DataDir = ServerConfig.DataDir, Endpoint = ServerConfig.IdmsEndpoint] { + return std::make_unique(DataDir / "cloud", Endpoint); + }); + } + } + ZEN_INFO("instantiating API service"); m_ApiService = std::make_unique(*m_Http); - ZEN_INFO("instantiating compute service"); - m_ComputeService = std::make_unique(ServerConfig.DataDir / "compute"); + ZEN_INFO("instantiating orchestrator service"); + m_OrchestratorService = + std::make_unique(ServerConfig.DataDir / "orch", ServerConfig.EnableWorkerWebSocket); + + ZEN_INFO("instantiating function service"); + m_ComputeService = std::make_unique(*m_CidStore, + m_StatsService, + ServerConfig.DataDir / "functions", + ServerConfig.MaxConcurrentActions); - // Ref Runner; - // Runner = zen::compute::CreateLocalRunner(*m_CidStore, ServerConfig.DataDir / "runner"); + m_FrontendService = std::make_unique(m_ContentRoot, m_StatusService); - // TODO: (re)implement default configuration here +# if ZEN_WITH_NOMAD + // Nomad provisioner + if (ServerConfig.NomadConfig.Enabled && !ServerConfig.NomadConfig.ServerUrl.empty()) + { + ZEN_INFO("instantiating Nomad provisioner (server: {})", ServerConfig.NomadConfig.ServerUrl); - ZEN_INFO("instantiating function service"); - m_FunctionService = - std::make_unique(*m_CidStore, m_StatsService, ServerConfig.DataDir / "functions"); + const auto& NomadCfg = ServerConfig.NomadConfig; + + if (!NomadCfg.Validate()) + { + ZEN_ERROR("invalid Nomad configuration"); + } + else + { + ExtendableStringBuilder<256> OrchestratorEndpoint; + OrchestratorEndpoint << m_Http->GetServiceUri(m_OrchestratorService.get()); + if (auto View = OrchestratorEndpoint.ToView(); !View.empty() && View.back() != '/') + { + OrchestratorEndpoint << '/'; + } + + m_NomadProvisioner = std::make_unique(NomadCfg, OrchestratorEndpoint); + } + } +# endif + +# if ZEN_WITH_HORDE + // Horde provisioner + if (ServerConfig.HordeConfig.Enabled && !ServerConfig.HordeConfig.ServerUrl.empty()) + { + ZEN_INFO("instantiating Horde provisioner (server: {})", ServerConfig.HordeConfig.ServerUrl); + + const auto& HordeConfig = ServerConfig.HordeConfig; + + if (!HordeConfig.Validate()) + { + ZEN_ERROR("invalid Horde configuration"); + } + else + { + ExtendableStringBuilder<256> OrchestratorEndpoint; + OrchestratorEndpoint << m_Http->GetServiceUri(m_OrchestratorService.get()); + if (auto View = OrchestratorEndpoint.ToView(); !View.empty() && View.back() != '/') + { + OrchestratorEndpoint << '/'; + } + + // If no binaries path is specified, just use the running executable's directory + std::filesystem::path BinariesPath = HordeConfig.BinariesPath.empty() ? GetRunningExecutablePath().parent_path() + : std::filesystem::path(HordeConfig.BinariesPath); + std::filesystem::path WorkingDir = ServerConfig.DataDir / "horde"; + + m_HordeProvisioner = std::make_unique(HordeConfig, BinariesPath, WorkingDir, OrchestratorEndpoint); + } + } +# endif +} + +void +ZenComputeServer::ResolveCloudMetadata() +{ + if (m_CloudMetadataFuture.valid()) + { + m_CloudMetadata = m_CloudMetadataFuture.get(); + } +} + +std::string +ZenComputeServer::GetInstanceId() const +{ + if (!m_InstanceId.empty()) + { + return m_InstanceId; + } + return fmt::format("{}-{}", GetMachineName(), GetCurrentProcessId()); +} + +std::string +ZenComputeServer::GetAnnounceUrl() const +{ + return m_Http->GetServiceUri(nullptr); } void ZenComputeServer::RegisterServices(const ZenComputeServerConfig& ServerConfig) { + ZEN_TRACE_CPU("ZenComputeServer::RegisterServices"); ZEN_UNUSED(ServerConfig); + m_Http->RegisterService(m_StatsService); + + if (m_ApiService) + { + m_Http->RegisterService(*m_ApiService); + } + + if (m_OrchestratorService) + { + m_Http->RegisterService(*m_OrchestratorService); + } + if (m_ComputeService) { m_Http->RegisterService(*m_ComputeService); } - if (m_ApiService) + if (m_FrontendService) { - m_Http->RegisterService(*m_ApiService); + m_Http->RegisterService(*m_FrontendService); + } +} + +CbObject +ZenComputeServer::BuildAnnounceBody() +{ + CbObjectWriter AnnounceBody; + AnnounceBody << "id" << GetInstanceId(); + AnnounceBody << "uri" << GetAnnounceUrl(); + AnnounceBody << "hostname" << GetMachineName(); + AnnounceBody << "platform" << GetRuntimePlatformName(); + + ExtendedSystemMetrics Sm = ApplyReportingOverrides(m_MetricsTracker.Query()); + + AnnounceBody.BeginObject("metrics"); + Describe(Sm, AnnounceBody); + AnnounceBody.EndObject(); + + AnnounceBody << "cpu_usage" << Sm.CpuUsagePercent; + AnnounceBody << "memory_total" << Sm.SystemMemoryMiB * 1024 * 1024; + AnnounceBody << "memory_used" << (Sm.SystemMemoryMiB - Sm.AvailSystemMemoryMiB) * 1024 * 1024; + + AnnounceBody << "bytes_received" << m_Http->GetTotalBytesReceived(); + AnnounceBody << "bytes_sent" << m_Http->GetTotalBytesSent(); + + auto Actions = m_ComputeService->GetActionCounts(); + AnnounceBody << "actions_pending" << Actions.Pending; + AnnounceBody << "actions_running" << Actions.Running; + AnnounceBody << "actions_completed" << Actions.Completed; + AnnounceBody << "active_queues" << Actions.ActiveQueues; + + // Derive provisioner from instance ID prefix (e.g. "horde-xxx" or "nomad-xxx") + if (m_InstanceId.starts_with("horde-")) + { + AnnounceBody << "provisioner" + << "horde"; + } + else if (m_InstanceId.starts_with("nomad-")) + { + AnnounceBody << "provisioner" + << "nomad"; + } + + ResolveCloudMetadata(); + if (m_CloudMetadata) + { + m_CloudMetadata->Describe(AnnounceBody); + } + + return AnnounceBody.Save(); +} + +void +ZenComputeServer::PostAnnounce() +{ + ZEN_TRACE_CPU("ZenComputeServer::PostAnnounce"); + + if (!m_ComputeService || m_CoordinatorEndpoint.empty()) + { + return; + } + + ZEN_INFO("notifying coordinator at '{}' of our availability at '{}'", m_CoordinatorEndpoint, GetAnnounceUrl()); + + try + { + CbObject Body = BuildAnnounceBody(); + + // If we have an active WebSocket connection, send via that instead of HTTP POST + if (m_OrchestratorWsClient && m_OrchestratorWsClient->IsOpen()) + { + MemoryView View = Body.GetView(); + m_OrchestratorWsClient->SendBinary(std::span(reinterpret_cast(View.GetData()), View.GetSize())); + ZEN_INFO("announced to coordinator via WebSocket"); + return; + } + + HttpClient CoordinatorHttp(m_CoordinatorEndpoint); + HttpClient::Response Result = CoordinatorHttp.Post("announce", std::move(Body)); + + if (Result.Error) + { + ZEN_ERROR("failed to notify coordinator at '{}': HTTP error {} - {}", + m_CoordinatorEndpoint, + Result.Error->ErrorCode, + Result.Error->ErrorMessage); + } + else if (!IsHttpOk(Result.StatusCode)) + { + ZEN_ERROR("failed to notify coordinator at '{}': unexpected HTTP status code {}", + m_CoordinatorEndpoint, + static_cast(Result.StatusCode)); + } + else + { + ZEN_INFO("successfully notified coordinator at '{}'", m_CoordinatorEndpoint); + } + } + catch (const std::exception& Ex) + { + ZEN_ERROR("failed to notify coordinator at '{}': {}", m_CoordinatorEndpoint, Ex.what()); + } +} + +void +ZenComputeServer::EnqueueAnnounceTimer() +{ + if (!m_ComputeService || m_CoordinatorEndpoint.empty()) + { + return; + } + + m_AnnounceTimer.expires_after(std::chrono::seconds(15)); + m_AnnounceTimer.async_wait([this](const asio::error_code& Ec) { + if (!Ec) + { + PostAnnounce(); + EnqueueAnnounceTimer(); + } + }); + EnsureIoRunner(); +} + +void +ZenComputeServer::InitializeOrchestratorWebSocket() +{ + if (!m_EnableWorkerWebSocket || m_CoordinatorEndpoint.empty()) + { + return; + } + + // Convert http://host:port → ws://host:port/orch/ws + std::string WsUrl = m_CoordinatorEndpoint; + if (WsUrl.starts_with("http://")) + { + WsUrl = "ws://" + WsUrl.substr(7); + } + else if (WsUrl.starts_with("https://")) + { + WsUrl = "wss://" + WsUrl.substr(8); + } + if (!WsUrl.empty() && WsUrl.back() != '/') + { + WsUrl += '/'; + } + WsUrl += "orch/ws"; + + ZEN_INFO("establishing WebSocket link to orchestrator at {}", WsUrl); + + m_OrchestratorWsHandler = std::make_unique(*this); + m_OrchestratorWsClient = + std::make_unique(WsUrl, *m_OrchestratorWsHandler, m_IoContext, HttpWsClientSettings{.LogCategory = "orch_ws"}); + + m_OrchestratorWsClient->Connect(); + EnsureIoRunner(); +} + +void +ZenComputeServer::EnqueueWsReconnect() +{ + m_WsReconnectTimer.expires_after(std::chrono::seconds(5)); + m_WsReconnectTimer.async_wait([this](const asio::error_code& Ec) { + if (!Ec && m_OrchestratorWsClient) + { + ZEN_INFO("attempting WebSocket reconnect to orchestrator"); + m_OrchestratorWsClient->Connect(); + } + }); + EnsureIoRunner(); +} + +void +ZenComputeServer::OrchestratorWsHandler::OnWsOpen() +{ + ZEN_INFO("WebSocket link to orchestrator established"); + + // Send initial announce immediately over the WebSocket + Server.PostAnnounce(); +} + +void +ZenComputeServer::OrchestratorWsHandler::OnWsMessage([[maybe_unused]] const WebSocketMessage& Msg) +{ + // Orchestrator does not push messages to workers; ignore +} + +void +ZenComputeServer::OrchestratorWsHandler::OnWsClose([[maybe_unused]] uint16_t Code, [[maybe_unused]] std::string_view Reason) +{ + ZEN_WARN("WebSocket link to orchestrator closed (code {}), falling back to HTTP announce", Code); + + // Trigger an immediate HTTP announce so the orchestrator has fresh state, + // then schedule a reconnect attempt. + Server.PostAnnounce(); + Server.EnqueueWsReconnect(); +} + +void +ZenComputeServer::ProvisionerMaintenanceTick() +{ +# if ZEN_WITH_HORDE + if (m_HordeProvisioner) + { + m_HordeProvisioner->SetTargetCoreCount(UINT32_MAX); + auto Stats = m_HordeProvisioner->GetStats(); + ZEN_DEBUG("Horde maintenance: target={}, estimated={}, active={}", + Stats.TargetCoreCount, + Stats.EstimatedCoreCount, + Stats.ActiveCoreCount); + } +# endif + +# if ZEN_WITH_NOMAD + if (m_NomadProvisioner) + { + m_NomadProvisioner->SetTargetCoreCount(UINT32_MAX); + auto Stats = m_NomadProvisioner->GetStats(); + ZEN_DEBUG("Nomad maintenance: target={}, estimated={}, running jobs={}", + Stats.TargetCoreCount, + Stats.EstimatedCoreCount, + Stats.RunningJobCount); } +# endif +} + +void +ZenComputeServer::EnqueueProvisionerMaintenanceTimer() +{ + bool HasProvisioner = false; +# if ZEN_WITH_HORDE + HasProvisioner = HasProvisioner || (m_HordeProvisioner != nullptr); +# endif +# if ZEN_WITH_NOMAD + HasProvisioner = HasProvisioner || (m_NomadProvisioner != nullptr); +# endif - if (m_FunctionService) + if (!HasProvisioner) { - m_Http->RegisterService(*m_FunctionService); + return; } + + m_ProvisionerMaintenanceTimer.expires_after(std::chrono::seconds(15)); + m_ProvisionerMaintenanceTimer.async_wait([this](const asio::error_code& Ec) { + if (!Ec) + { + ProvisionerMaintenanceTick(); + EnqueueProvisionerMaintenanceTimer(); + } + }); + EnsureIoRunner(); } void ZenComputeServer::Run() { + ZEN_TRACE_CPU("ZenComputeServer::Run"); + if (m_ProcessMonitor.IsActive()) { CheckOwnerPid(); @@ -236,6 +896,35 @@ ZenComputeServer::Run() OnReady(); + PostAnnounce(); + EnqueueAnnounceTimer(); + InitializeOrchestratorWebSocket(); + +# if ZEN_WITH_HORDE + // Start Horde provisioning if configured — request maximum allowed cores. + // SetTargetCoreCount clamps to HordeConfig::MaxCores internally. + if (m_HordeProvisioner) + { + ZEN_INFO("Horde provisioning starting"); + m_HordeProvisioner->SetTargetCoreCount(UINT32_MAX); + auto Stats = m_HordeProvisioner->GetStats(); + ZEN_INFO("Horde provisioning started (target cores: {})", Stats.TargetCoreCount); + } +# endif + +# if ZEN_WITH_NOMAD + // Start Nomad provisioning if configured — request maximum allowed cores. + // SetTargetCoreCount clamps to NomadConfig::MaxCores internally. + if (m_NomadProvisioner) + { + m_NomadProvisioner->SetTargetCoreCount(UINT32_MAX); + auto Stats = m_NomadProvisioner->GetStats(); + ZEN_INFO("Nomad provisioning started (target cores: {})", Stats.TargetCoreCount); + } +# endif + + EnqueueProvisionerMaintenanceTimer(); + m_Http->Run(IsInteractiveMode); SetNewState(kShuttingDown); @@ -254,6 +943,8 @@ ZenComputeServerMain::ZenComputeServerMain(ZenComputeServerConfig& ServerOptions void ZenComputeServerMain::DoRun(ZenServerState::ZenServerEntry* Entry) { + ZEN_TRACE_CPU("ZenComputeServerMain::DoRun"); + ZenComputeServer Server; Server.SetDataRoot(m_ServerOptions.DataDir); Server.SetContentRoot(m_ServerOptions.ContentDir); diff --git a/src/zenserver/compute/computeserver.h b/src/zenserver/compute/computeserver.h index 625140b23..e4a6b01d5 100644 --- a/src/zenserver/compute/computeserver.h +++ b/src/zenserver/compute/computeserver.h @@ -6,7 +6,11 @@ #if ZEN_WITH_COMPUTE_SERVICES +# include +# include +# include # include +# include "frontend/frontend.h" namespace cxxopts { class Options; @@ -16,19 +20,46 @@ struct Options; } namespace zen::compute { -class HttpFunctionService; -} +class CloudMetadata; +class HttpComputeService; +class HttpOrchestratorService; +} // namespace zen::compute + +# if ZEN_WITH_HORDE +# include +namespace zen::horde { +class HordeProvisioner; +} // namespace zen::horde +# endif + +# if ZEN_WITH_NOMAD +# include +namespace zen::nomad { +class NomadProvisioner; +} // namespace zen::nomad +# endif namespace zen { class CidStore; class HttpApiService; -class HttpComputeService; struct ZenComputeServerConfig : public ZenServerConfig { std::string UpstreamNotificationEndpoint; std::string InstanceId; // For use in notifications + std::string CoordinatorEndpoint; + std::string IdmsEndpoint; + int32_t MaxConcurrentActions = 0; // 0 = auto (LogicalProcessorCount * 2) + bool EnableWorkerWebSocket = false; // Use WebSocket for worker↔orchestrator link + +# if ZEN_WITH_HORDE + horde::HordeConfig HordeConfig; +# endif + +# if ZEN_WITH_NOMAD + nomad::NomadConfig NomadConfig; +# endif }; struct ZenComputeServerConfigurator : public ZenServerConfiguratorBase @@ -49,6 +80,16 @@ private: virtual void ValidateOptions() override; ZenComputeServerConfig& m_ServerOptions; + +# if ZEN_WITH_HORDE + std::string m_HordeModeStr = "direct"; + std::string m_HordeEncryptionStr = "none"; +# endif + +# if ZEN_WITH_NOMAD + std::string m_NomadDriverStr = "raw_exec"; + std::string m_NomadDistributionStr = "predeployed"; +# endif }; class ZenComputeServerMain : public ZenServerMain @@ -88,17 +129,59 @@ public: void Cleanup(); private: - HttpStatsService m_StatsService; - GcManager m_GcManager; - GcScheduler m_GcScheduler{m_GcManager}; - std::unique_ptr m_CidStore; - std::unique_ptr m_ComputeService; - std::unique_ptr m_ApiService; - std::unique_ptr m_FunctionService; - - void InitializeState(const ZenComputeServerConfig& ServerConfig); - void InitializeServices(const ZenComputeServerConfig& ServerConfig); - void RegisterServices(const ZenComputeServerConfig& ServerConfig); + HttpStatsService m_StatsService; + GcManager m_GcManager; + GcScheduler m_GcScheduler{m_GcManager}; + std::unique_ptr m_CidStore; + std::unique_ptr m_ApiService; + std::unique_ptr m_ComputeService; + std::unique_ptr m_OrchestratorService; + std::unique_ptr m_CloudMetadata; + std::future> m_CloudMetadataFuture; + std::unique_ptr m_FrontendService; +# if ZEN_WITH_HORDE + std::unique_ptr m_HordeProvisioner; +# endif +# if ZEN_WITH_NOMAD + std::unique_ptr m_NomadProvisioner; +# endif + SystemMetricsTracker m_MetricsTracker; + std::string m_CoordinatorEndpoint; + std::string m_InstanceId; + + asio::steady_timer m_AnnounceTimer{m_IoContext}; + asio::steady_timer m_ProvisionerMaintenanceTimer{m_IoContext}; + + void InitializeState(const ZenComputeServerConfig& ServerConfig); + void InitializeServices(const ZenComputeServerConfig& ServerConfig); + void RegisterServices(const ZenComputeServerConfig& ServerConfig); + void ResolveCloudMetadata(); + void PostAnnounce(); + void EnqueueAnnounceTimer(); + void EnqueueProvisionerMaintenanceTimer(); + void ProvisionerMaintenanceTick(); + std::string GetAnnounceUrl() const; + std::string GetInstanceId() const; + CbObject BuildAnnounceBody(); + + // Worker→orchestrator WebSocket client + struct OrchestratorWsHandler : public IWsClientHandler + { + ZenComputeServer& Server; + explicit OrchestratorWsHandler(ZenComputeServer& S) : Server(S) {} + + void OnWsOpen() override; + void OnWsMessage(const WebSocketMessage& Msg) override; + void OnWsClose(uint16_t Code, std::string_view Reason) override; + }; + + std::unique_ptr m_OrchestratorWsHandler; + std::unique_ptr m_OrchestratorWsClient; + asio::steady_timer m_WsReconnectTimer{m_IoContext}; + bool m_EnableWorkerWebSocket = false; + + void InitializeOrchestratorWebSocket(); + void EnqueueWsReconnect(); }; } // namespace zen diff --git a/src/zenserver/compute/computeservice.cpp b/src/zenserver/compute/computeservice.cpp deleted file mode 100644 index 2c0bc0ae9..000000000 --- a/src/zenserver/compute/computeservice.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#include "computeservice.h" - -#if ZEN_WITH_COMPUTE_SERVICES - -# include -# include -# include -# include -# include -# include - -ZEN_THIRD_PARTY_INCLUDES_START -# include -# include -ZEN_THIRD_PARTY_INCLUDES_END - -# include - -namespace zen { - -////////////////////////////////////////////////////////////////////////// - -struct ResourceMetrics -{ - uint64_t DiskUsageBytes = 0; - uint64_t MemoryUsageBytes = 0; -}; - -////////////////////////////////////////////////////////////////////////// - -struct HttpComputeService::Impl -{ - Impl(const Impl&) = delete; - Impl& operator=(const Impl&) = delete; - - Impl(); - ~Impl(); - - void Initialize(std::filesystem::path BaseDir) { ZEN_UNUSED(BaseDir); } - - void Cleanup() {} - -private: -}; - -HttpComputeService::Impl::Impl() -{ -} - -HttpComputeService::Impl::~Impl() -{ -} - -/////////////////////////////////////////////////////////////////////////// - -HttpComputeService::HttpComputeService(std::filesystem::path BaseDir) : m_Impl(std::make_unique()) -{ - using namespace std::literals; - - m_Impl->Initialize(BaseDir); - - m_Router.RegisterRoute( - "status", - [this](HttpRouterRequest& Req) { - CbObjectWriter Obj; - Obj.BeginArray("modules"); - Obj.EndArray(); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - }, - HttpVerb::kGet); - - m_Router.RegisterRoute( - "stats", - [this](HttpRouterRequest& Req) { - CbObjectWriter Obj; - Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); - }, - HttpVerb::kGet); -} - -HttpComputeService::~HttpComputeService() -{ -} - -const char* -HttpComputeService::BaseUri() const -{ - return "/compute/"; -} - -void -HttpComputeService::HandleRequest(zen::HttpServerRequest& Request) -{ - m_Router.HandleRequest(Request); -} - -} // namespace zen -#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zenserver/compute/computeservice.h b/src/zenserver/compute/computeservice.h deleted file mode 100644 index 339200dd8..000000000 --- a/src/zenserver/compute/computeservice.h +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include - -#if ZEN_WITH_COMPUTE_SERVICES -namespace zen { - -/** ZenServer Compute Service - * - * Manages a set of compute workers for use in UEFN content worker - * - */ -class HttpComputeService : public zen::HttpService -{ -public: - HttpComputeService(std::filesystem::path BaseDir); - ~HttpComputeService(); - - HttpComputeService(const HttpComputeService&) = delete; - HttpComputeService& operator=(const HttpComputeService&) = delete; - - virtual const char* BaseUri() const override; - virtual void HandleRequest(zen::HttpServerRequest& Request) override; - -private: - HttpRequestRouter m_Router; - - struct Impl; - - std::unique_ptr m_Impl; -}; - -} // namespace zen -#endif // ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index 4767029c0..c167cc70e 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ diff --git a/src/zenserver/frontend/html/404.html b/src/zenserver/frontend/html/404.html new file mode 100644 index 000000000..829ef2097 --- /dev/null +++ b/src/zenserver/frontend/html/404.html @@ -0,0 +1,486 @@ + + + + + +Ooops + + + + + + +
      + +
      + +
      +

      404 NOT FOUND

      +
      + + + + + + diff --git a/src/zenserver/frontend/html/compute.html b/src/zenserver/frontend/html/compute.html deleted file mode 100644 index 668189fe5..000000000 --- a/src/zenserver/frontend/html/compute.html +++ /dev/null @@ -1,991 +0,0 @@ - - - - - - Zen Compute Dashboard - - - - -
      -
      -
      -

      Zen Compute Dashboard

      -
      Last updated: Never
      -
      -
      -
      - Checking... -
      -
      - -
      - - -
      Action Queue
      -
      -
      -
      Pending Actions
      -
      -
      -
      Waiting to be scheduled
      -
      -
      -
      Running Actions
      -
      -
      -
      Currently executing
      -
      -
      -
      Completed Actions
      -
      -
      -
      Results available
      -
      -
      - - -
      -
      Action Queue History
      -
      - -
      -
      - - -
      Performance Metrics
      -
      -
      Completion Rate
      -
      -
      -
      -
      -
      1 min rate
      -
      -
      -
      -
      -
      5 min rate
      -
      -
      -
      -
      -
      15 min rate
      -
      -
      -
      -
      - Total Retired - - -
      -
      - Mean Rate - - -
      -
      -
      - - -
      Workers
      -
      -
      Worker Status
      -
      - Registered Workers - - -
      - -
      - - -
      Recent Actions
      -
      -
      Action History
      -
      No actions recorded yet.
      - -
      - - -
      System Resources
      -
      -
      -
      CPU Usage
      -
      -
      -
      Percent
      -
      -
      -
      -
      - -
      -
      -
      - Packages - - -
      -
      - Physical Cores - - -
      -
      - Logical Processors - - -
      -
      -
      -
      -
      Memory
      -
      - Used - - -
      -
      - Total - - -
      -
      -
      -
      -
      -
      -
      Disk
      -
      - Used - - -
      -
      - Total - - -
      -
      -
      -
      -
      -
      -
      - - - - diff --git a/src/zenserver/frontend/html/compute/banner.js b/src/zenserver/frontend/html/compute/banner.js new file mode 100644 index 000000000..61c7ce21f --- /dev/null +++ b/src/zenserver/frontend/html/compute/banner.js @@ -0,0 +1,321 @@ +/** + * zen-banner.js — Zen Compute dashboard banner Web Component + * + * Usage: + * + * + * + * + * + * + * Attributes: + * variant "full" (default) | "compact" + * cluster-status "nominal" (default) | "degraded" | "offline" + * load 0–100 integer, shown as a percentage (default: hidden) + * tagline custom tagline text (default: "Orchestrator Overview" / "Orchestrator") + * subtitle text after "ZEN" in the wordmark (default: "COMPUTE") + */ + +class ZenBanner extends HTMLElement { + + static get observedAttributes() { + return ['variant', 'cluster-status', 'load', 'tagline', 'subtitle']; + } + + attributeChangedCallback() { + if (this.shadowRoot) this._render(); + } + + connectedCallback() { + if (!this.shadowRoot) this.attachShadow({ mode: 'open' }); + this._render(); + } + + // ───────────────────────────────────────────── + // Derived values + // ───────────────────────────────────────────── + + get _variant() { return this.getAttribute('variant') || 'full'; } + get _status() { return (this.getAttribute('cluster-status') || 'nominal').toLowerCase(); } + get _load() { return this.getAttribute('load'); } // null → hidden + get _tagline() { return this.getAttribute('tagline'); } // null → default + get _subtitle() { return this.getAttribute('subtitle'); } // null → "COMPUTE" + + get _statusColor() { + return { nominal: '#7ecfb8', degraded: '#d4a84b', offline: '#c0504d' }[this._status] ?? '#7ecfb8'; + } + + get _statusLabel() { + return { nominal: 'NOMINAL', degraded: 'DEGRADED', offline: 'OFFLINE' }[this._status] ?? 'NOMINAL'; + } + + get _loadColor() { + const v = parseInt(this._load, 10); + if (isNaN(v)) return '#7ecfb8'; + if (v >= 85) return '#c0504d'; + if (v >= 60) return '#d4a84b'; + return '#7ecfb8'; + } + + // ───────────────────────────────────────────── + // Render + // ───────────────────────────────────────────── + + _render() { + const compact = this._variant === 'compact'; + this.shadowRoot.innerHTML = ` + + ${this._html(compact)} + `; + } + + // ───────────────────────────────────────────── + // CSS + // ───────────────────────────────────────────── + + _css(compact) { + const height = compact ? '60px' : '100px'; + const padding = compact ? '0 24px' : '0 32px'; + const gap = compact ? '16px' : '24px'; + const markSize = compact ? '34px' : '52px'; + const divH = compact ? '32px' : '48px'; + const nameSize = compact ? '15px' : '22px'; + const tagSize = compact ? '9px' : '11px'; + const sc = this._statusColor; + const lc = this._loadColor; + + return ` + @import url('https://fonts.googleapis.com/css2?family=Noto+Serif+JP:wght@300;400&family=Space+Mono:wght@400;700&display=swap'); + + *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + + :host { + display: block; + font-family: 'Space Mono', monospace; + } + + .banner { + width: 100%; + height: ${height}; + background: #0b0d10; + border: 1px solid #1e2330; + border-radius: 6px; + display: flex; + align-items: center; + padding: ${padding}; + gap: ${gap}; + position: relative; + overflow: hidden; + } + + /* scan-line texture */ + .banner::before { + content: ''; + position: absolute; + inset: 0; + background: repeating-linear-gradient( + 0deg, + transparent, transparent 3px, + rgba(255,255,255,0.012) 3px, rgba(255,255,255,0.012) 4px + ); + pointer-events: none; + } + + /* ambient glow */ + .banner::after { + content: ''; + position: absolute; + right: -60px; + top: 50%; + transform: translateY(-50%); + width: 280px; + height: 280px; + background: radial-gradient(circle, rgba(130,200,180,0.06) 0%, transparent 70%); + pointer-events: none; + } + + .logo-mark { + flex-shrink: 0; + width: ${markSize}; + height: ${markSize}; + } + + .logo-mark svg { width: 100%; height: 100%; } + + .divider { + width: 1px; + height: ${divH}; + background: linear-gradient(to bottom, transparent, #2a3040, transparent); + flex-shrink: 0; + } + + .text-block { + display: flex; + flex-direction: column; + gap: 4px; + } + + .wordmark { + font-weight: 700; + font-size: ${nameSize}; + letter-spacing: 0.12em; + color: #e8e4dc; + text-transform: uppercase; + line-height: 1; + } + + .wordmark span { color: #7ecfb8; } + + .tagline { + font-family: 'Noto Serif JP', serif; + font-weight: 300; + font-size: ${tagSize}; + letter-spacing: 0.3em; + color: #4a5a68; + text-transform: uppercase; + } + + .spacer { flex: 1; } + + /* ── right-side decorative circuit ── */ + .circuit { flex-shrink: 0; opacity: 0.22; } + + /* ── status cluster ── */ + .status-cluster { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 6px; + } + + .status-row { + display: flex; + align-items: center; + gap: 8px; + } + + .status-lbl { + font-size: 9px; + letter-spacing: 0.18em; + color: #3a4555; + text-transform: uppercase; + } + + .pill { + display: flex; + align-items: center; + gap: 5px; + border-radius: 20px; + padding: 2px 10px; + font-size: 10px; + letter-spacing: 0.1em; + } + + .pill.cluster { + color: ${sc}; + background: color-mix(in srgb, ${sc} 8%, transparent); + border: 1px solid color-mix(in srgb, ${sc} 28%, transparent); + } + + .pill.load-pill { + color: ${lc}; + background: color-mix(in srgb, ${lc} 8%, transparent); + border: 1px solid color-mix(in srgb, ${lc} 28%, transparent); + } + + .dot { + width: 5px; + height: 5px; + border-radius: 50%; + animation: pulse 2.4s ease-in-out infinite; + } + + .dot.cluster { background: ${sc}; } + .dot.load-dot { background: ${lc}; animation-delay: 0.5s; } + + @keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.25; } + } + `; + } + + // ───────────────────────────────────────────── + // HTML template + // ───────────────────────────────────────────── + + _html(compact) { + const loadAttr = this._load; + const showStatus = !compact; + + const rightSide = showStatus ? ` + + + + + + + + + +
      +
      + Cluster +
      +
      + ${this._statusLabel} +
      +
      + ${loadAttr !== null ? ` +
      + Load +
      +
      + ${parseInt(loadAttr, 10)} % +
      +
      ` : ''} +
      + ` : ''; + + return ` + + `; + } + + // ───────────────────────────────────────────── + // SVG logo mark + // ───────────────────────────────────────────── + + _svgMark() { + return ` + + + + + + + + + + + + + + + + + + `; + } +} + +customElements.define('zen-banner', ZenBanner); diff --git a/src/zenserver/frontend/html/compute/compute.html b/src/zenserver/frontend/html/compute/compute.html new file mode 100644 index 000000000..1e101d839 --- /dev/null +++ b/src/zenserver/frontend/html/compute/compute.html @@ -0,0 +1,1072 @@ + + + + + + Zen Compute Dashboard + + + + + + +
      + + + Node + Orchestrator + +
      Last updated: Never
      + +
      + + +
      Action Queue
      +
      +
      +
      Pending Actions
      +
      -
      +
      Waiting to be scheduled
      +
      +
      +
      Running Actions
      +
      -
      +
      Currently executing
      +
      +
      +
      Completed Actions
      +
      -
      +
      Results available
      +
      +
      + + +
      +
      Action Queue History
      +
      + +
      +
      + + +
      Performance Metrics
      +
      +
      Completion Rate
      +
      +
      +
      -
      +
      1 min rate
      +
      +
      +
      -
      +
      5 min rate
      +
      +
      +
      -
      +
      15 min rate
      +
      +
      +
      +
      + Total Retired + - +
      +
      + Mean Rate + - +
      +
      +
      + + +
      Workers
      +
      +
      Worker Status
      +
      + Registered Workers + - +
      + +
      + + +
      Queues
      +
      +
      Queue Status
      +
      No queues.
      + +
      + + +
      Recent Actions
      +
      +
      Action History
      +
      No actions recorded yet.
      + +
      + + +
      System Resources
      +
      +
      +
      CPU Usage
      +
      -
      +
      Percent
      +
      +
      +
      +
      + +
      +
      +
      + Packages + - +
      +
      + Physical Cores + - +
      +
      + Logical Processors + - +
      +
      +
      +
      +
      Memory
      +
      + Used + - +
      +
      + Total + - +
      +
      +
      +
      +
      +
      +
      Disk
      +
      + Used + - +
      +
      + Total + - +
      +
      +
      +
      +
      +
      +
      + + + + diff --git a/src/zenserver/frontend/html/compute/hub.html b/src/zenserver/frontend/html/compute/hub.html new file mode 100644 index 000000000..f66ba94d5 --- /dev/null +++ b/src/zenserver/frontend/html/compute/hub.html @@ -0,0 +1,310 @@ + + + + + + + + Zen Hub Dashboard + + + +
      + + + Hub + +
      Last updated: Never
      + +
      + +
      Capacity
      +
      +
      +
      Active Modules
      +
      -
      +
      Currently provisioned
      +
      +
      +
      Peak Modules
      +
      -
      +
      High watermark
      +
      +
      +
      Instance Limit
      +
      -
      +
      Maximum allowed
      +
      +
      +
      +
      +
      + +
      Modules
      +
      +
      Storage Server Instances
      +
      No modules provisioned.
      + + + + + + + + + +
      +
      + + + + diff --git a/src/zenserver/frontend/html/compute/index.html b/src/zenserver/frontend/html/compute/index.html new file mode 100644 index 000000000..9597fd7f3 --- /dev/null +++ b/src/zenserver/frontend/html/compute/index.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/zenserver/frontend/html/compute/nav.js b/src/zenserver/frontend/html/compute/nav.js new file mode 100644 index 000000000..8ec42abd0 --- /dev/null +++ b/src/zenserver/frontend/html/compute/nav.js @@ -0,0 +1,79 @@ +/** + * zen-nav.js — Zen dashboard navigation bar Web Component + * + * Usage: + * + * + * + * Node + * Orchestrator + * + * + * Each child becomes a nav link. The current page is + * highlighted automatically based on the href. + */ + +class ZenNav extends HTMLElement { + + connectedCallback() { + if (!this.shadowRoot) this.attachShadow({ mode: 'open' }); + this._render(); + } + + _render() { + const currentPath = window.location.pathname; + const items = Array.from(this.querySelectorAll(':scope > a')); + + const links = items.map(a => { + const href = a.getAttribute('href') || ''; + const label = a.textContent.trim(); + const active = currentPath.endsWith(href); + return `${label}`; + }).join(''); + + this.shadowRoot.innerHTML = ` + + + `; + } +} + +customElements.define('zen-nav', ZenNav); diff --git a/src/zenserver/frontend/html/compute/orchestrator.html b/src/zenserver/frontend/html/compute/orchestrator.html new file mode 100644 index 000000000..2ee57b6b3 --- /dev/null +++ b/src/zenserver/frontend/html/compute/orchestrator.html @@ -0,0 +1,831 @@ + + + + + + + + Zen Orchestrator Dashboard + + + +
      + + + Node + Orchestrator + +
      +
      +
      Last updated: Never
      +
      +
      + Agents: + - +
      +
      + +
      + +
      +
      Compute Agents
      +
      No agents registered.
      + + + + + + + + + + + + + + + + + + +
      +
      +
      Connected Clients
      +
      No clients connected.
      + + + + + + + + + + + + +
      +
      +
      +
      Event History
      +
      + + +
      +
      +
      +
      No provisioning events recorded.
      + + + + + + + + + + + +
      + +
      +
      + + + + diff --git a/src/zenserver/frontend/html/pages/page.js b/src/zenserver/frontend/html/pages/page.js index 3c2d3619a..592b699dc 100644 --- a/src/zenserver/frontend/html/pages/page.js +++ b/src/zenserver/frontend/html/pages/page.js @@ -3,6 +3,7 @@ "use strict"; import { WidgetHost } from "../util/widgets.js" +import { Fetcher } from "../util/fetcher.js" //////////////////////////////////////////////////////////////////////////////// export class PageBase extends WidgetHost @@ -63,6 +64,7 @@ export class ZenPage extends PageBase super(parent, ...args); super.set_title("zen"); this.add_branding(parent); + this.add_service_nav(parent); this.generate_crumbs(); } @@ -78,6 +80,40 @@ export class ZenPage extends PageBase root.tag("img").attr("src", "epicgames.ico").id("epic_logo"); } + add_service_nav(parent) + { + const nav = parent.tag().id("service_nav"); + + // Map service base URIs to dashboard links, this table is also used to detemine + // which links to show based on the services that are currently registered. + + const service_dashboards = [ + { base_uri: "/compute/", label: "Compute", href: "/dashboard/compute/compute.html" }, + { base_uri: "/orch/", label: "Orchestrator", href: "/dashboard/compute/orchestrator.html" }, + { base_uri: "/hub/", label: "Hub", href: "/dashboard/compute/hub.html" }, + ]; + + new Fetcher().resource("/api/").json().then((data) => { + const services = data.services || []; + const uris = new Set(services.map(s => s.base_uri)); + + const links = service_dashboards.filter(d => uris.has(d.base_uri)); + + if (links.length === 0) + { + nav.inner().style.display = "none"; + return; + } + + for (const link of links) + { + nav.tag("a").text(link.label).attr("href", link.href); + } + }).catch(() => { + nav.inner().style.display = "none"; + }); + } + set_title(...args) { super.set_title(...args); diff --git a/src/zenserver/frontend/html/zen.css b/src/zenserver/frontend/html/zen.css index 702bf9aa6..a80a1a4f6 100644 --- a/src/zenserver/frontend/html/zen.css +++ b/src/zenserver/frontend/html/zen.css @@ -80,6 +80,33 @@ input { } } +/* service nav -------------------------------------------------------------- */ + +#service_nav { + display: flex; + justify-content: center; + gap: 0.3em; + margin-bottom: 1.5em; + padding: 0.3em; + background-color: var(--theme_g3); + border: 1px solid var(--theme_g2); + border-radius: 0.4em; + + a { + padding: 0.3em 0.9em; + border-radius: 0.3em; + font-size: 0.85em; + color: var(--theme_g1); + text-decoration: none; + } + + a:hover { + background-color: var(--theme_p4); + color: var(--theme_g0); + text-decoration: none; + } +} + /* links -------------------------------------------------------------------- */ a { diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp index bf0e294c5..a757cd594 100644 --- a/src/zenserver/hub/hubservice.cpp +++ b/src/zenserver/hub/hubservice.cpp @@ -845,7 +845,7 @@ HttpHubService::HttpHubService(std::filesystem::path HubBaseDir, std::filesystem Obj << "currentInstanceCount" << m_Impl->GetInstanceCount(); Obj << "maxInstanceCount" << m_Impl->GetMaxInstanceCount(); Obj << "instanceLimit" << m_Impl->GetInstanceLimit(); - Req.ServerRequest().WriteResponse(HttpResponseCode::OK); + Req.ServerRequest().WriteResponse(HttpResponseCode::OK, Obj.Save()); }, HttpVerb::kGet); } diff --git a/src/zenserver/hub/zenhubserver.cpp b/src/zenserver/hub/zenhubserver.cpp index d0a0db417..c63c618df 100644 --- a/src/zenserver/hub/zenhubserver.cpp +++ b/src/zenserver/hub/zenhubserver.cpp @@ -143,6 +143,8 @@ ZenHubServer::InitializeServices(const ZenHubServerConfig& ServerConfig) ZEN_INFO("instantiating hub service"); m_HubService = std::make_unique(ServerConfig.DataDir / "hub", ServerConfig.DataDir / "servers"); m_HubService->SetNotificationEndpoint(ServerConfig.UpstreamNotificationEndpoint, ServerConfig.InstanceId); + + m_FrontendService = std::make_unique(m_ContentRoot, m_StatusService); } void @@ -159,6 +161,11 @@ ZenHubServer::RegisterServices(const ZenHubServerConfig& ServerConfig) { m_Http->RegisterService(*m_ApiService); } + + if (m_FrontendService) + { + m_Http->RegisterService(*m_FrontendService); + } } void diff --git a/src/zenserver/hub/zenhubserver.h b/src/zenserver/hub/zenhubserver.h index ac14362f0..4c56fdce5 100644 --- a/src/zenserver/hub/zenhubserver.h +++ b/src/zenserver/hub/zenhubserver.h @@ -2,6 +2,7 @@ #pragma once +#include "frontend/frontend.h" #include "zenserver.h" namespace cxxopts { @@ -81,8 +82,9 @@ private: std::filesystem::path m_ContentRoot; bool m_DebugOptionForcedCrash = false; - std::unique_ptr m_HubService; - std::unique_ptr m_ApiService; + std::unique_ptr m_HubService; + std::unique_ptr m_ApiService; + std::unique_ptr m_FrontendService; void InitializeState(const ZenHubServerConfig& ServerConfig); void InitializeServices(const ZenHubServerConfig& ServerConfig); diff --git a/src/zenserver/storage/zenstorageserver.cpp b/src/zenserver/storage/zenstorageserver.cpp index 3d81db656..bca26e87a 100644 --- a/src/zenserver/storage/zenstorageserver.cpp +++ b/src/zenserver/storage/zenstorageserver.cpp @@ -183,10 +183,15 @@ ZenStorageServer::RegisterServices() m_Http->RegisterService(*m_AdminService); + if (m_ApiService) + { + m_Http->RegisterService(*m_ApiService); + } + #if ZEN_WITH_COMPUTE_SERVICES - if (m_HttpFunctionService) + if (m_HttpComputeService) { - m_Http->RegisterService(*m_HttpFunctionService); + m_Http->RegisterService(*m_HttpComputeService); } #endif } @@ -279,8 +284,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions { ZEN_OTEL_SPAN("InitializeComputeService"); - m_HttpFunctionService = - std::make_unique(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions"); + m_HttpComputeService = + std::make_unique(*m_CidStore, m_StatsService, ServerOptions.DataDir / "functions"); } #endif @@ -316,6 +321,8 @@ ZenStorageServer::InitializeServices(const ZenStorageServerConfig& ServerOptions .AttachmentPassCount = ServerOptions.GcConfig.AttachmentPassCount}; m_GcScheduler.Initialize(GcConfig); + m_ApiService = std::make_unique(*m_Http); + // Create and register admin interface last to make sure all is properly initialized m_AdminService = std::make_unique( m_GcScheduler, @@ -832,7 +839,7 @@ ZenStorageServer::Cleanup() Flush(); #if ZEN_WITH_COMPUTE_SERVICES - m_HttpFunctionService.reset(); + m_HttpComputeService.reset(); #endif m_AdminService.reset(); diff --git a/src/zenserver/storage/zenstorageserver.h b/src/zenserver/storage/zenstorageserver.h index 456447a2a..5b163fc8e 100644 --- a/src/zenserver/storage/zenstorageserver.h +++ b/src/zenserver/storage/zenstorageserver.h @@ -25,7 +25,7 @@ #include "workspaces/httpworkspaces.h" #if ZEN_WITH_COMPUTE_SERVICES -# include +# include #endif namespace zen { @@ -93,7 +93,7 @@ private: std::unique_ptr m_ApiService; #if ZEN_WITH_COMPUTE_SERVICES - std::unique_ptr m_HttpFunctionService; + std::unique_ptr m_HttpComputeService; #endif }; diff --git a/src/zenserver/trace/tracerecorder.cpp b/src/zenserver/trace/tracerecorder.cpp new file mode 100644 index 000000000..5dec20e18 --- /dev/null +++ b/src/zenserver/trace/tracerecorder.cpp @@ -0,0 +1,565 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include "tracerecorder.h" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace zen { + +//////////////////////////////////////////////////////////////////////////////// + +struct TraceSession : public std::enable_shared_from_this +{ + TraceSession(asio::ip::tcp::socket&& Socket, const std::filesystem::path& OutputDir) + : m_Socket(std::move(Socket)) + , m_OutputDir(OutputDir) + , m_SessionId(Oid::NewOid()) + { + try + { + m_RemoteAddress = m_Socket.remote_endpoint().address().to_string(); + } + catch (...) + { + m_RemoteAddress = "unknown"; + } + + ZEN_INFO("Trace session {} started from {}", m_SessionId, m_RemoteAddress); + } + + ~TraceSession() + { + if (m_TraceFile.IsOpen()) + { + m_TraceFile.Close(); + } + + ZEN_INFO("Trace session {} ended, {} bytes recorded to '{}'", m_SessionId, m_TotalBytesRecorded, m_TraceFilePath); + } + + void Start() { ReadPreambleHeader(); } + + bool IsActive() const { return m_Socket.is_open(); } + + TraceSessionInfo GetInfo() const + { + TraceSessionInfo Info; + Info.SessionGuid = m_SessionGuid; + Info.TraceGuid = m_TraceGuid; + Info.ControlPort = m_ControlPort; + Info.TransportVersion = m_TransportVersion; + Info.ProtocolVersion = m_ProtocolVersion; + Info.RemoteAddress = m_RemoteAddress; + Info.BytesRecorded = m_TotalBytesRecorded; + Info.TraceFilePath = m_TraceFilePath; + return Info; + } + +private: + // Preamble format: + // [magic: 4 bytes][metadata_size: 2 bytes][metadata fields: variable][version: 2 bytes] + // + // Magic bytes: [0]=version_char ('2'-'9'), [1]='C', [2]='R', [3]='T' + // + // Metadata fields (repeated): + // [size: 1 byte][id: 1 byte][data: bytes] + // Field 0: ControlPort (uint16) + // Field 1: SessionGuid (16 bytes) + // Field 2: TraceGuid (16 bytes) + // + // Version: [transport: 1 byte][protocol: 1 byte] + + static constexpr size_t kMagicSize = 4; + static constexpr size_t kMetadataSizeFieldSize = 2; + static constexpr size_t kPreambleHeaderSize = kMagicSize + kMetadataSizeFieldSize; + static constexpr size_t kVersionSize = 2; + static constexpr size_t kPreambleBufferSize = 256; + static constexpr size_t kReadBufferSize = 64 * 1024; + + void ReadPreambleHeader() + { + auto Self = shared_from_this(); + + // Read the first 6 bytes: 4 magic + 2 metadata size + asio::async_read(m_Socket, + asio::buffer(m_PreambleBuffer, kPreambleHeaderSize), + [this, Self](const asio::error_code& Ec, std::size_t /*BytesRead*/) { + if (Ec) + { + HandleReadError("preamble header", Ec); + return; + } + + if (!ValidateMagic()) + { + ZEN_WARN("Trace session {}: invalid trace magic header", m_SessionId); + CloseSocket(); + return; + } + + ReadPreambleMetadata(); + }); + } + + bool ValidateMagic() + { + const uint8_t* Cursor = m_PreambleBuffer; + + // Validate magic: bytes are version, 'C', 'R', 'T' + if (Cursor[3] != 'T' || Cursor[2] != 'R' || Cursor[1] != 'C') + { + return false; + } + + if (Cursor[0] < '2' || Cursor[0] > '9') + { + return false; + } + + // Extract the metadata fields size (does not include the trailing version bytes) + std::memcpy(&m_MetadataFieldsSize, Cursor + kMagicSize, sizeof(m_MetadataFieldsSize)); + + if (m_MetadataFieldsSize + kVersionSize > kPreambleBufferSize - kPreambleHeaderSize) + { + return false; + } + + return true; + } + + void ReadPreambleMetadata() + { + auto Self = shared_from_this(); + size_t ReadSize = m_MetadataFieldsSize + kVersionSize; + + // Read metadata fields + 2 version bytes + asio::async_read(m_Socket, + asio::buffer(m_PreambleBuffer + kPreambleHeaderSize, ReadSize), + [this, Self](const asio::error_code& Ec, std::size_t /*BytesRead*/) { + if (Ec) + { + HandleReadError("preamble metadata", Ec); + return; + } + + if (!ParseMetadata()) + { + ZEN_WARN("Trace session {}: malformed trace metadata", m_SessionId); + CloseSocket(); + return; + } + + if (!CreateTraceFile()) + { + CloseSocket(); + return; + } + + // Write the full preamble to the trace file so it remains a valid .utrace + size_t PreambleSize = kPreambleHeaderSize + m_MetadataFieldsSize + kVersionSize; + std::error_code WriteEc; + m_TraceFile.Write(m_PreambleBuffer, PreambleSize, 0, WriteEc); + + if (WriteEc) + { + ZEN_ERROR("Trace session {}: failed to write preamble: {}", m_SessionId, WriteEc.message()); + CloseSocket(); + return; + } + + m_TotalBytesRecorded = PreambleSize; + + ZEN_INFO("Trace session {}: metadata - TransportV{} ProtocolV{} ControlPort:{} SessionGuid:{} TraceGuid:{}", + m_SessionId, + m_TransportVersion, + m_ProtocolVersion, + m_ControlPort, + m_SessionGuid, + m_TraceGuid); + + // Begin streaming trace data to disk + ReadMore(); + }); + } + + bool ParseMetadata() + { + const uint8_t* Cursor = m_PreambleBuffer + kPreambleHeaderSize; + int32_t Remaining = static_cast(m_MetadataFieldsSize); + + while (Remaining >= 2) + { + uint8_t FieldSize = Cursor[0]; + uint8_t FieldId = Cursor[1]; + Cursor += 2; + Remaining -= 2; + + if (Remaining < FieldSize) + { + return false; + } + + switch (FieldId) + { + case 0: // ControlPort + if (FieldSize >= sizeof(uint16_t)) + { + std::memcpy(&m_ControlPort, Cursor, sizeof(uint16_t)); + } + break; + case 1: // SessionGuid + if (FieldSize >= sizeof(Guid)) + { + std::memcpy(&m_SessionGuid, Cursor, sizeof(Guid)); + } + break; + case 2: // TraceGuid + if (FieldSize >= sizeof(Guid)) + { + std::memcpy(&m_TraceGuid, Cursor, sizeof(Guid)); + } + break; + } + + Cursor += FieldSize; + Remaining -= FieldSize; + } + + // Metadata should be fully consumed + if (Remaining != 0) + { + return false; + } + + // Version bytes follow immediately after the metadata fields + const uint8_t* VersionPtr = m_PreambleBuffer + kPreambleHeaderSize + m_MetadataFieldsSize; + m_TransportVersion = VersionPtr[0]; + m_ProtocolVersion = VersionPtr[1]; + + return true; + } + + bool CreateTraceFile() + { + m_TraceFilePath = m_OutputDir / fmt::format("{}.utrace", m_SessionId); + + try + { + m_TraceFile.Open(m_TraceFilePath, BasicFile::Mode::kTruncate); + ZEN_INFO("Trace session {} writing to '{}'", m_SessionId, m_TraceFilePath); + return true; + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Trace session {}: failed to create trace file '{}': {}", m_SessionId, m_TraceFilePath, Ex.what()); + return false; + } + } + + void ReadMore() + { + auto Self = shared_from_this(); + + m_Socket.async_read_some(asio::buffer(m_ReadBuffer, kReadBufferSize), + [this, Self](const asio::error_code& Ec, std::size_t BytesRead) { + if (!Ec) + { + if (BytesRead > 0 && m_TraceFile.IsOpen()) + { + std::error_code WriteEc; + const uint64_t FileOffset = m_TotalBytesRecorded; + m_TraceFile.Write(m_ReadBuffer, BytesRead, FileOffset, WriteEc); + + if (WriteEc) + { + ZEN_ERROR("Trace session {}: write error: {}", m_SessionId, WriteEc.message()); + CloseSocket(); + return; + } + + m_TotalBytesRecorded += BytesRead; + } + + ReadMore(); + } + else if (Ec == asio::error::eof) + { + ZEN_DEBUG("Trace session {} connection closed by peer", m_SessionId); + CloseSocket(); + } + else if (Ec == asio::error::operation_aborted) + { + ZEN_DEBUG("Trace session {} operation aborted", m_SessionId); + } + else + { + ZEN_WARN("Trace session {} read error: {}", m_SessionId, Ec.message()); + CloseSocket(); + } + }); + } + + void HandleReadError(const char* Phase, const asio::error_code& Ec) + { + if (Ec == asio::error::eof) + { + ZEN_DEBUG("Trace session {}: connection closed during {}", m_SessionId, Phase); + } + else if (Ec == asio::error::operation_aborted) + { + ZEN_DEBUG("Trace session {}: operation aborted during {}", m_SessionId, Phase); + } + else + { + ZEN_WARN("Trace session {}: error during {}: {}", m_SessionId, Phase, Ec.message()); + } + + CloseSocket(); + } + + void CloseSocket() + { + std::error_code Ec; + m_Socket.close(Ec); + + if (m_TraceFile.IsOpen()) + { + m_TraceFile.Close(); + } + } + + asio::ip::tcp::socket m_Socket; + std::filesystem::path m_OutputDir; + std::filesystem::path m_TraceFilePath; + BasicFile m_TraceFile; + Oid m_SessionId; + std::string m_RemoteAddress; + + // Preamble parsing + uint8_t m_PreambleBuffer[kPreambleBufferSize] = {}; + uint16_t m_MetadataFieldsSize = 0; + + // Extracted metadata + Guid m_SessionGuid{}; + Guid m_TraceGuid{}; + uint16_t m_ControlPort = 0; + uint8_t m_TransportVersion = 0; + uint8_t m_ProtocolVersion = 0; + + // Streaming + uint8_t m_ReadBuffer[kReadBufferSize]; + uint64_t m_TotalBytesRecorded = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// + +struct TraceRecorder::Impl +{ + Impl() : m_IoContext(), m_Acceptor(m_IoContext) {} + + ~Impl() { Shutdown(); } + + void Initialize(uint16_t InPort, const std::filesystem::path& OutputDir) + { + std::lock_guard Lock(m_Mutex); + + if (m_IsRunning) + { + ZEN_WARN("TraceRecorder already initialized"); + return; + } + + m_OutputDir = OutputDir; + + try + { + // Create output directory if it doesn't exist + CreateDirectories(m_OutputDir); + + // Configure acceptor + m_Acceptor.open(asio::ip::tcp::v4()); + m_Acceptor.set_option(asio::socket_base::reuse_address(true)); + m_Acceptor.bind(asio::ip::tcp::endpoint(asio::ip::tcp::v4(), InPort)); + m_Acceptor.listen(); + + m_Port = m_Acceptor.local_endpoint().port(); + + ZEN_INFO("TraceRecorder listening on port {}, output directory: '{}'", m_Port, m_OutputDir); + + m_IsRunning = true; + + // Start accepting connections + StartAccept(); + + // Start IO thread + m_IoThread = std::thread([this]() { + try + { + m_IoContext.run(); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("TraceRecorder IO thread exception: {}", Ex.what()); + } + }); + } + catch (const std::exception& Ex) + { + ZEN_ERROR("Failed to initialize TraceRecorder: {}", Ex.what()); + m_IsRunning = false; + throw; + } + } + + void Shutdown() + { + std::lock_guard Lock(m_Mutex); + + if (!m_IsRunning) + { + return; + } + + ZEN_INFO("TraceRecorder shutting down"); + + m_IsRunning = false; + + std::error_code Ec; + m_Acceptor.close(Ec); + + m_IoContext.stop(); + + if (m_IoThread.joinable()) + { + m_IoThread.join(); + } + + { + std::lock_guard SessionLock(m_SessionsMutex); + m_Sessions.clear(); + } + + ZEN_INFO("TraceRecorder shutdown complete"); + } + + bool IsRunning() const { return m_IsRunning; } + + uint16_t GetPort() const { return m_Port; } + + std::vector GetActiveSessions() const + { + std::lock_guard Lock(m_SessionsMutex); + + std::vector Result; + for (const auto& WeakSession : m_Sessions) + { + if (auto Session = WeakSession.lock()) + { + if (Session->IsActive()) + { + Result.push_back(Session->GetInfo()); + } + } + } + return Result; + } + +private: + void StartAccept() + { + auto Socket = std::make_shared(m_IoContext); + + m_Acceptor.async_accept(*Socket, [this, Socket](const asio::error_code& Ec) { + if (!Ec) + { + auto Session = std::make_shared(std::move(*Socket), m_OutputDir); + + { + std::lock_guard Lock(m_SessionsMutex); + + // Prune expired sessions while adding the new one + std::erase_if(m_Sessions, [](const std::weak_ptr& Wp) { return Wp.expired(); }); + m_Sessions.push_back(Session); + } + + Session->Start(); + } + else if (Ec != asio::error::operation_aborted) + { + ZEN_WARN("Accept error: {}", Ec.message()); + } + + // Continue accepting if still running + if (m_IsRunning) + { + StartAccept(); + } + }); + } + + asio::io_context m_IoContext; + asio::ip::tcp::acceptor m_Acceptor; + std::thread m_IoThread; + std::filesystem::path m_OutputDir; + std::mutex m_Mutex; + std::atomic m_IsRunning{false}; + uint16_t m_Port = 0; + + mutable std::mutex m_SessionsMutex; + std::vector> m_Sessions; +}; + +//////////////////////////////////////////////////////////////////////////////// + +TraceRecorder::TraceRecorder() : m_Impl(std::make_unique()) +{ +} + +TraceRecorder::~TraceRecorder() +{ + Shutdown(); +} + +void +TraceRecorder::Initialize(uint16_t InPort, const std::filesystem::path& OutputDir) +{ + m_Impl->Initialize(InPort, OutputDir); +} + +void +TraceRecorder::Shutdown() +{ + m_Impl->Shutdown(); +} + +bool +TraceRecorder::IsRunning() const +{ + return m_Impl->IsRunning(); +} + +uint16_t +TraceRecorder::GetPort() const +{ + return m_Impl->GetPort(); +} + +std::vector +TraceRecorder::GetActiveSessions() const +{ + return m_Impl->GetActiveSessions(); +} + +} // namespace zen diff --git a/src/zenserver/trace/tracerecorder.h b/src/zenserver/trace/tracerecorder.h new file mode 100644 index 000000000..48857aec8 --- /dev/null +++ b/src/zenserver/trace/tracerecorder.h @@ -0,0 +1,46 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace zen { + +struct TraceSessionInfo +{ + Guid SessionGuid{}; + Guid TraceGuid{}; + uint16_t ControlPort = 0; + uint8_t TransportVersion = 0; + uint8_t ProtocolVersion = 0; + std::string RemoteAddress; + uint64_t BytesRecorded = 0; + std::filesystem::path TraceFilePath; +}; + +class TraceRecorder +{ +public: + TraceRecorder(); + ~TraceRecorder(); + + void Initialize(uint16_t InPort, const std::filesystem::path& OutputDir); + void Shutdown(); + + bool IsRunning() const; + uint16_t GetPort() const; + + std::vector GetActiveSessions() const; + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +} // namespace zen \ No newline at end of file diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index 9ab51beb2..915b6a3b1 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -27,6 +27,7 @@ target("zenserver") add_packages("json11") add_packages("lua") add_packages("consul") + add_packages("nomad") if has_config("zenmimalloc") then add_packages("mimalloc") @@ -36,6 +37,14 @@ target("zenserver") add_packages("sentry-native") end + if has_config("zenhorde") then + add_deps("zenhorde") + end + + if has_config("zennomad") then + add_deps("zennomad") + end + if is_mode("release") then set_optimize("fastest") end @@ -145,4 +154,14 @@ target("zenserver") end copy_if_newer(path.join(installdir, "bin", consul_bin), path.join(target:targetdir(), consul_bin), consul_bin) end + + local nomad_pkg = target:pkg("nomad") + if nomad_pkg then + local installdir = nomad_pkg:installdir() + local nomad_bin = "nomad" + if is_plat("windows") then + nomad_bin = "nomad.exe" + end + copy_if_newer(path.join(installdir, "bin", nomad_bin), path.join(target:targetdir(), nomad_bin), nomad_bin) + end end) diff --git a/src/zentest-appstub/zentest-appstub.cpp b/src/zentest-appstub/zentest-appstub.cpp index 67fbef532..509629739 100644 --- a/src/zentest-appstub/zentest-appstub.cpp +++ b/src/zentest-appstub/zentest-appstub.cpp @@ -106,6 +106,11 @@ DescribeFunctions() << "Reverse"sv; Versions << "Version"sv << Guid::FromString("31313131-3131-3131-3131-313131313131"sv); Versions.EndObject(); + Versions.BeginObject(); + Versions << "Name"sv + << "Sleep"sv; + Versions << "Version"sv << Guid::FromString("88888888-8888-8888-8888-888888888888"sv); + Versions.EndObject(); Versions.EndArray(); return Versions.Save(); @@ -190,6 +195,12 @@ ExecuteFunction(CbObject Action, ContentResolver ChunkResolver) { return Apply(NullFunction); } + else if (Function == "Sleep"sv) + { + uint64_t SleepTimeMs = Action["Constants"sv].AsObjectView()["SleepTimeMs"sv].AsUInt64(); + zen::Sleep(static_cast(SleepTimeMs)); + return Apply(IdentityFunction); + } else { return {}; diff --git a/src/zenutil/include/zenutil/consoletui.h b/src/zenutil/include/zenutil/consoletui.h index 7dc68c126..5f74fa82b 100644 --- a/src/zenutil/include/zenutil/consoletui.h +++ b/src/zenutil/include/zenutil/consoletui.h @@ -2,6 +2,7 @@ #pragma once +#include #include #include #include diff --git a/src/zenutil/include/zenutil/zenserverprocess.h b/src/zenutil/include/zenutil/zenserverprocess.h index e81b154e8..2a8617162 100644 --- a/src/zenutil/include/zenutil/zenserverprocess.h +++ b/src/zenutil/include/zenutil/zenserverprocess.h @@ -84,6 +84,7 @@ struct ZenServerInstance { kStorageServer, // default kHubServer, + kComputeServer, }; ZenServerInstance(ZenServerEnvironment& TestEnvironment, ServerMode Mode = ServerMode::kStorageServer); diff --git a/src/zenutil/zenserverprocess.cpp b/src/zenutil/zenserverprocess.cpp index e127a92d7..b09c2d89a 100644 --- a/src/zenutil/zenserverprocess.cpp +++ b/src/zenutil/zenserverprocess.cpp @@ -787,6 +787,8 @@ ToString(ZenServerInstance::ServerMode Mode) return "storage"sv; case ZenServerInstance::ServerMode::kHubServer: return "hub"sv; + case ZenServerInstance::ServerMode::kComputeServer: + return "compute"sv; default: return "invalid"sv; } @@ -808,6 +810,10 @@ ZenServerInstance::SpawnServerInternal(int ChildId, std::string_view ServerArgs, { CommandLine << " hub"; } + else if (m_ServerMode == ServerMode::kComputeServer) + { + CommandLine << " compute"; + } CommandLine << " --child-id " << ChildEventName; diff --git a/xmake.lua b/xmake.lua index 7b9fee594..3454b264a 100644 --- a/xmake.lua +++ b/xmake.lua @@ -74,6 +74,7 @@ add_defines("EASTL_STD_ITERATOR_CATEGORY_ENABLED", "EASTL_DEPRECATIONS_FOR_2024_ add_requires("eastl", {system = false}) add_requires("consul", {system = false}) -- for hub tests +add_requires("nomad", {system = false}) -- for nomad provisioner tests if has_config("zenmimalloc") and not use_asan then add_requires("mimalloc", {system = false}) @@ -244,13 +245,29 @@ else add_defines("ZEN_WITH_HTTPSYS=0") end +local compute_default = false + option("zencompute") - set_default(false) + set_default(compute_default) set_showmenu(true) set_description("Enable compute services endpoint") option_end() add_define_by_config("ZEN_WITH_COMPUTE_SERVICES", "zencompute") +option("zenhorde") + set_default(compute_default) + set_showmenu(true) + set_description("Enable Horde worker provisioning") +option_end() +add_define_by_config("ZEN_WITH_HORDE", "zenhorde") + +option("zennomad") + set_default(compute_default) + set_showmenu(true) + set_description("Enable Nomad worker provisioning") +option_end() +add_define_by_config("ZEN_WITH_NOMAD", "zennomad") + if is_os("windows") then add_defines("UE_MEMORY_TRACE_AVAILABLE=1") @@ -304,6 +321,12 @@ includes("src/zenhttp", "src/zenhttp-test") includes("src/zennet", "src/zennet-test") includes("src/zenremotestore", "src/zenremotestore-test") includes("src/zencompute", "src/zencompute-test") +if has_config("zenhorde") then + includes("src/zenhorde") +end +if has_config("zennomad") then + includes("src/zennomad") +end includes("src/zenstore", "src/zenstore-test") includes("src/zentelemetry", "src/zentelemetry-test") includes("src/zenutil", "src/zenutil-test") -- cgit v1.2.3 From 6926c04dc4d7c5c0f0310b66c17c9a4e94d2e341 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Wed, 4 Mar 2026 16:07:14 +0100 Subject: more feedback during auth option parsing (#806) * remove stray std::unique_ptr Auth; causing crashes * add more feedback during parsing of auth options --- CHANGELOG.md | 1 + src/zen/authutils.cpp | 80 ++++++++++++++++++++++-------------- src/zen/cmds/builds_cmd.cpp | 2 - src/zencore/include/zencore/string.h | 2 + src/zencore/string.cpp | 56 +++++++++++++++++++++---- 5 files changed, 100 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4b484d53..72cfe3102 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ defaulting to WARN. The logging level can be changed via --log-debug=sentry-sdk or --log-info=sentry-sdk etc Though it should be noted that since Sentry is initialized very early there is a small period where the default will be WARN no matter what you pass on the command line. +- Improvement: Output more information about auth options for `zen` command - Bugfix: `--plain-progress` style progress bar should now show elapsed time correctly - Bugfix: Time spent indexing local and remote state during `zen builds download` now show the correct time - Bugfix: ObjectStore failed to correctly parse urls with sub folder paths causing 404 to be returned diff --git a/src/zen/authutils.cpp b/src/zen/authutils.cpp index 16427acf5..23ac70965 100644 --- a/src/zen/authutils.cpp +++ b/src/zen/authutils.cpp @@ -154,21 +154,34 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, ZEN_ASSERT(!SystemRootDir.empty()); if (!Auth) { - if (m_EncryptionKey.empty()) + static const std::string_view DefaultEncryptionKey("abcdefghijklmnopqrstuvxyz0123456"); + static const std::string_view DefaultEncryptionIV("0123456789abcdef"); + if (m_EncryptionKey.empty() && m_EncryptionIV.empty()) { - m_EncryptionKey = "abcdefghijklmnopqrstuvxyz0123456"; + m_EncryptionKey = DefaultEncryptionKey; + m_EncryptionIV = DefaultEncryptionIV; if (!Quiet) { - ZEN_CONSOLE_WARN("Using default encryption key"); + ZEN_CONSOLE_WARN("Auth: Using default encryption key and initialization vector for auth storage"); } } - - if (m_EncryptionIV.empty()) + else { - m_EncryptionIV = "0123456789abcdef"; - if (!Quiet) + if (m_EncryptionKey.empty()) + { + m_EncryptionKey = DefaultEncryptionKey; + if (!Quiet) + { + ZEN_CONSOLE_WARN("Auth: Using default encryption key for auth storage"); + } + } + if (m_EncryptionIV.empty()) { - ZEN_CONSOLE_WARN("Using default encryption initialization vector"); + m_EncryptionIV = DefaultEncryptionIV; + if (!Quiet) + { + ZEN_CONSOLE_WARN("Auth: Using default encryption initialization vector for auth storage"); + } } } @@ -187,9 +200,9 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, { ExtendableStringBuilder<128> SB; SB << "\n RootDirectory: " << AuthMgrConfig.RootDirectory.string(); - SB << "\n EncryptionKey: " << m_EncryptionKey; - SB << "\n EncryptionIV: " << m_EncryptionIV; - ZEN_CONSOLE("Creating auth manager with:{}", SB.ToString()); + SB << "\n EncryptionKey: " << HideSensitiveString(m_EncryptionKey); + SB << "\n EncryptionIV: " << HideSensitiveString(m_EncryptionIV); + ZEN_CONSOLE("Auth: Creating auth manager with:{}", SB.ToString()); } Auth = AuthMgr::Create(AuthMgrConfig); } @@ -204,13 +217,18 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, ExtendableStringBuilder<128> SB; SB << "\n Name: " << ProviderName; SB << "\n Url: " << m_OpenIdProviderUrl; - SB << "\n ClientId: " << m_OpenIdClientId; - ZEN_CONSOLE("Adding openid auth provider:{}", SB.ToString()); + SB << "\n ClientId: " << HideSensitiveString(m_OpenIdClientId); + ZEN_CONSOLE("Auth: Adding Open ID auth provider:{}", SB.ToString()); } Auth->AddOpenIdProvider({.Name = ProviderName, .Url = m_OpenIdProviderUrl, .ClientId = m_OpenIdClientId}); if (!m_OpenIdRefreshToken.empty()) { - ZEN_CONSOLE("Adding open id refresh token {} to provider {}", m_OpenIdRefreshToken, ProviderName); + if (!Quiet) + { + ZEN_CONSOLE("Auth: Adding open id refresh token {} to provider {}", + HideSensitiveString(m_OpenIdRefreshToken), + ProviderName); + } Auth->AddOpenIdToken({.ProviderName = ProviderName, .RefreshToken = m_OpenIdRefreshToken}); } } @@ -225,9 +243,9 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, if (!m_AccessToken.empty()) { - if (Verbose) + if (!Quiet) { - ZEN_CONSOLE("Adding static auth token: {}", m_AccessToken); + ZEN_CONSOLE("Auth: Using static auth token: {}", HideSensitiveString(m_AccessToken)); } ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(m_AccessToken); } @@ -237,9 +255,9 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, std::string ResolvedAccessToken = ReadAccessTokenFromJsonFile(m_AccessTokenPath); if (!ResolvedAccessToken.empty()) { - if (Verbose) + if (!Quiet) { - ZEN_CONSOLE("Adding static auth token from {}: {}", m_AccessTokenPath, ResolvedAccessToken); + ZEN_CONSOLE("Auth: Adding static auth token from {}: {}", m_AccessTokenPath, HideSensitiveString(ResolvedAccessToken)); } ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); } @@ -250,9 +268,9 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, { ExtendableStringBuilder<128> SB; SB << "\n Url: " << m_OAuthUrl; - SB << "\n ClientId: " << m_OAuthClientId; - SB << "\n ClientSecret: " << m_OAuthClientSecret; - ZEN_CONSOLE("Adding oauth provider:{}", SB.ToString()); + SB << "\n ClientId: " << HideSensitiveString(m_OAuthClientId); + SB << "\n ClientSecret: " << HideSensitiveString(m_OAuthClientSecret); + ZEN_CONSOLE("Auth: Adding oauth provider:{}", SB.ToString()); } ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOAuthClientCredentials( {.Url = m_OAuthUrl, .ClientId = m_OAuthClientId, .ClientSecret = m_OAuthClientSecret}); @@ -260,25 +278,27 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, else if (!m_OpenIdProviderName.empty()) { CreateAuthMgr(); - if (Verbose) + if (!Quiet) { - ZEN_CONSOLE("Using openid provider: {}", m_OpenIdProviderName); + ZEN_CONSOLE("Auth: Using OpenId provider: {}", m_OpenIdProviderName); } ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOpenIdProvider(*Auth, m_OpenIdProviderName); } else if (std::string ResolvedAccessToken = GetEnvAccessToken(m_AccessTokenEnv); !ResolvedAccessToken.empty()) { - if (Verbose) + if (!Quiet) { - ZEN_CONSOLE("Using environment variable '{}' as access token '{}'", m_AccessTokenEnv, ResolvedAccessToken); + ZEN_CONSOLE("Auth: Resolved environment variable '{}' to access token '{}'", + m_AccessTokenEnv, + HideSensitiveString(ResolvedAccessToken)); } ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); } - else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty()) + else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); OidcTokenExePath.empty()) { - if (Verbose) + if (!Quiet) { - ZEN_CONSOLE("Running oidctoken exe from path '{}'", m_OidcTokenAuthExecutablePath); + ZEN_CONSOLE("Auth: Using oidctoken exe from path '{}'", OidcTokenExePath); } ClientSettings.AccessTokenProvider = httpclientauth::CreateFromOidcTokenExecutable(OidcTokenExePath, HostUrl, Quiet, m_OidcTokenUnattended, Hidden); @@ -291,9 +311,9 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, if (!ClientSettings.AccessTokenProvider) { CreateAuthMgr(); - if (Verbose) + if (!Quiet) { - ZEN_CONSOLE("Using default openid provider"); + ZEN_CONSOLE("Auth: Using default Open ID provider"); } ClientSettings.AccessTokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(*Auth); } diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index ffdc5fe48..0722e9714 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -2808,8 +2808,6 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) .Verbose = m_VerboseHttp, .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}; - std::unique_ptr Auth; - std::string StorageDescription; std::string CacheDescription; diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index 5a12ba5d2..250eb9f56 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -1265,6 +1265,8 @@ private: uint64_t LoMask, HiMask; }; +std::string HideSensitiveString(std::string_view String); + ////////////////////////////////////////////////////////////////////////// void string_forcelink(); // internal diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index 27635a86c..3d0451e27 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -539,10 +539,33 @@ UrlDecode(std::string_view InUrl) return std::string(Url.ToView()); } -////////////////////////////////////////////////////////////////////////// -// -// Unit tests -// +std::string +HideSensitiveString(std::string_view String) +{ + const size_t Length = String.length(); + const size_t SourceLength = Length > 16 ? 4 : 0; + const size_t PadLength = Min(Length - SourceLength, 4u); + const bool AddEllipsis = (SourceLength + PadLength) < Length; + StringBuilder<16> SB; + if (SourceLength > 0) + { + SB << String.substr(0, SourceLength); + } + if (PadLength > 0) + { + SB << std::string(PadLength, 'X'); + } + if (AddEllipsis) + { + SB << "..."; + } + return SB.ToString(); +}; + + ////////////////////////////////////////////////////////////////////////// + // + // Unit tests + // #if ZEN_WITH_TESTS @@ -814,11 +837,6 @@ TEST_CASE("niceNum") } } -void -string_forcelink() -{ -} - TEST_CASE("StringBuilder") { StringBuilder<64> sb; @@ -1224,8 +1242,28 @@ TEST_CASE("string") } } +TEST_CASE("hidesensitivestring") +{ + using namespace std::literals; + + CHECK_EQ(HideSensitiveString(""sv), ""sv); + CHECK_EQ(HideSensitiveString("A"sv), "X"sv); + CHECK_EQ(HideSensitiveString("ABCD"sv), "XXXX"sv); + CHECK_EQ(HideSensitiveString("ABCDE"sv), "XXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGH"sv), "XXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGHIJKLMNOP"sv), "XXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGHIJKLMNOPQ"sv), "ABCDXXXX..."sv); + CHECK_EQ(HideSensitiveString("ABCDEFGHIJKLMNOPQRSTUVWXYZ012345"sv), "ABCDXXXX..."sv); + CHECK_EQ(HideSensitiveString("1234567890123456789"sv), "1234XXXX..."sv); +} + TEST_SUITE_END(); +void +string_forcelink() +{ +} + #endif } // namespace zen -- cgit v1.2.3 From 1f83b48a20bf90f41e18867620c5774f3be6280d Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Wed, 4 Mar 2026 17:23:36 +0100 Subject: Fixing various compiler issues (#807) Compile fixes for various versions of gcc,clang (non-UE) --- src/zencore/filesystem.cpp | 11 ++++++----- src/zencore/include/zencore/hashutils.h | 1 + src/zencore/xmake.lua | 7 ++++++- src/zenserver/xmake.lua | 6 ++++++ thirdparty/xmake.lua | 2 +- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/zencore/filesystem.cpp b/src/zencore/filesystem.cpp index 9885b2ada..8ed63565c 100644 --- a/src/zencore/filesystem.cpp +++ b/src/zencore/filesystem.cpp @@ -194,7 +194,7 @@ WipeDirectory(const wchar_t* DirPath, bool KeepDotFiles) FindClose(hFind); } - return true; + return Success; } bool @@ -1022,7 +1022,7 @@ TryCloneFile(const std::filesystem::path& FromPath, const std::filesystem::path& return false; } fchmod(ToFd, 0666); - ScopedFd $To = { FromFd }; + ScopedFd $To = { ToFd }; ioctl(ToFd, FICLONE, FromFd); @@ -1112,7 +1112,8 @@ CopyFile(const std::filesystem::path& FromPath, const std::filesystem::path& ToP size_t FileSizeBytes = Stat.st_size; - fchown(ToFd, Stat.st_uid, Stat.st_gid); + int $Ignore = fchown(ToFd, Stat.st_uid, Stat.st_gid); + ZEN_UNUSED($Ignore); // What's the appropriate error handling here? // Copy impl const size_t BufferSize = Min(FileSizeBytes, 64u << 10); @@ -1398,7 +1399,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer const uint64_t ChunkSize = Min(WriteSize, uint64_t(2) * 1024 * 1024 * 1024); #if ZEN_PLATFORM_WINDOWS - hRes = Outfile.Write(DataPtr, gsl::narrow_cast(WriteSize)); + hRes = Outfile.Write(DataPtr, gsl::narrow_cast(ChunkSize)); if (FAILED(hRes)) { Outfile.Close(); @@ -1407,7 +1408,7 @@ WriteFile(std::filesystem::path Path, const IoBuffer* const* Data, size_t Buffer ThrowSystemException(hRes, fmt::format("File write failed for '{}'", Path).c_str()); } #else - if (write(Fd, DataPtr, WriteSize) != int64_t(WriteSize)) + if (write(Fd, DataPtr, ChunkSize) != int64_t(ChunkSize)) { close(Fd); std::error_code DummyEc; diff --git a/src/zencore/include/zencore/hashutils.h b/src/zencore/include/zencore/hashutils.h index 6b9902b3a..8abfd4b6e 100644 --- a/src/zencore/include/zencore/hashutils.h +++ b/src/zencore/include/zencore/hashutils.h @@ -3,6 +3,7 @@ #pragma once #include +#include #include namespace zen { diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index 2f81b7ec8..ab842f6ed 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -14,7 +14,12 @@ target('zencore') end) set_configdir("include/zencore") add_files("**.cpp") - add_files("trace.cpp", {unity_ignored = true }) + if is_plat("linux") and not (get_config("toolchain") or ""):find("clang") then + -- GCC false positives in thirdparty trace.h (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100137) + add_files("trace.cpp", {unity_ignored = true, force = {cxxflags = {"-Wno-stringop-overread", "-Wno-dangling-pointer"}} }) + else + add_files("trace.cpp", {unity_ignored = true }) + end add_files("testing.cpp", {unity_ignored = true }) if has_config("zenrpmalloc") then diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index 915b6a3b1..7a9031782 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -19,6 +19,12 @@ target("zenserver") add_files("**.cpp") add_files("frontend/*.zip") add_files("zenserver.cpp", {unity_ignored = true }) + + if is_plat("linux") and not (get_config("toolchain") or ""):find("clang") then + -- GCC false positives in deeply inlined code (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100137) + add_files("storage/projectstore/httpprojectstore.cpp", {force = {cxxflags = "-Wno-stringop-overflow"} }) + add_files("storage/storageconfig.cpp", {force = {cxxflags = "-Wno-array-bounds"} }) + end add_includedirs(".") set_symbols("debug") diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua index 7c99d910e..a4a183910 100644 --- a/thirdparty/xmake.lua +++ b/thirdparty/xmake.lua @@ -66,7 +66,7 @@ target('cpr') if is_os("windows") then add_cxxflags("/wd4668") else - add_cxxflags("-Wno-undef") + add_cxxflags("-Wno-undef", "-Wno-deprecated-declarations") end add_packages("libcurl", {public=true}) -- cgit v1.2.3 From d8940b27c8a5c070c3b48ca9e575929df8d1d888 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 5 Mar 2026 00:08:19 +0100 Subject: added TEST_SUITE_BEGIN/END around some TEST_CASEs which didn't have them (#809) * added TEST_SUITE_BEGIN/END around some TEST_CASEs which didn't have them * fixed some stats issues * ScopedSpan should Initialize * annotated classes in stats.h with some documentation comments --- src/zencompute/cloudmetadata.cpp | 4 + src/zencompute/runners/deferreddeleter.cpp | 4 + src/zencore/xxhash.cpp | 4 + .../projectstore/remoteprojectstore.cpp | 4 + src/zenserver-test/logging-tests.cpp | 4 + src/zenserver-test/nomad-tests.cpp | 4 + src/zentelemetry/include/zentelemetry/otlptrace.h | 9 +- src/zentelemetry/include/zentelemetry/stats.h | 202 +++++++++++++++------ src/zentelemetry/stats.cpp | 2 +- 9 files changed, 180 insertions(+), 57 deletions(-) diff --git a/src/zencompute/cloudmetadata.cpp b/src/zencompute/cloudmetadata.cpp index b3b3210d9..65bac895f 100644 --- a/src/zencompute/cloudmetadata.cpp +++ b/src/zencompute/cloudmetadata.cpp @@ -622,6 +622,8 @@ CloudMetadata::PollGCPTermination() namespace zen::compute { +TEST_SUITE_BEGIN("compute.cloudmetadata"); + // --------------------------------------------------------------------------- // Test helper — spins up a local ASIO HTTP server hosting a MockImdsService // --------------------------------------------------------------------------- @@ -1000,6 +1002,8 @@ TEST_CASE("cloudmetadata.sentinel_files") } } +TEST_SUITE_END(); + void cloudmetadata_forcelink() { diff --git a/src/zencompute/runners/deferreddeleter.cpp b/src/zencompute/runners/deferreddeleter.cpp index 00977d9fa..4fad2cf70 100644 --- a/src/zencompute/runners/deferreddeleter.cpp +++ b/src/zencompute/runners/deferreddeleter.cpp @@ -231,6 +231,8 @@ deferreddeleter_forcelink() namespace zen::compute { +TEST_SUITE_BEGIN("compute.deferreddeleter"); + TEST_CASE("DeferredDirectoryDeleter.DeletesSingleDirectory") { ScopedTemporaryDirectory TempDir; @@ -331,6 +333,8 @@ TEST_CASE("DeferredDirectoryDeleter.MarkReadyShortensDeferral") CHECK(!std::filesystem::exists(Dir)); } +TEST_SUITE_END(); + } // namespace zen::compute #endif // ZEN_WITH_TESTS && ZEN_WITH_COMPUTE_SERVICES diff --git a/src/zencore/xxhash.cpp b/src/zencore/xxhash.cpp index 6d1050531..88a48dd68 100644 --- a/src/zencore/xxhash.cpp +++ b/src/zencore/xxhash.cpp @@ -59,6 +59,8 @@ xxhash_forcelink() { } +TEST_SUITE_BEGIN("core.xxhash"); + TEST_CASE("XXH3_128") { using namespace std::literals; @@ -96,6 +98,8 @@ TEST_CASE("XXH3_128") } } +TEST_SUITE_END(); + #endif } // namespace zen diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index 570025b6d..78f6014df 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -4240,6 +4240,8 @@ namespace projectstore_testutils { } // namespace projectstore_testutils +TEST_SUITE_BEGIN("remotestore.projectstore"); + struct ExportForceDisableBlocksTrue_ForceTempBlocksFalse { static const bool ForceDisableBlocks = true; @@ -4395,6 +4397,8 @@ TEST_CASE_TEMPLATE("project.store.export", CHECK(ImportForceCleanResult.ErrorCode == 0); } +TEST_SUITE_END(); + #endif // ZEN_WITH_TESTS void diff --git a/src/zenserver-test/logging-tests.cpp b/src/zenserver-test/logging-tests.cpp index fe39e14c0..f284f0371 100644 --- a/src/zenserver-test/logging-tests.cpp +++ b/src/zenserver-test/logging-tests.cpp @@ -15,6 +15,8 @@ namespace zen::tests { using namespace std::literals; +TEST_SUITE_BEGIN("server.logging"); + ////////////////////////////////////////////////////////////////////////// static bool @@ -252,6 +254,8 @@ TEST_CASE("logging.level.off_specific_logger") CHECK_MESSAGE(!LogContains(HttpLog, "server session id"), HttpLog); } +TEST_SUITE_END(); + } // namespace zen::tests #endif diff --git a/src/zenserver-test/nomad-tests.cpp b/src/zenserver-test/nomad-tests.cpp index 6eb99bc3a..f8f5a9a30 100644 --- a/src/zenserver-test/nomad-tests.cpp +++ b/src/zenserver-test/nomad-tests.cpp @@ -17,6 +17,8 @@ namespace zen::tests::nomad_tests { using namespace std::literals; +TEST_SUITE_BEGIN("server.nomad"); + TEST_CASE("nomad.client.lifecycle" * doctest::skip()) { zen::nomad::NomadProcess NomadProc; @@ -122,5 +124,7 @@ TEST_CASE("nomad.provisioner.integration" * doctest::skip()) NomadProc.StopNomadAgent(); } +TEST_SUITE_END(); + } // namespace zen::tests::nomad_tests #endif diff --git a/src/zentelemetry/include/zentelemetry/otlptrace.h b/src/zentelemetry/include/zentelemetry/otlptrace.h index 49dd90358..95718af55 100644 --- a/src/zentelemetry/include/zentelemetry/otlptrace.h +++ b/src/zentelemetry/include/zentelemetry/otlptrace.h @@ -317,6 +317,7 @@ public: ExtendableStringBuilder<128> NameBuilder; NamingFunction(NameBuilder); + Initialize(NameBuilder); } /** Construct a new span with a naming function AND initializer function @@ -350,7 +351,13 @@ public: // Execute a function with the span pointer if valid. This can // be used to add attributes or events to the span after creation - inline void WithSpan(auto Func) const { Func(*m_Span); } + inline void WithSpan(auto Func) const + { + if (m_Span) + { + Func(*m_Span); + } + } private: void Initialize(std::string_view Name); diff --git a/src/zentelemetry/include/zentelemetry/stats.h b/src/zentelemetry/include/zentelemetry/stats.h index 3e67bac1c..d58846a3b 100644 --- a/src/zentelemetry/include/zentelemetry/stats.h +++ b/src/zentelemetry/include/zentelemetry/stats.h @@ -16,6 +16,11 @@ class CbObjectWriter; namespace zen::metrics { +/** A single atomic value that can be set and read at any time. + * + * Useful for point-in-time readings such as queue depth, active connection count, + * or any value where only the current state matters rather than history. + */ template class Gauge { @@ -29,12 +34,12 @@ private: std::atomic m_Value; }; -/** Stats counter +/** Monotonically increasing (or decreasing) counter. * - * A counter is modified by adding or subtracting a value from a current value. - * This would typically be used to track number of requests in flight, number - * of active jobs etc + * Suitable for tracking quantities that go up and down over time, such as + * requests in flight or active jobs. All operations are lock-free via atomics. * + * Unlike a Meter, a Counter does not track rates — it only records a running total. */ class Counter { @@ -50,34 +55,56 @@ private: std::atomic m_count{0}; }; -/** Exponential Weighted Moving Average - - This is very raw, to use as little state as possible. If we - want to use this more broadly in user code we should perhaps - add a more user-friendly wrapper +/** Low-level exponential weighted moving average. + * + * Tracks a smoothed rate using the standard EWMA recurrence: + * + * rate = rate + alpha * (instantRate - rate) + * + * where instantRate = Count / Interval. The alpha value controls how quickly + * the average responds to changes — higher alpha means more weight on recent + * samples. Typical alphas are derived from a decay half-life (e.g. 1, 5, 15 + * minutes) and a fixed tick interval. + * + * This class is intentionally minimal to keep per-instance state to a single + * atomic double. See Meter for a more convenient wrapper. */ - class RawEWMA { public: - /// - /// Update EWMA with new measure - /// - /// Smoothing factor (between 0 and 1) - /// Elapsed time since last - /// Value - /// Whether this is the first update or not - void Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate); + /** Update the EWMA with a new observation. + * + * @param Alpha Smoothing factor in (0, 1). Smaller values give a + * slower-moving average; larger values track recent + * changes more aggressively. + * @param Interval Elapsed hi-freq timer ticks since the last Tick call. + * Used to compute the instantaneous rate as Count/Interval. + * @param Count Number of events observed during this interval. + * @param IsInitialUpdate True on the very first call: seeds the rate directly + * from the instantaneous rate rather than blending it in. + */ + void Tick(double Alpha, uint64_t Interval, uint64_t Count, bool IsInitialUpdate); + + /** Returns the current smoothed rate in events per second. */ double Rate() const; private: std::atomic m_Rate = 0; }; -/// -/// Tracks rate of events over time (i.e requests/sec), using -/// exponential moving averages -/// +/** Tracks the rate of events over time using exponential moving averages. + * + * Maintains three EWMA windows (1, 5, 15 minutes) in addition to a simple + * mean rate computed from the total count and elapsed wall time since + * construction. This mirrors the load-average conventions familiar from Unix. + * + * Rate updates are batched: Mark() accumulates a pending count and the EWMA + * is only advanced every ~5 seconds (controlled by kTickIntervalInSeconds), + * keeping contention low even under heavy call rates. Rates are returned in + * events per second. + * + * All operations are thread-safe via lock-free atomics. + */ class Meter { public: @@ -85,18 +112,18 @@ public: ~Meter(); inline uint64_t Count() const { return m_TotalCount; } - double Rate1(); // One-minute rate - double Rate5(); // Five-minute rate - double Rate15(); // Fifteen-minute rate - double MeanRate() const; // Mean rate since instantiation of this meter + double Rate1(); // One-minute EWMA rate (events/sec) + double Rate5(); // Five-minute EWMA rate (events/sec) + double Rate15(); // Fifteen-minute EWMA rate (events/sec) + double MeanRate() const; // Mean rate since instantiation (events/sec) void Mark(uint64_t Count = 1); // Register one or more events private: std::atomic m_TotalCount{0}; // Accumulator counting number of marks since beginning - std::atomic m_PendingCount{0}; // Pending EWMA update accumulator - std::atomic m_StartTick{0}; // Time this was instantiated (for mean) - std::atomic m_LastTick{0}; // Timestamp of last EWMA tick - std::atomic m_Remainder{0}; // Tracks the "modulo" of tick time + std::atomic m_PendingCount{0}; // Pending EWMA update accumulator; drained on each tick + std::atomic m_StartTick{0}; // Hi-freq timer value at construction (for MeanRate) + std::atomic m_LastTick{0}; // Hi-freq timer value of the last EWMA tick + std::atomic m_Remainder{0}; // Accumulated ticks not yet consumed by EWMA updates bool m_IsFirstTick = true; RawEWMA m_RateM1; RawEWMA m_RateM5; @@ -106,7 +133,14 @@ private: void Tick(); }; -/** Moment-in-time snapshot of a distribution +/** Immutable sorted snapshot of a reservoir sample. + * + * Constructed from a vector of sampled values which are sorted on construction. + * Percentiles are computed on demand via linear interpolation between adjacent + * sorted values, following the standard R-7 quantile method. + * + * Because this is a copy of the reservoir at a point in time, it can be held + * and queried without holding any locks on the source UniformSample. */ class SampleSnapshot { @@ -128,12 +162,19 @@ private: std::vector m_Values; }; -/** Randomly selects samples from a stream. Uses Vitter's - Algorithm R to produce a statistically representative sample. - - http://www.cs.umd.edu/~samir/498/vitter.pdf - Random Sampling with a Reservoir +/** Reservoir sampler for probabilistic distribution tracking. + * + * Maintains a fixed-size reservoir of samples drawn uniformly from the full + * history of values using Vitter's Algorithm R. This gives an unbiased + * statistical representation of the value distribution regardless of how many + * total values have been observed, at the cost of O(ReservoirSize) memory. + * + * A larger reservoir improves accuracy of tail percentiles (P99, P999) but + * increases memory and snapshot cost. The default of 1028 gives good accuracy + * for most telemetry uses. + * + * http://www.cs.umd.edu/~samir/498/vitter.pdf - Random Sampling with a Reservoir */ - class UniformSample { public: @@ -159,7 +200,14 @@ private: std::vector> m_Values; }; -/** Track (probabilistic) sample distribution along with min/max +/** Tracks the statistical distribution of a stream of values. + * + * Records exact min, max, count and mean across all values ever seen, plus a + * reservoir sample (via UniformSample) used to compute percentiles. Percentiles + * are therefore probabilistic — they reflect the distribution of a representative + * sample rather than the full history. + * + * All operations are thread-safe via lock-free atomics. */ class Histogram { @@ -183,11 +231,28 @@ private: std::atomic m_Count{0}; }; -/** Track timing and frequency of some operation - - Example usage would be to track frequency and duration of network - requests, or function calls. - +/** Combines a Histogram and a Meter to track both the distribution and rate + * of a recurring operation. + * + * Duration values are stored in hi-freq timer ticks. Use GetHifreqTimerToSeconds() + * when converting for display. + * + * Typical usage via the RAII Scope helper: + * + * OperationTiming MyTiming; + * + * { + * OperationTiming::Scope Scope(MyTiming); + * DoWork(); + * // Scope destructor calls Stop() automatically + * } + * + * // Or cancel if the operation should not be counted: + * { + * OperationTiming::Scope Scope(MyTiming); + * if (CacheHit) { Scope.Cancel(); return; } + * DoExpensiveWork(); + * } */ class OperationTiming { @@ -207,13 +272,19 @@ public: double Rate15() { return m_Meter.Rate15(); } double MeanRate() const { return m_Meter.MeanRate(); } + /** RAII helper that records duration from construction to Stop() or destruction. + * + * Call Cancel() to discard the measurement (e.g. for cache hits that should + * not skew latency statistics). After Stop() or Cancel() the destructor is a + * no-op. + */ struct Scope { Scope(OperationTiming& Outer); ~Scope(); - void Stop(); - void Cancel(); + void Stop(); // Record elapsed time and mark the meter + void Cancel(); // Discard this measurement; destructor becomes a no-op private: OperationTiming& m_Outer; @@ -225,6 +296,7 @@ private: Histogram m_Histogram; }; +/** Immutable snapshot of a Meter's state at a point in time. */ struct MeterSnapshot { uint64_t Count; @@ -234,6 +306,12 @@ struct MeterSnapshot double Rate15; }; +/** Immutable snapshot of a Histogram's state at a point in time. + * + * Count and all statistical values have been scaled by the ConversionFactor + * supplied when the snapshot was taken (e.g. GetHifreqTimerToSeconds() to + * convert timer ticks to seconds). + */ struct HistogramSnapshot { double Count; @@ -246,24 +324,29 @@ struct HistogramSnapshot double P999; }; +/** Combined snapshot of a Meter and Histogram pair. */ struct StatsSnapshot { MeterSnapshot Meter; HistogramSnapshot Histogram; }; +/** Combined snapshot of request timing and byte transfer statistics. */ struct RequestStatsSnapshot { StatsSnapshot Requests; StatsSnapshot Bytes; }; -/** Metrics for network requests - - Aggregates tracking of duration, payload sizes into a single - class - - */ +/** Tracks both the timing and payload size of network requests. + * + * Maintains two independent histogram+meter pairs: one for request duration + * (in hi-freq timer ticks) and one for transferred bytes. Both dimensions + * share the same request count — a single Update() call advances both. + * + * Duration accessors return values in hi-freq timer ticks. Multiply by + * GetHifreqTimerToSeconds() to convert to seconds. + */ class RequestStats { public: @@ -275,9 +358,9 @@ public: // Timing - int64_t MaxDuration() const { return m_BytesHistogram.Max(); } - int64_t MinDuration() const { return m_BytesHistogram.Min(); } - double MeanDuration() const { return m_BytesHistogram.Mean(); } + int64_t MaxDuration() const { return m_RequestTimeHistogram.Max(); } + int64_t MinDuration() const { return m_RequestTimeHistogram.Min(); } + double MeanDuration() const { return m_RequestTimeHistogram.Mean(); } SampleSnapshot DurationSnapshot() const { return m_RequestTimeHistogram.Snapshot(); } double Rate1() { return m_RequestMeter.Rate1(); } double Rate5() { return m_RequestMeter.Rate5(); } @@ -295,14 +378,23 @@ public: double ByteRate15() { return m_BytesMeter.Rate15(); } double ByteMeanRate() const { return m_BytesMeter.MeanRate(); } + /** RAII helper that records duration and byte count from construction to Stop() + * or destruction. + * + * The byte count can be supplied at construction or updated at any point via + * SetBytes() before the scope ends — useful when the response size is not + * known until the operation completes. + * + * Call Cancel() to discard the measurement entirely. + */ struct Scope { Scope(RequestStats& Outer, int64_t Bytes); ~Scope(); void SetBytes(int64_t Bytes) { m_Bytes = Bytes; } - void Stop(); - void Cancel(); + void Stop(); // Record elapsed time and byte count + void Cancel(); // Discard this measurement; destructor becomes a no-op private: RequestStats& m_Outer; diff --git a/src/zentelemetry/stats.cpp b/src/zentelemetry/stats.cpp index fcfcaf45e..a417bb52c 100644 --- a/src/zentelemetry/stats.cpp +++ b/src/zentelemetry/stats.cpp @@ -631,7 +631,7 @@ EmitSnapshot(const HistogramSnapshot& Snapshot, CbObjectWriter& Cbo) { Cbo << "t_count" << Snapshot.Count << "t_avg" << Snapshot.Avg; Cbo << "t_min" << Snapshot.Min << "t_max" << Snapshot.Max; - Cbo << "t_p75" << Snapshot.P75 << "t_p95" << Snapshot.P95 << "t_p99" << Snapshot.P999; + Cbo << "t_p75" << Snapshot.P75 << "t_p95" << Snapshot.P95 << "t_p99" << Snapshot.P99 << "t_p999" << Snapshot.P999; } void -- cgit v1.2.3 From 73bf948dc5e25fd8aecce864ba6527d2bdb18f81 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Thu, 5 Mar 2026 00:08:48 +0100 Subject: 5.7.22-pre0 --- VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION.txt b/VERSION.txt index cf3060b3b..9dae045da 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -5.7.21 \ No newline at end of file +5.7.22-pre0 \ No newline at end of file -- cgit v1.2.3 From 2f0d60cb431ffefecf3e0a383528691be74af21b Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Thu, 5 Mar 2026 14:31:27 +0100 Subject: oidctoken tool package (#810) * added OidcToken binary to the build process. The binary is mirrored from p4 and is placed next to the output of the build process. It is also placed in the release zip archives. * also fixed issue with Linux symbol stripping which was introduced in toolchain changes yesterday --- .github/workflows/create_release.yml | 10 +++++----- CHANGELOG.md | 1 + VERSION.txt | 2 +- repo/packages/o/oidctoken/xmake.lua | 23 +++++++++++++++++++++++ scripts/bundle.lua | 11 +++++++---- src/zenserver/xmake.lua | 11 +++++++++++ thirdparty/oidctoken/bin/linux-x64/OidcToken | Bin 0 -> 18835419 bytes thirdparty/oidctoken/bin/osx-x64/OidcToken | Bin 0 -> 18908752 bytes thirdparty/oidctoken/bin/win-x64/OidcToken.exe | Bin 0 -> 36700552 bytes toolchains/ue_clang.lua | 2 +- xmake.lua | 1 + 11 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 repo/packages/o/oidctoken/xmake.lua create mode 100644 thirdparty/oidctoken/bin/linux-x64/OidcToken create mode 100644 thirdparty/oidctoken/bin/osx-x64/OidcToken create mode 100644 thirdparty/oidctoken/bin/win-x64/OidcToken.exe diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 2eeedc0c2..3345573c0 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -26,7 +26,7 @@ jobs: run: | xmake config -v -y -m release --zensentry=yes - - name: Bundle + - name: Build and Bundle run: | xmake bundle -v -y --codesignidentity="Epic Games" @@ -62,13 +62,13 @@ jobs: run: | xmake config -v -y -m release --zensentry=yes --toolchain=ue-clang --sdk=${{ github.workspace }}/.tmp-ue-toolchain - - name: Bundle + - name: Build and Bundle run: | xmake bundle -v -y - name: Get Sentry CLI run: | - curl -sL https://sentry.io/get-cli/ | bash + curl -sL https://sentry.io/get-cli/ | SENTRY_CLI_VERSION="3.2.0" bash ls -la ./scripts env: INSTALL_DIR: ./scripts @@ -100,13 +100,13 @@ jobs: run: | xmake config -v -y -m release --zensentry=yes - - name: Bundle + - name: Build and Bundle run: | xmake bundle -v -y --codesignidentity="Developer ID Application" - name: Get Sentry CLI run: | - curl -sL https://sentry.io/get-cli/ | bash + curl -sL https://sentry.io/get-cli/ | SENTRY_CLI_VERSION="3.2.0" bash ls -la ./scripts env: INSTALL_DIR: ./scripts diff --git a/CHANGELOG.md b/CHANGELOG.md index 72cfe3102..39fa3747e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Feature: Added 404 page to dashboard, to make it easier to find your way back to a valid URL - Feature: HttpClient now properly handles multi-part request/response - Feature: Added `POST` variant to `/builds/{namespace}/{bucket}/{buildid}/blobs/{hash}` endpoint that accepts a CbObject payload with 1 or more ranges +- Feature: OidcToken helper executable is now bundled with the zen/zenserver binaries - Improvement: `zen builds download` now uses multi-range requests for blocks to reduce download size - Improvement: `zen oplog-import` now uses partial block with multi-range requests for blocks to reduce download size - Improvement: Improved feedback in log/console during `zen oplog-import` diff --git a/VERSION.txt b/VERSION.txt index 9dae045da..78852c09f 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -5.7.22-pre0 \ No newline at end of file +5.7.22-pre4 \ No newline at end of file diff --git a/repo/packages/o/oidctoken/xmake.lua b/repo/packages/o/oidctoken/xmake.lua new file mode 100644 index 000000000..76360e7bf --- /dev/null +++ b/repo/packages/o/oidctoken/xmake.lua @@ -0,0 +1,23 @@ +package("oidctoken") + set_description("OidcToken - OIDC token helper binary") + set_sourcedir(path.join(os.scriptdir(), "../../../../thirdparty/oidctoken")) + + on_install(function (package) + if is_plat("windows") then + os.cp("bin/win-x64/OidcToken.exe", package:installdir("bin")) + elseif is_plat("linux") then + os.cp("bin/linux-x64/OidcToken", package:installdir("bin")) + os.exec("chmod 755 %s", package:installdir("bin", "OidcToken")) + elseif is_plat("macosx") then + os.cp("bin/osx-x64/OidcToken", package:installdir("bin")) + os.exec("chmod 755 %s", package:installdir("bin", "OidcToken")) + end + end) + + on_test(function (package) + if is_plat("windows") then + os.run("%s --help", package:installdir("bin", "OidcToken.exe")) + else + os.run("%s --help", package:installdir("bin", "OidcToken")) + end + end) diff --git a/scripts/bundle.lua b/scripts/bundle.lua index b2a5e1e08..6f540c5b8 100644 --- a/scripts/bundle.lua +++ b/scripts/bundle.lua @@ -199,8 +199,9 @@ local function main_windows(signidentity) "build/windows/x64/release/zenserver.pdb", "build/windows/x64/release/zen.exe", "build/windows/x64/release/zen.pdb", - "build/windows/x64/release/crashpad_handler.exe") -end + "build/windows/x64/release/crashpad_handler.exe", + "build/windows/x64/release/oidctoken.exe") + end -------------------------------------------------------------------------------- local function main_mac(signidentity) @@ -277,7 +278,8 @@ local function main_mac(signidentity) "build/zenserver-macos.zip", "build/macosx/universal/release/zenserver", "build/macosx/universal/release/zen", - "build/macosx/universal/release/crashpad_handler") + "build/macosx/universal/release/crashpad_handler", + "build/macosx/x86_64/release/Oidctoken") end -------------------------------------------------------------------------------- @@ -288,7 +290,8 @@ local function main_linux() "build/zenserver-linux.zip", "build/linux/x86_64/release/zenserver", "build/linux/x86_64/release/zen", - "build/linux/x86_64/release/crashpad_handler") + "build/linux/x86_64/release/crashpad_handler", + "build/linux/x86_64/release/Oidctoken") end -------------------------------------------------------------------------------- diff --git a/src/zenserver/xmake.lua b/src/zenserver/xmake.lua index 7a9031782..f2ed17f05 100644 --- a/src/zenserver/xmake.lua +++ b/src/zenserver/xmake.lua @@ -33,6 +33,7 @@ target("zenserver") add_packages("json11") add_packages("lua") add_packages("consul") + add_packages("oidctoken") add_packages("nomad") if has_config("zenmimalloc") then @@ -161,6 +162,16 @@ target("zenserver") copy_if_newer(path.join(installdir, "bin", consul_bin), path.join(target:targetdir(), consul_bin), consul_bin) end + local oidctoken_pkg = target:pkg("oidctoken") + if oidctoken_pkg then + local installdir = oidctoken_pkg:installdir() + local oidctoken_bin = "OidcToken" + if is_plat("windows") then + oidctoken_bin = "OidcToken.exe" + end + copy_if_newer(path.join(installdir, "bin", oidctoken_bin), path.join(target:targetdir(), oidctoken_bin), oidctoken_bin) + end + local nomad_pkg = target:pkg("nomad") if nomad_pkg then local installdir = nomad_pkg:installdir() diff --git a/thirdparty/oidctoken/bin/linux-x64/OidcToken b/thirdparty/oidctoken/bin/linux-x64/OidcToken new file mode 100644 index 000000000..eb623bbab Binary files /dev/null and b/thirdparty/oidctoken/bin/linux-x64/OidcToken differ diff --git a/thirdparty/oidctoken/bin/osx-x64/OidcToken b/thirdparty/oidctoken/bin/osx-x64/OidcToken new file mode 100644 index 000000000..2c0ed61b5 Binary files /dev/null and b/thirdparty/oidctoken/bin/osx-x64/OidcToken differ diff --git a/thirdparty/oidctoken/bin/win-x64/OidcToken.exe b/thirdparty/oidctoken/bin/win-x64/OidcToken.exe new file mode 100644 index 000000000..48eed387b Binary files /dev/null and b/thirdparty/oidctoken/bin/win-x64/OidcToken.exe differ diff --git a/toolchains/ue_clang.lua b/toolchains/ue_clang.lua index 8a1efcc04..1fdb05ff9 100644 --- a/toolchains/ue_clang.lua +++ b/toolchains/ue_clang.lua @@ -38,7 +38,7 @@ toolchain("ue-clang") toolchain:set("toolset", "ld", path.join(bindir, "clang++")) toolchain:set("toolset", "sh", path.join(bindir, "clang++")) toolchain:set("toolset", "ar", path.join(bindir, "llvm-ar")) - toolchain:set("toolset", "strip", path.join(bindir, "llvm-strip")) + toolchain:set("toolset", "strip", path.join(bindir, "x86_64-unknown-linux-gnu-strip")) toolchain:set("toolset", "objcopy", path.join(bindir, "llvm-objcopy")) toolchain:set("toolset", "as", path.join(bindir, "clang")) toolchain:set("toolset", "ranlib", path.join(bindir, "llvm-ranlib")) diff --git a/xmake.lua b/xmake.lua index 3454b264a..dfe383c1a 100644 --- a/xmake.lua +++ b/xmake.lua @@ -75,6 +75,7 @@ add_requires("eastl", {system = false}) add_requires("consul", {system = false}) -- for hub tests add_requires("nomad", {system = false}) -- for nomad provisioner tests +add_requires("oidctoken", {system = false}) if has_config("zenmimalloc") and not use_asan then add_requires("mimalloc", {system = false}) -- cgit v1.2.3 From 2275a88da7d0dbcfbc70c6050b7a1417036ea98d Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Fri, 6 Mar 2026 07:45:02 +0100 Subject: fix oidctoken exe lookup check (#811) --- src/zen/authutils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zen/authutils.cpp b/src/zen/authutils.cpp index 23ac70965..534f7952b 100644 --- a/src/zen/authutils.cpp +++ b/src/zen/authutils.cpp @@ -294,7 +294,7 @@ AuthCommandLineOptions::ParseOptions(cxxopts::Options& Ops, } ClientSettings.AccessTokenProvider = httpclientauth::CreateFromStaticToken(ResolvedAccessToken); } - else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); OidcTokenExePath.empty()) + else if (std::filesystem::path OidcTokenExePath = FindOidcTokenExePath(m_OidcTokenAuthExecutablePath); !OidcTokenExePath.empty()) { if (!Quiet) { -- cgit v1.2.3 From 1e731796187ad73b2dee44b48fcecdd487616394 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 6 Mar 2026 10:11:51 +0100 Subject: Claude config, some bug fixes (#813) * Claude config updates * Bug fixes and hardening across `zencore` and `zenhttp`, identified via static analysis. ### zencore - **`ZEN_ASSERT` macro** -- extended to accept an optional string message literal; added `ZEN_ASSERT_MSG_` helper for message formatting. Callers needing runtime fmt-style formatting should use `ZEN_ASSERT_FORMAT`. - **`MpscQueue`** -- fixed `TypeCompatibleStorage` to use a properly-sized `char Storage[sizeof(T)]` array instead of a single `char`; corrected `Data()` to cast `&Storage` rather than `this`; switched cache-line alignment to a fixed constant to avoid GCC's `-Winterference-size` warning. Enabled previously-disabled tests. - **`StringBuilderImpl`** -- initialized `m_Base`/`m_CurPos`/`m_End` to `nullptr`. Fixed `StringCompare` return type (`bool` -> `int`). Fixed `ParseInt` to reject strings with trailing non-numeric characters. Removed deprecated `` include. - **`NiceNumGeneral`** -- replaced `powl()` with integer `IntPow()` to avoid floating-point precision issues. - **`RwLock::ExclusiveLockScope`** -- added move constructor/assignment; initialized `m_Lock` to `nullptr`. - **`Latch::AddCount`** -- fixed variable type (`std::atomic_ptrdiff_t` -> `std::ptrdiff_t` for the return value of `fetch_add`). - **`thread.cpp`** -- fixed Linux `pthread_setname_np` 16-byte name truncation; added null check before dereferencing in `Event::Close()`; fixed `NamedEvent::Close()` to call `close(Fd)` outside the lock region; added null guard in `NamedMutex` destructor; `Sleep()` now returns early for non-positive durations. - **`MD5Stream`** -- was entirely stubbed out (no-op); now correctly calls `MD5Init`/`MD5Update`/`MD5Final`. Fixed `ToHexString` to use the correct string length. Fixed forward declarations. Fixed tests to compare `compare() == 0`. - **`sentryintegration.cpp`** -- guard against null `filename`/`funcname` in spdlog message handler to prevent a crash in `fmt::format`. - **`jobqueue.cpp`** -- fixed lost job ID when `IdGenerator` wraps around zero; fixed raw `Job*` in `RunningJobs` map (potential use-after-free) to `RefPtr`; fixed range-loop copies; fixed format string typo. - **`trace.cpp`** -- suppress GCC false-positive warnings in third-party `trace.h` include. ### zenhttp - **WebSocket close race** (`wsasio`, `wshttpsys`, `httpwsclient`) -- `m_CloseSent` promoted from `bool` to `std::atomic`; close check changed to `exchange(true)` to eliminate the check-then-set data race. - **`wsframecodec.cpp`** -- reject WebSocket frames with payload > 256 MB to prevent OOM from malformed/malicious frames. - **`oidc.cpp`** -- URL-encode refresh token and client ID in token requests (`FormUrlEncode`); parse `end_session_endpoint` and `device_authorization_endpoint` from OIDC discovery document. - **`httpclientcommon.cpp`** -- propagate error code from `AppendData` when flushing the cache buffer. - **`httpclient.h`** -- initialize all uninitialized members (`ErrorCode`, `UploadedBytes`, `DownloadedBytes`, `ElapsedSeconds`, `MultipartBoundary` fields). - **`httpserver.h`** -- fix `operator=` return type for `HttpRpcHandler` (missing `&`). - **`packageformat.h`** -- fix `~0u` (32-bit truncation) to `~uint64_t(0)` for a `uint64_t` field. - **`httpparser`** -- initialize `m_RequestVerb` in both declaration and `ResetState()`. - **`httpplugin.cpp`** -- initialize `m_BasePort`; fix format string missing quotes around connection name. - **`httptracer.h`** -- move `#pragma once` before includes. - **`websocket.h`** -- initialize `WebSocketMessage::Opcode`. ### zenserver - **`hubservice.cpp`** -- fix two `ZEN_ASSERT` calls that incorrectly used fmt-style format args; converted to `ZEN_ASSERT_FORMAT`. --- .clang-format | 5 ++++ .claude/settings.json | 6 +++++ .gitignore | 13 ++------- src/zencore/blake3.cpp | 2 +- src/zencore/commandline.cpp | 1 + src/zencore/include/zencore/md5.h | 2 ++ src/zencore/include/zencore/mpscqueue.h | 20 +++++++------- src/zencore/include/zencore/string.h | 22 +++++++-------- src/zencore/include/zencore/thread.h | 16 ++++++++--- src/zencore/include/zencore/xxhash.h | 2 +- src/zencore/include/zencore/zencore.h | 34 ++++++++++++++--------- src/zencore/jobqueue.cpp | 20 ++++++-------- src/zencore/logging.cpp | 2 +- src/zencore/md5.cpp | 24 ++++++++++------- src/zencore/memtrack/tagtrace.cpp | 2 +- src/zencore/mpscqueue.cpp | 4 +-- src/zencore/sentryintegration.cpp | 9 +++++-- src/zencore/string.cpp | 15 +++++++++-- src/zencore/thread.cpp | 42 +++++++++++++++++++---------- src/zencore/trace.cpp | 9 +++++++ src/zencore/xmake.lua | 7 +---- src/zenhttp/auth/oidc.cpp | 24 ++++++++++++++++- src/zenhttp/clients/httpclientcommon.cpp | 5 +++- src/zenhttp/clients/httpwsclient.cpp | 8 +++--- src/zenhttp/include/zenhttp/httpclient.h | 14 +++++----- src/zenhttp/include/zenhttp/httpserver.h | 2 +- src/zenhttp/include/zenhttp/packageformat.h | 2 +- src/zenhttp/include/zenhttp/websocket.h | 2 +- src/zenhttp/servers/httpparser.cpp | 8 +----- src/zenhttp/servers/httpparser.h | 2 +- src/zenhttp/servers/httpplugin.cpp | 4 +-- src/zenhttp/servers/httptracer.h | 4 +-- src/zenhttp/servers/wsasio.cpp | 6 ++--- src/zenhttp/servers/wsasio.h | 2 +- src/zenhttp/servers/wsframecodec.cpp | 7 +++++ src/zenhttp/servers/wshttpsys.cpp | 6 ++--- src/zenhttp/servers/wshttpsys.h | 2 +- src/zenserver/hub/hubservice.cpp | 5 ++-- 38 files changed, 221 insertions(+), 139 deletions(-) create mode 100644 .claude/settings.json diff --git a/.clang-format b/.clang-format index 46f378928..22fb0d846 100644 --- a/.clang-format +++ b/.clang-format @@ -148,5 +148,10 @@ WhitespaceSensitiveMacros: - STRINGIZE ... --- +Language: Json +IndentWidth: 2 +ColumnLimit: 0 +UseTab: Never +--- Language: JavaScript DisableFormat: true diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 000000000..0c03c6a34 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,6 @@ +{ + "attribution": { + "commit": "", + "pr": "" + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index eaf1656a6..0aa028930 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,8 @@ ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore .DS_Store -.claude/ +.claude/settings.local.json +.profile/ # User-specific files *.suo @@ -12,18 +13,8 @@ *.userosscache *.sln.docstates -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - # Build results -[Dd]ebug/ -[Rr]elease/ -x64/ -x86/ build/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ vsxmake20*/ vs20*/ diff --git a/src/zencore/blake3.cpp b/src/zencore/blake3.cpp index 123918de5..55f9b74af 100644 --- a/src/zencore/blake3.cpp +++ b/src/zencore/blake3.cpp @@ -123,7 +123,7 @@ BLAKE3::ToHexString(StringBuilderBase& outBuilder) const char str[65]; ToHexString(str); - outBuilder.AppendRange(str, &str[65]); + outBuilder.AppendRange(str, &str[StringLength]); return outBuilder; } diff --git a/src/zencore/commandline.cpp b/src/zencore/commandline.cpp index 426cf23d6..718ef9678 100644 --- a/src/zencore/commandline.cpp +++ b/src/zencore/commandline.cpp @@ -14,6 +14,7 @@ ZEN_THIRD_PARTY_INCLUDES_END # include #endif +#include #include namespace zen { diff --git a/src/zencore/include/zencore/md5.h b/src/zencore/include/zencore/md5.h index d934dd86b..3b0b7cae6 100644 --- a/src/zencore/include/zencore/md5.h +++ b/src/zencore/include/zencore/md5.h @@ -43,6 +43,8 @@ public: MD5 GetHash(); private: + // Opaque storage for MD5_CTX (104 bytes, aligned to uint32_t) + alignas(4) uint8_t m_Context[104]; }; void md5_forcelink(); // internal diff --git a/src/zencore/include/zencore/mpscqueue.h b/src/zencore/include/zencore/mpscqueue.h index 19e410d85..d97c433fd 100644 --- a/src/zencore/include/zencore/mpscqueue.h +++ b/src/zencore/include/zencore/mpscqueue.h @@ -22,10 +22,10 @@ namespace zen { template struct TypeCompatibleStorage { - ElementType* Data() { return (ElementType*)this; } - const ElementType* Data() const { return (const ElementType*)this; } + ElementType* Data() { return reinterpret_cast(&Storage); } + const ElementType* Data() const { return reinterpret_cast(&Storage); } - alignas(ElementType) char DataMember; + alignas(ElementType) char Storage[sizeof(ElementType)]; }; /** Fast multi-producer/single-consumer unbounded concurrent queue. @@ -58,7 +58,7 @@ public: Tail = Next; Next = Tail->Next.load(std::memory_order_relaxed); - std::destroy_at((ElementType*)&Tail->Value); + std::destroy_at(Tail->Value.Data()); delete Tail; } } @@ -67,7 +67,7 @@ public: void Enqueue(ArgTypes&&... Args) { Node* New = new Node; - new (&New->Value) ElementType(std::forward(Args)...); + new (New->Value.Data()) ElementType(std::forward(Args)...); Node* Prev = Head.exchange(New, std::memory_order_acq_rel); Prev->Next.store(New, std::memory_order_release); @@ -82,7 +82,7 @@ public: return {}; } - ElementType* ValuePtr = (ElementType*)&Next->Value; + ElementType* ValuePtr = Next->Value.Data(); std::optional Res{std::move(*ValuePtr)}; std::destroy_at(ValuePtr); @@ -100,9 +100,11 @@ private: }; private: - std::atomic Head; // accessed only by producers - alignas(hardware_constructive_interference_size) - Node* Tail; // accessed only by consumer, hence should be on a different cache line than `Head` + // Use a fixed constant to avoid GCC's -Winterference-size warning with std::hardware_destructive_interference_size + static constexpr std::size_t CacheLineSize = 64; + + alignas(CacheLineSize) std::atomic Head; // accessed only by producers + alignas(CacheLineSize) Node* Tail; // accessed only by consumer, separate cache line from Head }; void mpscqueue_forcelink(); diff --git a/src/zencore/include/zencore/string.h b/src/zencore/include/zencore/string.h index 250eb9f56..4deca63ed 100644 --- a/src/zencore/include/zencore/string.h +++ b/src/zencore/include/zencore/string.h @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -51,7 +50,7 @@ StringLength(const wchar_t* str) return wcslen(str); } -inline bool +inline int StringCompare(const char16_t* s1, const char16_t* s2) { char16_t c1, c2; @@ -66,7 +65,7 @@ StringCompare(const char16_t* s1, const char16_t* s2) ++s1; ++s2; } - return uint16_t(c1) - uint16_t(c2); + return int(uint16_t(c1)) - int(uint16_t(c2)); } inline bool @@ -122,10 +121,10 @@ public: StringBuilderImpl() = default; ~StringBuilderImpl(); - StringBuilderImpl(const StringBuilderImpl&) = delete; - StringBuilderImpl(const StringBuilderImpl&&) = delete; + StringBuilderImpl(const StringBuilderImpl&) = delete; + StringBuilderImpl(StringBuilderImpl&&) = delete; const StringBuilderImpl& operator=(const StringBuilderImpl&) = delete; - const StringBuilderImpl& operator=(const StringBuilderImpl&&) = delete; + StringBuilderImpl& operator=(StringBuilderImpl&&) = delete; inline size_t AddUninitialized(size_t Count) { @@ -374,9 +373,9 @@ protected: [[noreturn]] void Fail(const char* FailReason); // note: throws exception - C* m_Base; - C* m_CurPos; - C* m_End; + C* m_Base = nullptr; + C* m_CurPos = nullptr; + C* m_End = nullptr; bool m_IsDynamic = false; bool m_IsExtendable = false; }; @@ -773,8 +772,9 @@ std::optional ParseInt(const std::string_view& Input) { T Out = 0; - const std::from_chars_result Result = std::from_chars(Input.data(), Input.data() + Input.size(), Out); - if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range) + const char* End = Input.data() + Input.size(); + const std::from_chars_result Result = std::from_chars(Input.data(), End, Out); + if (Result.ec == std::errc::invalid_argument || Result.ec == std::errc::result_out_of_range || Result.ptr != End) { return std::nullopt; } diff --git a/src/zencore/include/zencore/thread.h b/src/zencore/include/zencore/thread.h index a1c68b0b2..d0d710ee8 100644 --- a/src/zencore/include/zencore/thread.h +++ b/src/zencore/include/zencore/thread.h @@ -58,7 +58,7 @@ public: } private: - RwLock* m_Lock; + RwLock* m_Lock = nullptr; }; inline auto WithSharedLock(auto&& Fun) @@ -69,6 +69,16 @@ public: struct ExclusiveLockScope { + ExclusiveLockScope(const ExclusiveLockScope& Rhs) = delete; + ExclusiveLockScope(ExclusiveLockScope&& Rhs) : m_Lock(Rhs.m_Lock) { Rhs.m_Lock = nullptr; } + ExclusiveLockScope& operator=(ExclusiveLockScope&& Rhs) + { + ReleaseNow(); + m_Lock = Rhs.m_Lock; + Rhs.m_Lock = nullptr; + return *this; + } + ExclusiveLockScope& operator=(const ExclusiveLockScope& Rhs) = delete; ExclusiveLockScope(RwLock& Lock) : m_Lock(&Lock) { Lock.AcquireExclusive(); } ~ExclusiveLockScope() { ReleaseNow(); } @@ -82,7 +92,7 @@ public: } private: - RwLock* m_Lock; + RwLock* m_Lock = nullptr; }; inline auto WithExclusiveLock(auto&& Fun) @@ -195,7 +205,7 @@ public: // false positive completion results. void AddCount(std::ptrdiff_t Count) { - std::atomic_ptrdiff_t Old = Counter.fetch_add(Count); + std::ptrdiff_t Old = Counter.fetch_add(Count); ZEN_ASSERT(Old > 0); } diff --git a/src/zencore/include/zencore/xxhash.h b/src/zencore/include/zencore/xxhash.h index fc55b513b..f79d39b61 100644 --- a/src/zencore/include/zencore/xxhash.h +++ b/src/zencore/include/zencore/xxhash.h @@ -87,7 +87,7 @@ struct XXH3_128Stream } private: - XXH3_state_s m_State; + XXH3_state_s m_State{}; }; struct XXH3_128Stream_deprecated diff --git a/src/zencore/include/zencore/zencore.h b/src/zencore/include/zencore/zencore.h index 177a19fff..a31950b0b 100644 --- a/src/zencore/include/zencore/zencore.h +++ b/src/zencore/include/zencore/zencore.h @@ -70,26 +70,36 @@ protected: } // namespace zen -#define ZEN_ASSERT(x, ...) \ - do \ - { \ - if (x) [[unlikely]] \ - break; \ - zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ +#define ZEN_ASSERT(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, ZEN_ASSERT_MSG_(#x, ##__VA_ARGS__)); \ } while (false) #ifndef NDEBUG -# define ZEN_ASSERT_SLOW(x, ...) \ - do \ - { \ - if (x) [[unlikely]] \ - break; \ - zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, #x); \ +# define ZEN_ASSERT_SLOW(x, ...) \ + do \ + { \ + if (x) [[unlikely]] \ + break; \ + zen::AssertImpl::ExecAssert(__FILE__, __LINE__, __FUNCTION__, ZEN_ASSERT_MSG_(#x, ##__VA_ARGS__)); \ } while (false) #else # define ZEN_ASSERT_SLOW(x, ...) #endif +// Internal: select between "expr" and "expr: message" forms. +// With no extra args: ZEN_ASSERT_MSG_("expr") -> "expr" +// With a message arg: ZEN_ASSERT_MSG_("expr", "msg") -> "expr" ": " "msg" +// With fmt-style args: ZEN_ASSERT_MSG_("expr", "msg", args...) -> "expr" ": " "msg" +// The extra fmt args are silently discarded here — use ZEN_ASSERT_FORMAT for those. +#define ZEN_ASSERT_MSG_SELECT_(_1, _2, N, ...) N +#define ZEN_ASSERT_MSG_1_(expr) expr +#define ZEN_ASSERT_MSG_2_(expr, msg, ...) expr ": " msg +#define ZEN_ASSERT_MSG_(expr, ...) ZEN_ASSERT_MSG_SELECT_(unused, ##__VA_ARGS__, ZEN_ASSERT_MSG_2_, ZEN_ASSERT_MSG_1_)(expr, ##__VA_ARGS__) + ////////////////////////////////////////////////////////////////////////// #define ZEN_NOT_IMPLEMENTED(...) ZEN_ASSERT(false, __VA_ARGS__) diff --git a/src/zencore/jobqueue.cpp b/src/zencore/jobqueue.cpp index 35724b07a..d6a8a6479 100644 --- a/src/zencore/jobqueue.cpp +++ b/src/zencore/jobqueue.cpp @@ -90,7 +90,7 @@ public: uint64_t NewJobId = IdGenerator.fetch_add(1); if (NewJobId == 0) { - IdGenerator.fetch_add(1); + NewJobId = IdGenerator.fetch_add(1); } RefPtr NewJob(new Job()); NewJob->Queue = this; @@ -129,7 +129,7 @@ public: QueuedJobs.erase(It); } }); - ZEN_ERROR("Failed to schedule job {}:'{}' to job queue. Reason: ''", NewJob->Id.Id, NewJob->Name, Ex.what()); + ZEN_ERROR("Failed to schedule job {}:'{}' to job queue. Reason: '{}'", NewJob->Id.Id, NewJob->Name, Ex.what()); throw; } } @@ -221,11 +221,11 @@ public: std::vector Jobs; QueueLock.WithSharedLock([&]() { - for (auto It : RunningJobs) + for (const auto& It : RunningJobs) { Jobs.push_back({.Id = JobId{It.first}, .Status = JobStatus::Running}); } - for (auto It : CompletedJobs) + for (const auto& It : CompletedJobs) { if (IsStale(It.second->EndTick)) { @@ -234,7 +234,7 @@ public: } Jobs.push_back({.Id = JobId{It.first}, .Status = JobStatus::Completed}); } - for (auto It : AbortedJobs) + for (const auto& It : AbortedJobs) { if (IsStale(It.second->EndTick)) { @@ -243,7 +243,7 @@ public: } Jobs.push_back({.Id = JobId{It.first}, .Status = JobStatus::Aborted}); } - for (auto It : QueuedJobs) + for (const auto& It : QueuedJobs) { Jobs.push_back({.Id = It->Id, .Status = JobStatus::Queued}); } @@ -337,7 +337,7 @@ public: std::atomic_bool InitializedFlag = false; RwLock QueueLock; std::deque> QueuedJobs; - std::unordered_map RunningJobs; + std::unordered_map> RunningJobs; std::unordered_map> CompletedJobs; std::unordered_map> AbortedJobs; @@ -429,20 +429,16 @@ JobQueue::ToString(JobStatus Status) { case JobQueue::JobStatus::Queued: return "Queued"sv; - break; case JobQueue::JobStatus::Running: return "Running"sv; - break; case JobQueue::JobStatus::Aborted: return "Aborted"sv; - break; case JobQueue::JobStatus::Completed: return "Completed"sv; - break; default: ZEN_ASSERT(false); + return ""sv; } - return ""sv; } std::unique_ptr diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index e960a2729..ebd68de09 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -303,7 +303,7 @@ GetLogLevel() LoggerRef Default() { - ZEN_ASSERT(TheDefaultLogger); + ZEN_ASSERT(TheDefaultLogger, "logging::InitializeLogging() must be called before using the logger"); return TheDefaultLogger; } diff --git a/src/zencore/md5.cpp b/src/zencore/md5.cpp index 83ed53fc8..f8cfee3ac 100644 --- a/src/zencore/md5.cpp +++ b/src/zencore/md5.cpp @@ -56,9 +56,9 @@ struct MD5_CTX unsigned char digest[16]; /* actual digest after MD5Final call */ }; -void MD5Init(); -void MD5Update(); -void MD5Final(); +void MD5Init(MD5_CTX* mdContext); +void MD5Update(MD5_CTX* mdContext, unsigned char* inBuf, unsigned int inLen); +void MD5Final(MD5_CTX* mdContext); /* ********************************************************************** @@ -370,28 +370,32 @@ MD5 MD5::Zero; // Initialized to all zeroes MD5Stream::MD5Stream() { + static_assert(sizeof(MD5_CTX) <= sizeof(m_Context)); Reset(); } void MD5Stream::Reset() { + MD5Init(reinterpret_cast(m_Context)); } MD5Stream& MD5Stream::Append(const void* Data, size_t ByteCount) { - ZEN_UNUSED(Data); - ZEN_UNUSED(ByteCount); - + MD5Update(reinterpret_cast(m_Context), (unsigned char*)Data, (unsigned int)ByteCount); return *this; } MD5 MD5Stream::GetHash() { - MD5 md5{}; + MD5_CTX FinalCtx; + memcpy(&FinalCtx, m_Context, sizeof(MD5_CTX)); + MD5Final(&FinalCtx); + MD5 md5{}; + memcpy(md5.Hash, FinalCtx.digest, 16); return md5; } @@ -428,7 +432,7 @@ MD5::ToHexString(StringBuilderBase& outBuilder) const char str[41]; ToHexString(str); - outBuilder.AppendRange(str, &str[40]); + outBuilder.AppendRange(str, &str[StringLength]); return outBuilder; } @@ -470,11 +474,11 @@ TEST_CASE("MD5") MD5::String_t Buffer; Result.ToHexString(Buffer); - CHECK(Output.compare(Buffer)); + CHECK(Output.compare(Buffer) == 0); MD5 Reresult = MD5::FromHexString(Buffer); Reresult.ToHexString(Buffer); - CHECK(Output.compare(Buffer)); + CHECK(Output.compare(Buffer) == 0); } TEST_SUITE_END(); diff --git a/src/zencore/memtrack/tagtrace.cpp b/src/zencore/memtrack/tagtrace.cpp index 70a74365d..fca4a2ec3 100644 --- a/src/zencore/memtrack/tagtrace.cpp +++ b/src/zencore/memtrack/tagtrace.cpp @@ -186,7 +186,7 @@ FTagTrace::AnnounceSpecialTags() const { auto EmitTag = [](const char16_t* DisplayString, int32_t Tag, int32_t ParentTag) { const uint32_t DisplayLen = (uint32_t)StringLength(DisplayString); - UE_TRACE_LOG(Memory, TagSpec, MemAllocChannel, DisplayLen * sizeof(ANSICHAR)) + UE_TRACE_LOG(Memory, TagSpec, MemAllocChannel, DisplayLen * sizeof(char16_t)) << TagSpec.Tag(Tag) << TagSpec.Parent(ParentTag) << TagSpec.Display(DisplayString, DisplayLen); }; diff --git a/src/zencore/mpscqueue.cpp b/src/zencore/mpscqueue.cpp index f749f1c90..bdd22e20c 100644 --- a/src/zencore/mpscqueue.cpp +++ b/src/zencore/mpscqueue.cpp @@ -7,7 +7,7 @@ namespace zen { -#if ZEN_WITH_TESTS && 0 +#if ZEN_WITH_TESTS TEST_SUITE_BEGIN("core.mpscqueue"); TEST_CASE("mpsc") { @@ -24,4 +24,4 @@ mpscqueue_forcelink() { } -} // namespace zen \ No newline at end of file +} // namespace zen diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index 636e182b4..bfff114c3 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -81,8 +81,13 @@ sentry_sink::sink_it_(const spdlog::details::log_msg& msg) } try { - std::string Message = fmt::format("{}\n{}({}) [{}]", msg.payload, msg.source.filename, msg.source.line, msg.source.funcname); - sentry_value_t event = sentry_value_new_message_event( + auto MaybeNullString = [](const char* Ptr) { return Ptr ? Ptr : ""; }; + std::string Message = fmt::format("{}\n{}({}) [{}]", + msg.payload, + MaybeNullString(msg.source.filename), + msg.source.line, + MaybeNullString(msg.source.funcname)); + sentry_value_t event = sentry_value_new_message_event( /* level */ MapToSentryLevel[msg.level], /* logger */ nullptr, /* message */ Message.c_str()); diff --git a/src/zencore/string.cpp b/src/zencore/string.cpp index 3d0451e27..ed0ba6f46 100644 --- a/src/zencore/string.cpp +++ b/src/zencore/string.cpp @@ -268,6 +268,17 @@ namespace { /* kNicenumTime */ 1000}; } // namespace +uint64_t +IntPow(uint64_t Base, int Exp) +{ + uint64_t Result = 1; + for (int I = 0; I < Exp; ++I) + { + Result *= Base; + } + return Result; +} + /* * Convert a number to an appropriately human-readable output. */ @@ -315,7 +326,7 @@ NiceNumGeneral(uint64_t Num, std::span Buffer, NicenumFormat Format) const char* u = UnitStrings[Format][Index]; - if ((Index == 0) || ((Num % (uint64_t)powl((int)KiloUnit[Format], Index)) == 0)) + if ((Index == 0) || ((Num % IntPow(KiloUnit[Format], Index)) == 0)) { /* * If this is an even multiple of the base, always display @@ -339,7 +350,7 @@ NiceNumGeneral(uint64_t Num, std::span Buffer, NicenumFormat Format) for (int i = 2; i >= 0; i--) { - double Value = (double)Num / (uint64_t)powl((int)KiloUnit[Format], Index); + double Value = (double)Num / IntPow(KiloUnit[Format], Index); /* * Don't print floating point values for time. Note, diff --git a/src/zencore/thread.cpp b/src/zencore/thread.cpp index 9e3486e49..54459cbaa 100644 --- a/src/zencore/thread.cpp +++ b/src/zencore/thread.cpp @@ -133,7 +133,10 @@ SetCurrentThreadName([[maybe_unused]] std::string_view ThreadName) #elif ZEN_PLATFORM_MAC pthread_setname_np(ThreadNameZ.c_str()); #else - pthread_setname_np(pthread_self(), ThreadNameZ.c_str()); + // Linux pthread_setname_np has a 16-byte limit (15 chars + NUL) + StringBuilder<16> LinuxThreadName; + LinuxThreadName << LimitedThreadName.substr(0, 15); + pthread_setname_np(pthread_self(), LinuxThreadName.c_str()); #endif } // namespace zen @@ -233,12 +236,15 @@ Event::Close() #else std::atomic_thread_fence(std::memory_order_acquire); auto* Inner = (EventInner*)m_EventHandle.load(); + if (Inner) { - std::unique_lock Lock(Inner->Mutex); - Inner->bSet.store(true); - m_EventHandle = nullptr; + { + std::unique_lock Lock(Inner->Mutex); + Inner->bSet.store(true); + m_EventHandle = nullptr; + } + delete Inner; } - delete Inner; #endif } @@ -351,7 +357,7 @@ NamedEvent::NamedEvent(std::string_view EventName) intptr_t Packed; Packed = intptr_t(Sem) << 32; Packed |= intptr_t(Fd) & 0xffff'ffff; - m_EventHandle = (void*)Packed; + m_EventHandle = (void*)Packed; #endif ZEN_ASSERT(m_EventHandle != nullptr); } @@ -372,7 +378,9 @@ NamedEvent::Close() #if ZEN_PLATFORM_WINDOWS CloseHandle(m_EventHandle); #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Fd = int(intptr_t(m_EventHandle.load()) & 0xffff'ffff); + const intptr_t Handle = intptr_t(m_EventHandle.load()); + const int Fd = int(Handle & 0xffff'ffff); + const int Sem = int(Handle >> 32); if (flock(Fd, LOCK_EX | LOCK_NB) == 0) { @@ -388,11 +396,10 @@ NamedEvent::Close() } flock(Fd, LOCK_UN | LOCK_NB); - close(Fd); - - int Sem = int(intptr_t(m_EventHandle.load()) >> 32); semctl(Sem, 0, IPC_RMID); } + + close(Fd); #endif m_EventHandle = nullptr; @@ -481,9 +488,12 @@ NamedMutex::~NamedMutex() CloseHandle(m_MutexHandle); } #elif ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC - int Inner = int(intptr_t(m_MutexHandle)); - flock(Inner, LOCK_UN); - close(Inner); + if (m_MutexHandle) + { + int Inner = int(intptr_t(m_MutexHandle)); + flock(Inner, LOCK_UN); + close(Inner); + } #endif } @@ -516,7 +526,6 @@ NamedMutex::Create(std::string_view MutexName) if (flock(Inner, LOCK_EX) != 0) { close(Inner); - Inner = 0; return false; } @@ -583,6 +592,11 @@ GetCurrentThreadId() void Sleep(int ms) { + if (ms <= 0) + { + return; + } + #if ZEN_PLATFORM_WINDOWS ::Sleep(ms); #else diff --git a/src/zencore/trace.cpp b/src/zencore/trace.cpp index a026974c0..7c195e69f 100644 --- a/src/zencore/trace.cpp +++ b/src/zencore/trace.cpp @@ -10,7 +10,16 @@ # define TRACE_IMPLEMENT 1 # undef _WINSOCK_DEPRECATED_NO_WARNINGS +// GCC false positives in thirdparty trace.h (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100137) +# if ZEN_COMPILER_GCC +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wstringop-overread" +# pragma GCC diagnostic ignored "-Wdangling-pointer" +# endif # include +# if ZEN_COMPILER_GCC +# pragma GCC diagnostic pop +# endif # include # include diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index ab842f6ed..2f81b7ec8 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -14,12 +14,7 @@ target('zencore') end) set_configdir("include/zencore") add_files("**.cpp") - if is_plat("linux") and not (get_config("toolchain") or ""):find("clang") then - -- GCC false positives in thirdparty trace.h (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100137) - add_files("trace.cpp", {unity_ignored = true, force = {cxxflags = {"-Wno-stringop-overread", "-Wno-dangling-pointer"}} }) - else - add_files("trace.cpp", {unity_ignored = true }) - end + add_files("trace.cpp", {unity_ignored = true }) add_files("testing.cpp", {unity_ignored = true }) if has_config("zenrpmalloc") then diff --git a/src/zenhttp/auth/oidc.cpp b/src/zenhttp/auth/oidc.cpp index 38e7586ad..23bbc17e8 100644 --- a/src/zenhttp/auth/oidc.cpp +++ b/src/zenhttp/auth/oidc.cpp @@ -32,6 +32,25 @@ namespace details { using namespace std::literals; +static std::string +FormUrlEncode(std::string_view Input) +{ + std::string Result; + Result.reserve(Input.size()); + for (char C : Input) + { + if ((C >= 'A' && C <= 'Z') || (C >= 'a' && C <= 'z') || (C >= '0' && C <= '9') || C == '-' || C == '_' || C == '.' || C == '~') + { + Result.push_back(C); + } + else + { + Result.append(fmt::format("%{:02X}", static_cast(C))); + } + } + return Result; +} + OidcClient::OidcClient(const OidcClient::Options& Options) { m_BaseUrl = std::string(Options.BaseUrl); @@ -67,6 +86,8 @@ OidcClient::Initialize() .TokenEndpoint = Json["token_endpoint"].string_value(), .UserInfoEndpoint = Json["userinfo_endpoint"].string_value(), .RegistrationEndpoint = Json["registration_endpoint"].string_value(), + .EndSessionEndpoint = Json["end_session_endpoint"].string_value(), + .DeviceAuthorizationEndpoint = Json["device_authorization_endpoint"].string_value(), .JwksUri = Json["jwks_uri"].string_value(), .SupportedResponseTypes = details::ToStringArray(Json["response_types_supported"]), .SupportedResponseModes = details::ToStringArray(Json["response_modes_supported"]), @@ -81,7 +102,8 @@ OidcClient::Initialize() OidcClient::RefreshTokenResult OidcClient::RefreshToken(std::string_view RefreshToken) { - const std::string Body = fmt::format("grant_type=refresh_token&refresh_token={}&client_id={}", RefreshToken, m_ClientId); + const std::string Body = + fmt::format("grant_type=refresh_token&refresh_token={}&client_id={}", FormUrlEncode(RefreshToken), FormUrlEncode(m_ClientId)); HttpClient Http{m_Config.TokenEndpoint}; diff --git a/src/zenhttp/clients/httpclientcommon.cpp b/src/zenhttp/clients/httpclientcommon.cpp index 9ded23375..6f4c67dd0 100644 --- a/src/zenhttp/clients/httpclientcommon.cpp +++ b/src/zenhttp/clients/httpclientcommon.cpp @@ -142,7 +142,10 @@ namespace detail { DataSize -= CopySize; if (m_CacheBufferOffset == CacheBufferSize) { - AppendData(m_CacheBuffer, CacheBufferSize); + if (std::error_code Ec = AppendData(m_CacheBuffer, CacheBufferSize)) + { + return Ec; + } if (DataSize > 0) { ZEN_ASSERT(DataSize < CacheBufferSize); diff --git a/src/zenhttp/clients/httpwsclient.cpp b/src/zenhttp/clients/httpwsclient.cpp index 36a6f081b..9497dadb8 100644 --- a/src/zenhttp/clients/httpwsclient.cpp +++ b/src/zenhttp/clients/httpwsclient.cpp @@ -351,9 +351,8 @@ struct HttpWsClient::Impl } // Echo masked close frame if we haven't sent one yet - if (!m_CloseSent) + if (!m_CloseSent.exchange(true)) { - m_CloseSent = true; std::vector CloseFrame = WsFrameCodec::BuildMaskedCloseFrame(Code); EnqueueWrite(std::move(CloseFrame)); } @@ -479,9 +478,8 @@ struct HttpWsClient::Impl return; } - if (!m_CloseSent) + if (!m_CloseSent.exchange(true)) { - m_CloseSent = true; std::vector CloseFrame = WsFrameCodec::BuildMaskedCloseFrame(Code, Reason); EnqueueWrite(std::move(CloseFrame)); } @@ -515,7 +513,7 @@ struct HttpWsClient::Impl bool m_IsWriting = false; std::atomic m_IsOpen{false}; - bool m_CloseSent = false; + std::atomic m_CloseSent{false}; }; ////////////////////////////////////////////////////////////////////////// diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index d87082d10..bec4984db 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -128,7 +128,7 @@ public: struct ErrorContext { - int ErrorCode; + int ErrorCode = 0; std::string ErrorMessage; /** True when the error is a transport-level connection failure (connect timeout, refused, DNS) */ @@ -179,19 +179,19 @@ public: KeyValueMap Header; // The number of bytes sent as part of the request - int64_t UploadedBytes; + int64_t UploadedBytes = 0; // The number of bytes received as part of the response - int64_t DownloadedBytes; + int64_t DownloadedBytes = 0; // The elapsed time in seconds for the request to execute - double ElapsedSeconds; + double ElapsedSeconds = 0.0; struct MultipartBoundary { - uint64_t OffsetInPayload; - uint64_t RangeOffset; - uint64_t RangeLength; + uint64_t OffsetInPayload = 0; + uint64_t RangeOffset = 0; + uint64_t RangeLength = 0; HttpContentType ContentType; }; diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index 02cccc540..c1152dc3e 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -440,7 +440,7 @@ public: ~HttpRpcHandler(); HttpRpcHandler(const HttpRpcHandler&) = delete; - HttpRpcHandler operator=(const HttpRpcHandler&) = delete; + HttpRpcHandler& operator=(const HttpRpcHandler&) = delete; void AddRpc(std::string_view RpcId, std::function HandlerFunction); diff --git a/src/zenhttp/include/zenhttp/packageformat.h b/src/zenhttp/include/zenhttp/packageformat.h index c90b840da..1a5068580 100644 --- a/src/zenhttp/include/zenhttp/packageformat.h +++ b/src/zenhttp/include/zenhttp/packageformat.h @@ -68,7 +68,7 @@ struct CbAttachmentEntry struct CbAttachmentReferenceHeader { uint64_t PayloadByteOffset = 0; - uint64_t PayloadByteSize = ~0u; + uint64_t PayloadByteSize = ~uint64_t(0); uint16_t AbsolutePathLength = 0; // This header will be followed by UTF8 encoded absolute path to backing file diff --git a/src/zenhttp/include/zenhttp/websocket.h b/src/zenhttp/include/zenhttp/websocket.h index 7a6fb33dd..bc3293282 100644 --- a/src/zenhttp/include/zenhttp/websocket.h +++ b/src/zenhttp/include/zenhttp/websocket.h @@ -22,7 +22,7 @@ enum class WebSocketOpcode : uint8_t struct WebSocketMessage { - WebSocketOpcode Opcode; + WebSocketOpcode Opcode = WebSocketOpcode::kText; IoBuffer Payload; uint16_t CloseCode = 0; }; diff --git a/src/zenhttp/servers/httpparser.cpp b/src/zenhttp/servers/httpparser.cpp index 3b1229375..918b55dc6 100644 --- a/src/zenhttp/servers/httpparser.cpp +++ b/src/zenhttp/servers/httpparser.cpp @@ -245,13 +245,6 @@ NormalizeUrlPath(std::string_view InUrl, std::string& NormalizedUrl) NormalizedUrl.reserve(UrlLength); NormalizedUrl.append(Url, UrlIndex); } - - // NOTE: this check is redundant given the enclosing if, - // need to verify the intent of this code - if (!LastCharWasSeparator) - { - NormalizedUrl.push_back('/'); - } } else if (!NormalizedUrl.empty()) { @@ -389,6 +382,7 @@ HttpRequestParser::ResetState() m_UpgradeHeaderIndex = -1; m_SecWebSocketKeyHeaderIndex = -1; m_SecWebSocketVersionHeaderIndex = -1; + m_RequestVerb = HttpVerb::kGet; m_Expect100Continue = false; m_BodyBuffer = {}; m_BodyPosition = 0; diff --git a/src/zenhttp/servers/httpparser.h b/src/zenhttp/servers/httpparser.h index d40a5aeb0..23ad9d8fb 100644 --- a/src/zenhttp/servers/httpparser.h +++ b/src/zenhttp/servers/httpparser.h @@ -93,7 +93,7 @@ private: int8_t m_UpgradeHeaderIndex; int8_t m_SecWebSocketKeyHeaderIndex; int8_t m_SecWebSocketVersionHeaderIndex; - HttpVerb m_RequestVerb; + HttpVerb m_RequestVerb = HttpVerb::kGet; std::atomic_bool m_KeepAlive{false}; bool m_Expect100Continue = false; int m_RequestId = -1; diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index 8564826d6..850dafdca 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -123,7 +123,7 @@ struct HttpPluginServerImpl : public HttpPluginServer, TransportServer bool m_IsRequestLoggingEnabled = false; LoggerRef m_RequestLog; std::atomic_uint32_t m_ConnectionIdCounter{0}; - int m_BasePort; + int m_BasePort = 0; HttpServerTracer m_RequestTracer; @@ -294,7 +294,7 @@ HttpPluginConnectionHandler::Initialize(TransportConnection* Transport, HttpPlug ConnectionName = "anonymous"; } - ZEN_LOG_TRACE(m_Server->m_RequestLog, "NEW connection #{} ('')", m_ConnectionId, ConnectionName); + ZEN_LOG_TRACE(m_Server->m_RequestLog, "NEW connection #{} ('{}')", m_ConnectionId, ConnectionName); } uint32_t diff --git a/src/zenhttp/servers/httptracer.h b/src/zenhttp/servers/httptracer.h index da72c79c9..a9a45f162 100644 --- a/src/zenhttp/servers/httptracer.h +++ b/src/zenhttp/servers/httptracer.h @@ -1,9 +1,9 @@ // Copyright Epic Games, Inc. All Rights Reserved. -#include - #pragma once +#include + namespace zen { /** Helper class for HTTP server implementations diff --git a/src/zenhttp/servers/wsasio.cpp b/src/zenhttp/servers/wsasio.cpp index dfc1eac38..3e31b58bc 100644 --- a/src/zenhttp/servers/wsasio.cpp +++ b/src/zenhttp/servers/wsasio.cpp @@ -140,9 +140,8 @@ WsAsioConnection::ProcessReceivedData() } // Echo close frame back if we haven't sent one yet - if (!m_CloseSent) + if (!m_CloseSent.exchange(true)) { - m_CloseSent = true; std::vector CloseFrame = WsFrameCodec::BuildCloseFrame(Code); EnqueueWrite(std::move(CloseFrame)); } @@ -208,9 +207,8 @@ WsAsioConnection::DoClose(uint16_t Code, std::string_view Reason) return; } - if (!m_CloseSent) + if (!m_CloseSent.exchange(true)) { - m_CloseSent = true; std::vector CloseFrame = WsFrameCodec::BuildCloseFrame(Code, Reason); EnqueueWrite(std::move(CloseFrame)); } diff --git a/src/zenhttp/servers/wsasio.h b/src/zenhttp/servers/wsasio.h index a638ea836..d8ffdc00a 100644 --- a/src/zenhttp/servers/wsasio.h +++ b/src/zenhttp/servers/wsasio.h @@ -65,7 +65,7 @@ private: bool m_IsWriting = false; std::atomic m_IsOpen{true}; - bool m_CloseSent = false; + std::atomic m_CloseSent{false}; }; } // namespace zen::asio_http diff --git a/src/zenhttp/servers/wsframecodec.cpp b/src/zenhttp/servers/wsframecodec.cpp index a4c5e0f16..e452141fe 100644 --- a/src/zenhttp/servers/wsframecodec.cpp +++ b/src/zenhttp/servers/wsframecodec.cpp @@ -51,6 +51,13 @@ WsFrameCodec::TryParseFrame(const uint8_t* Data, size_t Size) HeaderSize = 10; } + // Reject frames with unreasonable payload sizes to prevent OOM + static constexpr uint64_t kMaxPayloadSize = 256 * 1024 * 1024; // 256 MB + if (PayloadLen > kMaxPayloadSize) + { + return {}; + } + const size_t MaskSize = Masked ? 4 : 0; const size_t TotalFrame = HeaderSize + MaskSize + PayloadLen; diff --git a/src/zenhttp/servers/wshttpsys.cpp b/src/zenhttp/servers/wshttpsys.cpp index 3f0f0b447..3408b64b3 100644 --- a/src/zenhttp/servers/wshttpsys.cpp +++ b/src/zenhttp/servers/wshttpsys.cpp @@ -217,9 +217,8 @@ WsHttpSysConnection::ProcessReceivedData() bool ShouldSendClose = false; { RwLock::ExclusiveLockScope _(m_WriteLock); - if (!m_CloseSent) + if (!m_CloseSent.exchange(true)) { - m_CloseSent = true; ShouldSendClose = true; } } @@ -412,9 +411,8 @@ WsHttpSysConnection::DoClose(uint16_t Code, std::string_view Reason) bool ShouldSendClose = false; { RwLock::ExclusiveLockScope _(m_WriteLock); - if (!m_CloseSent) + if (!m_CloseSent.exchange(true)) { - m_CloseSent = true; ShouldSendClose = true; } } diff --git a/src/zenhttp/servers/wshttpsys.h b/src/zenhttp/servers/wshttpsys.h index ab0ca381a..d854289e0 100644 --- a/src/zenhttp/servers/wshttpsys.h +++ b/src/zenhttp/servers/wshttpsys.h @@ -96,7 +96,7 @@ private: Ref m_SelfRef; std::atomic m_ShutdownRequested{false}; std::atomic m_IsOpen{true}; - bool m_CloseSent = false; + std::atomic m_CloseSent{false}; }; } // namespace zen diff --git a/src/zenserver/hub/hubservice.cpp b/src/zenserver/hub/hubservice.cpp index a757cd594..7b999ae20 100644 --- a/src/zenserver/hub/hubservice.cpp +++ b/src/zenserver/hub/hubservice.cpp @@ -4,6 +4,7 @@ #include "hydration.h" +#include #include #include #include @@ -195,7 +196,7 @@ StorageServerInstance::~StorageServerInstance() void StorageServerInstance::SpawnServerProcess() { - ZEN_ASSERT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId); + ZEN_ASSERT_FORMAT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId); m_ServerInstance.SetServerExecutablePath(GetRunningExecutablePath()); m_ServerInstance.SetDataDir(m_BaseDir); @@ -322,7 +323,7 @@ StorageServerInstance::WakeLocked() return; } - ZEN_ASSERT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId); + ZEN_ASSERT_FORMAT(!m_ServerInstance.IsRunning(), "Storage server instance for module '{}' is already running", m_ModuleId); try { -- cgit v1.2.3 From 5115b419cefd41e8d5cc465c8c7ae5140cde71d4 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Fri, 6 Mar 2026 12:39:06 +0100 Subject: zenstore bug-fixes from static analysis pass (#815) **Bug fixes across zenstore, zenremotestore, and related subsystems, primarily surfaced by static analysis.** ## Cache subsystem (cachedisklayer.cpp) - Fixed tombstone scoping bug: tombstone flag and missing entry were recorded outside the block where data was removed, causing non-missing entries to be incorrectly tombstoned - Fixed use-after-overwrite: `RemoveMemCachedData`/`RemoveMetaData` were called after `Payload` was overwritten on cache put, leaking stale data - Fixed incorrect retry sleep formula (`100 - (3 - RetriesLeft) * 100` always produced the same or negative value; corrected to `(3 - RetriesLeft) * 100`) - Fixed broken `break` missing from sidecar file read loop, causing reads past valid data - Fixed missing format argument in three `ZEN_WARN`/`ZEN_ERROR` log calls (format string had `{}` placeholders with no corresponding argument, or vice versa) - Fixed elapsed timer being accumulated inside the wrong scope in `HandleRpcGetCacheRecords` - Fixed test asserting against unserialized `RecordPolicy` instead of the deserialized `Loaded` copy - Initialized `AbortFlag`/`PauseFlag` atomics at declaration (UB if read before first write) ## Build store (buildstore.cpp / buildstore.h) - Fixed wrong variable used in warning log: used loop index `ResultIndex` instead of `Index`/`MetaLocationResultIndexes[Index]`, logging wrong hash values - Fixed `sizeof(AccessTimesHeader)` used instead of `sizeof(AccessTimeRecord)` when advancing write offset, corrupting the access times file if the sizes differ - Initialized `m_LastAccessTimeUpdateCount` atomic member (was uninitialized) - Changed map iteration loops to use `const auto&` to avoid unnecessary copies ## Project store (projectstore.cpp / projectstore.h) - Fixed wrong iterator dereferenced in `IterateChunks`: used `ChunkIt->second` (from a different map lookup) instead of `MetaIt->second` - Fixed wrong assert variable: `Sizes[Index]` should be `RawSizes[Index]` - Fixed `MakeTombstone`/`IsTombstone` inconsistency: `MakeTombstone` was zeroing `OpLsn` but `IsTombstone` checks `OpLsn.Number != 0`; tombstone creation now preserves `OpLsn` - Fixed uninitialized `InvalidEntries` counter - Fixed format string mismatch in warning log - Initialized `AbortFlag`/`PauseFlag` atomics; changed map iteration to `const auto&` ## Workspaces (workspaces.cpp) - Fixed missing alias registration when a workspace share is updated: alias was deleted but never re-inserted - Fixed integer overflow in range clamping: `(RequestedOffset + RequestedSize) > Size` could wrap; corrected to `RequestedSize > Size - RequestedOffset` - Changed map iteration loops to `const auto&` ## CAS subsystem (cas.cpp, caslog.cpp, compactcas.cpp, filecas.cpp) - Fixed `IterateChunks` passing original `Payload` buffer instead of the modified `Chunk` buffer (content type was set on the copy but the original was sent to the callback) - Fixed invalid `std::future::get()` call on default-constructed futures - Fixed sign-comparison in `CasLogFile::Replay` loop (`int i` vs `size_t`) - Changed `CasLogFile::IsValid` and `Open` to take `const std::filesystem::path&` instead of by value - Fixed format string in `~CasContainerStrategy` error log ## Remote store (zenremotestore) - Fixed `FolderContent::operator==` always returning true: loop variable `PathCount` was initialized to 0 instead of `Paths.size()` - Fixed `GetChunkIndexForRawHash` looking up from wrong map (`RawHashToSequenceIndex` instead of `ChunkHashToChunkIndex`) - Fixed double-counted `UniqueSequencesFound` stat (incremented in both branches of an if/else) - Fixed `RawSize` sentinel value truncation: `(uint32_t)-1` assigned to a `uint64_t` field; corrected to `(uint64_t)-1` - Initialized uninitialized atomic and struct members across `buildstorageoperations.h`, `chunkblock.h`, and `remoteprojectstore.h` --- src/zenremotestore/chunking/chunkedcontent.cpp | 4 +-- .../zenremotestore/builds/buildstorageoperations.h | 6 ++-- .../include/zenremotestore/chunking/chunkblock.h | 2 +- .../zenremotestore/chunking/chunkedcontent.h | 2 +- .../projectstore/remoteprojectstore.h | 20 ++++++------- src/zenstore/buildstore/buildstore.cpp | 16 +++++----- src/zenstore/cache/cachedisklayer.cpp | 32 ++++++++++---------- src/zenstore/cache/cachepolicy.cpp | 14 ++++----- src/zenstore/cache/cacherpc.cpp | 4 +-- src/zenstore/cache/structuredcachestore.cpp | 8 ++--- src/zenstore/cas.cpp | 11 ++++--- src/zenstore/caslog.cpp | 6 ++-- src/zenstore/cidstore.cpp | 3 +- src/zenstore/compactcas.cpp | 16 +++++----- src/zenstore/filecas.cpp | 14 ++++----- src/zenstore/filecas.h | 2 +- .../include/zenstore/buildstore/buildstore.h | 2 +- .../include/zenstore/cache/cachedisklayer.h | 34 +++++++++++----------- src/zenstore/include/zenstore/cache/cacheshared.h | 6 ++-- .../include/zenstore/cache/structuredcachestore.h | 12 ++++---- src/zenstore/include/zenstore/caslog.h | 10 +++---- src/zenstore/include/zenstore/gc.h | 6 ++-- src/zenstore/include/zenstore/projectstore.h | 10 ++----- src/zenstore/projectstore.cpp | 26 ++++++++--------- src/zenstore/workspaces.cpp | 16 ++++++---- 25 files changed, 144 insertions(+), 138 deletions(-) diff --git a/src/zenremotestore/chunking/chunkedcontent.cpp b/src/zenremotestore/chunking/chunkedcontent.cpp index 62c927508..c09ab9d3a 100644 --- a/src/zenremotestore/chunking/chunkedcontent.cpp +++ b/src/zenremotestore/chunking/chunkedcontent.cpp @@ -166,7 +166,6 @@ namespace { if (Chunked.Info.ChunkSequence.empty()) { AddChunkSequence(Stats, OutChunkedContent.ChunkedContent, ChunkHashToChunkIndex, Chunked.Info.RawHash, RawSize); - Stats.UniqueSequencesFound++; } else { @@ -186,7 +185,6 @@ namespace { Chunked.Info.ChunkHashes, ChunkSizes); } - Stats.UniqueSequencesFound++; } }); Stats.FilesChunked++; @@ -253,7 +251,7 @@ FolderContent::operator==(const FolderContent& Rhs) const if ((Platform == Rhs.Platform) && (RawSizes == Rhs.RawSizes) && (Attributes == Rhs.Attributes) && (ModificationTicks == Rhs.ModificationTicks) && (Paths.size() == Rhs.Paths.size())) { - size_t PathCount = 0; + size_t PathCount = Paths.size(); for (size_t PathIndex = 0; PathIndex < PathCount; PathIndex++) { if (Paths[PathIndex].generic_string() != Rhs.Paths[PathIndex].generic_string()) diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h index 875b8593b..0d2eded58 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageoperations.h @@ -161,7 +161,7 @@ public: DownloadStatistics m_DownloadStats; WriteChunkStatistics m_WriteChunkStats; RebuildFolderStateStatistics m_RebuildFolderStateStats; - std::atomic m_WrittenChunkByteCount; + std::atomic m_WrittenChunkByteCount = 0; private: struct BlockWriteOps @@ -186,7 +186,7 @@ private: uint32_t ScavengedContentIndex = (uint32_t)-1; uint32_t ScavengedPathIndex = (uint32_t)-1; uint32_t RemoteSequenceIndex = (uint32_t)-1; - uint64_t RawSize = (uint32_t)-1; + uint64_t RawSize = (uint64_t)-1; }; struct CopyChunkData @@ -362,7 +362,7 @@ private: const std::filesystem::path m_TempDownloadFolderPath; const std::filesystem::path m_TempBlockFolderPath; - std::atomic m_ValidatedChunkByteCount; + std::atomic m_ValidatedChunkByteCount = 0; }; struct FindBlocksStatistics diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h index 7aae1442e..20b6fd371 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h @@ -24,7 +24,7 @@ struct ThinChunkBlockDescription struct ChunkBlockDescription : public ThinChunkBlockDescription { - uint64_t HeaderSize; + uint64_t HeaderSize = 0; std::vector ChunkRawLengths; std::vector ChunkCompressedLengths; }; diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h index d402bd3f0..f44381e42 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkedcontent.h @@ -231,7 +231,7 @@ GetSequenceIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& Raw inline uint32_t GetChunkIndexForRawHash(const ChunkedContentLookup& Lookup, const IoHash& RawHash) { - return Lookup.RawHashToSequenceIndex.at(RawHash); + return Lookup.ChunkHashToChunkIndex.at(RawHash); } inline uint32_t diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h index 2cf10c664..42786d0f2 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h @@ -92,22 +92,22 @@ public: struct RemoteStoreInfo { - bool CreateBlocks; - bool UseTempBlockFiles; - bool AllowChunking; + bool CreateBlocks = false; + bool UseTempBlockFiles = false; + bool AllowChunking = false; std::string ContainerName; std::string Description; }; struct Stats { - std::uint64_t m_SentBytes; - std::uint64_t m_ReceivedBytes; - std::uint64_t m_RequestTimeNS; - std::uint64_t m_RequestCount; - std::uint64_t m_PeakSentBytes; - std::uint64_t m_PeakReceivedBytes; - std::uint64_t m_PeakBytesPerSec; + std::uint64_t m_SentBytes = 0; + std::uint64_t m_ReceivedBytes = 0; + std::uint64_t m_RequestTimeNS = 0; + std::uint64_t m_RequestCount = 0; + std::uint64_t m_PeakSentBytes = 0; + std::uint64_t m_PeakReceivedBytes = 0; + std::uint64_t m_PeakBytesPerSec = 0; }; struct ExtendedStats diff --git a/src/zenstore/buildstore/buildstore.cpp b/src/zenstore/buildstore/buildstore.cpp index 49ed7cdd2..dff1c3c61 100644 --- a/src/zenstore/buildstore/buildstore.cpp +++ b/src/zenstore/buildstore/buildstore.cpp @@ -373,8 +373,8 @@ BuildStore::PutMetadatas(std::span BlobHashes, std::span AbortFlag; - std::atomic PauseFlag; + std::atomic AbortFlag{false}; + std::atomic PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); for (size_t Index = 0; Index < Metadatas.size(); Index++) { @@ -505,8 +505,8 @@ BuildStore::GetMetadatas(std::span BlobHashes, WorkerThreadPool* O else { ZEN_WARN("Metadata {} for blob {} is malformed (not a compressed binary format)", - MetadataHashes[ResultIndex], - BlobHashes[ResultIndex]); + MetadataHashes[Index], + BlobHashes[MetaLocationResultIndexes[Index]]); } } } @@ -561,7 +561,7 @@ BuildStore::GetStorageStats() const RwLock::SharedLockScope _(m_Lock); Result.EntryCount = m_BlobLookup.size(); - for (auto LookupIt : m_BlobLookup) + for (const auto& LookupIt : m_BlobLookup) { const BlobIndex ReadBlobIndex = LookupIt.second; const BlobEntry& ReadBlobEntry = m_BlobEntries[ReadBlobIndex]; @@ -634,7 +634,7 @@ BuildStore::CompactState() const size_t MetadataCount = m_MetadataEntries.size(); MetadataEntries.reserve(MetadataCount); - for (auto LookupIt : m_BlobLookup) + for (const auto& LookupIt : m_BlobLookup) { const IoHash& BlobHash = LookupIt.first; const BlobIndex ReadBlobIndex = LookupIt.second; @@ -955,7 +955,7 @@ BuildStore::WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::files std::vector AccessRecords; AccessRecords.reserve(Header.AccessTimeCount); - for (auto It : m_BlobLookup) + for (const auto& It : m_BlobLookup) { const IoHash& Key = It.first; const BlobIndex Index = It.second; @@ -965,7 +965,7 @@ BuildStore::WriteAccessTimes(const RwLock::ExclusiveLockScope&, const std::files } uint64_t RecordsSize = sizeof(AccessTimeRecord) * Header.AccessTimeCount; TempFile.Write(AccessRecords.data(), RecordsSize, Offset); - Offset += sizeof(AccessTimesHeader) * Header.AccessTimeCount; + Offset += sizeof(AccessTimeRecord) * Header.AccessTimeCount; } if (TempFile.MoveTemporaryIntoPlace(AccessTimesPath, Ec); Ec) { diff --git a/src/zenstore/cache/cachedisklayer.cpp b/src/zenstore/cache/cachedisklayer.cpp index b73b3e6fb..d53f9f369 100644 --- a/src/zenstore/cache/cachedisklayer.cpp +++ b/src/zenstore/cache/cachedisklayer.cpp @@ -602,7 +602,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B if (FileSize < sizeof(BucketMetaHeader)) { - ZEN_WARN("Failed to read sidecar file '{}'. Minimum size {} expected, actual size: ", + ZEN_WARN("Failed to read sidecar file '{}'. Minimum size {} expected, actual size: {}", SidecarPath, sizeof(BucketMetaHeader), FileSize); @@ -654,6 +654,7 @@ BucketManifestSerializer::ReadSidecarFile(RwLock::ExclusiveLockScope& B SidecarPath, sizeof(ManifestData), CurrentReadOffset); + break; } CurrentReadOffset += sizeof(ManifestData); @@ -1011,7 +1012,7 @@ ZenCacheDiskLayer::CacheBucket::WriteIndexSnapshotLocked(uint64_t LogPosi { // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in // the end it will be the same result - ZEN_WARN("snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); + ZEN_WARN("snapshot failed to clean log file '{}', reason: '{}'", LogPath, Ec.message()); } m_SlogFile.Open(LogPath, CasLogFile::Mode::kWrite); } @@ -1267,10 +1268,10 @@ ZenCacheDiskLayer::CacheBucket::InitializeIndexFromDisk(RwLock::ExclusiveLockSco { RemoveMemCachedData(IndexLock, Payload); RemoveMetaData(IndexLock, Payload); + Location.Flags |= DiskLocation::kTombStone; + MissingEntries.push_back(DiskIndexEntry{.Key = It.first, .Location = Location}); } } - Location.Flags |= DiskLocation::kTombStone; - MissingEntries.push_back(DiskIndexEntry{.Key = It.first, .Location = Location}); } ZEN_ASSERT(!MissingEntries.empty()); @@ -2812,7 +2813,7 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c m_BucketDir, Ec.message(), RetriesLeft); - Sleep(100 - (3 - RetriesLeft) * 100); // Total 600 ms + Sleep((3 - RetriesLeft) * 100); // Total 600 ms Ec.clear(); DataFile.MoveTemporaryIntoPlace(FsPath, Ec); RetriesLeft--; @@ -2866,11 +2867,12 @@ ZenCacheDiskLayer::CacheBucket::PutStandaloneCacheValue(const IoHash& HashKey, c { EntryIndex = It.value(); ZEN_ASSERT_SLOW(EntryIndex < PayloadIndex(m_AccessTimes.size())); - BucketPayload& Payload = m_Payloads[EntryIndex]; - uint64_t OldSize = Payload.Location.Size(); + BucketPayload& Payload = m_Payloads[EntryIndex]; + uint64_t OldSize = Payload.Location.Size(); + RemoveMemCachedData(IndexLock, Payload); + RemoveMetaData(IndexLock, Payload); Payload = BucketPayload{.Location = Loc}; m_AccessTimes[EntryIndex] = GcClock::TickCount(); - RemoveMemCachedData(IndexLock, Payload); m_StandaloneSize.fetch_sub(OldSize, std::memory_order::relaxed); } if ((Value.RawSize != 0 || Value.RawHash != IoHash::Zero) && Value.RawSize <= std::numeric_limits::max()) @@ -3521,7 +3523,7 @@ ZenCacheDiskLayer::CacheBucket::GetReferences(const LoggerRef& Logger, } else { - ZEN_WARN("Cache record {} payload is malformed. Reason: ", RawHash, ToString(Error)); + ZEN_WARN("Cache record {} payload is malformed. Reason: {}", RawHash, ToString(Error)); } return false; }; @@ -4282,8 +4284,8 @@ ZenCacheDiskLayer::DiscoverBuckets() RwLock SyncLock; WorkerThreadPool& Pool = GetLargeWorkerPool(EWorkloadType::Burst); - std::atomic AbortFlag; - std::atomic PauseFlag; + std::atomic AbortFlag{false}; + std::atomic PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -4454,8 +4456,8 @@ ZenCacheDiskLayer::Flush() } { WorkerThreadPool& Pool = GetMediumWorkerPool(EWorkloadType::Burst); - std::atomic AbortFlag; - std::atomic PauseFlag; + std::atomic AbortFlag{false}; + std::atomic PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -4496,8 +4498,8 @@ ZenCacheDiskLayer::Scrub(ScrubContext& Ctx) RwLock::SharedLockScope _(m_Lock); - std::atomic Abort; - std::atomic Pause; + std::atomic Abort{false}; + std::atomic Pause{false}; ParallelWork Work(Abort, Pause, WorkerThreadPool::EMode::DisableBacklog); try diff --git a/src/zenstore/cache/cachepolicy.cpp b/src/zenstore/cache/cachepolicy.cpp index ce6a14bd9..c1e7dc5b3 100644 --- a/src/zenstore/cache/cachepolicy.cpp +++ b/src/zenstore/cache/cachepolicy.cpp @@ -403,13 +403,13 @@ TEST_CASE("cacherecordpolicy") RecordPolicy.Save(Writer); CbObject Saved = Writer.Save()->AsObject(); CacheRecordPolicy Loaded = CacheRecordPolicy::Load(Saved).Get(); - CHECK(!RecordPolicy.IsUniform()); - CHECK(RecordPolicy.GetRecordPolicy() == UnionPolicy); - CHECK(RecordPolicy.GetBasePolicy() == DefaultPolicy); - CHECK(RecordPolicy.GetValuePolicy(PartialOid) == PartialOverlap); - CHECK(RecordPolicy.GetValuePolicy(NoOverlapOid) == NoOverlap); - CHECK(RecordPolicy.GetValuePolicy(OtherOid) == DefaultValuePolicy); - CHECK(RecordPolicy.GetValuePolicies().size() == 2); + CHECK(!Loaded.IsUniform()); + CHECK(Loaded.GetRecordPolicy() == UnionPolicy); + CHECK(Loaded.GetBasePolicy() == DefaultPolicy); + CHECK(Loaded.GetValuePolicy(PartialOid) == PartialOverlap); + CHECK(Loaded.GetValuePolicy(NoOverlapOid) == NoOverlap); + CHECK(Loaded.GetValuePolicy(OtherOid) == DefaultValuePolicy); + CHECK(Loaded.GetValuePolicies().size() == 2); } } diff --git a/src/zenstore/cache/cacherpc.cpp b/src/zenstore/cache/cacherpc.cpp index e1fd0a3e6..90c5a5e60 100644 --- a/src/zenstore/cache/cacherpc.cpp +++ b/src/zenstore/cache/cacherpc.cpp @@ -866,8 +866,8 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb Request.Complete = false; } } - Request.ElapsedTimeUs += Timer.GetElapsedTimeUs(); } + Request.ElapsedTimeUs += Timer.GetElapsedTimeUs(); }; m_UpstreamCache.GetCacheRecords(*Namespace, UpstreamRequests, std::move(OnCacheRecordGetComplete)); @@ -934,7 +934,7 @@ CacheRpcHandler::HandleRpcGetCacheRecords(const CacheRequestContext& Context, Cb *Namespace, Key.Bucket, Key.Hash, - Request.RecordObject ? ""sv : " (PARTIAL)"sv, + Request.RecordObject ? " (PARTIAL)"sv : ""sv, Request.Source ? Request.Source->Url : "LOCAL"sv, NiceLatencyNs(Request.ElapsedTimeUs * 1000)); m_CacheStats.MissCount++; diff --git a/src/zenstore/cache/structuredcachestore.cpp b/src/zenstore/cache/structuredcachestore.cpp index 18023e2d6..cff0e9a35 100644 --- a/src/zenstore/cache/structuredcachestore.cpp +++ b/src/zenstore/cache/structuredcachestore.cpp @@ -686,8 +686,8 @@ ZenCacheStore::Get(const CacheRequestContext& Context, return false; } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get [{}], bucket '{}', key '{}'", - Context, Namespace, + Context, Bucket, HashKey.ToHexString()); @@ -722,8 +722,8 @@ ZenCacheStore::Get(const CacheRequestContext& Context, } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Get [{}], bucket '{}', key '{}'", - Context, Namespace, + Context, Bucket, HashKey.ToHexString()); @@ -790,8 +790,8 @@ ZenCacheStore::Put(const CacheRequestContext& Context, } ZEN_WARN("request for unknown namespace '{}' in ZenCacheStore::Put [{}] bucket '{}', key '{}'", - Context, Namespace, + Context, Bucket, HashKey.ToHexString()); @@ -816,7 +816,7 @@ ZenCacheStore::DropNamespace(std::string_view InNamespace) { std::function PostDropOp; { - RwLock::SharedLockScope _(m_NamespacesLock); + RwLock::ExclusiveLockScope _(m_NamespacesLock); if (auto It = m_Namespaces.find(std::string(InNamespace)); It != m_Namespaces.end()) { ZenCacheNamespace& Namespace = *It->second; diff --git a/src/zenstore/cas.cpp b/src/zenstore/cas.cpp index f80952322..8855c87d8 100644 --- a/src/zenstore/cas.cpp +++ b/src/zenstore/cas.cpp @@ -153,7 +153,10 @@ CasImpl::Initialize(const CidStoreConfiguration& InConfig) } for (std::future& Result : Work) { - Result.get(); + if (Result.valid()) + { + Result.get(); + } } } } @@ -426,7 +429,7 @@ CasImpl::IterateChunks(std::span DecompressedIds, [&](size_t Index, const IoBuffer& Payload) { IoBuffer Chunk(Payload); Chunk.SetContentType(ZenContentType::kCompressedBinary); - return AsyncCallback(Index, Payload); + return AsyncCallback(Index, Chunk); }, OptionalWorkerPool, LargeSizeLimit == 0 ? m_Config.HugeValueThreshold : Min(LargeSizeLimit, m_Config.HugeValueThreshold))) @@ -439,7 +442,7 @@ CasImpl::IterateChunks(std::span DecompressedIds, [&](size_t Index, const IoBuffer& Payload) { IoBuffer Chunk(Payload); Chunk.SetContentType(ZenContentType::kCompressedBinary); - return AsyncCallback(Index, Payload); + return AsyncCallback(Index, Chunk); }, OptionalWorkerPool, LargeSizeLimit == 0 ? m_Config.TinyValueThreshold : Min(LargeSizeLimit, m_Config.TinyValueThreshold))) @@ -452,7 +455,7 @@ CasImpl::IterateChunks(std::span DecompressedIds, [&](size_t Index, const IoBuffer& Payload) { IoBuffer Chunk(Payload); Chunk.SetContentType(ZenContentType::kCompressedBinary); - return AsyncCallback(Index, Payload); + return AsyncCallback(Index, Chunk); }, OptionalWorkerPool)) { diff --git a/src/zenstore/caslog.cpp b/src/zenstore/caslog.cpp index 492ce9317..44664dac2 100644 --- a/src/zenstore/caslog.cpp +++ b/src/zenstore/caslog.cpp @@ -35,7 +35,7 @@ CasLogFile::~CasLogFile() } bool -CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize) +CasLogFile::IsValid(const std::filesystem::path& FileName, size_t RecordSize) { if (!IsFile(FileName)) { @@ -71,7 +71,7 @@ CasLogFile::IsValid(std::filesystem::path FileName, size_t RecordSize) } void -CasLogFile::Open(std::filesystem::path FileName, size_t RecordSize, Mode Mode) +CasLogFile::Open(const std::filesystem::path& FileName, size_t RecordSize, Mode Mode) { m_RecordSize = RecordSize; @@ -205,7 +205,7 @@ CasLogFile::Replay(std::function&& Handler, uint64_t SkipEntr m_File.Read(ReadBuffer.data(), BytesToRead, LogBaseOffset + ReadOffset); - for (int i = 0; i < int(EntriesToRead); ++i) + for (size_t i = 0; i < EntriesToRead; ++i) { Handler(ReadBuffer.data() + (i * m_RecordSize)); } diff --git a/src/zenstore/cidstore.cpp b/src/zenstore/cidstore.cpp index bedf91287..b20d8f565 100644 --- a/src/zenstore/cidstore.cpp +++ b/src/zenstore/cidstore.cpp @@ -48,13 +48,13 @@ struct CidStore::Impl std::vector AddChunks(std::span ChunkDatas, std::span RawHashes, CidStore::InsertMode Mode) { + ZEN_ASSERT(ChunkDatas.size() == RawHashes.size()); if (ChunkDatas.size() == 1) { std::vector Result(1); Result[0] = AddChunk(ChunkDatas[0], RawHashes[0], Mode); return Result; } - ZEN_ASSERT(ChunkDatas.size() == RawHashes.size()); std::vector Chunks; Chunks.reserve(ChunkDatas.size()); #if ZEN_BUILD_DEBUG @@ -81,6 +81,7 @@ struct CidStore::Impl m_CasStore.InsertChunks(Chunks, RawHashes, static_cast(Mode)); ZEN_ASSERT(CasResults.size() == ChunkDatas.size()); std::vector Result; + Result.reserve(CasResults.size()); for (const CasStore::InsertResult& CasResult : CasResults) { if (CasResult.New) diff --git a/src/zenstore/compactcas.cpp b/src/zenstore/compactcas.cpp index 21411df59..b09892687 100644 --- a/src/zenstore/compactcas.cpp +++ b/src/zenstore/compactcas.cpp @@ -153,7 +153,7 @@ CasContainerStrategy::~CasContainerStrategy() } catch (const std::exception& Ex) { - ZEN_ERROR("~CasContainerStrategy failed with: ", Ex.what()); + ZEN_ERROR("~CasContainerStrategy failed with: {}", Ex.what()); } m_Gc.RemoveGcReferenceStore(*this); m_Gc.RemoveGcStorage(this); @@ -440,9 +440,9 @@ CasContainerStrategy::IterateChunks(std::span ChunkHas return true; } - std::atomic AbortFlag; + std::atomic AbortFlag{false}; { - std::atomic PauseFlag; + std::atomic PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -559,8 +559,8 @@ CasContainerStrategy::ScrubStorage(ScrubContext& Ctx) std::vector ChunkLocations; std::vector ChunkIndexToChunkHash; - std::atomic Abort; - std::atomic Pause; + std::atomic Abort{false}; + std::atomic Pause{false}; ParallelWork Work(Abort, Pause, WorkerThreadPool::EMode::DisableBacklog); try @@ -1007,7 +1007,7 @@ CasContainerStrategy::CompactIndex(RwLock::ExclusiveLockScope&) std::vector Locations; Locations.reserve(EntryCount); LocationMap.reserve(EntryCount); - for (auto It : m_LocationMap) + for (const auto& It : m_LocationMap) { size_t EntryIndex = Locations.size(); Locations.push_back(m_Locations[It.second]); @@ -1106,7 +1106,7 @@ CasContainerStrategy::MakeIndexSnapshot(bool ResetLog) { // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in // the end it will be the same result - ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); + ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, Ec.message()); } m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite); } @@ -1136,7 +1136,7 @@ CasContainerStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint uint64_t Size = ObjectIndexFile.FileSize(); if (Size >= sizeof(CasDiskIndexHeader)) { - uint64_t ExpectedEntryCount = (Size - sizeof(sizeof(CasDiskIndexHeader))) / sizeof(CasDiskIndexEntry); + uint64_t ExpectedEntryCount = (Size - sizeof(CasDiskIndexHeader)) / sizeof(CasDiskIndexEntry); CasDiskIndexHeader Header; ObjectIndexFile.Read(&Header, sizeof(Header), 0); if ((Header.Magic == CasDiskIndexHeader::ExpectedMagic) && (Header.Version == CasDiskIndexHeader::CurrentVersion) && diff --git a/src/zenstore/filecas.cpp b/src/zenstore/filecas.cpp index 295451818..0088afe6e 100644 --- a/src/zenstore/filecas.cpp +++ b/src/zenstore/filecas.cpp @@ -383,7 +383,7 @@ FileCasStrategy::InsertChunk(IoBuffer Chunk, const IoHash& ChunkHash, CasStore:: HRESULT WriteRes = PayloadFile.Write(Cursor, Size); if (FAILED(WriteRes)) { - ThrowSystemException(hRes, fmt::format("failed to write {} bytes to shard file '{}'", ChunkSize, ChunkPath)); + ThrowSystemException(WriteRes, fmt::format("failed to write {} bytes to shard file '{}'", ChunkSize, ChunkPath)); } }; #else @@ -669,8 +669,8 @@ FileCasStrategy::IterateChunks(std::span ChunkHashes, return true; }; - std::atomic AbortFlag; - std::atomic PauseFlag; + std::atomic AbortFlag{false}; + std::atomic PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -823,8 +823,8 @@ FileCasStrategy::ScrubStorage(ScrubContext& Ctx) ZEN_INFO("discovered {} files @ '{}' ({} not in index), scrubbing", m_Index.size(), m_RootDirectory, DiscoveredFilesNotInIndex); - std::atomic Abort; - std::atomic Pause; + std::atomic Abort{false}; + std::atomic Pause{false}; ParallelWork Work(Abort, Pause, WorkerThreadPool::EMode::DisableBacklog); try @@ -1016,7 +1016,7 @@ FileCasStrategy::MakeIndexSnapshot(bool ResetLog) { // This is non-critical, it only means that we will replay the events of the log over the snapshot - inefficent but in // the end it will be the same result - ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, IndexPath, Ec.message()); + ZEN_WARN("Snapshot failed to clean log file '{}', reason: '{}'", LogPath, Ec.message()); } m_CasLog.Open(LogPath, CasLogFile::Mode::kWrite); } @@ -1052,7 +1052,7 @@ FileCasStrategy::ReadIndexFile(const std::filesystem::path& IndexPath, uint32_t& uint64_t Size = ObjectIndexFile.FileSize(); if (Size >= sizeof(FileCasIndexHeader)) { - uint64_t ExpectedEntryCount = (Size - sizeof(sizeof(FileCasIndexHeader))) / sizeof(FileCasIndexEntry); + uint64_t ExpectedEntryCount = (Size - sizeof(FileCasIndexHeader)) / sizeof(FileCasIndexEntry); FileCasIndexHeader Header; ObjectIndexFile.Read(&Header, sizeof(Header), 0); if ((Header.Magic == FileCasIndexHeader::ExpectedMagic) && (Header.Version == FileCasIndexHeader::CurrentVersion) && diff --git a/src/zenstore/filecas.h b/src/zenstore/filecas.h index e93356927..41756b65f 100644 --- a/src/zenstore/filecas.h +++ b/src/zenstore/filecas.h @@ -74,7 +74,7 @@ private: { static const uint32_t kTombStone = 0x0000'0001; - bool IsFlagSet(const uint32_t Flag) const { return (Flags & kTombStone) == Flag; } + bool IsFlagSet(const uint32_t Flag) const { return (Flags & Flag) == Flag; } IoHash Key; uint32_t Flags = 0; diff --git a/src/zenstore/include/zenstore/buildstore/buildstore.h b/src/zenstore/include/zenstore/buildstore/buildstore.h index bfc83ba0d..ea2ef7f89 100644 --- a/src/zenstore/include/zenstore/buildstore/buildstore.h +++ b/src/zenstore/include/zenstore/buildstore/buildstore.h @@ -223,7 +223,7 @@ private: uint64_t m_MetaLogFlushPosition = 0; std::unique_ptr> m_TrackedBlobKeys; - std::atomic m_LastAccessTimeUpdateCount; + std::atomic m_LastAccessTimeUpdateCount{0}; friend class BuildStoreGcReferenceChecker; friend class BuildStoreGcReferencePruner; diff --git a/src/zenstore/include/zenstore/cache/cachedisklayer.h b/src/zenstore/include/zenstore/cache/cachedisklayer.h index 3d684587d..393e289ac 100644 --- a/src/zenstore/include/zenstore/cache/cachedisklayer.h +++ b/src/zenstore/include/zenstore/cache/cachedisklayer.h @@ -153,14 +153,14 @@ public: struct BucketStats { - uint64_t DiskSize; - uint64_t MemorySize; - uint64_t DiskHitCount; - uint64_t DiskMissCount; - uint64_t DiskWriteCount; - uint64_t MemoryHitCount; - uint64_t MemoryMissCount; - uint64_t MemoryWriteCount; + uint64_t DiskSize = 0; + uint64_t MemorySize = 0; + uint64_t DiskHitCount = 0; + uint64_t DiskMissCount = 0; + uint64_t DiskWriteCount = 0; + uint64_t MemoryHitCount = 0; + uint64_t MemoryMissCount = 0; + uint64_t MemoryWriteCount = 0; metrics::RequestStatsSnapshot PutOps; metrics::RequestStatsSnapshot GetOps; }; @@ -174,8 +174,8 @@ public: struct DiskStats { std::vector BucketStats; - uint64_t DiskSize; - uint64_t MemorySize; + uint64_t DiskSize = 0; + uint64_t MemorySize = 0; }; struct PutResult @@ -395,12 +395,12 @@ public: TCasLogFile m_SlogFile; uint64_t m_LogFlushPosition = 0; - std::atomic m_DiskHitCount; - std::atomic m_DiskMissCount; - std::atomic m_DiskWriteCount; - std::atomic m_MemoryHitCount; - std::atomic m_MemoryMissCount; - std::atomic m_MemoryWriteCount; + std::atomic m_DiskHitCount{0}; + std::atomic m_DiskMissCount{0}; + std::atomic m_DiskWriteCount{0}; + std::atomic m_MemoryHitCount{0}; + std::atomic m_MemoryMissCount{0}; + std::atomic m_MemoryWriteCount{0}; metrics::RequestStats m_PutOps; metrics::RequestStats m_GetOps; @@ -540,7 +540,7 @@ private: Configuration m_Configuration; std::atomic_uint64_t m_TotalMemCachedSize{}; std::atomic_bool m_IsMemCacheTrimming = false; - std::atomic m_NextAllowedTrimTick; + std::atomic m_NextAllowedTrimTick{}; mutable RwLock m_Lock; BucketMap_t m_Buckets; std::vector> m_DroppedBuckets; diff --git a/src/zenstore/include/zenstore/cache/cacheshared.h b/src/zenstore/include/zenstore/cache/cacheshared.h index 791720589..8e9cd7fd7 100644 --- a/src/zenstore/include/zenstore/cache/cacheshared.h +++ b/src/zenstore/include/zenstore/cache/cacheshared.h @@ -40,12 +40,12 @@ struct CacheValueDetails { struct ValueDetails { - uint64_t Size; - uint64_t RawSize; + uint64_t Size = 0; + uint64_t RawSize = 0; IoHash RawHash; GcClock::Tick LastAccess{}; std::vector Attachments; - ZenContentType ContentType; + ZenContentType ContentType = ZenContentType::kBinary; }; struct BucketDetails diff --git a/src/zenstore/include/zenstore/cache/structuredcachestore.h b/src/zenstore/include/zenstore/cache/structuredcachestore.h index 5a0a8b069..3722a0d31 100644 --- a/src/zenstore/include/zenstore/cache/structuredcachestore.h +++ b/src/zenstore/include/zenstore/cache/structuredcachestore.h @@ -70,9 +70,9 @@ public: struct NamespaceStats { - uint64_t HitCount; - uint64_t MissCount; - uint64_t WriteCount; + uint64_t HitCount = 0; + uint64_t MissCount = 0; + uint64_t WriteCount = 0; metrics::RequestStatsSnapshot PutOps; metrics::RequestStatsSnapshot GetOps; ZenCacheDiskLayer::DiskStats DiskStats; @@ -342,11 +342,11 @@ private: void LogWorker(); RwLock m_LogQueueLock; std::vector m_LogQueue; - std::atomic_bool m_ExitLogging; + std::atomic_bool m_ExitLogging{false}; Event m_LogEvent; std::thread m_AsyncLoggingThread; - std::atomic_bool m_WriteLogEnabled; - std::atomic_bool m_AccessLogEnabled; + std::atomic_bool m_WriteLogEnabled{false}; + std::atomic_bool m_AccessLogEnabled{false}; friend class CacheStoreReferenceChecker; }; diff --git a/src/zenstore/include/zenstore/caslog.h b/src/zenstore/include/zenstore/caslog.h index f3dd32fb1..7967d9dae 100644 --- a/src/zenstore/include/zenstore/caslog.h +++ b/src/zenstore/include/zenstore/caslog.h @@ -20,8 +20,8 @@ public: kTruncate }; - static bool IsValid(std::filesystem::path FileName, size_t RecordSize); - void Open(std::filesystem::path FileName, size_t RecordSize, Mode Mode); + static bool IsValid(const std::filesystem::path& FileName, size_t RecordSize); + void Open(const std::filesystem::path& FileName, size_t RecordSize, Mode Mode); void Append(const void* DataPointer, uint64_t DataSize); void Replay(std::function&& Handler, uint64_t SkipEntryCount); void Flush(); @@ -48,7 +48,7 @@ private: static_assert(sizeof(FileHeader) == 64); private: - void Open(std::filesystem::path FileName, size_t RecordSize, BasicFile::Mode Mode); + void Open(const std::filesystem::path& FileName, size_t RecordSize, BasicFile::Mode Mode); BasicFile m_File; FileHeader m_Header; @@ -60,8 +60,8 @@ template class TCasLogFile : public CasLogFile { public: - static bool IsValid(std::filesystem::path FileName) { return CasLogFile::IsValid(FileName, sizeof(T)); } - void Open(std::filesystem::path FileName, Mode Mode) { CasLogFile::Open(FileName, sizeof(T), Mode); } + static bool IsValid(const std::filesystem::path& FileName) { return CasLogFile::IsValid(FileName, sizeof(T)); } + void Open(const std::filesystem::path& FileName, Mode Mode) { CasLogFile::Open(FileName, sizeof(T), Mode); } // This should be called before the Replay() is called to do some basic sanity checking bool Initialize() { return true; } diff --git a/src/zenstore/include/zenstore/gc.h b/src/zenstore/include/zenstore/gc.h index 794f50d96..67cf852f9 100644 --- a/src/zenstore/include/zenstore/gc.h +++ b/src/zenstore/include/zenstore/gc.h @@ -443,8 +443,8 @@ struct GcSchedulerState uint64_t DiskFree = 0; GcClock::TimePoint LastFullGcTime{}; GcClock::TimePoint LastLightweightGcTime{}; - std::chrono::seconds RemainingTimeUntilLightweightGc; - std::chrono::seconds RemainingTimeUntilFullGc; + std::chrono::seconds RemainingTimeUntilLightweightGc{}; + std::chrono::seconds RemainingTimeUntilFullGc{}; uint64_t RemainingSpaceUntilFullGC = 0; std::chrono::milliseconds LastFullGcDuration{}; @@ -562,7 +562,7 @@ private: GcClock::TimePoint m_LastGcExpireTime{}; IoHash m_LastFullAttachmentRangeMin = IoHash::Zero; IoHash m_LastFullAttachmentRangeMax = IoHash::Max; - uint8_t m_AttachmentPassIndex; + uint8_t m_AttachmentPassIndex = 0; std::chrono::milliseconds m_LastFullGcDuration{}; GcStorageSize m_LastFullGCDiff; diff --git a/src/zenstore/include/zenstore/projectstore.h b/src/zenstore/include/zenstore/projectstore.h index 33ef996db..6f49cd024 100644 --- a/src/zenstore/include/zenstore/projectstore.h +++ b/src/zenstore/include/zenstore/projectstore.h @@ -67,8 +67,8 @@ public: struct OplogEntryAddress { - uint32_t Offset; // note: Multiple of m_OpsAlign! - uint32_t Size; + uint32_t Offset = 0; // note: Multiple of m_OpsAlign! + uint32_t Size = 0; }; struct OplogEntry @@ -80,11 +80,7 @@ public: uint32_t Reserved; inline bool IsTombstone() const { return OpCoreAddress.Offset == 0 && OpCoreAddress.Size == 0 && OpLsn.Number; } - inline void MakeTombstone() - { - OpLsn = {}; - OpCoreAddress.Offset = OpCoreAddress.Size = OpCoreHash = Reserved = 0; - } + inline void MakeTombstone() { OpCoreAddress.Offset = OpCoreAddress.Size = OpCoreHash = Reserved = 0; } }; static_assert(IsPow2(sizeof(OplogEntry))); diff --git a/src/zenstore/projectstore.cpp b/src/zenstore/projectstore.cpp index 217336eec..3f705d12c 100644 --- a/src/zenstore/projectstore.cpp +++ b/src/zenstore/projectstore.cpp @@ -1488,7 +1488,7 @@ ProjectStore::Oplog::Read() else { std::vector OpLogEntries; - uint64_t InvalidEntries; + uint64_t InvalidEntries = 0; m_Storage->ReadOplogEntriesFromLog(OpLogEntries, InvalidEntries, m_LogFlushPosition); for (const OplogEntry& OpEntry : OpLogEntries) { @@ -1750,8 +1750,8 @@ ProjectStore::Oplog::Validate(const std::filesystem::path& ProjectRootDir, } }; - std::atomic AbortFlag; - std::atomic PauseFlag; + std::atomic AbortFlag{false}; + std::atomic PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -2373,7 +2373,7 @@ ProjectStore::Oplog::IterateChunks(const std::filesystem::path& P else if (auto MetaIt = m_MetaMap.find(ChunkId); MetaIt != m_MetaMap.end()) { CidChunkIndexes.push_back(ChunkIndex); - CidChunkHashes.push_back(ChunkIt->second); + CidChunkHashes.push_back(MetaIt->second); } else if (auto FileIt = m_FileMap.find(ChunkId); FileIt != m_FileMap.end()) { @@ -2384,8 +2384,8 @@ ProjectStore::Oplog::IterateChunks(const std::filesystem::path& P } if (OptionalWorkerPool) { - std::atomic AbortFlag; - std::atomic PauseFlag; + std::atomic AbortFlag{false}; + std::atomic PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -3817,7 +3817,7 @@ ProjectStore::Project::OpenOplog(std::string_view OplogId, bool AllowCompact, bo std::filesystem::path DeletePath; if (!RemoveOplog(OplogId, DeletePath)) { - ZEN_WARN("Failed to clean up deleted oplog {}/{}", Identifier, OplogId, OplogBasePath); + ZEN_WARN("Failed to clean up deleted oplog {}/{} at '{}'", Identifier, OplogId, OplogBasePath); } ReOpen = true; @@ -4053,8 +4053,8 @@ ProjectStore::Project::Scrub(ScrubContext& Ctx) RwLock::SharedLockScope _(m_ProjectLock); - std::atomic Abort; - std::atomic Pause; + std::atomic Abort{false}; + std::atomic Pause{false}; ParallelWork Work(Abort, Pause, WorkerThreadPool::EMode::DisableBacklog); try @@ -4433,8 +4433,8 @@ ProjectStore::Flush() } WorkerThreadPool& WorkerPool = GetSmallWorkerPool(EWorkloadType::Burst); - std::atomic AbortFlag; - std::atomic PauseFlag; + std::atomic AbortFlag{false}; + std::atomic PauseFlag{false}; ParallelWork Work(AbortFlag, PauseFlag, WorkerThreadPool::EMode::DisableBacklog); try { @@ -4974,7 +4974,7 @@ ProjectStore::GetProjectChunkInfos(LoggerRef InLog, Project& Project, Oplog& Opl } if (WantsRawSizeField) { - ZEN_ASSERT_SLOW(Sizes[Index] == (uint64_t)-1); + ZEN_ASSERT_SLOW(RawSizes[Index] == (uint64_t)-1); RawSizes[Index] = Payload.GetSize(); } } @@ -5762,7 +5762,7 @@ public: } } - for (auto ProjectIt : m_ProjectStore.m_Projects) + for (const auto& ProjectIt : m_ProjectStore.m_Projects) { Ref Project = ProjectIt.second; std::vector OplogsToCompact = Project->GetOplogsToCompact(); diff --git a/src/zenstore/workspaces.cpp b/src/zenstore/workspaces.cpp index df3cd31ef..ad21bbc68 100644 --- a/src/zenstore/workspaces.cpp +++ b/src/zenstore/workspaces.cpp @@ -383,7 +383,7 @@ Workspace::GetShares() const { std::vector> Shares; Shares.reserve(m_Shares.size()); - for (auto It : m_Shares) + for (const auto& It : m_Shares) { Shares.push_back(It.second); } @@ -435,7 +435,7 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) Workspace = FindWorkspace(Lock, WorkspaceId); if (Workspace) { - for (auto Share : Workspace->GetShares()) + for (const auto& Share : Workspace->GetShares()) { DeletedShares.insert(Share->GetConfig().Id); } @@ -482,6 +482,12 @@ Workspaces::RefreshWorkspaceShares(const Oid& WorkspaceId) m_ShareAliases.erase(Share->GetConfig().Alias); } Workspace->SetShare(Configuration.Id, std::move(NewShare)); + if (!Configuration.Alias.empty()) + { + m_ShareAliases.insert_or_assign( + Configuration.Alias, + ShareAlias{.WorkspaceId = WorkspaceId, .ShareId = Configuration.Id}); + } } } else @@ -602,7 +608,7 @@ Workspaces::GetWorkspaceShareChunks(const Oid& WorkspaceId, { RequestedOffset = Size; } - if ((RequestedOffset + RequestedSize) > Size) + if (RequestedSize > Size - RequestedOffset) { RequestedSize = Size - RequestedOffset; } @@ -649,7 +655,7 @@ Workspaces::GetWorkspaces() const { std::vector Workspaces; RwLock::SharedLockScope Lock(m_Lock); - for (auto It : m_Workspaces) + for (const auto& It : m_Workspaces) { Workspaces.push_back(It.first); } @@ -679,7 +685,7 @@ Workspaces::GetWorkspaceShares(const Oid& WorkspaceId) const if (Workspace) { std::vector Shares; - for (auto Share : Workspace->GetShares()) + for (const auto& Share : Workspace->GetShares()) { Shares.push_back(Share->GetConfig().Id); } -- cgit v1.2.3 From 19a117889c2db6b817af9458c04c04f324162e75 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 9 Mar 2026 10:50:47 +0100 Subject: Eliminate spdlog dependency (#773) Removes the vendored spdlog library (~12,000 lines) and replaces it with a purpose-built logging system in zencore (~1,800 lines). The new implementation provides the same functionality with fewer abstractions, no shared_ptr overhead, and full control over the logging pipeline. ### What changed **New logging core in zencore/logging/:** - LogMessage, Formatter, Sink, Logger, Registry - core abstractions matching spdlog's model but simplified - AnsiColorStdoutSink - ANSI color console output (replaces spdlog stdout_color_sink) - MsvcSink - OutputDebugString on Windows (replaces spdlog msvc_sink) - AsyncSink - async logging via BlockingQueue worker thread (replaces spdlog async_logger) - NullSink, MessageOnlyFormatter - utility types - Thread-safe timestamp caching in formatters using RwLock **Moved to zenutil/logging/:** - FullFormatter - full log formatting with timestamp, logger name, level, source location, multiline alignment - JsonFormatter - structured JSON log output - RotatingFileSink - rotating file sink with atomic size tracking **API changes:** - Log levels are now an enum (LogLevel) instead of int, eliminating the zen::logging::level namespace - LoggerRef no longer wraps shared_ptr - it holds a raw pointer with the registry owning lifetime - Logger error handler is wired through Registry and propagated to all loggers on registration - Logger::Log() now populates ThreadId on every message **Cleanup:** - Deleted thirdparty/spdlog/ entirely (110+ files) - Deleted full_test_formatter (was ~80% duplicate of FullFormatter) - Renamed snake_case classes to PascalCase (full_formatter -> FullFormatter, json_formatter -> JsonFormatter, sentry_sink -> SentrySink) - Removed spdlog from xmake dependency graph ### Build / test impact - zencore no longer depends on spdlog - zenutil and zenvfs xmake.lua updated to drop spdlog dep - zentelemetry xmake.lua updated to drop spdlog dep - All existing tests pass, no test changes required beyond formatter class renames --- src/zen/cmds/builds_cmd.cpp | 8 +- src/zen/cmds/wipe_cmd.cpp | 8 +- src/zen/progressbar.cpp | 12 +- src/zen/zen.cpp | 1 - src/zencore/include/zencore/blockingqueue.h | 2 + src/zencore/include/zencore/logbase.h | 113 +- src/zencore/include/zencore/logging.h | 214 ++-- .../include/zencore/logging/ansicolorsink.h | 26 + src/zencore/include/zencore/logging/asyncsink.h | 30 + src/zencore/include/zencore/logging/formatter.h | 20 + src/zencore/include/zencore/logging/helpers.h | 122 ++ src/zencore/include/zencore/logging/logger.h | 63 + src/zencore/include/zencore/logging/logmsg.h | 66 + src/zencore/include/zencore/logging/memorybuffer.h | 11 + .../include/zencore/logging/messageonlyformatter.h | 22 + src/zencore/include/zencore/logging/msvcsink.h | 30 + src/zencore/include/zencore/logging/nullsink.h | 17 + src/zencore/include/zencore/logging/registry.h | 70 + src/zencore/include/zencore/logging/sink.h | 34 + src/zencore/include/zencore/logging/tracesink.h | 23 + src/zencore/include/zencore/sentryintegration.h | 8 +- src/zencore/logging.cpp | 328 +++-- src/zencore/logging/ansicolorsink.cpp | 178 +++ src/zencore/logging/asyncsink.cpp | 212 ++++ src/zencore/logging/logger.cpp | 142 +++ src/zencore/logging/msvcsink.cpp | 80 ++ src/zencore/logging/registry.cpp | 330 +++++ src/zencore/logging/tracesink.cpp | 88 ++ src/zencore/sentryintegration.cpp | 128 +- src/zencore/testing.cpp | 4 +- src/zencore/xmake.lua | 1 - src/zencore/zencore.cpp | 2 +- src/zenhttp/servers/httpasio.cpp | 4 +- src/zenhttp/servers/httpplugin.cpp | 4 +- src/zenhttp/transports/dlltransport.cpp | 38 +- .../include/zenremotestore/operationlogoutput.h | 24 +- src/zenremotestore/operationlogoutput.cpp | 14 +- .../projectstore/remoteprojectstore.cpp | 5 +- src/zenserver-test/logging-tests.cpp | 2 +- src/zenserver-test/zenserver-test.cpp | 8 +- src/zenserver/diag/diagsvcs.cpp | 6 +- src/zenserver/diag/logging.cpp | 51 +- src/zenserver/diag/otlphttp.cpp | 4 +- src/zenserver/diag/otlphttp.h | 15 +- src/zenserver/main.cpp | 2 +- src/zenserver/storage/admin/admin.cpp | 6 +- src/zenstore/projectstore.cpp | 2 +- .../include/zentelemetry/otlpencoder.h | 8 +- src/zentelemetry/otlpencoder.cpp | 44 +- src/zentelemetry/xmake.lua | 2 +- src/zenutil/config/commandlineoptions.cpp | 1 + src/zenutil/config/loggingconfig.cpp | 22 +- src/zenutil/include/zenutil/config/loggingconfig.h | 2 +- src/zenutil/include/zenutil/logging.h | 11 +- .../include/zenutil/logging/fullformatter.h | 89 +- .../include/zenutil/logging/jsonformatter.h | 168 ++- .../include/zenutil/logging/rotatingfilesink.h | 89 +- .../include/zenutil/logging/testformatter.h | 160 --- src/zenutil/logging.cpp | 144 +-- src/zenutil/xmake.lua | 2 +- src/zenvfs/xmake.lua | 2 +- thirdparty/spdlog/.clang-format | 19 - thirdparty/spdlog/.clang-tidy | 53 - thirdparty/spdlog/.gitattributes | 1 - thirdparty/spdlog/.gitignore | 98 -- thirdparty/spdlog/LICENSE | 26 - thirdparty/spdlog/README.md | 553 -------- thirdparty/spdlog/include/spdlog/async.h | 99 -- .../spdlog/include/spdlog/async_logger-inl.h | 84 -- thirdparty/spdlog/include/spdlog/async_logger.h | 74 -- thirdparty/spdlog/include/spdlog/cfg/argv.h | 40 - thirdparty/spdlog/include/spdlog/cfg/env.h | 36 - thirdparty/spdlog/include/spdlog/cfg/helpers-inl.h | 106 -- thirdparty/spdlog/include/spdlog/cfg/helpers.h | 29 - thirdparty/spdlog/include/spdlog/common-inl.h | 68 - thirdparty/spdlog/include/spdlog/common.h | 406 ------ .../spdlog/include/spdlog/details/backtracer-inl.h | 63 - .../spdlog/include/spdlog/details/backtracer.h | 45 - .../spdlog/include/spdlog/details/circular_q.h | 115 -- .../include/spdlog/details/console_globals.h | 28 - .../include/spdlog/details/file_helper-inl.h | 151 --- .../spdlog/include/spdlog/details/file_helper.h | 61 - .../spdlog/include/spdlog/details/fmt_helper.h | 141 -- .../spdlog/include/spdlog/details/log_msg-inl.h | 44 - thirdparty/spdlog/include/spdlog/details/log_msg.h | 40 - .../include/spdlog/details/log_msg_buffer-inl.h | 54 - .../spdlog/include/spdlog/details/log_msg_buffer.h | 32 - .../include/spdlog/details/mpmc_blocking_q.h | 177 --- .../spdlog/include/spdlog/details/null_mutex.h | 35 - thirdparty/spdlog/include/spdlog/details/os-inl.h | 605 --------- thirdparty/spdlog/include/spdlog/details/os.h | 127 -- .../include/spdlog/details/periodic_worker-inl.h | 26 - .../include/spdlog/details/periodic_worker.h | 58 - .../spdlog/include/spdlog/details/registry-inl.h | 270 ---- .../spdlog/include/spdlog/details/registry.h | 131 -- .../include/spdlog/details/synchronous_factory.h | 22 - .../include/spdlog/details/thread_pool-inl.h | 125 -- .../spdlog/include/spdlog/details/thread_pool.h | 117 -- .../include/spdlog/details/windows_include.h | 11 - thirdparty/spdlog/include/spdlog/fmt/bin_to_hex.h | 224 ---- thirdparty/spdlog/include/spdlog/fmt/chrono.h | 23 - thirdparty/spdlog/include/spdlog/fmt/compile.h | 23 - thirdparty/spdlog/include/spdlog/fmt/fmt.h | 26 - thirdparty/spdlog/include/spdlog/fmt/ostr.h | 23 - thirdparty/spdlog/include/spdlog/fmt/ranges.h | 23 - thirdparty/spdlog/include/spdlog/fmt/std.h | 24 - thirdparty/spdlog/include/spdlog/fmt/xchar.h | 23 - thirdparty/spdlog/include/spdlog/formatter.h | 17 - thirdparty/spdlog/include/spdlog/fwd.h | 18 - thirdparty/spdlog/include/spdlog/logger-inl.h | 198 --- thirdparty/spdlog/include/spdlog/logger.h | 379 ------ thirdparty/spdlog/include/spdlog/mdc.h | 52 - .../spdlog/include/spdlog/pattern_formatter-inl.h | 1340 -------------------- .../spdlog/include/spdlog/pattern_formatter.h | 118 -- .../include/spdlog/sinks/ansicolor_sink-inl.h | 142 --- .../spdlog/include/spdlog/sinks/ansicolor_sink.h | 116 -- .../spdlog/include/spdlog/sinks/base_sink-inl.h | 59 - thirdparty/spdlog/include/spdlog/sinks/base_sink.h | 51 - .../include/spdlog/sinks/basic_file_sink-inl.h | 48 - .../spdlog/include/spdlog/sinks/basic_file_sink.h | 66 - .../spdlog/include/spdlog/sinks/callback_sink.h | 56 - .../spdlog/include/spdlog/sinks/daily_file_sink.h | 254 ---- thirdparty/spdlog/include/spdlog/sinks/dist_sink.h | 81 -- .../spdlog/include/spdlog/sinks/hourly_file_sink.h | 193 --- thirdparty/spdlog/include/spdlog/sinks/msvc_sink.h | 68 - thirdparty/spdlog/include/spdlog/sinks/null_sink.h | 41 - thirdparty/spdlog/include/spdlog/sinks/sink-inl.h | 22 - thirdparty/spdlog/include/spdlog/sinks/sink.h | 34 - .../include/spdlog/sinks/stdout_color_sinks-inl.h | 38 - .../include/spdlog/sinks/stdout_color_sinks.h | 49 - .../spdlog/include/spdlog/sinks/stdout_sinks-inl.h | 127 -- .../spdlog/include/spdlog/sinks/stdout_sinks.h | 84 -- .../spdlog/include/spdlog/sinks/syslog_sink.h | 104 -- .../spdlog/include/spdlog/sinks/systemd_sink.h | 121 -- .../include/spdlog/sinks/win_eventlog_sink.h | 260 ---- .../include/spdlog/sinks/wincolor_sink-inl.h | 172 --- .../spdlog/include/spdlog/sinks/wincolor_sink.h | 82 -- thirdparty/spdlog/include/spdlog/spdlog-inl.h | 96 -- thirdparty/spdlog/include/spdlog/spdlog.h | 357 ------ thirdparty/spdlog/include/spdlog/stopwatch.h | 66 - thirdparty/spdlog/include/spdlog/tweakme.h | 148 --- thirdparty/spdlog/include/spdlog/version.h | 11 - thirdparty/spdlog/tests/includes.h | 42 - thirdparty/spdlog/tests/main.cpp | 10 - thirdparty/spdlog/tests/test_async.cpp | 200 --- thirdparty/spdlog/tests/test_backtrace.cpp | 73 -- thirdparty/spdlog/tests/test_bin_to_hex.cpp | 97 -- thirdparty/spdlog/tests/test_cfg.cpp | 178 --- thirdparty/spdlog/tests/test_circular_q.cpp | 50 - thirdparty/spdlog/tests/test_create_dir.cpp | 144 --- thirdparty/spdlog/tests/test_custom_callbacks.cpp | 37 - thirdparty/spdlog/tests/test_daily_logger.cpp | 169 --- thirdparty/spdlog/tests/test_dup_filter.cpp | 83 -- thirdparty/spdlog/tests/test_errors.cpp | 112 -- thirdparty/spdlog/tests/test_eventlog.cpp | 75 -- thirdparty/spdlog/tests/test_file_helper.cpp | 169 --- thirdparty/spdlog/tests/test_file_logging.cpp | 187 --- thirdparty/spdlog/tests/test_fmt_helper.cpp | 82 -- thirdparty/spdlog/tests/test_macros.cpp | 53 - thirdparty/spdlog/tests/test_misc.cpp | 224 ---- thirdparty/spdlog/tests/test_mpmc_q.cpp | 114 -- thirdparty/spdlog/tests/test_pattern_formatter.cpp | 660 ---------- thirdparty/spdlog/tests/test_registry.cpp | 125 -- thirdparty/spdlog/tests/test_ringbuffer.cpp | 52 - thirdparty/spdlog/tests/test_sink.h | 70 - thirdparty/spdlog/tests/test_stdout_api.cpp | 90 -- thirdparty/spdlog/tests/test_stopwatch.cpp | 42 - thirdparty/spdlog/tests/test_systemd.cpp | 14 - thirdparty/spdlog/tests/test_time_point.cpp | 35 - thirdparty/spdlog/tests/utils.cpp | 102 -- thirdparty/spdlog/tests/utils.h | 18 - thirdparty/xmake.lua | 6 - 172 files changed, 2297 insertions(+), 13965 deletions(-) create mode 100644 src/zencore/include/zencore/logging/ansicolorsink.h create mode 100644 src/zencore/include/zencore/logging/asyncsink.h create mode 100644 src/zencore/include/zencore/logging/formatter.h create mode 100644 src/zencore/include/zencore/logging/helpers.h create mode 100644 src/zencore/include/zencore/logging/logger.h create mode 100644 src/zencore/include/zencore/logging/logmsg.h create mode 100644 src/zencore/include/zencore/logging/memorybuffer.h create mode 100644 src/zencore/include/zencore/logging/messageonlyformatter.h create mode 100644 src/zencore/include/zencore/logging/msvcsink.h create mode 100644 src/zencore/include/zencore/logging/nullsink.h create mode 100644 src/zencore/include/zencore/logging/registry.h create mode 100644 src/zencore/include/zencore/logging/sink.h create mode 100644 src/zencore/include/zencore/logging/tracesink.h create mode 100644 src/zencore/logging/ansicolorsink.cpp create mode 100644 src/zencore/logging/asyncsink.cpp create mode 100644 src/zencore/logging/logger.cpp create mode 100644 src/zencore/logging/msvcsink.cpp create mode 100644 src/zencore/logging/registry.cpp create mode 100644 src/zencore/logging/tracesink.cpp delete mode 100644 src/zenutil/include/zenutil/logging/testformatter.h delete mode 100644 thirdparty/spdlog/.clang-format delete mode 100644 thirdparty/spdlog/.clang-tidy delete mode 100644 thirdparty/spdlog/.gitattributes delete mode 100644 thirdparty/spdlog/.gitignore delete mode 100644 thirdparty/spdlog/LICENSE delete mode 100644 thirdparty/spdlog/README.md delete mode 100644 thirdparty/spdlog/include/spdlog/async.h delete mode 100644 thirdparty/spdlog/include/spdlog/async_logger-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/async_logger.h delete mode 100644 thirdparty/spdlog/include/spdlog/cfg/argv.h delete mode 100644 thirdparty/spdlog/include/spdlog/cfg/env.h delete mode 100644 thirdparty/spdlog/include/spdlog/cfg/helpers-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/cfg/helpers.h delete mode 100644 thirdparty/spdlog/include/spdlog/common-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/common.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/backtracer-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/backtracer.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/circular_q.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/console_globals.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/file_helper-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/file_helper.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/fmt_helper.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/log_msg-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/log_msg.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/log_msg_buffer-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/log_msg_buffer.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/mpmc_blocking_q.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/null_mutex.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/os-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/os.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/periodic_worker-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/periodic_worker.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/registry-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/registry.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/synchronous_factory.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/thread_pool-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/thread_pool.h delete mode 100644 thirdparty/spdlog/include/spdlog/details/windows_include.h delete mode 100644 thirdparty/spdlog/include/spdlog/fmt/bin_to_hex.h delete mode 100644 thirdparty/spdlog/include/spdlog/fmt/chrono.h delete mode 100644 thirdparty/spdlog/include/spdlog/fmt/compile.h delete mode 100644 thirdparty/spdlog/include/spdlog/fmt/fmt.h delete mode 100644 thirdparty/spdlog/include/spdlog/fmt/ostr.h delete mode 100644 thirdparty/spdlog/include/spdlog/fmt/ranges.h delete mode 100644 thirdparty/spdlog/include/spdlog/fmt/std.h delete mode 100644 thirdparty/spdlog/include/spdlog/fmt/xchar.h delete mode 100644 thirdparty/spdlog/include/spdlog/formatter.h delete mode 100644 thirdparty/spdlog/include/spdlog/fwd.h delete mode 100644 thirdparty/spdlog/include/spdlog/logger-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/logger.h delete mode 100644 thirdparty/spdlog/include/spdlog/mdc.h delete mode 100644 thirdparty/spdlog/include/spdlog/pattern_formatter-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/pattern_formatter.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/ansicolor_sink-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/ansicolor_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/base_sink-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/base_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/basic_file_sink-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/basic_file_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/callback_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/daily_file_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/dist_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/hourly_file_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/msvc_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/null_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/sink-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/stdout_color_sinks-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/stdout_color_sinks.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/stdout_sinks-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/stdout_sinks.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/syslog_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/systemd_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/win_eventlog_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/wincolor_sink-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/sinks/wincolor_sink.h delete mode 100644 thirdparty/spdlog/include/spdlog/spdlog-inl.h delete mode 100644 thirdparty/spdlog/include/spdlog/spdlog.h delete mode 100644 thirdparty/spdlog/include/spdlog/stopwatch.h delete mode 100644 thirdparty/spdlog/include/spdlog/tweakme.h delete mode 100644 thirdparty/spdlog/include/spdlog/version.h delete mode 100644 thirdparty/spdlog/tests/includes.h delete mode 100644 thirdparty/spdlog/tests/main.cpp delete mode 100644 thirdparty/spdlog/tests/test_async.cpp delete mode 100644 thirdparty/spdlog/tests/test_backtrace.cpp delete mode 100644 thirdparty/spdlog/tests/test_bin_to_hex.cpp delete mode 100644 thirdparty/spdlog/tests/test_cfg.cpp delete mode 100644 thirdparty/spdlog/tests/test_circular_q.cpp delete mode 100644 thirdparty/spdlog/tests/test_create_dir.cpp delete mode 100644 thirdparty/spdlog/tests/test_custom_callbacks.cpp delete mode 100644 thirdparty/spdlog/tests/test_daily_logger.cpp delete mode 100644 thirdparty/spdlog/tests/test_dup_filter.cpp delete mode 100644 thirdparty/spdlog/tests/test_errors.cpp delete mode 100644 thirdparty/spdlog/tests/test_eventlog.cpp delete mode 100644 thirdparty/spdlog/tests/test_file_helper.cpp delete mode 100644 thirdparty/spdlog/tests/test_file_logging.cpp delete mode 100644 thirdparty/spdlog/tests/test_fmt_helper.cpp delete mode 100644 thirdparty/spdlog/tests/test_macros.cpp delete mode 100644 thirdparty/spdlog/tests/test_misc.cpp delete mode 100644 thirdparty/spdlog/tests/test_mpmc_q.cpp delete mode 100644 thirdparty/spdlog/tests/test_pattern_formatter.cpp delete mode 100644 thirdparty/spdlog/tests/test_registry.cpp delete mode 100644 thirdparty/spdlog/tests/test_ringbuffer.cpp delete mode 100644 thirdparty/spdlog/tests/test_sink.h delete mode 100644 thirdparty/spdlog/tests/test_stdout_api.cpp delete mode 100644 thirdparty/spdlog/tests/test_stopwatch.cpp delete mode 100644 thirdparty/spdlog/tests/test_systemd.cpp delete mode 100644 thirdparty/spdlog/tests/test_time_point.cpp delete mode 100644 thirdparty/spdlog/tests/utils.cpp delete mode 100644 thirdparty/spdlog/tests/utils.h diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index 0722e9714..e5cbafbea 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -269,10 +269,10 @@ namespace builds_impl { static ProgressBar::Mode ProgressMode = ProgressBar::Mode::Pretty; #undef ZEN_CONSOLE_VERBOSE -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \ +#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ + if (IsVerbose) \ + { \ + ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ } const std::string DefaultAccessTokenEnvVariableName( diff --git a/src/zen/cmds/wipe_cmd.cpp b/src/zen/cmds/wipe_cmd.cpp index fd9e28a80..10f5ad8e1 100644 --- a/src/zen/cmds/wipe_cmd.cpp +++ b/src/zen/cmds/wipe_cmd.cpp @@ -50,10 +50,10 @@ namespace wipe_impl { } #undef ZEN_CONSOLE_VERBOSE -#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ - if (IsVerbose) \ - { \ - ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__); \ +#define ZEN_CONSOLE_VERBOSE(fmtstr, ...) \ + if (IsVerbose) \ + { \ + ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__); \ } static void SignalCallbackHandler(int SigNum) diff --git a/src/zen/progressbar.cpp b/src/zen/progressbar.cpp index 9467ed60d..b758c061b 100644 --- a/src/zen/progressbar.cpp +++ b/src/zen/progressbar.cpp @@ -390,19 +390,19 @@ class ConsoleOpLogOutput : public OperationLogOutput { public: ConsoleOpLogOutput(zen::ProgressBar::Mode InMode) : m_Mode(InMode) {} - virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) + virtual void EmitLogMessage(const logging::LogPoint& Point, fmt::format_args Args) override { - logging::EmitConsoleLogMessage(LogLevel, Format, Args); + logging::EmitConsoleLogMessage(Point, Args); } - virtual void SetLogOperationName(std::string_view Name) { zen::ProgressBar::SetLogOperationName(m_Mode, Name); } - virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) + virtual void SetLogOperationName(std::string_view Name) override { zen::ProgressBar::SetLogOperationName(m_Mode, Name); } + virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override { zen::ProgressBar::SetLogOperationProgress(m_Mode, StepIndex, StepCount); } - virtual uint32_t GetProgressUpdateDelayMS() { return GetUpdateDelayMS(m_Mode); } + virtual uint32_t GetProgressUpdateDelayMS() override { return GetUpdateDelayMS(m_Mode); } - virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) { return new ConsoleOpLogProgressBar(m_Mode, InSubTask); } + virtual ProgressBar* CreateProgressBar(std::string_view InSubTask) override { return new ConsoleOpLogProgressBar(m_Mode, InSubTask); } private: zen::ProgressBar::Mode m_Mode; diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index ba8a76bc3..7f7afa322 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -689,7 +689,6 @@ main(int argc, char** argv) const LoggingOptions LogOptions = {.IsDebug = GlobalOptions.IsDebug, .IsVerbose = GlobalOptions.IsVerbose, .IsTest = false, - .AllowAsync = false, .NoConsoleOutput = GlobalOptions.LoggingConfig.NoConsoleOutput, .QuietConsole = GlobalOptions.LoggingConfig.QuietConsole, .AbsLogFile = GlobalOptions.LoggingConfig.AbsLogFile, diff --git a/src/zencore/include/zencore/blockingqueue.h b/src/zencore/include/zencore/blockingqueue.h index e91fdc659..b6c93e937 100644 --- a/src/zencore/include/zencore/blockingqueue.h +++ b/src/zencore/include/zencore/blockingqueue.h @@ -2,6 +2,8 @@ #pragma once +#include // For ZEN_ASSERT + #include #include #include diff --git a/src/zencore/include/zencore/logbase.h b/src/zencore/include/zencore/logbase.h index 00af68b0a..ece17a85e 100644 --- a/src/zencore/include/zencore/logbase.h +++ b/src/zencore/include/zencore/logbase.h @@ -4,96 +4,85 @@ #include -#define ZEN_LOG_LEVEL_TRACE 0 -#define ZEN_LOG_LEVEL_DEBUG 1 -#define ZEN_LOG_LEVEL_INFO 2 -#define ZEN_LOG_LEVEL_WARN 3 -#define ZEN_LOG_LEVEL_ERROR 4 -#define ZEN_LOG_LEVEL_CRITICAL 5 -#define ZEN_LOG_LEVEL_OFF 6 - -#define ZEN_LEVEL_NAME_TRACE std::string_view("trace", 5) -#define ZEN_LEVEL_NAME_DEBUG std::string_view("debug", 5) -#define ZEN_LEVEL_NAME_INFO std::string_view("info", 4) -#define ZEN_LEVEL_NAME_WARNING std::string_view("warning", 7) -#define ZEN_LEVEL_NAME_ERROR std::string_view("error", 5) -#define ZEN_LEVEL_NAME_CRITICAL std::string_view("critical", 8) -#define ZEN_LEVEL_NAME_OFF std::string_view("off", 3) - -namespace zen::logging::level { +namespace zen::logging { enum LogLevel : int { - Trace = ZEN_LOG_LEVEL_TRACE, - Debug = ZEN_LOG_LEVEL_DEBUG, - Info = ZEN_LOG_LEVEL_INFO, - Warn = ZEN_LOG_LEVEL_WARN, - Err = ZEN_LOG_LEVEL_ERROR, - Critical = ZEN_LOG_LEVEL_CRITICAL, - Off = ZEN_LOG_LEVEL_OFF, + Trace, + Debug, + Info, + Warn, + Err, + Critical, + Off, LogLevelCount }; LogLevel ParseLogLevelString(std::string_view String); std::string_view ToStringView(LogLevel Level); -} // namespace zen::logging::level - -namespace zen::logging { - -void SetLogLevel(level::LogLevel NewLogLevel); -level::LogLevel GetLogLevel(); +void SetLogLevel(LogLevel NewLogLevel); +LogLevel GetLogLevel(); -} // namespace zen::logging +struct SourceLocation +{ + constexpr SourceLocation() = default; + constexpr SourceLocation(const char* InFilename, int InLine) : Filename(InFilename), Line(InLine) {} -namespace spdlog { -class logger; -} + constexpr operator bool() const noexcept { return Line != 0; } -namespace zen::logging { + const char* Filename{nullptr}; + int Line{0}; +}; -struct SourceLocation +/** This encodes the constant parts of a log message which can be emitted once + * and then referred to by log events. + * + * It's *critical* that instances of this struct are permanent and never + * destroyed, as log messages will refer to them by pointer. The easiest way + * to ensure this is to create them as function-local statics. + * + * The logging macros already do this for you so this should not be something + * you normally would need to worry about. + */ +struct LogPoint { - constexpr SourceLocation() = default; - constexpr SourceLocation(const char* filename_in, int line_in, const char* funcname_in) - : filename(filename_in) - , line(line_in) - , funcname(funcname_in) - { - } - - constexpr bool empty() const noexcept { return line == 0; } - - // IMPORTANT NOTE: the layout of this class must match the spdlog::source_loc class - // since we currently pass a pointer to it into spdlog after casting it to - // spdlog::source_loc* - // - // This is intended to be an intermediate state, before we (probably) transition off - // spdlog entirely - - const char* filename{nullptr}; - int line{0}; - const char* funcname{nullptr}; + SourceLocation Location; + LogLevel Level; + std::string_view FormatString; }; +class Logger; + } // namespace zen::logging namespace zen { +// Lightweight non-owning handle to a Logger. Loggers are owned by the Registry +// via Ref; LoggerRef exists as a cheap (raw pointer) handle that can be +// stored in members and passed through logging macros without requiring the +// complete Logger type or incurring refcount overhead on every log call. struct LoggerRef { LoggerRef() = default; - LoggerRef(spdlog::logger& InLogger) : SpdLogger(&InLogger) {} + LoggerRef(logging::Logger& InLogger) : m_Logger(&InLogger) {} + // This exists so that logging macros can pass LoggerRef or LogCategory + // to ZEN_LOG without needing to know which one it is LoggerRef Logger() { return *this; } - bool ShouldLog(int Level) const; - inline operator bool() const { return SpdLogger != nullptr; } + bool ShouldLog(logging::LogLevel Level) const; + inline operator bool() const { return m_Logger != nullptr; } + + inline logging::Logger* operator->() const { return m_Logger; } + inline logging::Logger& operator*() const { return *m_Logger; } - void SetLogLevel(logging::level::LogLevel NewLogLevel); - logging::level::LogLevel GetLogLevel(); + void SetLogLevel(logging::LogLevel NewLogLevel); + logging::LogLevel GetLogLevel(); + void Flush(); - spdlog::logger* SpdLogger = nullptr; +private: + logging::Logger* m_Logger = nullptr; }; } // namespace zen diff --git a/src/zencore/include/zencore/logging.h b/src/zencore/include/zencore/logging.h index 74a44d028..4b593c19e 100644 --- a/src/zencore/include/zencore/logging.h +++ b/src/zencore/include/zencore/logging.h @@ -9,16 +9,9 @@ #if ZEN_PLATFORM_WINDOWS # define ZEN_LOG_SECTION(Id) ZEN_DATA_SECTION(Id) -# pragma section(".zlog$f", read) # pragma section(".zlog$l", read) -# pragma section(".zlog$m", read) -# pragma section(".zlog$s", read) -# define ZEN_DECLARE_FUNCTION static constinit ZEN_LOG_SECTION(".zlog$f") char FuncName[] = __FUNCTION__; -# define ZEN_LOG_FUNCNAME FuncName #else # define ZEN_LOG_SECTION(Id) -# define ZEN_DECLARE_FUNCTION -# define ZEN_LOG_FUNCNAME static_cast(__func__) #endif namespace zen::logging { @@ -37,34 +30,29 @@ LoggerRef ErrorLog(); void SetErrorLog(std::string_view LoggerId); LoggerRef Get(std::string_view Name); -void ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers); +void ConfigureLogLevels(LogLevel Level, std::string_view Loggers); void RefreshLogLevels(); -void RefreshLogLevels(level::LogLevel DefaultLevel); - +void RefreshLogLevels(LogLevel DefaultLevel); + +/** LogCategory allows for the creation of log categories that can be used with + * the logging macros just like a logger reference. The main purpose of this is + * to allow for static log categories in global scope where we can't actually + * go ahead and instantiate a logger immediately because the logging system may + * not be initialized yet. + */ struct LogCategory { - inline LogCategory(std::string_view InCategory) : CategoryName(InCategory) {} - - inline zen::LoggerRef Logger() - { - if (LoggerRef) - { - return LoggerRef; - } + inline LogCategory(std::string_view InCategory) : m_CategoryName(InCategory) {} - LoggerRef = zen::logging::Get(CategoryName); - return LoggerRef; - } + LoggerRef Logger(); - std::string CategoryName; - zen::LoggerRef LoggerRef; +private: + std::string m_CategoryName; + LoggerRef m_LoggerRef; }; -void EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args); -void EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Message); -void EmitLogMessage(LoggerRef& Logger, const SourceLocation& Location, int LogLevel, std::string_view Message); -void EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Format, fmt::format_args Args); -void EmitLogMessage(LoggerRef& Logger, const SourceLocation& Location, int LogLevel, std::string_view Format, fmt::format_args Args); +void EmitConsoleLogMessage(const LogPoint& Lp, fmt::format_args Args); +void EmitLogMessage(LoggerRef& Logger, const LogPoint& Lp, fmt::format_args Args); template auto @@ -79,15 +67,14 @@ namespace zen { extern LoggerRef TheDefaultLogger; -inline LoggerRef -Log() -{ - if (TheDefaultLogger) - { - return TheDefaultLogger; - } - return zen::logging::ConsoleLog(); -} +/** + * This is the default logger, which any ZEN_INFO et al will get if there's + * no Log() function declared in the current scope. + * + * Typically, classes which want to log to its own channel will declare a Log() + * member function which returns a LoggerRef created at construction time. + */ +LoggerRef Log(); using logging::ConsoleLog; using logging::ErrorLog; @@ -98,12 +85,6 @@ using zen::ConsoleLog; using zen::ErrorLog; using zen::Log; -inline consteval bool -LogIsErrorLevel(int LogLevel) -{ - return (LogLevel == zen::logging::level::Err || LogLevel == zen::logging::level::Critical); -}; - #if ZEN_BUILD_DEBUG # define ZEN_CHECK_FORMAT_STRING(fmtstr, ...) \ while (false) \ @@ -117,75 +98,66 @@ LogIsErrorLevel(int LogLevel) } #endif -#define ZEN_LOG_WITH_LOCATION(InLogger, InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - ZEN_DECLARE_FUNCTION \ - static constinit ZEN_LOG_SECTION(".zlog$s") char FileName[] = __FILE__; \ - static constinit ZEN_LOG_SECTION(".zlog$m") char FormatString[] = fmtstr; \ - static constinit ZEN_LOG_SECTION(".zlog$l") zen::logging::SourceLocation Location{FileName, __LINE__, ZEN_LOG_FUNCNAME}; \ - zen::LoggerRef Logger = InLogger; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - if (Logger.ShouldLog(InLevel)) \ - { \ - zen::logging::EmitLogMessage(Logger, \ - Location, \ - InLevel, \ - std::string_view(FormatString, sizeof FormatString - 1), \ - zen::logging::LogCaptureArguments(__VA_ARGS__)); \ - } \ +#define ZEN_LOG_WITH_LOCATION(InLogger, InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit ZEN_LOG_SECTION(".zlog$l") \ + zen::logging::LogPoint LogPoint{zen::logging::SourceLocation{__FILE__, __LINE__}, InLevel, std::string_view(fmtstr)}; \ + zen::LoggerRef Logger = InLogger; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + if (Logger.ShouldLog(InLevel)) \ + { \ + zen::logging::EmitLogMessage(Logger, LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ + } \ } while (false); -#define ZEN_LOG(InLogger, InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - static constinit ZEN_LOG_SECTION(".zlog$m") char FormatString[] = fmtstr; \ - zen::LoggerRef Logger = InLogger; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - if (Logger.ShouldLog(InLevel)) \ - { \ - zen::logging::EmitLogMessage(Logger, \ - InLevel, \ - std::string_view(FormatString, sizeof FormatString - 1), \ - zen::logging::LogCaptureArguments(__VA_ARGS__)); \ - } \ +#define ZEN_LOG(InLogger, InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit ZEN_LOG_SECTION(".zlog$l") zen::logging::LogPoint LogPoint{{}, InLevel, std::string_view(fmtstr)}; \ + zen::LoggerRef Logger = InLogger; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + if (Logger.ShouldLog(InLevel)) \ + { \ + zen::logging::EmitLogMessage(Logger, LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ + } \ } while (false); #define ZEN_DEFINE_LOG_CATEGORY_STATIC(Category, Name) \ static zen::logging::LogCategory Category { Name } -#define ZEN_LOG_TRACE(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Trace, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_DEBUG(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_INFO(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_WARN(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_ERROR(Category, fmtstr, ...) ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::level::Err, fmtstr, ##__VA_ARGS__) -#define ZEN_LOG_CRITICAL(Category, fmtstr, ...) \ - ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::level::Critical, fmtstr, ##__VA_ARGS__) - -#define ZEN_TRACE(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Trace, fmtstr, ##__VA_ARGS__) -#define ZEN_DEBUG(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_INFO(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_WARN(fmtstr, ...) ZEN_LOG(Log(), zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -#define ZEN_ERROR(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Err, fmtstr, ##__VA_ARGS__) -#define ZEN_CRITICAL(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Critical, fmtstr, ##__VA_ARGS__) - -#define ZEN_CONSOLE_LOG(InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - zen::logging::EmitConsoleLogMessage(InLevel, fmtstr, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ +#define ZEN_LOG_TRACE(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Trace, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_DEBUG(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_INFO(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_WARN(Category, fmtstr, ...) ZEN_LOG(Category.Logger(), zen::logging::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_ERROR(Category, fmtstr, ...) ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::Err, fmtstr, ##__VA_ARGS__) +#define ZEN_LOG_CRITICAL(Category, fmtstr, ...) ZEN_LOG_WITH_LOCATION(Category.Logger(), zen::logging::Critical, fmtstr, ##__VA_ARGS__) + +#define ZEN_TRACE(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Trace, fmtstr, ##__VA_ARGS__) +#define ZEN_DEBUG(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_INFO(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_WARN(fmtstr, ...) ZEN_LOG(Log(), zen::logging::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_ERROR(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Err, fmtstr, ##__VA_ARGS__) +#define ZEN_CRITICAL(fmtstr, ...) ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Critical, fmtstr, ##__VA_ARGS__) + +#define ZEN_CONSOLE_LOG(InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit ZEN_LOG_SECTION(".zlog$l") zen::logging::LogPoint LogPoint{{}, InLevel, std::string_view(fmtstr)}; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + zen::logging::EmitConsoleLogMessage(LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ } while (false) -#define ZEN_CONSOLE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_TRACE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Trace, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_DEBUG(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_INFO(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_WARN(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_ERROR(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Err, fmtstr, ##__VA_ARGS__) -#define ZEN_CONSOLE_CRITICAL(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::level::Critical, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_TRACE(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Trace, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_DEBUG(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_INFO(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_WARN(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_ERROR(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Err, fmtstr, ##__VA_ARGS__) +#define ZEN_CONSOLE_CRITICAL(fmtstr, ...) ZEN_CONSOLE_LOG(zen::logging::Critical, fmtstr, ##__VA_ARGS__) ////////////////////////////////////////////////////////////////////////// @@ -240,28 +212,28 @@ std::string_view EmitActivitiesForLogging(StringBuilderBase& OutString); #define ZEN_LOG_SCOPE(...) ScopedLazyActivity $Activity##__LINE__([&](StringBuilderBase& Out) { Out << fmt::format(__VA_ARGS__); }) -#define ZEN_SCOPED_WARN(fmtstr, ...) \ - do \ - { \ - ExtendableStringBuilder<256> ScopeString; \ - const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ - ZEN_LOG(Log(), zen::logging::level::Warn, fmtstr "{}", ##__VA_ARGS__, Scopes); \ +#define ZEN_SCOPED_WARN(fmtstr, ...) \ + do \ + { \ + ExtendableStringBuilder<256> ScopeString; \ + const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ + ZEN_LOG(Log(), zen::logging::Warn, fmtstr "{}", ##__VA_ARGS__, Scopes); \ } while (false) -#define ZEN_SCOPED_ERROR(fmtstr, ...) \ - do \ - { \ - ExtendableStringBuilder<256> ScopeString; \ - const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ - ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Err, fmtstr "{}", ##__VA_ARGS__, Scopes); \ +#define ZEN_SCOPED_ERROR(fmtstr, ...) \ + do \ + { \ + ExtendableStringBuilder<256> ScopeString; \ + const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ + ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Err, fmtstr "{}", ##__VA_ARGS__, Scopes); \ } while (false) -#define ZEN_SCOPED_CRITICAL(fmtstr, ...) \ - do \ - { \ - ExtendableStringBuilder<256> ScopeString; \ - const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ - ZEN_LOG_WITH_LOCATION(Log(), zen::logging::level::Critical, fmtstr "{}", ##__VA_ARGS__, Scopes); \ +#define ZEN_SCOPED_CRITICAL(fmtstr, ...) \ + do \ + { \ + ExtendableStringBuilder<256> ScopeString; \ + const std::string_view Scopes = EmitActivitiesForLogging(ScopeString); \ + ZEN_LOG_WITH_LOCATION(Log(), zen::logging::Critical, fmtstr "{}", ##__VA_ARGS__, Scopes); \ } while (false) ScopedActivityBase* GetThreadActivity(); diff --git a/src/zencore/include/zencore/logging/ansicolorsink.h b/src/zencore/include/zencore/logging/ansicolorsink.h new file mode 100644 index 000000000..9f859e8d7 --- /dev/null +++ b/src/zencore/include/zencore/logging/ansicolorsink.h @@ -0,0 +1,26 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include + +namespace zen::logging { + +class AnsiColorStdoutSink : public Sink +{ +public: + AnsiColorStdoutSink(); + ~AnsiColorStdoutSink() override; + + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr InFormatter) override; + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/asyncsink.h b/src/zencore/include/zencore/logging/asyncsink.h new file mode 100644 index 000000000..c49a1ccce --- /dev/null +++ b/src/zencore/include/zencore/logging/asyncsink.h @@ -0,0 +1,30 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include + +namespace zen::logging { + +class AsyncSink : public Sink +{ +public: + explicit AsyncSink(std::vector InSinks); + ~AsyncSink() override; + + AsyncSink(const AsyncSink&) = delete; + AsyncSink& operator=(const AsyncSink&) = delete; + + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr InFormatter) override; + +private: + struct Impl; + std::unique_ptr m_Impl; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/formatter.h b/src/zencore/include/zencore/logging/formatter.h new file mode 100644 index 000000000..11904d71d --- /dev/null +++ b/src/zencore/include/zencore/logging/formatter.h @@ -0,0 +1,20 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include + +namespace zen::logging { + +class Formatter +{ +public: + virtual ~Formatter() = default; + virtual void Format(const LogMessage& Msg, MemoryBuffer& Dest) = 0; + virtual std::unique_ptr Clone() const = 0; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/helpers.h b/src/zencore/include/zencore/logging/helpers.h new file mode 100644 index 000000000..ce021e1a5 --- /dev/null +++ b/src/zencore/include/zencore/logging/helpers.h @@ -0,0 +1,122 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +#include +#include +#include + +namespace zen::logging::helpers { + +inline void +AppendStringView(std::string_view Sv, MemoryBuffer& Dest) +{ + Dest.append(Sv.data(), Sv.data() + Sv.size()); +} + +inline void +AppendInt(int N, MemoryBuffer& Dest) +{ + fmt::format_int Formatted(N); + Dest.append(Formatted.data(), Formatted.data() + Formatted.size()); +} + +inline void +Pad2(int N, MemoryBuffer& Dest) +{ + if (N >= 0 && N < 100) + { + Dest.push_back(static_cast('0' + N / 10)); + Dest.push_back(static_cast('0' + N % 10)); + } + else + { + fmt::format_int Formatted(N); + Dest.append(Formatted.data(), Formatted.data() + Formatted.size()); + } +} + +inline void +Pad3(uint32_t N, MemoryBuffer& Dest) +{ + if (N < 1000) + { + Dest.push_back(static_cast('0' + N / 100)); + Dest.push_back(static_cast('0' + (N / 10) % 10)); + Dest.push_back(static_cast('0' + N % 10)); + } + else + { + AppendInt(static_cast(N), Dest); + } +} + +inline void +PadUint(size_t N, unsigned int Width, MemoryBuffer& Dest) +{ + fmt::format_int Formatted(N); + auto StrLen = static_cast(Formatted.size()); + if (Width > StrLen) + { + for (unsigned int Pad = 0; Pad < Width - StrLen; ++Pad) + { + Dest.push_back('0'); + } + } + Dest.append(Formatted.data(), Formatted.data() + Formatted.size()); +} + +template +inline ToDuration +TimeFraction(std::chrono::system_clock::time_point Tp) +{ + using std::chrono::duration_cast; + using std::chrono::seconds; + auto Duration = Tp.time_since_epoch(); + auto Secs = duration_cast(Duration); + return duration_cast(Duration) - duration_cast(Secs); +} + +inline std::tm +SafeLocaltime(std::time_t Time) +{ + std::tm Result{}; +#if defined(_WIN32) + localtime_s(&Result, &Time); +#else + localtime_r(&Time, &Result); +#endif + return Result; +} + +inline const char* +ShortFilename(const char* Path) +{ + if (Path == nullptr) + { + return Path; + } + + const char* It = Path; + const char* LastSep = Path; + while (*It) + { + if (*It == '/' || *It == '\\') + { + LastSep = It + 1; + } + ++It; + } + return LastSep; +} + +inline std::string_view +LevelToShortString(LogLevel Level) +{ + return ToStringView(Level); +} + +} // namespace zen::logging::helpers diff --git a/src/zencore/include/zencore/logging/logger.h b/src/zencore/include/zencore/logging/logger.h new file mode 100644 index 000000000..39d1139a5 --- /dev/null +++ b/src/zencore/include/zencore/logging/logger.h @@ -0,0 +1,63 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include + +namespace zen::logging { + +class ErrorHandler +{ +public: + virtual ~ErrorHandler() = default; + virtual void HandleError(const std::string_view& Msg) = 0; +}; + +class Logger : public RefCounted +{ +public: + Logger(std::string_view InName, SinkPtr InSink); + Logger(std::string_view InName, std::span InSinks); + ~Logger(); + + Logger(const Logger&) = delete; + Logger& operator=(const Logger&) = delete; + + void Log(const LogPoint& Point, fmt::format_args Args); + + bool ShouldLog(LogLevel InLevel) const { return InLevel >= m_Level.load(std::memory_order_relaxed); } + + void SetLevel(LogLevel InLevel) { m_Level.store(InLevel, std::memory_order_relaxed); } + LogLevel GetLevel() const { return m_Level.load(std::memory_order_relaxed); } + + void SetFlushLevel(LogLevel InLevel) { m_FlushLevel.store(InLevel, std::memory_order_relaxed); } + LogLevel GetFlushLevel() const { return m_FlushLevel.load(std::memory_order_relaxed); } + + std::string_view Name() const; + + void SetSinks(std::vector InSinks); + void AddSink(SinkPtr InSink); + + void SetFormatter(std::unique_ptr InFormatter); + + void SetErrorHandler(ErrorHandler* Handler); + + void Flush(); + + Ref Clone(std::string_view NewName) const; + +private: + void SinkIt(const LogMessage& Msg); + void FlushIfNeeded(LogLevel InLevel); + + struct Impl; + std::unique_ptr m_Impl; + std::atomic m_Level{Info}; + std::atomic m_FlushLevel{Off}; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/logmsg.h b/src/zencore/include/zencore/logging/logmsg.h new file mode 100644 index 000000000..1d8b6b1b7 --- /dev/null +++ b/src/zencore/include/zencore/logging/logmsg.h @@ -0,0 +1,66 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include + +namespace zen::logging { + +using LogClock = std::chrono::system_clock; + +struct LogMessage +{ + LogMessage() = default; + + LogMessage(const LogPoint& InPoint, std::string_view InLoggerName, std::string_view InPayload) + : m_LoggerName(InLoggerName) + , m_Level(InPoint.Level) + , m_Time(LogClock::now()) + , m_Source(InPoint.Location) + , m_Payload(InPayload) + , m_Point(&InPoint) + { + } + + std::string_view GetPayload() const { return m_Payload; } + int GetThreadId() const { return m_ThreadId; } + LogClock::time_point GetTime() const { return m_Time; } + LogLevel GetLevel() const { return m_Level; } + std::string_view GetLoggerName() const { return m_LoggerName; } + const SourceLocation& GetSource() const { return m_Source; } + const LogPoint& GetLogPoint() const { return *m_Point; } + + void SetThreadId(int InThreadId) { m_ThreadId = InThreadId; } + void SetPayload(std::string_view InPayload) { m_Payload = InPayload; } + void SetLoggerName(std::string_view InName) { m_LoggerName = InName; } + void SetLevel(LogLevel InLevel) { m_Level = InLevel; } + void SetTime(LogClock::time_point InTime) { m_Time = InTime; } + void SetSource(const SourceLocation& InSource) { m_Source = InSource; } + + mutable size_t ColorRangeStart = 0; + mutable size_t ColorRangeEnd = 0; + +private: + static constexpr LogPoint s_DefaultPoints[LogLevelCount] = { + {{}, Trace, {}}, + {{}, Debug, {}}, + {{}, Info, {}}, + {{}, Warn, {}}, + {{}, Err, {}}, + {{}, Critical, {}}, + {{}, Off, {}}, + }; + + std::string_view m_LoggerName; + LogLevel m_Level = Off; + std::chrono::system_clock::time_point m_Time; + SourceLocation m_Source; + std::string_view m_Payload; + const LogPoint* m_Point = &s_DefaultPoints[Off]; + int m_ThreadId = 0; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/memorybuffer.h b/src/zencore/include/zencore/logging/memorybuffer.h new file mode 100644 index 000000000..cd0ff324f --- /dev/null +++ b/src/zencore/include/zencore/logging/memorybuffer.h @@ -0,0 +1,11 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen::logging { + +using MemoryBuffer = fmt::basic_memory_buffer; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/messageonlyformatter.h b/src/zencore/include/zencore/logging/messageonlyformatter.h new file mode 100644 index 000000000..ce25fe9a6 --- /dev/null +++ b/src/zencore/include/zencore/logging/messageonlyformatter.h @@ -0,0 +1,22 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include + +namespace zen::logging { + +class MessageOnlyFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr Clone() const override { return std::make_unique(); } +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/msvcsink.h b/src/zencore/include/zencore/logging/msvcsink.h new file mode 100644 index 000000000..48ea1b915 --- /dev/null +++ b/src/zencore/include/zencore/logging/msvcsink.h @@ -0,0 +1,30 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#if ZEN_PLATFORM_WINDOWS + +# include + +namespace zen::logging { + +class MsvcSink : public Sink +{ +public: + MsvcSink(); + ~MsvcSink() override = default; + + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr InFormatter) override; + +private: + std::mutex m_Mutex; + std::unique_ptr m_Formatter; +}; + +} // namespace zen::logging + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/src/zencore/include/zencore/logging/nullsink.h b/src/zencore/include/zencore/logging/nullsink.h new file mode 100644 index 000000000..7ac5677c6 --- /dev/null +++ b/src/zencore/include/zencore/logging/nullsink.h @@ -0,0 +1,17 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen::logging { + +class NullSink : public Sink +{ +public: + void Log(const LogMessage& /*Msg*/) override {} + void Flush() override {} + void SetFormatter(std::unique_ptr /*InFormatter*/) override {} +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/registry.h b/src/zencore/include/zencore/logging/registry.h new file mode 100644 index 000000000..a4d3692d2 --- /dev/null +++ b/src/zencore/include/zencore/logging/registry.h @@ -0,0 +1,70 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace zen::logging { + +class Registry +{ +public: + using LogLevels = std::span>; + + static Registry& Instance(); + void Shutdown(); + + void Register(Ref InLogger); + void Drop(const std::string& Name); + Ref Get(const std::string& Name); + + void SetDefaultLogger(Ref InLogger); + Logger* DefaultLoggerRaw(); + Ref DefaultLogger(); + + void SetGlobalLevel(LogLevel Level); + LogLevel GetGlobalLevel() const; + void SetLevels(LogLevels Levels, LogLevel* DefaultLevel); + + void FlushAll(); + void FlushOn(LogLevel Level); + void FlushEvery(std::chrono::seconds Interval); + + // Change formatter on all registered loggers + void SetFormatter(std::unique_ptr InFormatter); + + // Apply function to all registered loggers. Note that the function will + // be called while the registry mutex is held, so it should be fast and + // not attempt to call back into the registry. + template + void ApplyAll(Func&& F) + { + ApplyAllImpl([](void* Ctx, Ref L) { (*static_cast*>(Ctx))(std::move(L)); }, &F); + } + + // Set error handler for all loggers in the registry. The handler is called + // if any logger encounters an error during logging or flushing. + // The caller must ensure the handler outlives the registry. + void SetErrorHandler(ErrorHandler* Handler); + +private: + void ApplyAllImpl(void (*Func)(void*, Ref), void* Context); + + Registry(); + ~Registry(); + + Registry(const Registry&) = delete; + Registry& operator=(const Registry&) = delete; + + struct Impl; + std::unique_ptr m_Impl; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/sink.h b/src/zencore/include/zencore/logging/sink.h new file mode 100644 index 000000000..172176a4e --- /dev/null +++ b/src/zencore/include/zencore/logging/sink.h @@ -0,0 +1,34 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include +#include +#include + +#include +#include + +namespace zen::logging { + +class Sink : public RefCounted +{ +public: + virtual ~Sink() = default; + + virtual void Log(const LogMessage& Msg) = 0; + virtual void Flush() = 0; + + virtual void SetFormatter(std::unique_ptr InFormatter) = 0; + + bool ShouldLog(LogLevel InLevel) const { return InLevel >= m_Level.load(std::memory_order_relaxed); } + void SetLevel(LogLevel InLevel) { m_Level.store(InLevel, std::memory_order_relaxed); } + LogLevel GetLevel() const { return m_Level.load(std::memory_order_relaxed); } + +protected: + std::atomic m_Level{Trace}; +}; + +using SinkPtr = Ref; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/logging/tracesink.h b/src/zencore/include/zencore/logging/tracesink.h new file mode 100644 index 000000000..e63d838b4 --- /dev/null +++ b/src/zencore/include/zencore/logging/tracesink.h @@ -0,0 +1,23 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#pragma once + +#include + +namespace zen::logging { + +/** + * A logging sink that forwards log messages to the trace system. + * + * Work-in-progress, not fully implemented. + */ + +class TraceSink : public Sink +{ +public: + void Log(const LogMessage& Msg) override; + void Flush() override; + void SetFormatter(std::unique_ptr InFormatter) override; +}; + +} // namespace zen::logging diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index faf1238b7..a4e33d69e 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -11,11 +11,9 @@ #if ZEN_USE_SENTRY -# include +# include -ZEN_THIRD_PARTY_INCLUDES_START -# include -ZEN_THIRD_PARTY_INCLUDES_END +# include namespace sentry { @@ -53,7 +51,7 @@ private: std::string m_SentryUserName; std::string m_SentryHostName; std::string m_SentryId; - std::shared_ptr m_SentryLogger; + Ref m_SentryLogger; }; } // namespace zen diff --git a/src/zencore/logging.cpp b/src/zencore/logging.cpp index ebd68de09..099518637 100644 --- a/src/zencore/logging.cpp +++ b/src/zencore/logging.cpp @@ -2,208 +2,128 @@ #include "zencore/logging.h" +#include +#include +#include +#include +#include #include #include #include #include -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END +#include #if ZEN_PLATFORM_WINDOWS # pragma section(".zlog$a", read) -# pragma section(".zlog$f", read) -# pragma section(".zlog$m", read) -# pragma section(".zlog$s", read) +# pragma section(".zlog$l", read) # pragma section(".zlog$z", read) #endif namespace zen { -// We shadow the underlying spdlog default logger, in order to avoid a bunch of overhead LoggerRef TheDefaultLogger; +LoggerRef +Log() +{ + if (TheDefaultLogger) + { + return TheDefaultLogger; + } + return zen::logging::ConsoleLog(); +} + } // namespace zen namespace zen::logging { -using MemoryBuffer_t = fmt::basic_memory_buffer; - -struct LoggingContext -{ - inline LoggingContext(); - inline ~LoggingContext(); - - zen::logging::MemoryBuffer_t MessageBuffer; - - inline std::string_view Message() const { return std::string_view(MessageBuffer.data(), MessageBuffer.size()); } -}; +////////////////////////////////////////////////////////////////////////// -LoggingContext::LoggingContext() +LoggerRef +LogCategory::Logger() { -} + // This should be thread safe since zen::logging::Get() will return + // the same logger instance for the same category name. Also the + // LoggerRef is simply a pointer. + if (!m_LoggerRef) + { + m_LoggerRef = zen::logging::Get(m_CategoryName); + } -LoggingContext::~LoggingContext() -{ + return m_LoggerRef; } -////////////////////////////////////////////////////////////////////////// - static inline bool -IsErrorLevel(int LogLevel) +IsErrorLevel(LogLevel InLevel) { - return (LogLevel == zen::logging::level::Err || LogLevel == zen::logging::level::Critical); + return (InLevel == Err || InLevel == Critical); }; -static_assert(sizeof(spdlog::source_loc) == sizeof(SourceLocation)); -static_assert(offsetof(spdlog::source_loc, filename) == offsetof(SourceLocation, filename)); -static_assert(offsetof(spdlog::source_loc, line) == offsetof(SourceLocation, line)); -static_assert(offsetof(spdlog::source_loc, funcname) == offsetof(SourceLocation, funcname)); - void -EmitLogMessage(LoggerRef& Logger, int LogLevel, const std::string_view Message) +EmitLogMessage(LoggerRef& Logger, const LogPoint& Lp, fmt::format_args Args) { ZEN_MEMSCOPE(ELLMTag::Logging); - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - Logger.SpdLogger->log(InLevel, Message); - if (IsErrorLevel(LogLevel)) - { - if (LoggerRef ErrLogger = zen::logging::ErrorLog()) - { - ErrLogger.SpdLogger->log(InLevel, Message); - } - } -} -void -EmitLogMessage(LoggerRef& Logger, int LogLevel, std::string_view Format, fmt::format_args Args) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - zen::logging::LoggingContext LogCtx; - fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); - zen::logging::EmitLogMessage(Logger, LogLevel, LogCtx.Message()); -} + Logger->Log(Lp, Args); -void -EmitLogMessage(LoggerRef& Logger, const SourceLocation& InLocation, int LogLevel, const std::string_view Message) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - const spdlog::source_loc& Location = *reinterpret_cast(&InLocation); - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - Logger.SpdLogger->log(Location, InLevel, Message); - if (IsErrorLevel(LogLevel)) + if (IsErrorLevel(Lp.Level)) { if (LoggerRef ErrLogger = zen::logging::ErrorLog()) { - ErrLogger.SpdLogger->log(Location, InLevel, Message); + ErrLogger->Log(Lp, Args); } } } void -EmitLogMessage(LoggerRef& Logger, const SourceLocation& InLocation, int LogLevel, std::string_view Format, fmt::format_args Args) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - zen::logging::LoggingContext LogCtx; - fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); - zen::logging::EmitLogMessage(Logger, InLocation, LogLevel, LogCtx.Message()); -} - -void -EmitConsoleLogMessage(int LogLevel, const std::string_view Message) -{ - ZEN_MEMSCOPE(ELLMTag::Logging); - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - ConsoleLog().SpdLogger->log(InLevel, Message); -} - -#define ZEN_COLOR_YELLOW "\033[0;33m" -#define ZEN_COLOR_RED "\033[0;31m" -#define ZEN_BRIGHT_COLOR_RED "\033[1;31m" -#define ZEN_COLOR_RESET "\033[0m" - -void -EmitConsoleLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) +EmitConsoleLogMessage(const LogPoint& Lp, fmt::format_args Args) { ZEN_MEMSCOPE(ELLMTag::Logging); - zen::logging::LoggingContext LogCtx; - - // We are not using a format option for console which include log level since it would interfere with normal console output - - const spdlog::level::level_enum InLevel = (spdlog::level::level_enum)LogLevel; - switch (InLevel) - { - case spdlog::level::level_enum::warn: - fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_YELLOW "Warning: " ZEN_COLOR_RESET); - break; - case spdlog::level::level_enum::err: - fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_BRIGHT_COLOR_RED "Error: " ZEN_COLOR_RESET); - break; - case spdlog::level::level_enum::critical: - fmt::format_to(fmt::appender(LogCtx.MessageBuffer), ZEN_COLOR_RED "Critical: " ZEN_COLOR_RESET); - break; - default: - break; - } - fmt::vformat_to(fmt::appender(LogCtx.MessageBuffer), Format, Args); - zen::logging::EmitConsoleLogMessage(LogLevel, LogCtx.Message()); + ConsoleLog()->Log(Lp, Args); } } // namespace zen::logging -namespace zen::logging::level { +namespace zen::logging { -spdlog::level::level_enum -to_spdlog_level(LogLevel NewLogLevel) -{ - return static_cast((int)NewLogLevel); -} +constinit std::string_view LevelNames[] = {std::string_view("trace", 5), + std::string_view("debug", 5), + std::string_view("info", 4), + std::string_view("warning", 7), + std::string_view("error", 5), + std::string_view("critical", 8), + std::string_view("off", 3)}; LogLevel -to_logging_level(spdlog::level::level_enum NewLogLevel) -{ - return static_cast((int)NewLogLevel); -} - -constinit std::string_view LevelNames[] = {ZEN_LEVEL_NAME_TRACE, - ZEN_LEVEL_NAME_DEBUG, - ZEN_LEVEL_NAME_INFO, - ZEN_LEVEL_NAME_WARNING, - ZEN_LEVEL_NAME_ERROR, - ZEN_LEVEL_NAME_CRITICAL, - ZEN_LEVEL_NAME_OFF}; - -level::LogLevel ParseLogLevelString(std::string_view Name) { - for (int Level = 0; Level < level::LogLevelCount; ++Level) + for (int Level = 0; Level < LogLevelCount; ++Level) { if (LevelNames[Level] == Name) - return static_cast(Level); + { + return static_cast(Level); + } } if (Name == "warn") { - return level::Warn; + return Warn; } if (Name == "err") { - return level::Err; + return Err; } - return level::Off; + return Off; } std::string_view -ToStringView(level::LogLevel Level) +ToStringView(LogLevel Level) { - if (int(Level) < level::LogLevelCount) + if (int(Level) < LogLevelCount) { return LevelNames[int(Level)]; } @@ -211,17 +131,17 @@ ToStringView(level::LogLevel Level) return "None"; } -} // namespace zen::logging::level +} // namespace zen::logging ////////////////////////////////////////////////////////////////////////// namespace zen::logging { RwLock LogLevelsLock; -std::string LogLevels[level::LogLevelCount]; +std::string LogLevels[LogLevelCount]; void -ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers) +ConfigureLogLevels(LogLevel Level, std::string_view Loggers) { ZEN_MEMSCOPE(ELLMTag::Logging); @@ -230,18 +150,18 @@ ConfigureLogLevels(level::LogLevel Level, std::string_view Loggers) } void -RefreshLogLevels(level::LogLevel* DefaultLevel) +RefreshLogLevels(LogLevel* DefaultLevel) { ZEN_MEMSCOPE(ELLMTag::Logging); - spdlog::details::registry::log_levels Levels; + std::vector> Levels; { RwLock::SharedLockScope _(LogLevelsLock); - for (int i = 0; i < level::LogLevelCount; ++i) + for (int i = 0; i < LogLevelCount; ++i) { - level::LogLevel CurrentLevel{i}; + LogLevel CurrentLevel{i}; std::string_view Spec = LogLevels[i]; @@ -260,24 +180,16 @@ RefreshLogLevels(level::LogLevel* DefaultLevel) Spec = {}; } - Levels[LoggerName] = to_spdlog_level(CurrentLevel); + Levels.emplace_back(std::move(LoggerName), CurrentLevel); } } } - if (DefaultLevel) - { - spdlog::level::level_enum SpdDefaultLevel = to_spdlog_level(*DefaultLevel); - spdlog::details::registry::instance().set_levels(Levels, &SpdDefaultLevel); - } - else - { - spdlog::details::registry::instance().set_levels(Levels, nullptr); - } + Registry::Instance().SetLevels(Levels, DefaultLevel); } void -RefreshLogLevels(level::LogLevel DefaultLevel) +RefreshLogLevels(LogLevel DefaultLevel) { RefreshLogLevels(&DefaultLevel); } @@ -289,15 +201,15 @@ RefreshLogLevels() } void -SetLogLevel(level::LogLevel NewLogLevel) +SetLogLevel(LogLevel NewLogLevel) { - spdlog::set_level(to_spdlog_level(NewLogLevel)); + Registry::Instance().SetGlobalLevel(NewLogLevel); } -level::LogLevel +LogLevel GetLogLevel() { - return level::to_logging_level(spdlog::get_level()); + return Registry::Instance().GetGlobalLevel(); } LoggerRef @@ -312,10 +224,10 @@ SetDefault(std::string_view NewDefaultLoggerId) { ZEN_MEMSCOPE(ELLMTag::Logging); - auto NewDefaultLogger = spdlog::get(std::string(NewDefaultLoggerId)); + Ref NewDefaultLogger = Registry::Instance().Get(std::string(NewDefaultLoggerId)); ZEN_ASSERT(NewDefaultLogger); - spdlog::set_default_logger(NewDefaultLogger); + Registry::Instance().SetDefaultLogger(NewDefaultLogger); TheDefaultLogger = LoggerRef(*NewDefaultLogger); } @@ -338,11 +250,11 @@ SetErrorLog(std::string_view NewErrorLoggerId) } else { - auto NewErrorLogger = spdlog::get(std::string(NewErrorLoggerId)); + Ref NewErrorLogger = Registry::Instance().Get(std::string(NewErrorLoggerId)); ZEN_ASSERT(NewErrorLogger); - TheErrorLogger = LoggerRef(*NewErrorLogger.get()); + TheErrorLogger = LoggerRef(*NewErrorLogger.Get()); } } @@ -353,39 +265,75 @@ Get(std::string_view Name) { ZEN_MEMSCOPE(ELLMTag::Logging); - std::shared_ptr Logger = spdlog::get(std::string(Name)); + Ref FoundLogger = Registry::Instance().Get(std::string(Name)); - if (!Logger) + if (!FoundLogger) { g_LoggerMutex.WithExclusiveLock([&] { - Logger = spdlog::get(std::string(Name)); + FoundLogger = Registry::Instance().Get(std::string(Name)); - if (!Logger) + if (!FoundLogger) { - Logger = Default().SpdLogger->clone(std::string(Name)); - spdlog::apply_logger_env_levels(Logger); - spdlog::register_logger(Logger); + FoundLogger = Default()->Clone(std::string(Name)); + Registry::Instance().Register(FoundLogger); } }); } - return *Logger; + return *FoundLogger; } -std::once_flag ConsoleInitFlag; -std::shared_ptr ConLogger; +std::once_flag ConsoleInitFlag; +Ref ConLogger; void SuppressConsoleLog() { + ZEN_MEMSCOPE(ELLMTag::Logging); + if (ConLogger) { - spdlog::drop("console"); + Registry::Instance().Drop("console"); ConLogger = {}; } - ConLogger = spdlog::null_logger_mt("console"); + + SinkPtr NullSinkPtr(new NullSink()); + ConLogger = Ref(new Logger("console", std::vector{NullSinkPtr})); + Registry::Instance().Register(ConLogger); } +#define ZEN_COLOR_YELLOW "\033[0;33m" +#define ZEN_COLOR_RED "\033[0;31m" +#define ZEN_BRIGHT_COLOR_RED "\033[1;31m" +#define ZEN_COLOR_RESET "\033[0m" + +class ConsoleFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + switch (Msg.GetLevel()) + { + case Warn: + fmt::format_to(fmt::appender(Dest), ZEN_COLOR_YELLOW "Warning: " ZEN_COLOR_RESET); + break; + case Err: + fmt::format_to(fmt::appender(Dest), ZEN_BRIGHT_COLOR_RED "Error: " ZEN_COLOR_RESET); + break; + case Critical: + fmt::format_to(fmt::appender(Dest), ZEN_COLOR_RED "Critical: " ZEN_COLOR_RESET); + break; + default: + break; + } + + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr Clone() const override { return std::make_unique(); } +}; + LoggerRef ConsoleLog() { @@ -394,10 +342,10 @@ ConsoleLog() std::call_once(ConsoleInitFlag, [&] { if (!ConLogger) { - ConLogger = spdlog::stdout_color_mt("console"); - spdlog::apply_logger_env_levels(ConLogger); - - ConLogger->set_pattern("%v"); + SinkPtr ConsoleSink(new AnsiColorStdoutSink()); + ConsoleSink->SetFormatter(std::make_unique()); + ConLogger = Ref(new Logger("console", std::vector{ConsoleSink})); + Registry::Instance().Register(ConLogger); } }); @@ -407,9 +355,11 @@ ConsoleLog() void ResetConsoleLog() { + ZEN_MEMSCOPE(ELLMTag::Logging); + LoggerRef ConLog = ConsoleLog(); - ConLog.SpdLogger->set_pattern("%v"); + ConLog->SetFormatter(std::make_unique()); } void @@ -417,13 +367,15 @@ InitializeLogging() { ZEN_MEMSCOPE(ELLMTag::Logging); - TheDefaultLogger = *spdlog::default_logger_raw(); + TheDefaultLogger = *Registry::Instance().DefaultLoggerRaw(); } void ShutdownLogging() { - spdlog::shutdown(); + ZEN_MEMSCOPE(ELLMTag::Logging); + + Registry::Instance().Shutdown(); TheDefaultLogger = {}; } @@ -457,7 +409,7 @@ EnableVTMode() void FlushLogging() { - spdlog::details::registry::instance().flush_all(); + Registry::Instance().FlushAll(); } } // namespace zen::logging @@ -465,21 +417,27 @@ FlushLogging() namespace zen { bool -LoggerRef::ShouldLog(int Level) const +LoggerRef::ShouldLog(logging::LogLevel Level) const { - return SpdLogger->should_log(static_cast(Level)); + return m_Logger->ShouldLog(Level); } void -LoggerRef::SetLogLevel(logging::level::LogLevel NewLogLevel) +LoggerRef::SetLogLevel(logging::LogLevel NewLogLevel) { - SpdLogger->set_level(to_spdlog_level(NewLogLevel)); + m_Logger->SetLevel(NewLogLevel); } -logging::level::LogLevel +logging::LogLevel LoggerRef::GetLogLevel() { - return logging::level::to_logging_level(SpdLogger->level()); + return m_Logger->GetLevel(); +} + +void +LoggerRef::Flush() +{ + m_Logger->Flush(); } thread_local ScopedActivityBase* t_ScopeStack = nullptr; diff --git a/src/zencore/logging/ansicolorsink.cpp b/src/zencore/logging/ansicolorsink.cpp new file mode 100644 index 000000000..9b9959862 --- /dev/null +++ b/src/zencore/logging/ansicolorsink.cpp @@ -0,0 +1,178 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include +#include + +#include +#include + +namespace zen::logging { + +// Default formatter replicating spdlog's %+ pattern: +// [YYYY-MM-DD HH:MM:SS.mmm] [logger_name] [level] message\n +class DefaultConsoleFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + // timestamp + auto Secs = std::chrono::duration_cast(Msg.GetTime().time_since_epoch()); + if (Secs != m_LastLogSecs) + { + m_LastLogSecs = Secs; + m_CachedLocalTm = helpers::SafeLocaltime(LogClock::to_time_t(Msg.GetTime())); + } + + Dest.push_back('['); + helpers::AppendInt(m_CachedLocalTm.tm_year + 1900, Dest); + Dest.push_back('-'); + helpers::Pad2(m_CachedLocalTm.tm_mon + 1, Dest); + Dest.push_back('-'); + helpers::Pad2(m_CachedLocalTm.tm_mday, Dest); + Dest.push_back(' '); + helpers::Pad2(m_CachedLocalTm.tm_hour, Dest); + Dest.push_back(':'); + helpers::Pad2(m_CachedLocalTm.tm_min, Dest); + Dest.push_back(':'); + helpers::Pad2(m_CachedLocalTm.tm_sec, Dest); + Dest.push_back('.'); + auto Millis = helpers::TimeFraction(Msg.GetTime()); + helpers::Pad3(static_cast(Millis.count()), Dest); + Dest.push_back(']'); + Dest.push_back(' '); + + // logger name + if (Msg.GetLoggerName().size() > 0) + { + Dest.push_back('['); + helpers::AppendStringView(Msg.GetLoggerName(), Dest); + Dest.push_back(']'); + Dest.push_back(' '); + } + + // level (colored range) + Dest.push_back('['); + Msg.ColorRangeStart = Dest.size(); + helpers::AppendStringView(helpers::LevelToShortString(Msg.GetLevel()), Dest); + Msg.ColorRangeEnd = Dest.size(); + Dest.push_back(']'); + Dest.push_back(' '); + + // message + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr Clone() const override { return std::make_unique(); } + +private: + std::chrono::seconds m_LastLogSecs{0}; + std::tm m_CachedLocalTm{}; +}; + +static constexpr std::string_view s_Reset = "\033[m"; + +static std::string_view +GetColorForLevel(LogLevel InLevel) +{ + using namespace std::string_view_literals; + switch (InLevel) + { + case Trace: + return "\033[37m"sv; // white + case Debug: + return "\033[36m"sv; // cyan + case Info: + return "\033[32m"sv; // green + case Warn: + return "\033[33m\033[1m"sv; // bold yellow + case Err: + return "\033[31m\033[1m"sv; // bold red + case Critical: + return "\033[1m\033[41m"sv; // bold on red background + default: + return s_Reset; + } +} + +struct AnsiColorStdoutSink::Impl +{ + Impl() : m_Formatter(std::make_unique()) {} + + void Log(const LogMessage& Msg) + { + std::lock_guard Lock(m_Mutex); + + MemoryBuffer Formatted; + m_Formatter->Format(Msg, Formatted); + + if (Msg.ColorRangeEnd > Msg.ColorRangeStart) + { + // Print pre-color range + fwrite(Formatted.data(), 1, Msg.ColorRangeStart, m_File); + + // Print color + std::string_view Color = GetColorForLevel(Msg.GetLevel()); + fwrite(Color.data(), 1, Color.size(), m_File); + + // Print colored range + fwrite(Formatted.data() + Msg.ColorRangeStart, 1, Msg.ColorRangeEnd - Msg.ColorRangeStart, m_File); + + // Reset color + fwrite(s_Reset.data(), 1, s_Reset.size(), m_File); + + // Print remainder + fwrite(Formatted.data() + Msg.ColorRangeEnd, 1, Formatted.size() - Msg.ColorRangeEnd, m_File); + } + else + { + fwrite(Formatted.data(), 1, Formatted.size(), m_File); + } + + fflush(m_File); + } + + void Flush() + { + std::lock_guard Lock(m_Mutex); + fflush(m_File); + } + + void SetFormatter(std::unique_ptr InFormatter) + { + std::lock_guard Lock(m_Mutex); + m_Formatter = std::move(InFormatter); + } + +private: + std::mutex m_Mutex; + std::unique_ptr m_Formatter; + FILE* m_File = stdout; +}; + +AnsiColorStdoutSink::AnsiColorStdoutSink() : m_Impl(std::make_unique()) +{ +} + +AnsiColorStdoutSink::~AnsiColorStdoutSink() = default; + +void +AnsiColorStdoutSink::Log(const LogMessage& Msg) +{ + m_Impl->Log(Msg); +} + +void +AnsiColorStdoutSink::Flush() +{ + m_Impl->Flush(); +} + +void +AnsiColorStdoutSink::SetFormatter(std::unique_ptr InFormatter) +{ + m_Impl->SetFormatter(std::move(InFormatter)); +} + +} // namespace zen::logging diff --git a/src/zencore/logging/asyncsink.cpp b/src/zencore/logging/asyncsink.cpp new file mode 100644 index 000000000..02bf9f3ba --- /dev/null +++ b/src/zencore/logging/asyncsink.cpp @@ -0,0 +1,212 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include +#include + +#include +#include +#include + +namespace zen::logging { + +struct AsyncLogMessage +{ + enum class Type : uint8_t + { + Log, + Flush, + Shutdown + }; + + Type MsgType = Type::Log; + + // Points to the LogPoint from upstream logging code. LogMessage guarantees + // this is always valid (either a static LogPoint from ZEN_LOG macros or one + // of the per-level default LogPoints). + const LogPoint* Point = nullptr; + + int ThreadId = 0; + std::string OwnedPayload; + std::string OwnedLoggerName; + std::chrono::system_clock::time_point Time; + + std::shared_ptr> FlushPromise; +}; + +struct AsyncSink::Impl +{ + explicit Impl(std::vector InSinks) : m_Sinks(std::move(InSinks)) + { + m_WorkerThread = std::thread([this]() { + zen::SetCurrentThreadName("AsyncLog"); + WorkerLoop(); + }); + } + + ~Impl() + { + AsyncLogMessage ShutdownMsg; + ShutdownMsg.MsgType = AsyncLogMessage::Type::Shutdown; + m_Queue.Enqueue(std::move(ShutdownMsg)); + + if (m_WorkerThread.joinable()) + { + m_WorkerThread.join(); + } + } + + void Log(const LogMessage& Msg) + { + AsyncLogMessage AsyncMsg; + AsyncMsg.OwnedPayload = std::string(Msg.GetPayload()); + AsyncMsg.OwnedLoggerName = std::string(Msg.GetLoggerName()); + AsyncMsg.ThreadId = Msg.GetThreadId(); + AsyncMsg.Time = Msg.GetTime(); + AsyncMsg.Point = &Msg.GetLogPoint(); + AsyncMsg.MsgType = AsyncLogMessage::Type::Log; + + m_Queue.Enqueue(std::move(AsyncMsg)); + } + + void Flush() + { + auto Promise = std::make_shared>(); + auto Future = Promise->get_future(); + + AsyncLogMessage FlushMsg; + FlushMsg.MsgType = AsyncLogMessage::Type::Flush; + FlushMsg.FlushPromise = std::move(Promise); + + m_Queue.Enqueue(std::move(FlushMsg)); + + Future.get(); + } + + void SetFormatter(std::unique_ptr InFormatter) + { + for (auto& CurrentSink : m_Sinks) + { + CurrentSink->SetFormatter(InFormatter->Clone()); + } + } + +private: + void ForwardLogToSinks(const AsyncLogMessage& AsyncMsg) + { + LogMessage Reconstructed(*AsyncMsg.Point, AsyncMsg.OwnedLoggerName, AsyncMsg.OwnedPayload); + Reconstructed.SetTime(AsyncMsg.Time); + Reconstructed.SetThreadId(AsyncMsg.ThreadId); + + for (auto& CurrentSink : m_Sinks) + { + if (CurrentSink->ShouldLog(Reconstructed.GetLevel())) + { + try + { + CurrentSink->Log(Reconstructed); + } + catch (const std::exception&) + { + } + } + } + } + + void FlushSinks() + { + for (auto& CurrentSink : m_Sinks) + { + try + { + CurrentSink->Flush(); + } + catch (const std::exception&) + { + } + } + } + + void WorkerLoop() + { + AsyncLogMessage Msg; + while (m_Queue.WaitAndDequeue(Msg)) + { + switch (Msg.MsgType) + { + case AsyncLogMessage::Type::Log: + { + ForwardLogToSinks(Msg); + break; + } + + case AsyncLogMessage::Type::Flush: + { + FlushSinks(); + if (Msg.FlushPromise) + { + Msg.FlushPromise->set_value(); + } + break; + } + + case AsyncLogMessage::Type::Shutdown: + { + m_Queue.CompleteAdding(); + + AsyncLogMessage Remaining; + while (m_Queue.WaitAndDequeue(Remaining)) + { + if (Remaining.MsgType == AsyncLogMessage::Type::Log) + { + ForwardLogToSinks(Remaining); + } + else if (Remaining.MsgType == AsyncLogMessage::Type::Flush) + { + FlushSinks(); + if (Remaining.FlushPromise) + { + Remaining.FlushPromise->set_value(); + } + } + } + + FlushSinks(); + return; + } + } + } + } + + std::vector m_Sinks; + BlockingQueue m_Queue; + std::thread m_WorkerThread; +}; + +AsyncSink::AsyncSink(std::vector InSinks) : m_Impl(std::make_unique(std::move(InSinks))) +{ +} + +AsyncSink::~AsyncSink() = default; + +void +AsyncSink::Log(const LogMessage& Msg) +{ + m_Impl->Log(Msg); +} + +void +AsyncSink::Flush() +{ + m_Impl->Flush(); +} + +void +AsyncSink::SetFormatter(std::unique_ptr InFormatter) +{ + m_Impl->SetFormatter(std::move(InFormatter)); +} + +} // namespace zen::logging diff --git a/src/zencore/logging/logger.cpp b/src/zencore/logging/logger.cpp new file mode 100644 index 000000000..dd1675bb1 --- /dev/null +++ b/src/zencore/logging/logger.cpp @@ -0,0 +1,142 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include + +#include +#include + +namespace zen::logging { + +struct Logger::Impl +{ + std::string m_Name; + std::vector m_Sinks; + ErrorHandler* m_ErrorHandler = nullptr; +}; + +Logger::Logger(std::string_view InName, SinkPtr InSink) : m_Impl(std::make_unique()) +{ + m_Impl->m_Name = InName; + m_Impl->m_Sinks.push_back(std::move(InSink)); +} + +Logger::Logger(std::string_view InName, std::span InSinks) : m_Impl(std::make_unique()) +{ + m_Impl->m_Name = InName; + m_Impl->m_Sinks.assign(InSinks.begin(), InSinks.end()); +} + +Logger::~Logger() = default; + +void +Logger::Log(const LogPoint& Point, fmt::format_args Args) +{ + if (!ShouldLog(Point.Level)) + { + return; + } + + fmt::basic_memory_buffer Buffer; + fmt::vformat_to(fmt::appender(Buffer), Point.FormatString, Args); + + LogMessage LogMsg(Point, m_Impl->m_Name, std::string_view(Buffer.data(), Buffer.size())); + LogMsg.SetThreadId(GetCurrentThreadId()); + SinkIt(LogMsg); + FlushIfNeeded(Point.Level); +} + +void +Logger::SinkIt(const LogMessage& Msg) +{ + for (auto& CurrentSink : m_Impl->m_Sinks) + { + if (CurrentSink->ShouldLog(Msg.GetLevel())) + { + try + { + CurrentSink->Log(Msg); + } + catch (const std::exception& Ex) + { + if (m_Impl->m_ErrorHandler) + { + m_Impl->m_ErrorHandler->HandleError(Ex.what()); + } + } + } + } +} + +void +Logger::FlushIfNeeded(LogLevel InLevel) +{ + if (InLevel >= m_FlushLevel.load(std::memory_order_relaxed)) + { + Flush(); + } +} + +void +Logger::Flush() +{ + for (auto& CurrentSink : m_Impl->m_Sinks) + { + try + { + CurrentSink->Flush(); + } + catch (const std::exception& Ex) + { + if (m_Impl->m_ErrorHandler) + { + m_Impl->m_ErrorHandler->HandleError(Ex.what()); + } + } + } +} + +void +Logger::SetSinks(std::vector InSinks) +{ + m_Impl->m_Sinks = std::move(InSinks); +} + +void +Logger::AddSink(SinkPtr InSink) +{ + m_Impl->m_Sinks.push_back(std::move(InSink)); +} + +void +Logger::SetErrorHandler(ErrorHandler* Handler) +{ + m_Impl->m_ErrorHandler = Handler; +} + +void +Logger::SetFormatter(std::unique_ptr InFormatter) +{ + for (auto& CurrentSink : m_Impl->m_Sinks) + { + CurrentSink->SetFormatter(InFormatter->Clone()); + } +} + +std::string_view +Logger::Name() const +{ + return m_Impl->m_Name; +} + +Ref +Logger::Clone(std::string_view NewName) const +{ + Ref Cloned(new Logger(NewName, m_Impl->m_Sinks)); + Cloned->SetLevel(m_Level.load(std::memory_order_relaxed)); + Cloned->SetFlushLevel(m_FlushLevel.load(std::memory_order_relaxed)); + Cloned->SetErrorHandler(m_Impl->m_ErrorHandler); + return Cloned; +} + +} // namespace zen::logging diff --git a/src/zencore/logging/msvcsink.cpp b/src/zencore/logging/msvcsink.cpp new file mode 100644 index 000000000..457a4d6e1 --- /dev/null +++ b/src/zencore/logging/msvcsink.cpp @@ -0,0 +1,80 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#if ZEN_PLATFORM_WINDOWS + +# include +# include +# include + +ZEN_THIRD_PARTY_INCLUDES_START +# include +ZEN_THIRD_PARTY_INCLUDES_END + +namespace zen::logging { + +// Default formatter for MSVC debug output: [level] message\n +// For error/critical messages with source info, prepends file(line): so that +// the message is clickable in the Visual Studio Output window. +class DefaultMsvcFormatter : public Formatter +{ +public: + void Format(const LogMessage& Msg, MemoryBuffer& Dest) override + { + const auto& Source = Msg.GetSource(); + if (Msg.GetLevel() >= LogLevel::Err && Source) + { + helpers::AppendStringView(Source.Filename, Dest); + Dest.push_back('('); + helpers::AppendInt(Source.Line, Dest); + Dest.push_back(')'); + Dest.push_back(':'); + Dest.push_back(' '); + } + + Dest.push_back('['); + helpers::AppendStringView(helpers::LevelToShortString(Msg.GetLevel()), Dest); + Dest.push_back(']'); + Dest.push_back(' '); + helpers::AppendStringView(Msg.GetPayload(), Dest); + Dest.push_back('\n'); + } + + std::unique_ptr Clone() const override { return std::make_unique(); } +}; + +MsvcSink::MsvcSink() : m_Formatter(std::make_unique()) +{ +} + +void +MsvcSink::Log(const LogMessage& Msg) +{ + std::lock_guard Lock(m_Mutex); + + MemoryBuffer Formatted; + m_Formatter->Format(Msg, Formatted); + + // Null-terminate for OutputDebugStringA + Formatted.push_back('\0'); + + OutputDebugStringA(Formatted.data()); +} + +void +MsvcSink::Flush() +{ + // Nothing to flush for OutputDebugString +} + +void +MsvcSink::SetFormatter(std::unique_ptr InFormatter) +{ + std::lock_guard Lock(m_Mutex); + m_Formatter = std::move(InFormatter); +} + +} // namespace zen::logging + +#endif // ZEN_PLATFORM_WINDOWS diff --git a/src/zencore/logging/registry.cpp b/src/zencore/logging/registry.cpp new file mode 100644 index 000000000..3ed1fb0df --- /dev/null +++ b/src/zencore/logging/registry.cpp @@ -0,0 +1,330 @@ +// Copyright Epic Games, Inc. All Rights Reserved. + +#include + +#include +#include + +#include +#include +#include +#include +#include + +namespace zen::logging { + +struct Registry::Impl +{ + Impl() + { + // Create default logger with a stdout color sink + SinkPtr DefaultSink(new AnsiColorStdoutSink()); + m_DefaultLogger = Ref(new Logger("", DefaultSink)); + m_Loggers[""] = m_DefaultLogger; + } + + ~Impl() { StopPeriodicFlush(); } + + void Register(Ref InLogger) + { + std::lock_guard Lock(m_Mutex); + if (m_ErrorHandler) + { + InLogger->SetErrorHandler(m_ErrorHandler); + } + m_Loggers[std::string(InLogger->Name())] = std::move(InLogger); + } + + void Drop(const std::string& Name) + { + std::lock_guard Lock(m_Mutex); + m_Loggers.erase(Name); + } + + Ref Get(const std::string& Name) + { + std::lock_guard Lock(m_Mutex); + auto It = m_Loggers.find(Name); + if (It != m_Loggers.end()) + { + return It->second; + } + return {}; + } + + void SetDefaultLogger(Ref InLogger) + { + std::lock_guard Lock(m_Mutex); + if (InLogger) + { + m_Loggers[std::string(InLogger->Name())] = InLogger; + } + m_DefaultLogger = std::move(InLogger); + } + + Logger* DefaultLoggerRaw() { return m_DefaultLogger.Get(); } + + Ref DefaultLogger() + { + std::lock_guard Lock(m_Mutex); + return m_DefaultLogger; + } + + void SetGlobalLevel(LogLevel Level) + { + m_GlobalLevel.store(Level, std::memory_order_relaxed); + std::lock_guard Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetLevel(Level); + } + } + + LogLevel GetGlobalLevel() const { return m_GlobalLevel.load(std::memory_order_relaxed); } + + void SetLevels(Registry::LogLevels Levels, LogLevel* DefaultLevel) + { + std::lock_guard Lock(m_Mutex); + + if (DefaultLevel) + { + m_GlobalLevel.store(*DefaultLevel, std::memory_order_relaxed); + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetLevel(*DefaultLevel); + } + } + + for (auto& [LoggerName, Level] : Levels) + { + auto It = m_Loggers.find(LoggerName); + if (It != m_Loggers.end()) + { + It->second->SetLevel(Level); + } + } + } + + void FlushAll() + { + std::lock_guard Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + try + { + CurLogger->Flush(); + } + catch (const std::exception&) + { + } + } + } + + void FlushOn(LogLevel Level) + { + std::lock_guard Lock(m_Mutex); + m_FlushLevel = Level; + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetFlushLevel(Level); + } + } + + void FlushEvery(std::chrono::seconds Interval) + { + StopPeriodicFlush(); + + m_PeriodicFlushRunning.store(true, std::memory_order_relaxed); + + m_FlushThread = std::thread([this, Interval] { + while (m_PeriodicFlushRunning.load(std::memory_order_relaxed)) + { + { + std::unique_lock Lock(m_PeriodicFlushMutex); + m_PeriodicFlushCv.wait_for(Lock, Interval, [this] { return !m_PeriodicFlushRunning.load(std::memory_order_relaxed); }); + } + + if (m_PeriodicFlushRunning.load(std::memory_order_relaxed)) + { + FlushAll(); + } + } + }); + } + + void SetFormatter(std::unique_ptr InFormatter) + { + std::lock_guard Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetFormatter(InFormatter->Clone()); + } + } + + void ApplyAll(void (*Func)(void*, Ref), void* Context) + { + std::lock_guard Lock(m_Mutex); + for (auto& [Name, CurLogger] : m_Loggers) + { + Func(Context, CurLogger); + } + } + + void SetErrorHandler(ErrorHandler* Handler) + { + std::lock_guard Lock(m_Mutex); + m_ErrorHandler = Handler; + for (auto& [Name, CurLogger] : m_Loggers) + { + CurLogger->SetErrorHandler(Handler); + } + } + + void Shutdown() + { + StopPeriodicFlush(); + FlushAll(); + + std::lock_guard Lock(m_Mutex); + m_Loggers.clear(); + m_DefaultLogger = nullptr; + } + +private: + void StopPeriodicFlush() + { + if (m_FlushThread.joinable()) + { + m_PeriodicFlushRunning.store(false, std::memory_order_relaxed); + { + std::lock_guard Lock(m_PeriodicFlushMutex); + m_PeriodicFlushCv.notify_one(); + } + m_FlushThread.join(); + } + } + + std::mutex m_Mutex; + std::unordered_map> m_Loggers; + Ref m_DefaultLogger; + std::atomic m_GlobalLevel{Trace}; + LogLevel m_FlushLevel{Off}; + ErrorHandler* m_ErrorHandler = nullptr; + + // Periodic flush + std::atomic m_PeriodicFlushRunning{false}; + std::mutex m_PeriodicFlushMutex; + std::condition_variable m_PeriodicFlushCv; + std::thread m_FlushThread; +}; + +Registry& +Registry::Instance() +{ + static Registry s_Instance; + return s_Instance; +} + +Registry::Registry() : m_Impl(std::make_unique()) +{ +} + +Registry::~Registry() = default; + +void +Registry::Register(Ref InLogger) +{ + m_Impl->Register(std::move(InLogger)); +} + +void +Registry::Drop(const std::string& Name) +{ + m_Impl->Drop(Name); +} + +Ref +Registry::Get(const std::string& Name) +{ + return m_Impl->Get(Name); +} + +void +Registry::SetDefaultLogger(Ref InLogger) +{ + m_Impl->SetDefaultLogger(std::move(InLogger)); +} + +Logger* +Registry::DefaultLoggerRaw() +{ + return m_Impl->DefaultLoggerRaw(); +} + +Ref +Registry::DefaultLogger() +{ + return m_Impl->DefaultLogger(); +} + +void +Registry::SetGlobalLevel(LogLevel Level) +{ + m_Impl->SetGlobalLevel(Level); +} + +LogLevel +Registry::GetGlobalLevel() const +{ + return m_Impl->GetGlobalLevel(); +} + +void +Registry::SetLevels(LogLevels Levels, LogLevel* DefaultLevel) +{ + m_Impl->SetLevels(Levels, DefaultLevel); +} + +void +Registry::FlushAll() +{ + m_Impl->FlushAll(); +} + +void +Registry::FlushOn(LogLevel Level) +{ + m_Impl->FlushOn(Level); +} + +void +Registry::FlushEvery(std::chrono::seconds Interval) +{ + m_Impl->FlushEvery(Interval); +} + +void +Registry::SetFormatter(std::unique_ptr InFormatter) +{ + m_Impl->SetFormatter(std::move(InFormatter)); +} + +void +Registry::ApplyAllImpl(void (*Func)(void*, Ref), void* Context) +{ + m_Impl->ApplyAll(Func, Context); +} + +void +Registry::SetErrorHandler(ErrorHandler* Handler) +{ + m_Impl->SetErrorHandler(Handler); +} + +void +Registry::Shutdown() +{ + m_Impl->Shutdown(); +} + +} // namespace zen::logging diff --git a/src/zencore/logging/tracesink.cpp b/src/zencore/logging/tracesink.cpp new file mode 100644 index 000000000..e3533327b --- /dev/null +++ b/src/zencore/logging/tracesink.cpp @@ -0,0 +1,88 @@ + +// Copyright Epic Games, Inc. All Rights Reserved. + +#include +#include +#include +#include +#include + +namespace zen::logging { + +UE_TRACE_CHANNEL_DEFINE(LogChannel) + +UE_TRACE_EVENT_BEGIN(Logging, LogCategory, NoSync | Important) + UE_TRACE_EVENT_FIELD(const void*, CategoryPointer) + UE_TRACE_EVENT_FIELD(uint8_t, DefaultVerbosity) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, Name) +UE_TRACE_EVENT_END() + +UE_TRACE_EVENT_BEGIN(Logging, LogMessageSpec, NoSync | Important) + UE_TRACE_EVENT_FIELD(const void*, LogPoint) + UE_TRACE_EVENT_FIELD(const void*, CategoryPointer) + UE_TRACE_EVENT_FIELD(int32_t, Line) + UE_TRACE_EVENT_FIELD(uint8_t, Verbosity) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, FileName) + UE_TRACE_EVENT_FIELD(UE::Trace::AnsiString, FormatString) +UE_TRACE_EVENT_END() + +UE_TRACE_EVENT_BEGIN(Logging, LogMessage, NoSync) + UE_TRACE_EVENT_FIELD(const void*, LogPoint) + UE_TRACE_EVENT_FIELD(uint64_t, Cycle) + UE_TRACE_EVENT_FIELD(uint8_t[], FormatArgs) +UE_TRACE_EVENT_END() + +void +TraceLogCategory(const logging::Logger* Category, const char* Name, logging::LogLevel DefaultVerbosity) +{ + uint16_t NameLen = uint16_t(strlen(Name)); + UE_TRACE_LOG(Logging, LogCategory, LogChannel, NameLen * sizeof(ANSICHAR)) + << LogCategory.CategoryPointer(Category) << LogCategory.DefaultVerbosity(uint8_t(DefaultVerbosity)) + << LogCategory.Name(Name, NameLen); +} + +void +TraceLogMessageSpec(const void* LogPoint, + const logging::Logger* Category, + logging::LogLevel Verbosity, + const std::string_view File, + int32_t Line, + const std::string_view Format) +{ + uint16_t FileNameLen = uint16_t(File.size()); + uint16_t FormatStringLen = uint16_t(Format.size()); + uint32_t DataSize = (FileNameLen * sizeof(ANSICHAR)) + (FormatStringLen * sizeof(ANSICHAR)); + UE_TRACE_LOG(Logging, LogMessageSpec, LogChannel, DataSize) + << LogMessageSpec.LogPoint(LogPoint) << LogMessageSpec.CategoryPointer(Category) << LogMessageSpec.Line(Line) + << LogMessageSpec.Verbosity(uint8_t(Verbosity)) << LogMessageSpec.FileName(File.data(), FileNameLen) + << LogMessageSpec.FormatString(Format.data(), FormatStringLen); +} + +void +TraceLogMessageInternal(const void* LogPoint, int32_t EncodedFormatArgsSize, const uint8_t* EncodedFormatArgs) +{ + UE_TRACE_LOG(Logging, LogMessage, LogChannel) << LogMessage.LogPoint(LogPoint) << LogMessage.Cycle(GetHifreqTimerValue()) + << LogMessage.FormatArgs(EncodedFormatArgs, EncodedFormatArgsSize); +} + +////////////////////////////////////////////////////////////////////////// + +void +TraceSink::Log(const LogMessage& Msg) +{ + ZEN_UNUSED(Msg); +} + +void +TraceSink::Flush() +{ +} + +void +TraceSink::SetFormatter(std::unique_ptr /*InFormatter*/) +{ + // This sink doesn't use a formatter since it just forwards the raw format + // args to the trace system +} + +} // namespace zen::logging diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index bfff114c3..e39b8438d 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -4,29 +4,23 @@ #include #include +#include +#include #include #include #include #include -#if ZEN_PLATFORM_LINUX +#if ZEN_PLATFORM_LINUX || ZEN_PLATFORM_MAC # include +# include #endif -#if ZEN_PLATFORM_MAC -# include -#endif - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - #if ZEN_USE_SENTRY # define SENTRY_BUILD_STATIC 1 ZEN_THIRD_PARTY_INCLUDES_START # include -# include ZEN_THIRD_PARTY_INCLUDES_END namespace sentry { @@ -44,76 +38,58 @@ struct SentryAssertImpl : zen::AssertImpl const zen::CallstackFrames* Callstack) override; }; -class sentry_sink final : public spdlog::sinks::base_sink +static constexpr sentry_level_t MapToSentryLevel[zen::logging::LogLevelCount] = {SENTRY_LEVEL_DEBUG, + SENTRY_LEVEL_DEBUG, + SENTRY_LEVEL_INFO, + SENTRY_LEVEL_WARNING, + SENTRY_LEVEL_ERROR, + SENTRY_LEVEL_FATAL, + SENTRY_LEVEL_DEBUG}; + +class SentrySink final : public zen::logging::Sink { public: - sentry_sink(); - ~sentry_sink(); + SentrySink() = default; + ~SentrySink() = default; + + void Log(const zen::logging::LogMessage& Msg) override + { + if (Msg.GetLevel() != zen::logging::Err && Msg.GetLevel() != zen::logging::Critical) + { + return; + } + try + { + std::string Message = fmt::format("{}\n{}({})", Msg.GetPayload(), Msg.GetSource().Filename, Msg.GetSource().Line); + sentry_value_t Event = sentry_value_new_message_event( + /* level */ MapToSentryLevel[Msg.GetLevel()], + /* logger */ nullptr, + /* message */ Message.c_str()); + sentry_event_value_add_stacktrace(Event, NULL, 0); + sentry_capture_event(Event); + } + catch (const std::exception&) + { + // If our logging with Message formatting fails we do a non-allocating version and just post the payload raw + char TmpBuffer[256]; + size_t MaxCopy = zen::Min(Msg.GetPayload().size(), size_t(255)); + memcpy(TmpBuffer, Msg.GetPayload().data(), MaxCopy); + TmpBuffer[MaxCopy] = '\0'; + sentry_value_t Event = sentry_value_new_message_event( + /* level */ SENTRY_LEVEL_ERROR, + /* logger */ nullptr, + /* message */ TmpBuffer); + sentry_event_value_add_stacktrace(Event, NULL, 0); + sentry_capture_event(Event); + } + } -protected: - void sink_it_(const spdlog::details::log_msg& msg) override; - void flush_() override; + void Flush() override {} + void SetFormatter(std::unique_ptr) override {} }; ////////////////////////////////////////////////////////////////////////// -static constexpr sentry_level_t MapToSentryLevel[spdlog::level::level_enum::n_levels] = {SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_DEBUG, - SENTRY_LEVEL_INFO, - SENTRY_LEVEL_WARNING, - SENTRY_LEVEL_ERROR, - SENTRY_LEVEL_FATAL, - SENTRY_LEVEL_DEBUG}; - -sentry_sink::sentry_sink() -{ -} -sentry_sink::~sentry_sink() -{ -} - -void -sentry_sink::sink_it_(const spdlog::details::log_msg& msg) -{ - if (msg.level != spdlog::level::err && msg.level != spdlog::level::critical) - { - return; - } - try - { - auto MaybeNullString = [](const char* Ptr) { return Ptr ? Ptr : ""; }; - std::string Message = fmt::format("{}\n{}({}) [{}]", - msg.payload, - MaybeNullString(msg.source.filename), - msg.source.line, - MaybeNullString(msg.source.funcname)); - sentry_value_t event = sentry_value_new_message_event( - /* level */ MapToSentryLevel[msg.level], - /* logger */ nullptr, - /* message */ Message.c_str()); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } - catch (const std::exception&) - { - // If our logging with Message formatting fails we do a non-allocating version and just post the msg.payload raw - char TmpBuffer[256]; - size_t MaxCopy = zen::Min(msg.payload.size(), size_t(255)); - memcpy(TmpBuffer, msg.payload.data(), MaxCopy); - TmpBuffer[MaxCopy] = '\0'; - sentry_value_t event = sentry_value_new_message_event( - /* level */ SENTRY_LEVEL_ERROR, - /* logger */ nullptr, - /* message */ TmpBuffer); - sentry_event_value_add_stacktrace(event, NULL, 0); - sentry_capture_event(event); - } -} -void -sentry_sink::flush_() -{ -} - void SentryAssertImpl::OnAssert(const char* Filename, int LineNumber, @@ -340,7 +316,9 @@ SentryIntegration::Initialize(const Config& Conf, const std::string& CommandLine sentry_set_user(SentryUserObject); - m_SentryLogger = spdlog::create("sentry"); + logging::SinkPtr SentrySink(new sentry::SentrySink()); + m_SentryLogger = Ref(new logging::Logger("sentry", std::vector{SentrySink})); + logging::Registry::Instance().Register(m_SentryLogger); logging::SetErrorLog("sentry"); m_SentryAssert = std::make_unique(); @@ -354,7 +332,7 @@ SentryIntegration::LogStartupInformation() { // Initialize the sentry-sdk log category at Warn level to reduce startup noise. // The level can be overridden via --log-debug=sentry-sdk or --log-info=sentry-sdk - LogSentry.Logger().SetLogLevel(logging::level::Warn); + LogSentry.Logger().SetLogLevel(logging::Warn); if (m_IsInitialized) { diff --git a/src/zencore/testing.cpp b/src/zencore/testing.cpp index 0bae139bd..089e376bb 100644 --- a/src/zencore/testing.cpp +++ b/src/zencore/testing.cpp @@ -143,11 +143,11 @@ TestRunner::ApplyCommandLine(int Argc, char const* const* Argv) { if (Argv[i] == "--debug"sv) { - zen::logging::SetLogLevel(zen::logging::level::Debug); + zen::logging::SetLogLevel(zen::logging::Debug); } else if (Argv[i] == "--verbose"sv) { - zen::logging::SetLogLevel(zen::logging::level::Trace); + zen::logging::SetLogLevel(zen::logging::Trace); } } diff --git a/src/zencore/xmake.lua b/src/zencore/xmake.lua index 2f81b7ec8..171f4c533 100644 --- a/src/zencore/xmake.lua +++ b/src/zencore/xmake.lua @@ -26,7 +26,6 @@ target('zencore') end add_deps("zenbase") - add_deps("spdlog") add_deps("utfcpp") add_deps("oodle") add_deps("blake3") diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp index d82474705..8c29a8962 100644 --- a/src/zencore/zencore.cpp +++ b/src/zencore/zencore.cpp @@ -147,7 +147,7 @@ AssertImpl::OnAssert(const char* Filename, int LineNumber, const char* FunctionN Message.push_back('\0'); // We use direct ZEN_LOG here instead of ZEN_ERROR as we don't care about *this* code location in the log - ZEN_LOG(Log(), zen::logging::level::Err, "{}", Message.data()); + ZEN_LOG(Log(), zen::logging::Err, "{}", Message.data()); zen::logging::FlushLogging(); } diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 33f182df9..2cf051d14 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -150,7 +150,7 @@ inline LoggerRef InitLogger() { LoggerRef Logger = logging::Get("asio"); - // Logger.SetLogLevel(logging::level::Trace); + // Logger.SetLogLevel(logging::Trace); return Logger; } @@ -1256,7 +1256,7 @@ HttpServerConnection::HandleRequest() const HttpVerb RequestVerb = Request.RequestVerb(); const std::string_view Uri = Request.RelativeUri(); - if (m_Server.m_RequestLog.ShouldLog(logging::level::Trace)) + if (m_Server.m_RequestLog.ShouldLog(logging::Trace)) { ZEN_LOG_TRACE(m_Server.m_RequestLog, "connection #{} Handling Request: {} {} ({} bytes ({}), accept: {})", diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index 850dafdca..021b941bd 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -383,7 +383,7 @@ HttpPluginConnectionHandler::HandleRequest() const HttpVerb RequestVerb = Request.RequestVerb(); const std::string_view Uri = Request.RelativeUri(); - if (m_Server->m_RequestLog.ShouldLog(logging::level::Trace)) + if (m_Server->m_RequestLog.ShouldLog(logging::Trace)) { ZEN_LOG_TRACE(m_Server->m_RequestLog, "connection #{} Handling Request: {} {} ({} bytes ({}), accept: {})", @@ -480,7 +480,7 @@ HttpPluginConnectionHandler::HandleRequest() const std::vector& ResponseBuffers = Response->ResponseBuffers(); - if (m_Server->m_RequestLog.ShouldLog(logging::level::Trace)) + if (m_Server->m_RequestLog.ShouldLog(logging::Trace)) { m_Server->m_RequestTracer.WriteDebugPayload(fmt::format("response_{}_{}.bin", m_ConnectionId, RequestNumber), ResponseBuffers); diff --git a/src/zenhttp/transports/dlltransport.cpp b/src/zenhttp/transports/dlltransport.cpp index 9135d5425..489324aba 100644 --- a/src/zenhttp/transports/dlltransport.cpp +++ b/src/zenhttp/transports/dlltransport.cpp @@ -72,20 +72,36 @@ DllTransportLogger::DllTransportLogger(std::string_view PluginName) : m_PluginNa void DllTransportLogger::LogMessage(LogLevel PluginLogLevel, const char* Message) { - logging::level::LogLevel Level; - // clang-format off switch (PluginLogLevel) { - case LogLevel::Trace: Level = logging::level::Trace; break; - case LogLevel::Debug: Level = logging::level::Debug; break; - case LogLevel::Info: Level = logging::level::Info; break; - case LogLevel::Warn: Level = logging::level::Warn; break; - case LogLevel::Err: Level = logging::level::Err; break; - case LogLevel::Critical: Level = logging::level::Critical; break; - default: Level = logging::level::Off; break; + case LogLevel::Trace: + ZEN_TRACE("[{}] {}", m_PluginName, Message); + return; + + case LogLevel::Debug: + ZEN_DEBUG("[{}] {}", m_PluginName, Message); + return; + + case LogLevel::Info: + ZEN_INFO("[{}] {}", m_PluginName, Message); + return; + + case LogLevel::Warn: + ZEN_WARN("[{}] {}", m_PluginName, Message); + return; + + case LogLevel::Err: + ZEN_ERROR("[{}] {}", m_PluginName, Message); + return; + + case LogLevel::Critical: + ZEN_CRITICAL("[{}] {}", m_PluginName, Message); + return; + + default: + ZEN_UNUSED(Message); + break; } - // clang-format on - ZEN_LOG(Log(), Level, "[{}] {}", m_PluginName, Message) } uint32_t diff --git a/src/zenremotestore/include/zenremotestore/operationlogoutput.h b/src/zenremotestore/include/zenremotestore/operationlogoutput.h index 6f10ab156..32b95f50f 100644 --- a/src/zenremotestore/include/zenremotestore/operationlogoutput.h +++ b/src/zenremotestore/include/zenremotestore/operationlogoutput.h @@ -11,7 +11,7 @@ class OperationLogOutput { public: virtual ~OperationLogOutput() {} - virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) = 0; + virtual void EmitLogMessage(const logging::LogPoint& Point, fmt::format_args Args) = 0; virtual void SetLogOperationName(std::string_view Name) = 0; virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) = 0; @@ -60,19 +60,17 @@ public: OperationLogOutput* CreateStandardLogOutput(LoggerRef Log); -#define ZEN_OPERATION_LOG(OutputTarget, InLevel, fmtstr, ...) \ - do \ - { \ - using namespace std::literals; \ - ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ - OutputTarget.EmitLogMessage(InLevel, fmtstr, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ +#define ZEN_OPERATION_LOG(OutputTarget, InLevel, fmtstr, ...) \ + do \ + { \ + using namespace std::literals; \ + static constinit zen::logging::LogPoint LogPoint{{}, InLevel, std::string_view(fmtstr)}; \ + ZEN_CHECK_FORMAT_STRING(fmtstr##sv, ##__VA_ARGS__); \ + (OutputTarget).EmitLogMessage(LogPoint, zen::logging::LogCaptureArguments(__VA_ARGS__)); \ } while (false) -#define ZEN_OPERATION_LOG_INFO(OutputTarget, fmtstr, ...) \ - ZEN_OPERATION_LOG((OutputTarget), zen::logging::level::Info, fmtstr, ##__VA_ARGS__) -#define ZEN_OPERATION_LOG_DEBUG(OutputTarget, fmtstr, ...) \ - ZEN_OPERATION_LOG((OutputTarget), zen::logging::level::Debug, fmtstr, ##__VA_ARGS__) -#define ZEN_OPERATION_LOG_WARN(OutputTarget, fmtstr, ...) \ - ZEN_OPERATION_LOG((OutputTarget), zen::logging::level::Warn, fmtstr, ##__VA_ARGS__) +#define ZEN_OPERATION_LOG_INFO(OutputTarget, fmtstr, ...) ZEN_OPERATION_LOG(OutputTarget, zen::logging::Info, fmtstr, ##__VA_ARGS__) +#define ZEN_OPERATION_LOG_DEBUG(OutputTarget, fmtstr, ...) ZEN_OPERATION_LOG(OutputTarget, zen::logging::Debug, fmtstr, ##__VA_ARGS__) +#define ZEN_OPERATION_LOG_WARN(OutputTarget, fmtstr, ...) ZEN_OPERATION_LOG(OutputTarget, zen::logging::Warn, fmtstr, ##__VA_ARGS__) } // namespace zen diff --git a/src/zenremotestore/operationlogoutput.cpp b/src/zenremotestore/operationlogoutput.cpp index 7ed93c947..5ed844c9d 100644 --- a/src/zenremotestore/operationlogoutput.cpp +++ b/src/zenremotestore/operationlogoutput.cpp @@ -3,6 +3,7 @@ #include #include +#include ZEN_THIRD_PARTY_INCLUDES_START #include @@ -30,13 +31,11 @@ class StandardLogOutput : public OperationLogOutput { public: StandardLogOutput(LoggerRef& Log) : m_Log(Log) {} - virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) override + virtual void EmitLogMessage(const logging::LogPoint& Point, fmt::format_args Args) override { - if (m_Log.ShouldLog(LogLevel)) + if (m_Log.ShouldLog(Point.Level)) { - fmt::basic_memory_buffer MessageBuffer; - fmt::vformat_to(fmt::appender(MessageBuffer), Format, Args); - ZEN_LOG(m_Log, LogLevel, "{}", std::string_view(MessageBuffer.data(), MessageBuffer.size())); + m_Log->Log(Point, Args); } } @@ -47,7 +46,7 @@ public: } virtual void SetLogOperationProgress(uint32_t StepIndex, uint32_t StepCount) override { - const size_t PercentDone = StepCount > 0u ? gsl::narrow((100 * StepIndex) / StepCount) : 0u; + [[maybe_unused]] const size_t PercentDone = StepCount > 0u ? gsl::narrow((100 * StepIndex) / StepCount) : 0u; ZEN_OPERATION_LOG_INFO(*this, "{}: {}%", m_LogOperationName, PercentDone); } virtual uint32_t GetProgressUpdateDelayMS() override { return 2000; } @@ -59,13 +58,14 @@ public: private: LoggerRef m_Log; std::string m_LogOperationName; + LoggerRef Log() { return m_Log; } }; void StandardLogOutputProgressBar::UpdateState(const State& NewState, bool DoLinebreak) { ZEN_UNUSED(DoLinebreak); - const size_t PercentDone = + [[maybe_unused]] const size_t PercentDone = NewState.TotalCount > 0u ? gsl::narrow((100 * (NewState.TotalCount - NewState.RemainingCount)) / NewState.TotalCount) : 0u; std::string Task = NewState.Task; switch (NewState.Status) diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index 78f6014df..c8c5f201d 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -246,13 +246,12 @@ namespace remotestore_impl { { public: JobContextLogOutput(JobContext* OptionalContext) : m_OptionalContext(OptionalContext) {} - virtual void EmitLogMessage(int LogLevel, std::string_view Format, fmt::format_args Args) override + virtual void EmitLogMessage(const logging::LogPoint& Point, fmt::format_args Args) override { - ZEN_UNUSED(LogLevel); if (m_OptionalContext) { fmt::basic_memory_buffer MessageBuffer; - fmt::vformat_to(fmt::appender(MessageBuffer), Format, Args); + fmt::vformat_to(fmt::appender(MessageBuffer), Point.FormatString, Args); remotestore_impl::ReportMessage(m_OptionalContext, std::string_view(MessageBuffer.data(), MessageBuffer.size())); } } diff --git a/src/zenserver-test/logging-tests.cpp b/src/zenserver-test/logging-tests.cpp index f284f0371..2e530ff92 100644 --- a/src/zenserver-test/logging-tests.cpp +++ b/src/zenserver-test/logging-tests.cpp @@ -71,7 +71,7 @@ TEST_CASE("logging.file.default") // entry written by the default logger's console sink must therefore not appear // in captured stdout. (The "console" named logger — used by ZEN_CONSOLE_* // macros — may still emit plain-text messages without a level marker, so we -// check for the absence of the full_formatter "[info]" prefix rather than the +// check for the absence of the FullFormatter "[info]" prefix rather than the // message text itself.) TEST_CASE("logging.console.quiet") { diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp index bd36d731f..8d5400294 100644 --- a/src/zenserver-test/zenserver-test.cpp +++ b/src/zenserver-test/zenserver-test.cpp @@ -9,6 +9,7 @@ # include # include # include +# include # include # include # include @@ -17,7 +18,7 @@ # include # include # include -# include +# include # include # include @@ -85,8 +86,9 @@ main(int argc, char** argv) zen::logging::InitializeLogging(); - zen::logging::SetLogLevel(zen::logging::level::Debug); - spdlog::set_formatter(std::make_unique("test", std::chrono::system_clock::now())); + zen::logging::SetLogLevel(zen::logging::Debug); + zen::logging::Registry::Instance().SetFormatter( + std::make_unique("test", std::chrono::system_clock::now())); std::filesystem::path ProgramBaseDir = GetRunningExecutablePath().parent_path(); std::filesystem::path TestBaseDir = std::filesystem::current_path() / ".test"; diff --git a/src/zenserver/diag/diagsvcs.cpp b/src/zenserver/diag/diagsvcs.cpp index d8d53b0e3..5fa81ff9f 100644 --- a/src/zenserver/diag/diagsvcs.cpp +++ b/src/zenserver/diag/diagsvcs.cpp @@ -12,9 +12,7 @@ #include #include -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END +#include namespace zen { @@ -64,7 +62,7 @@ HttpHealthService::HttpHealthService() [this](HttpRouterRequest& RoutedReq) { HttpServerRequest& HttpReq = RoutedReq.ServerRequest(); - zen::Log().SpdLogger->flush(); + zen::Log().Flush(); std::filesystem::path Path = [&] { RwLock::SharedLockScope _(m_InfoLock); diff --git a/src/zenserver/diag/logging.cpp b/src/zenserver/diag/logging.cpp index 75a8efc09..178c3d3b5 100644 --- a/src/zenserver/diag/logging.cpp +++ b/src/zenserver/diag/logging.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include #include @@ -14,10 +16,6 @@ #include "otlphttp.h" -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - namespace zen { void @@ -43,13 +41,12 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService) std::filesystem::path HttpLogPath = InOptions.DataDir / "logs" / "http.log"; zen::CreateDirectories(HttpLogPath.parent_path()); - auto HttpSink = std::make_shared(HttpLogPath, - /* max size */ 128 * 1024 * 1024, - /* max files */ 16, - /* rotate on open */ true); - auto HttpLogger = std::make_shared("http_requests", HttpSink); - spdlog::apply_logger_env_levels(HttpLogger); - spdlog::register_logger(HttpLogger); + logging::SinkPtr HttpSink(new zen::logging::RotatingFileSink(HttpLogPath, + /* max size */ 128 * 1024 * 1024, + /* max files */ 16, + /* rotate on open */ true)); + Ref HttpLogger(new logging::Logger("http_requests", std::vector{HttpSink})); + logging::Registry::Instance().Register(HttpLogger); if (WithCacheService) { @@ -57,33 +54,30 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService) std::filesystem::path CacheLogPath = InOptions.DataDir / "logs" / "z$.log"; zen::CreateDirectories(CacheLogPath.parent_path()); - auto CacheSink = std::make_shared(CacheLogPath, - /* max size */ 128 * 1024 * 1024, - /* max files */ 16, - /* rotate on open */ false); - auto CacheLogger = std::make_shared("z$", CacheSink); - spdlog::apply_logger_env_levels(CacheLogger); - spdlog::register_logger(CacheLogger); + logging::SinkPtr CacheSink(new zen::logging::RotatingFileSink(CacheLogPath, + /* max size */ 128 * 1024 * 1024, + /* max files */ 16, + /* rotate on open */ false)); + Ref CacheLogger(new logging::Logger("z$", std::vector{CacheSink})); + logging::Registry::Instance().Register(CacheLogger); // Jupiter - only log upstream HTTP traffic to file - auto JupiterLogger = std::make_shared("jupiter", FileSink); - spdlog::apply_logger_env_levels(JupiterLogger); - spdlog::register_logger(JupiterLogger); + Ref JupiterLogger(new logging::Logger("jupiter", std::vector{FileSink})); + logging::Registry::Instance().Register(JupiterLogger); // Zen - only log upstream HTTP traffic to file - auto ZenClientLogger = std::make_shared("zenclient", FileSink); - spdlog::apply_logger_env_levels(ZenClientLogger); - spdlog::register_logger(ZenClientLogger); + Ref ZenClientLogger(new logging::Logger("zenclient", std::vector{FileSink})); + logging::Registry::Instance().Register(ZenClientLogger); } #if ZEN_WITH_OTEL if (!InOptions.LoggingConfig.OtelEndpointUri.empty()) { // TODO: Should sanity check that endpoint is reachable? Also, a valid URI? - auto OtelSink = std::make_shared(InOptions.LoggingConfig.OtelEndpointUri); - zen::logging::Default().SpdLogger->sinks().push_back(std::move(OtelSink)); + logging::SinkPtr OtelSink(new zen::logging::OtelHttpProtobufSink(InOptions.LoggingConfig.OtelEndpointUri)); + zen::logging::Default()->AddSink(std::move(OtelSink)); } #endif @@ -91,9 +85,10 @@ InitializeServerLogging(const ZenServerConfig& InOptions, bool WithCacheService) const zen::Oid ServerSessionId = zen::GetSessionId(); - spdlog::apply_all([&](auto Logger) { + static constinit logging::LogPoint SessionIdPoint{{}, logging::Info, "server session id: {}"}; + logging::Registry::Instance().ApplyAll([&](auto Logger) { ZEN_MEMSCOPE(ELLMTag::Logging); - Logger->info("server session id: {}", ServerSessionId); + Logger->Log(SessionIdPoint, fmt::make_format_args(ServerSessionId)); }); } diff --git a/src/zenserver/diag/otlphttp.cpp b/src/zenserver/diag/otlphttp.cpp index d62ccccb6..1434c9331 100644 --- a/src/zenserver/diag/otlphttp.cpp +++ b/src/zenserver/diag/otlphttp.cpp @@ -53,7 +53,7 @@ OtelHttpProtobufSink::TraceRecorder::RecordSpans(zen::otel::TraceId Trace, std:: } void -OtelHttpProtobufSink::log(const spdlog::details::log_msg& Msg) +OtelHttpProtobufSink::Log(const LogMessage& Msg) { { std::string Data = m_Encoder.FormatOtelProtobuf(Msg); @@ -74,7 +74,7 @@ OtelHttpProtobufSink::log(const spdlog::details::log_msg& Msg) } } void -OtelHttpProtobufSink::flush() +OtelHttpProtobufSink::Flush() { } diff --git a/src/zenserver/diag/otlphttp.h b/src/zenserver/diag/otlphttp.h index 2281bdcc0..8254af04d 100644 --- a/src/zenserver/diag/otlphttp.h +++ b/src/zenserver/diag/otlphttp.h @@ -3,7 +3,7 @@ #pragma once -#include +#include #include #include #include @@ -14,12 +14,12 @@ namespace zen::logging { /** - * OTLP/HTTP sink for spdlog + * OTLP/HTTP sink for logging * * Sends log messages and traces to an OpenTelemetry collector via OTLP over HTTP */ -class OtelHttpProtobufSink : public spdlog::sinks::sink +class OtelHttpProtobufSink : public Sink { public: // Note that this URI should be the base URI of the OTLP HTTP endpoint, e.g. @@ -31,10 +31,9 @@ public: OtelHttpProtobufSink& operator=(const OtelHttpProtobufSink&) = delete; private: - virtual void log(const spdlog::details::log_msg& Msg) override; - virtual void flush() override; - virtual void set_pattern(const std::string& pattern) override { ZEN_UNUSED(pattern); } - virtual void set_formatter(std::unique_ptr sink_formatter) override { ZEN_UNUSED(sink_formatter); } + virtual void Log(const LogMessage& Msg) override; + virtual void Flush() override; + virtual void SetFormatter(std::unique_ptr) override {} void RecordSpans(zen::otel::TraceId Trace, std::span Spans); @@ -61,4 +60,4 @@ private: } // namespace zen::logging -#endif \ No newline at end of file +#endif diff --git a/src/zenserver/main.cpp b/src/zenserver/main.cpp index c764cbde6..09ecc48e5 100644 --- a/src/zenserver/main.cpp +++ b/src/zenserver/main.cpp @@ -246,7 +246,7 @@ test_main(int argc, char** argv) # endif // ZEN_PLATFORM_WINDOWS zen::logging::InitializeLogging(); - zen::logging::SetLogLevel(zen::logging::level::Debug); + zen::logging::SetLogLevel(zen::logging::Debug); zen::MaximizeOpenFileCount(); diff --git a/src/zenserver/storage/admin/admin.cpp b/src/zenserver/storage/admin/admin.cpp index 19155e02b..c9f999c69 100644 --- a/src/zenserver/storage/admin/admin.cpp +++ b/src/zenserver/storage/admin/admin.cpp @@ -716,7 +716,7 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, "logs", [this](HttpRouterRequest& Req) { CbObjectWriter Obj; - auto LogLevel = logging::level::ToStringView(logging::GetLogLevel()); + auto LogLevel = logging::ToStringView(logging::GetLogLevel()); Obj.AddString("loglevel", std::string_view(LogLevel.data(), LogLevel.size())); Obj.AddString("Logfile", PathToUtf8(m_LogPaths.AbsLogPath)); Obj.BeginObject("cache"); @@ -767,8 +767,8 @@ HttpAdminService::HttpAdminService(GcScheduler& Scheduler, } if (std::string Param(Params.GetValue("loglevel")); Param.empty() == false) { - logging::level::LogLevel NewLevel = logging::level::ParseLogLevelString(Param); - std::string_view LogLevel = logging::level::ToStringView(NewLevel); + logging::LogLevel NewLevel = logging::ParseLogLevelString(Param); + std::string_view LogLevel = logging::ToStringView(NewLevel); if (LogLevel != Param) { return Req.ServerRequest().WriteResponse(HttpResponseCode::BadRequest, diff --git a/src/zenstore/projectstore.cpp b/src/zenstore/projectstore.cpp index 3f705d12c..1706c9105 100644 --- a/src/zenstore/projectstore.cpp +++ b/src/zenstore/projectstore.cpp @@ -4360,7 +4360,7 @@ ProjectStore::ProjectStore(CidStore& Store, std::filesystem::path BasePath, GcMa , m_DiskWriteBlocker(Gc.GetDiskWriteBlocker()) { ZEN_INFO("initializing project store at '{}'", m_ProjectBasePath); - // m_Log.set_level(spdlog::level::debug); + // m_Log.SetLogLevel(zen::logging::Debug); m_Gc.AddGcStorage(this); m_Gc.AddGcReferencer(*this); m_Gc.AddGcReferenceLocker(*this); diff --git a/src/zentelemetry/include/zentelemetry/otlpencoder.h b/src/zentelemetry/include/zentelemetry/otlpencoder.h index ed6665781..f280aa9ec 100644 --- a/src/zentelemetry/include/zentelemetry/otlpencoder.h +++ b/src/zentelemetry/include/zentelemetry/otlpencoder.h @@ -13,9 +13,9 @@ # include # include -namespace spdlog { namespace details { - struct log_msg; -}} // namespace spdlog::details +namespace zen::logging { +struct LogMessage; +} // namespace zen::logging namespace zen::otel { enum class Resource : protozero::pbf_tag_type; @@ -46,7 +46,7 @@ public: void AddResourceAttribute(const std::string_view& Key, const std::string_view& Value); void AddResourceAttribute(const std::string_view& Key, int64_t Value); - std::string FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const; + std::string FormatOtelProtobuf(const logging::LogMessage& Msg) const; std::string FormatOtelMetrics() const; std::string FormatOtelTrace(zen::otel::TraceId Trace, std::span Spans) const; diff --git a/src/zentelemetry/otlpencoder.cpp b/src/zentelemetry/otlpencoder.cpp index 677545066..5477c5381 100644 --- a/src/zentelemetry/otlpencoder.cpp +++ b/src/zentelemetry/otlpencoder.cpp @@ -3,9 +3,9 @@ #include "zentelemetry/otlpencoder.h" #include +#include #include -#include #include #include @@ -29,49 +29,49 @@ OtlpEncoder::~OtlpEncoder() } static int -MapSeverity(const spdlog::level::level_enum Level) +MapSeverity(const logging::LogLevel Level) { switch (Level) { - case spdlog::level::critical: + case logging::Critical: return otel::SEVERITY_NUMBER_FATAL; - case spdlog::level::err: + case logging::Err: return otel::SEVERITY_NUMBER_ERROR; - case spdlog::level::warn: + case logging::Warn: return otel::SEVERITY_NUMBER_WARN; - case spdlog::level::info: + case logging::Info: return otel::SEVERITY_NUMBER_INFO; - case spdlog::level::debug: + case logging::Debug: return otel::SEVERITY_NUMBER_DEBUG; default: - case spdlog::level::trace: + case logging::Trace: return otel::SEVERITY_NUMBER_TRACE; } } static const char* -MapSeverityText(const spdlog::level::level_enum Level) +MapSeverityText(const logging::LogLevel Level) { switch (Level) { - case spdlog::level::critical: + case logging::Critical: return "fatal"; - case spdlog::level::err: + case logging::Err: return "error"; - case spdlog::level::warn: + case logging::Warn: return "warn"; - case spdlog::level::info: + case logging::Info: return "info"; - case spdlog::level::debug: + case logging::Debug: return "debug"; default: - case spdlog::level::trace: + case logging::Trace: return "trace"; } } std::string -OtlpEncoder::FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const +OtlpEncoder::FormatOtelProtobuf(const logging::LogMessage& Msg) const { std::string Data; @@ -98,7 +98,7 @@ OtlpEncoder::FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const protozero::pbf_builder IsBuilder{SlBuilder, otel::ScopeLogs::required_InstrumentationScope_scope}; - IsBuilder.add_string(otel::InstrumentationScope::string_name, Msg.logger_name.data(), Msg.logger_name.size()); + IsBuilder.add_string(otel::InstrumentationScope::string_name, Msg.GetLoggerName().data(), Msg.GetLoggerName().size()); } // LogRecord log_records @@ -106,13 +106,13 @@ OtlpEncoder::FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const protozero::pbf_builder LrBuilder{SlBuilder, otel::ScopeLogs::required_repeated_LogRecord_log_records}; LrBuilder.add_fixed64(otel::LogRecord::required_fixed64_time_unix_nano, - std::chrono::duration_cast(Msg.time.time_since_epoch()).count()); + std::chrono::duration_cast(Msg.GetTime().time_since_epoch()).count()); - const int Severity = MapSeverity(Msg.level); + const int Severity = MapSeverity(Msg.GetLevel()); LrBuilder.add_enum(otel::LogRecord::optional_SeverityNumber_severity_number, Severity); - LrBuilder.add_string(otel::LogRecord::optional_string_severity_text, MapSeverityText(Msg.level)); + LrBuilder.add_string(otel::LogRecord::optional_string_severity_text, MapSeverityText(Msg.GetLevel())); otel::TraceId TraceId; const otel::SpanId SpanId = otel::Span::GetCurrentSpanId(TraceId); @@ -127,7 +127,7 @@ OtlpEncoder::FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const { protozero::pbf_builder BodyBuilder{LrBuilder, otel::LogRecord::optional_anyvalue_body}; - BodyBuilder.add_string(otel::AnyValue::string_string_value, Msg.payload.data(), Msg.payload.size()); + BodyBuilder.add_string(otel::AnyValue::string_string_value, Msg.GetPayload().data(), Msg.GetPayload().size()); } // attributes @@ -139,7 +139,7 @@ OtlpEncoder::FormatOtelProtobuf(const spdlog::details::log_msg& Msg) const { protozero::pbf_builder AvBuilder{KvBuilder, otel::KeyValue::AnyValue_value}; - AvBuilder.add_int64(otel::AnyValue::int64_int_value, Msg.thread_id); + AvBuilder.add_int64(otel::AnyValue::int64_int_value, Msg.GetThreadId()); } } } diff --git a/src/zentelemetry/xmake.lua b/src/zentelemetry/xmake.lua index 7739c0a08..cd9a18ec4 100644 --- a/src/zentelemetry/xmake.lua +++ b/src/zentelemetry/xmake.lua @@ -6,5 +6,5 @@ target('zentelemetry') add_headerfiles("**.h") add_files("**.cpp") add_includedirs("include", {public=true}) - add_deps("zencore", "protozero", "spdlog") + add_deps("zencore", "protozero") add_deps("robin-map") diff --git a/src/zenutil/config/commandlineoptions.cpp b/src/zenutil/config/commandlineoptions.cpp index 2344354b3..25f5522d8 100644 --- a/src/zenutil/config/commandlineoptions.cpp +++ b/src/zenutil/config/commandlineoptions.cpp @@ -2,6 +2,7 @@ #include +#include #include #include diff --git a/src/zenutil/config/loggingconfig.cpp b/src/zenutil/config/loggingconfig.cpp index 9ec816b1b..5092c60aa 100644 --- a/src/zenutil/config/loggingconfig.cpp +++ b/src/zenutil/config/loggingconfig.cpp @@ -21,13 +21,13 @@ ZenLoggingCmdLineOptions::AddCliOptions(cxxopts::Options& options, ZenLoggingCon ("log-id", "Specify id for adding context to log output", cxxopts::value(LoggingConfig.LogId)) ("quiet", "Configure console logger output to level WARN", cxxopts::value(LoggingConfig.QuietConsole)->default_value("false")) ("noconsole", "Disable console logging", cxxopts::value(LoggingConfig.NoConsoleOutput)->default_value("false")) - ("log-trace", "Change selected loggers to level TRACE", cxxopts::value(LoggingConfig.Loggers[logging::level::Trace])) - ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value(LoggingConfig.Loggers[logging::level::Debug])) - ("log-info", "Change selected loggers to level INFO", cxxopts::value(LoggingConfig.Loggers[logging::level::Info])) - ("log-warn", "Change selected loggers to level WARN", cxxopts::value(LoggingConfig.Loggers[logging::level::Warn])) - ("log-error", "Change selected loggers to level ERROR", cxxopts::value(LoggingConfig.Loggers[logging::level::Err])) - ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value(LoggingConfig.Loggers[logging::level::Critical])) - ("log-off", "Change selected loggers to level OFF", cxxopts::value(LoggingConfig.Loggers[logging::level::Off])) + ("log-trace", "Change selected loggers to level TRACE", cxxopts::value(LoggingConfig.Loggers[logging::Trace])) + ("log-debug", "Change selected loggers to level DEBUG", cxxopts::value(LoggingConfig.Loggers[logging::Debug])) + ("log-info", "Change selected loggers to level INFO", cxxopts::value(LoggingConfig.Loggers[logging::Info])) + ("log-warn", "Change selected loggers to level WARN", cxxopts::value(LoggingConfig.Loggers[logging::Warn])) + ("log-error", "Change selected loggers to level ERROR", cxxopts::value(LoggingConfig.Loggers[logging::Err])) + ("log-critical", "Change selected loggers to level CRITICAL", cxxopts::value(LoggingConfig.Loggers[logging::Critical])) + ("log-off", "Change selected loggers to level OFF", cxxopts::value(LoggingConfig.Loggers[logging::Off])) ("otlp-endpoint", "OpenTelemetry endpoint URI (e.g http://localhost:4318)", cxxopts::value(LoggingConfig.OtelEndpointUri)) ; // clang-format on @@ -47,7 +47,7 @@ ApplyLoggingOptions(cxxopts::Options& options, ZenLoggingConfig& LoggingConfig) if (LoggingConfig.QuietConsole) { bool HasExplicitConsoleLevel = false; - for (int i = 0; i < logging::level::LogLevelCount; ++i) + for (int i = 0; i < logging::LogLevelCount; ++i) { if (LoggingConfig.Loggers[i].find("console") != std::string::npos) { @@ -58,7 +58,7 @@ ApplyLoggingOptions(cxxopts::Options& options, ZenLoggingConfig& LoggingConfig) if (!HasExplicitConsoleLevel) { - std::string& WarnLoggers = LoggingConfig.Loggers[logging::level::Warn]; + std::string& WarnLoggers = LoggingConfig.Loggers[logging::Warn]; if (!WarnLoggers.empty()) { WarnLoggers += ","; @@ -67,9 +67,9 @@ ApplyLoggingOptions(cxxopts::Options& options, ZenLoggingConfig& LoggingConfig) } } - for (int i = 0; i < logging::level::LogLevelCount; ++i) + for (int i = 0; i < logging::LogLevelCount; ++i) { - logging::ConfigureLogLevels(logging::level::LogLevel(i), LoggingConfig.Loggers[i]); + logging::ConfigureLogLevels(logging::LogLevel(i), LoggingConfig.Loggers[i]); } logging::RefreshLogLevels(); } diff --git a/src/zenutil/include/zenutil/config/loggingconfig.h b/src/zenutil/include/zenutil/config/loggingconfig.h index 6d6f64b30..b55b2d9f7 100644 --- a/src/zenutil/include/zenutil/config/loggingconfig.h +++ b/src/zenutil/include/zenutil/config/loggingconfig.h @@ -17,7 +17,7 @@ struct ZenLoggingConfig bool NoConsoleOutput = false; // Control default use of stdout for diagnostics bool QuietConsole = false; // Configure console logger output to level WARN std::filesystem::path AbsLogFile; // Absolute path to main log file - std::string Loggers[logging::level::LogLevelCount]; + std::string Loggers[logging::LogLevelCount]; std::string LogId; // Id for tagging log output std::string OtelEndpointUri; // OpenTelemetry endpoint URI }; diff --git a/src/zenutil/include/zenutil/logging.h b/src/zenutil/include/zenutil/logging.h index 85ddc86cd..95419c274 100644 --- a/src/zenutil/include/zenutil/logging.h +++ b/src/zenutil/include/zenutil/logging.h @@ -3,19 +3,12 @@ #pragma once #include +#include #include #include #include -namespace spdlog::sinks { -class sink; -} - -namespace spdlog { -using sink_ptr = std::shared_ptr; -} - ////////////////////////////////////////////////////////////////////////// // // Logging utilities @@ -45,6 +38,6 @@ void FinishInitializeLogging(const LoggingOptions& LoggingOptions); void InitializeLogging(const LoggingOptions& LoggingOptions); void ShutdownLogging(); -spdlog::sink_ptr GetFileSink(); +logging::SinkPtr GetFileSink(); } // namespace zen diff --git a/src/zenutil/include/zenutil/logging/fullformatter.h b/src/zenutil/include/zenutil/logging/fullformatter.h index 9f245becd..33cb94dae 100644 --- a/src/zenutil/include/zenutil/logging/fullformatter.h +++ b/src/zenutil/include/zenutil/logging/fullformatter.h @@ -2,21 +2,19 @@ #pragma once +#include +#include #include #include #include -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END - namespace zen::logging { -class full_formatter final : public spdlog::formatter +class FullFormatter final : public Formatter { public: - full_formatter(std::string_view LogId, std::chrono::time_point Epoch) + FullFormatter(std::string_view LogId, std::chrono::time_point Epoch) : m_Epoch(Epoch) , m_LogId(LogId) , m_LinePrefix(128, ' ') @@ -24,16 +22,19 @@ public: { } - full_formatter(std::string_view LogId) : m_LogId(LogId), m_LinePrefix(128, ' '), m_UseFullDate(true) {} + FullFormatter(std::string_view LogId) : m_LogId(LogId), m_LinePrefix(128, ' '), m_UseFullDate(true) {} - virtual std::unique_ptr clone() const override + virtual std::unique_ptr Clone() const override { ZEN_MEMSCOPE(ELLMTag::Logging); - // Note: this does not properly clone m_UseFullDate - return std::make_unique(m_LogId, m_Epoch); + if (m_UseFullDate) + { + return std::make_unique(m_LogId); + } + return std::make_unique(m_LogId, m_Epoch); } - virtual void format(const spdlog::details::log_msg& msg, spdlog::memory_buf_t& OutBuffer) override + virtual void Format(const LogMessage& Msg, MemoryBuffer& OutBuffer) override { ZEN_MEMSCOPE(ELLMTag::Logging); @@ -44,38 +45,38 @@ public: std::chrono::seconds TimestampSeconds; - std::chrono::milliseconds millis; + std::chrono::milliseconds Millis; if (m_UseFullDate) { - TimestampSeconds = std::chrono::duration_cast(msg.time.time_since_epoch()); + TimestampSeconds = std::chrono::duration_cast(Msg.GetTime().time_since_epoch()); if (TimestampSeconds != m_LastLogSecs) { RwLock::ExclusiveLockScope _(m_TimestampLock); m_LastLogSecs = TimestampSeconds; - m_CachedLocalTm = spdlog::details::os::localtime(spdlog::log_clock::to_time_t(msg.time)); + m_CachedLocalTm = helpers::SafeLocaltime(LogClock::to_time_t(Msg.GetTime())); m_CachedDatetime.clear(); m_CachedDatetime.push_back('['); - spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_year % 100, m_CachedDatetime); + helpers::Pad2(m_CachedLocalTm.tm_year % 100, m_CachedDatetime); m_CachedDatetime.push_back('-'); - spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_mon + 1, m_CachedDatetime); + helpers::Pad2(m_CachedLocalTm.tm_mon + 1, m_CachedDatetime); m_CachedDatetime.push_back('-'); - spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_mday, m_CachedDatetime); + helpers::Pad2(m_CachedLocalTm.tm_mday, m_CachedDatetime); m_CachedDatetime.push_back(' '); - spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_hour, m_CachedDatetime); + helpers::Pad2(m_CachedLocalTm.tm_hour, m_CachedDatetime); m_CachedDatetime.push_back(':'); - spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_min, m_CachedDatetime); + helpers::Pad2(m_CachedLocalTm.tm_min, m_CachedDatetime); m_CachedDatetime.push_back(':'); - spdlog::details::fmt_helper::pad2(m_CachedLocalTm.tm_sec, m_CachedDatetime); + helpers::Pad2(m_CachedLocalTm.tm_sec, m_CachedDatetime); m_CachedDatetime.push_back('.'); } - millis = spdlog::details::fmt_helper::time_fraction(msg.time); + Millis = helpers::TimeFraction(Msg.GetTime()); } else { - auto ElapsedTime = msg.time - m_Epoch; + auto ElapsedTime = Msg.GetTime() - m_Epoch; TimestampSeconds = std::chrono::duration_cast(ElapsedTime); if (m_CacheTimestamp.load() != TimestampSeconds) @@ -93,15 +94,15 @@ public: m_CachedDatetime.clear(); m_CachedDatetime.push_back('['); - spdlog::details::fmt_helper::pad2(LogHours, m_CachedDatetime); + helpers::Pad2(LogHours, m_CachedDatetime); m_CachedDatetime.push_back(':'); - spdlog::details::fmt_helper::pad2(LogMins, m_CachedDatetime); + helpers::Pad2(LogMins, m_CachedDatetime); m_CachedDatetime.push_back(':'); - spdlog::details::fmt_helper::pad2(LogSecs, m_CachedDatetime); + helpers::Pad2(LogSecs, m_CachedDatetime); m_CachedDatetime.push_back('.'); } - millis = std::chrono::duration_cast(ElapsedTime - TimestampSeconds); + Millis = std::chrono::duration_cast(ElapsedTime - TimestampSeconds); } { @@ -109,44 +110,43 @@ public: OutBuffer.append(m_CachedDatetime.begin(), m_CachedDatetime.end()); } - spdlog::details::fmt_helper::pad3(static_cast(millis.count()), OutBuffer); + helpers::Pad3(static_cast(Millis.count()), OutBuffer); OutBuffer.push_back(']'); OutBuffer.push_back(' '); if (!m_LogId.empty()) { OutBuffer.push_back('['); - spdlog::details::fmt_helper::append_string_view(m_LogId, OutBuffer); + helpers::AppendStringView(m_LogId, OutBuffer); OutBuffer.push_back(']'); OutBuffer.push_back(' '); } // append logger name if exists - if (msg.logger_name.size() > 0) + if (Msg.GetLoggerName().size() > 0) { OutBuffer.push_back('['); - spdlog::details::fmt_helper::append_string_view(msg.logger_name, OutBuffer); + helpers::AppendStringView(Msg.GetLoggerName(), OutBuffer); OutBuffer.push_back(']'); OutBuffer.push_back(' '); } OutBuffer.push_back('['); // wrap the level name with color - msg.color_range_start = OutBuffer.size(); - spdlog::details::fmt_helper::append_string_view(spdlog::level::to_string_view(msg.level), OutBuffer); - msg.color_range_end = OutBuffer.size(); + Msg.ColorRangeStart = OutBuffer.size(); + helpers::AppendStringView(helpers::LevelToShortString(Msg.GetLevel()), OutBuffer); + Msg.ColorRangeEnd = OutBuffer.size(); OutBuffer.push_back(']'); OutBuffer.push_back(' '); // add source location if present - if (!msg.source.empty()) + if (Msg.GetSource()) { OutBuffer.push_back('['); - const char* filename = - spdlog::details::short_filename_formatter::basename(msg.source.filename); - spdlog::details::fmt_helper::append_string_view(filename, OutBuffer); + const char* Filename = helpers::ShortFilename(Msg.GetSource().Filename); + helpers::AppendStringView(Filename, OutBuffer); OutBuffer.push_back(':'); - spdlog::details::fmt_helper::append_int(msg.source.line, OutBuffer); + helpers::AppendInt(Msg.GetSource().Line, OutBuffer); OutBuffer.push_back(']'); OutBuffer.push_back(' '); } @@ -156,8 +156,9 @@ public: const size_t LinePrefixCount = Min(OutBuffer.size(), m_LinePrefix.size()); - auto ItLineBegin = msg.payload.begin(); - auto ItMessageEnd = msg.payload.end(); + auto MsgPayload = Msg.GetPayload(); + auto ItLineBegin = MsgPayload.begin(); + auto ItMessageEnd = MsgPayload.end(); bool IsFirstline = true; { @@ -170,9 +171,9 @@ public: } else { - spdlog::details::fmt_helper::append_string_view(std::string_view(m_LinePrefix.data(), LinePrefixCount), OutBuffer); + helpers::AppendStringView(std::string_view(m_LinePrefix.data(), LinePrefixCount), OutBuffer); } - spdlog::details::fmt_helper::append_string_view(spdlog::string_view_t(&*ItLineBegin, ItLineEnd - ItLineBegin), OutBuffer); + helpers::AppendStringView(std::string_view(&*ItLineBegin, ItLineEnd - ItLineBegin), OutBuffer); }; while (ItLineEnd != ItMessageEnd) @@ -187,7 +188,7 @@ public: if (ItLineBegin != ItMessageEnd) { EmitLine(); - spdlog::details::fmt_helper::append_string_view("\n"sv, OutBuffer); + helpers::AppendStringView("\n"sv, OutBuffer); } } } @@ -197,7 +198,7 @@ private: std::tm m_CachedLocalTm; std::chrono::seconds m_LastLogSecs{std::chrono::seconds(87654321)}; std::atomic m_CacheTimestamp{std::chrono::seconds(87654321)}; - spdlog::memory_buf_t m_CachedDatetime; + MemoryBuffer m_CachedDatetime; std::string m_LogId; std::string m_LinePrefix; bool m_UseFullDate = true; diff --git a/src/zenutil/include/zenutil/logging/jsonformatter.h b/src/zenutil/include/zenutil/logging/jsonformatter.h index 3f660e421..216b1b5e5 100644 --- a/src/zenutil/include/zenutil/logging/jsonformatter.h +++ b/src/zenutil/include/zenutil/logging/jsonformatter.h @@ -2,27 +2,26 @@ #pragma once +#include +#include #include #include #include - -ZEN_THIRD_PARTY_INCLUDES_START -#include -ZEN_THIRD_PARTY_INCLUDES_END +#include namespace zen::logging { using namespace std::literals; -class json_formatter final : public spdlog::formatter +class JsonFormatter final : public Formatter { public: - json_formatter(std::string_view LogId) : m_LogId(LogId) {} + JsonFormatter(std::string_view LogId) : m_LogId(LogId) {} - virtual std::unique_ptr clone() const override { return std::make_unique(m_LogId); } + virtual std::unique_ptr Clone() const override { return std::make_unique(m_LogId); } - virtual void format(const spdlog::details::log_msg& msg, spdlog::memory_buf_t& dest) override + virtual void Format(const LogMessage& Msg, MemoryBuffer& Dest) override { ZEN_MEMSCOPE(ELLMTag::Logging); @@ -30,141 +29,132 @@ public: using std::chrono::milliseconds; using std::chrono::seconds; - auto secs = std::chrono::duration_cast(msg.time.time_since_epoch()); - if (secs != m_LastLogSecs) + auto Secs = std::chrono::duration_cast(Msg.GetTime().time_since_epoch()); + if (Secs != m_LastLogSecs) { - m_CachedTm = spdlog::details::os::localtime(spdlog::log_clock::to_time_t(msg.time)); - m_LastLogSecs = secs; - } - - const auto& tm_time = m_CachedTm; + RwLock::ExclusiveLockScope _(m_TimestampLock); + m_CachedTm = helpers::SafeLocaltime(LogClock::to_time_t(Msg.GetTime())); + m_LastLogSecs = Secs; - // cache the date/time part for the next second. - - if (m_CacheTimestamp != secs || m_CachedDatetime.size() == 0) - { + // cache the date/time part for the next second. m_CachedDatetime.clear(); - spdlog::details::fmt_helper::append_int(tm_time.tm_year + 1900, m_CachedDatetime); + helpers::AppendInt(m_CachedTm.tm_year + 1900, m_CachedDatetime); m_CachedDatetime.push_back('-'); - spdlog::details::fmt_helper::pad2(tm_time.tm_mon + 1, m_CachedDatetime); + helpers::Pad2(m_CachedTm.tm_mon + 1, m_CachedDatetime); m_CachedDatetime.push_back('-'); - spdlog::details::fmt_helper::pad2(tm_time.tm_mday, m_CachedDatetime); + helpers::Pad2(m_CachedTm.tm_mday, m_CachedDatetime); m_CachedDatetime.push_back(' '); - spdlog::details::fmt_helper::pad2(tm_time.tm_hour, m_CachedDatetime); + helpers::Pad2(m_CachedTm.tm_hour, m_CachedDatetime); m_CachedDatetime.push_back(':'); - spdlog::details::fmt_helper::pad2(tm_time.tm_min, m_CachedDatetime); + helpers::Pad2(m_CachedTm.tm_min, m_CachedDatetime); m_CachedDatetime.push_back(':'); - spdlog::details::fmt_helper::pad2(tm_time.tm_sec, m_CachedDatetime); + helpers::Pad2(m_CachedTm.tm_sec, m_CachedDatetime); m_CachedDatetime.push_back('.'); - - m_CacheTimestamp = secs; } - dest.append("{"sv); - dest.append("\"time\": \""sv); - dest.append(m_CachedDatetime.begin(), m_CachedDatetime.end()); - auto millis = spdlog::details::fmt_helper::time_fraction(msg.time); - spdlog::details::fmt_helper::pad3(static_cast(millis.count()), dest); - dest.append("\", "sv); + helpers::AppendStringView("{"sv, Dest); + helpers::AppendStringView("\"time\": \""sv, Dest); + { + RwLock::SharedLockScope _(m_TimestampLock); + Dest.append(m_CachedDatetime.begin(), m_CachedDatetime.end()); + } + auto Millis = helpers::TimeFraction(Msg.GetTime()); + helpers::Pad3(static_cast(Millis.count()), Dest); + helpers::AppendStringView("\", "sv, Dest); - dest.append("\"status\": \""sv); - dest.append(spdlog::level::to_string_view(msg.level)); - dest.append("\", "sv); + helpers::AppendStringView("\"status\": \""sv, Dest); + helpers::AppendStringView(helpers::LevelToShortString(Msg.GetLevel()), Dest); + helpers::AppendStringView("\", "sv, Dest); - dest.append("\"source\": \""sv); - dest.append("zenserver"sv); - dest.append("\", "sv); + helpers::AppendStringView("\"source\": \""sv, Dest); + helpers::AppendStringView("zenserver"sv, Dest); + helpers::AppendStringView("\", "sv, Dest); - dest.append("\"service\": \""sv); - dest.append("zencache"sv); - dest.append("\", "sv); + helpers::AppendStringView("\"service\": \""sv, Dest); + helpers::AppendStringView("zencache"sv, Dest); + helpers::AppendStringView("\", "sv, Dest); if (!m_LogId.empty()) { - dest.append("\"id\": \""sv); - dest.append(m_LogId); - dest.append("\", "sv); + helpers::AppendStringView("\"id\": \""sv, Dest); + helpers::AppendStringView(m_LogId, Dest); + helpers::AppendStringView("\", "sv, Dest); } - if (msg.logger_name.size() > 0) + if (Msg.GetLoggerName().size() > 0) { - dest.append("\"logger.name\": \""sv); - dest.append(msg.logger_name); - dest.append("\", "sv); + helpers::AppendStringView("\"logger.name\": \""sv, Dest); + helpers::AppendStringView(Msg.GetLoggerName(), Dest); + helpers::AppendStringView("\", "sv, Dest); } - if (msg.thread_id != 0) + if (Msg.GetThreadId() != 0) { - dest.append("\"logger.thread_name\": \""sv); - spdlog::details::fmt_helper::pad_uint(msg.thread_id, 0, dest); - dest.append("\", "sv); + helpers::AppendStringView("\"logger.thread_name\": \""sv, Dest); + helpers::PadUint(Msg.GetThreadId(), 0, Dest); + helpers::AppendStringView("\", "sv, Dest); } - if (!msg.source.empty()) + if (Msg.GetSource()) { - dest.append("\"file\": \""sv); - WriteEscapedString( - dest, - spdlog::details::short_filename_formatter::basename(msg.source.filename)); - dest.append("\","sv); - - dest.append("\"line\": \""sv); - dest.append(fmt::format("{}", msg.source.line)); - dest.append("\","sv); - - dest.append("\"logger.method_name\": \""sv); - WriteEscapedString(dest, msg.source.funcname); - dest.append("\", "sv); + helpers::AppendStringView("\"file\": \""sv, Dest); + WriteEscapedString(Dest, helpers::ShortFilename(Msg.GetSource().Filename)); + helpers::AppendStringView("\","sv, Dest); + + helpers::AppendStringView("\"line\": \""sv, Dest); + helpers::AppendInt(Msg.GetSource().Line, Dest); + helpers::AppendStringView("\","sv, Dest); } - dest.append("\"message\": \""sv); - WriteEscapedString(dest, msg.payload); - dest.append("\""sv); + helpers::AppendStringView("\"message\": \""sv, Dest); + WriteEscapedString(Dest, Msg.GetPayload()); + helpers::AppendStringView("\""sv, Dest); - dest.append("}\n"sv); + helpers::AppendStringView("}\n"sv, Dest); } private: - static inline const std::unordered_map SpecialCharacterMap{{'\b', "\\b"sv}, - {'\f', "\\f"sv}, - {'\n', "\\n"sv}, - {'\r', "\\r"sv}, - {'\t', "\\t"sv}, - {'"', "\\\""sv}, - {'\\', "\\\\"sv}}; - - static void WriteEscapedString(spdlog::memory_buf_t& dest, const spdlog::string_view_t& payload) + static inline const std::unordered_map s_SpecialCharacterMap{{'\b', "\\b"sv}, + {'\f', "\\f"sv}, + {'\n', "\\n"sv}, + {'\r', "\\r"sv}, + {'\t', "\\t"sv}, + {'"', "\\\""sv}, + {'\\', "\\\\"sv}}; + + static void WriteEscapedString(MemoryBuffer& Dest, const std::string_view& Text) { - const char* RangeStart = payload.begin(); - for (const char* It = RangeStart; It != payload.end(); ++It) + const char* RangeStart = Text.data(); + const char* End = Text.data() + Text.size(); + for (const char* It = RangeStart; It != End; ++It) { - if (auto SpecialIt = SpecialCharacterMap.find(*It); SpecialIt != SpecialCharacterMap.end()) + if (auto SpecialIt = s_SpecialCharacterMap.find(*It); SpecialIt != s_SpecialCharacterMap.end()) { if (RangeStart != It) { - dest.append(RangeStart, It); + Dest.append(RangeStart, It); } - dest.append(SpecialIt->second); + helpers::AppendStringView(SpecialIt->second, Dest); RangeStart = It + 1; } } - if (RangeStart != payload.end()) + if (RangeStart != End) { - dest.append(RangeStart, payload.end()); + Dest.append(RangeStart, End); } }; std::tm m_CachedTm{0, 0, 0, 0, 0, 0, 0, 0, 0}; std::chrono::seconds m_LastLogSecs{0}; - std::chrono::seconds m_CacheTimestamp{0}; - spdlog::memory_buf_t m_CachedDatetime; + MemoryBuffer m_CachedDatetime; std::string m_LogId; + RwLock m_TimestampLock; }; } // namespace zen::logging diff --git a/src/zenutil/include/zenutil/logging/rotatingfilesink.h b/src/zenutil/include/zenutil/logging/rotatingfilesink.h index 8901b7779..cebc5b110 100644 --- a/src/zenutil/include/zenutil/logging/rotatingfilesink.h +++ b/src/zenutil/include/zenutil/logging/rotatingfilesink.h @@ -3,14 +3,11 @@ #pragma once #include +#include +#include +#include #include -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - #include #include @@ -19,13 +16,14 @@ namespace zen::logging { // Basically the same functionality as spdlog::sinks::rotating_file_sink with the biggest difference // being that it just ignores any errors when writing/rotating files and keeps chugging on. // It will keep trying to log, and if it starts to work it will continue to log. -class RotatingFileSink : public spdlog::sinks::sink +class RotatingFileSink : public Sink { public: RotatingFileSink(const std::filesystem::path& BaseFilename, std::size_t MaxSize, std::size_t MaxFiles, bool RotateOnOpen = false) : m_BaseFilename(BaseFilename) , m_MaxSize(MaxSize) , m_MaxFiles(MaxFiles) + , m_Formatter(std::make_unique()) { ZEN_MEMSCOPE(ELLMTag::Logging); @@ -76,18 +74,21 @@ public: RotatingFileSink& operator=(const RotatingFileSink&) = delete; RotatingFileSink& operator=(RotatingFileSink&&) = delete; - virtual void log(const spdlog::details::log_msg& msg) override + virtual void Log(const LogMessage& Msg) override { ZEN_MEMSCOPE(ELLMTag::Logging); try { - spdlog::memory_buf_t Formatted; - if (TrySinkIt(msg, Formatted)) + MemoryBuffer Formatted; + if (TrySinkIt(Msg, Formatted)) { return; } - while (true) + + // This intentionally has no limit on the number of retries, see + // comment above. + for (;;) { { RwLock::ExclusiveLockScope RotateLock(m_Lock); @@ -113,7 +114,7 @@ public: // Silently eat errors } } - virtual void flush() override + virtual void Flush() override { if (!m_NeedFlush) { @@ -138,28 +139,14 @@ public: m_NeedFlush = false; } - virtual void set_pattern(const std::string& pattern) override + virtual void SetFormatter(std::unique_ptr InFormatter) override { ZEN_MEMSCOPE(ELLMTag::Logging); try { RwLock::ExclusiveLockScope _(m_Lock); - m_Formatter = spdlog::details::make_unique(pattern); - } - catch (const std::exception&) - { - // Silently eat errors - } - } - virtual void set_formatter(std::unique_ptr sink_formatter) override - { - ZEN_MEMSCOPE(ELLMTag::Logging); - - try - { - RwLock::ExclusiveLockScope _(m_Lock); - m_Formatter = std::move(sink_formatter); + m_Formatter = std::move(InFormatter); } catch (const std::exception&) { @@ -186,11 +173,17 @@ private: return; } - // If we fail to rotate, try extending the current log file m_CurrentSize = m_CurrentFile.FileSize(OutEc); + if (OutEc) + { + // FileSize failed but we have an open file — reset to 0 + // so we can at least attempt writes from the start + m_CurrentSize = 0; + OutEc.clear(); + } } - bool TrySinkIt(const spdlog::details::log_msg& msg, spdlog::memory_buf_t& OutFormatted) + bool TrySinkIt(const LogMessage& Msg, MemoryBuffer& OutFormatted) { ZEN_MEMSCOPE(ELLMTag::Logging); @@ -199,15 +192,15 @@ private: { return false; } - m_Formatter->format(msg, OutFormatted); - size_t add_size = OutFormatted.size(); - size_t write_pos = m_CurrentSize.fetch_add(add_size); - if (write_pos + add_size > m_MaxSize) + m_Formatter->Format(Msg, OutFormatted); + size_t AddSize = OutFormatted.size(); + size_t WritePos = m_CurrentSize.fetch_add(AddSize); + if (WritePos + AddSize > m_MaxSize) { return false; } std::error_code Ec; - m_CurrentFile.Write(OutFormatted.data(), OutFormatted.size(), write_pos, Ec); + m_CurrentFile.Write(OutFormatted.data(), OutFormatted.size(), WritePos, Ec); if (Ec) { return false; @@ -216,7 +209,7 @@ private: return true; } - bool TrySinkIt(const spdlog::memory_buf_t& Formatted) + bool TrySinkIt(const MemoryBuffer& Formatted) { ZEN_MEMSCOPE(ELLMTag::Logging); @@ -225,15 +218,15 @@ private: { return false; } - size_t add_size = Formatted.size(); - size_t write_pos = m_CurrentSize.fetch_add(add_size); - if (write_pos + add_size > m_MaxSize) + size_t AddSize = Formatted.size(); + size_t WritePos = m_CurrentSize.fetch_add(AddSize); + if (WritePos + AddSize > m_MaxSize) { return false; } std::error_code Ec; - m_CurrentFile.Write(Formatted.data(), Formatted.size(), write_pos, Ec); + m_CurrentFile.Write(Formatted.data(), Formatted.size(), WritePos, Ec); if (Ec) { return false; @@ -242,14 +235,14 @@ private: return true; } - RwLock m_Lock; - const std::filesystem::path m_BaseFilename; - std::unique_ptr m_Formatter; - std::atomic_size_t m_CurrentSize; - const std::size_t m_MaxSize; - const std::size_t m_MaxFiles; - BasicFile m_CurrentFile; - std::atomic m_NeedFlush = false; + RwLock m_Lock; + const std::filesystem::path m_BaseFilename; + const std::size_t m_MaxSize; + const std::size_t m_MaxFiles; + std::unique_ptr m_Formatter; + std::atomic_size_t m_CurrentSize; + BasicFile m_CurrentFile; + std::atomic m_NeedFlush = false; }; } // namespace zen::logging diff --git a/src/zenutil/include/zenutil/logging/testformatter.h b/src/zenutil/include/zenutil/logging/testformatter.h deleted file mode 100644 index 0b0c191fb..000000000 --- a/src/zenutil/include/zenutil/logging/testformatter.h +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright Epic Games, Inc. All Rights Reserved. - -#pragma once - -#include - -#include - -namespace zen::logging { - -class full_test_formatter final : public spdlog::formatter -{ -public: - full_test_formatter(std::string_view LogId, std::chrono::time_point Epoch) : m_Epoch(Epoch), m_LogId(LogId) - { - } - - virtual std::unique_ptr clone() const override - { - ZEN_MEMSCOPE(ELLMTag::Logging); - return std::make_unique(m_LogId, m_Epoch); - } - - static constexpr bool UseDate = false; - - virtual void format(const spdlog::details::log_msg& msg, spdlog::memory_buf_t& dest) override - { - ZEN_MEMSCOPE(ELLMTag::Logging); - - using namespace std::literals; - - if constexpr (UseDate) - { - auto secs = std::chrono::duration_cast(msg.time.time_since_epoch()); - if (secs != m_LastLogSecs) - { - m_CachedTm = spdlog::details::os::localtime(spdlog::log_clock::to_time_t(msg.time)); - m_LastLogSecs = secs; - } - } - - const auto& tm_time = m_CachedTm; - - // cache the date/time part for the next second. - auto duration = msg.time - m_Epoch; - auto secs = std::chrono::duration_cast(duration); - - if (m_CacheTimestamp != secs) - { - RwLock::ExclusiveLockScope _(m_TimestampLock); - - m_CachedDatetime.clear(); - m_CachedDatetime.push_back('['); - - if constexpr (UseDate) - { - spdlog::details::fmt_helper::append_int(tm_time.tm_year + 1900, m_CachedDatetime); - m_CachedDatetime.push_back('-'); - - spdlog::details::fmt_helper::pad2(tm_time.tm_mon + 1, m_CachedDatetime); - m_CachedDatetime.push_back('-'); - - spdlog::details::fmt_helper::pad2(tm_time.tm_mday, m_CachedDatetime); - m_CachedDatetime.push_back(' '); - - spdlog::details::fmt_helper::pad2(tm_time.tm_hour, m_CachedDatetime); - m_CachedDatetime.push_back(':'); - - spdlog::details::fmt_helper::pad2(tm_time.tm_min, m_CachedDatetime); - m_CachedDatetime.push_back(':'); - - spdlog::details::fmt_helper::pad2(tm_time.tm_sec, m_CachedDatetime); - } - else - { - int Count = int(secs.count()); - - const int LogSecs = Count % 60; - Count /= 60; - - const int LogMins = Count % 60; - Count /= 60; - - const int LogHours = Count; - - spdlog::details::fmt_helper::pad2(LogHours, m_CachedDatetime); - m_CachedDatetime.push_back(':'); - spdlog::details::fmt_helper::pad2(LogMins, m_CachedDatetime); - m_CachedDatetime.push_back(':'); - spdlog::details::fmt_helper::pad2(LogSecs, m_CachedDatetime); - } - - m_CachedDatetime.push_back('.'); - - m_CacheTimestamp = secs; - } - - { - RwLock::SharedLockScope _(m_TimestampLock); - dest.append(m_CachedDatetime.begin(), m_CachedDatetime.end()); - } - - auto millis = spdlog::details::fmt_helper::time_fraction(msg.time); - spdlog::details::fmt_helper::pad3(static_cast(millis.count()), dest); - dest.push_back(']'); - dest.push_back(' '); - - if (!m_LogId.empty()) - { - dest.push_back('['); - spdlog::details::fmt_helper::append_string_view(m_LogId, dest); - dest.push_back(']'); - dest.push_back(' '); - } - - // append logger name if exists - if (msg.logger_name.size() > 0) - { - dest.push_back('['); - spdlog::details::fmt_helper::append_string_view(msg.logger_name, dest); - dest.push_back(']'); - dest.push_back(' '); - } - - dest.push_back('['); - // wrap the level name with color - msg.color_range_start = dest.size(); - spdlog::details::fmt_helper::append_string_view(spdlog::level::to_string_view(msg.level), dest); - msg.color_range_end = dest.size(); - dest.push_back(']'); - dest.push_back(' '); - - // add source location if present - if (!msg.source.empty()) - { - dest.push_back('['); - const char* filename = - spdlog::details::short_filename_formatter::basename(msg.source.filename); - spdlog::details::fmt_helper::append_string_view(filename, dest); - dest.push_back(':'); - spdlog::details::fmt_helper::append_int(msg.source.line, dest); - dest.push_back(']'); - dest.push_back(' '); - } - - spdlog::details::fmt_helper::append_string_view(msg.payload, dest); - spdlog::details::fmt_helper::append_string_view("\n"sv, dest); - } - -private: - std::chrono::time_point m_Epoch; - std::tm m_CachedTm; - std::chrono::seconds m_LastLogSecs{std::chrono::seconds(87654321)}; - std::chrono::seconds m_CacheTimestamp{std::chrono::seconds(87654321)}; - spdlog::memory_buf_t m_CachedDatetime; - std::string m_LogId; - RwLock m_TimestampLock; -}; - -} // namespace zen::logging diff --git a/src/zenutil/logging.cpp b/src/zenutil/logging.cpp index 54ac30c5d..1258ca155 100644 --- a/src/zenutil/logging.cpp +++ b/src/zenutil/logging.cpp @@ -2,18 +2,15 @@ #include "zenutil/logging.h" -ZEN_THIRD_PARTY_INCLUDES_START -#include -#include -#include -#include -#include -ZEN_THIRD_PARTY_INCLUDES_END - #include #include #include #include +#include +#include +#include +#include +#include #include #include #include @@ -27,9 +24,9 @@ ZEN_THIRD_PARTY_INCLUDES_END namespace zen { static bool g_IsLoggingInitialized; -spdlog::sink_ptr g_FileSink; +logging::SinkPtr g_FileSink; -spdlog::sink_ptr +logging::SinkPtr GetFileSink() { return g_FileSink; @@ -52,33 +49,9 @@ BeginInitializeLogging(const LoggingOptions& LogOptions) zen::logging::InitializeLogging(); zen::logging::EnableVTMode(); - bool IsAsync = LogOptions.AllowAsync; - - if (LogOptions.IsDebug) - { - IsAsync = false; - } - - if (LogOptions.IsTest) - { - IsAsync = false; - } - - if (IsAsync) - { - const int QueueSize = 8192; - const int ThreadCount = 1; - spdlog::init_thread_pool(QueueSize, ThreadCount, [&] { SetCurrentThreadName("spdlog_async"); }); - - auto AsyncSink = spdlog::create_async("main"); - zen::logging::SetDefault("main"); - } - // Sinks - spdlog::sink_ptr FileSink; - - // spdlog can't create directories that starts with `\\?\` so we make sure the folder exists before creating the logger instance + logging::SinkPtr FileSink; if (!LogOptions.AbsLogFile.empty()) { @@ -87,17 +60,17 @@ BeginInitializeLogging(const LoggingOptions& LogOptions) zen::CreateDirectories(LogOptions.AbsLogFile.parent_path()); } - FileSink = std::make_shared(LogOptions.AbsLogFile, - /* max size */ 128 * 1024 * 1024, - /* max files */ 16, - /* rotate on open */ true); + FileSink = logging::SinkPtr(new zen::logging::RotatingFileSink(LogOptions.AbsLogFile, + /* max size */ 128 * 1024 * 1024, + /* max files */ 16, + /* rotate on open */ true)); if (LogOptions.AbsLogFile.extension() == ".json") { - FileSink->set_formatter(std::make_unique(LogOptions.LogId)); + FileSink->SetFormatter(std::make_unique(LogOptions.LogId)); } else { - FileSink->set_formatter(std::make_unique(LogOptions.LogId)); // this will have a date prefix + FileSink->SetFormatter(std::make_unique(LogOptions.LogId)); // this will have a date prefix } } @@ -127,7 +100,7 @@ BeginInitializeLogging(const LoggingOptions& LogOptions) Message.push_back('\0'); // We use direct ZEN_LOG here instead of ZEN_ERROR as we don't care about *this* code location in the log - ZEN_LOG(Log(), zen::logging::level::Critical, "{}", Message.data()); + ZEN_LOG(Log(), zen::logging::Critical, "{}", Message.data()); zen::logging::FlushLogging(); } catch (const std::exception&) @@ -143,9 +116,9 @@ BeginInitializeLogging(const LoggingOptions& LogOptions) // Default LoggerRef DefaultLogger = zen::logging::Default(); - auto& Sinks = DefaultLogger.SpdLogger->sinks(); - Sinks.clear(); + // Collect sinks into a local vector first so we can optionally wrap them + std::vector Sinks; if (LogOptions.NoConsoleOutput) { @@ -153,10 +126,10 @@ BeginInitializeLogging(const LoggingOptions& LogOptions) } else { - auto ConsoleSink = std::make_shared(); + logging::SinkPtr ConsoleSink(new logging::AnsiColorStdoutSink()); if (LogOptions.QuietConsole) { - ConsoleSink->set_level(spdlog::level::warn); + ConsoleSink->SetLevel(logging::Warn); } Sinks.push_back(ConsoleSink); } @@ -169,40 +142,54 @@ BeginInitializeLogging(const LoggingOptions& LogOptions) #if ZEN_PLATFORM_WINDOWS if (zen::IsDebuggerPresent() && LogOptions.IsDebug) { - auto DebugSink = std::make_shared(); - DebugSink->set_level(spdlog::level::debug); + logging::SinkPtr DebugSink(new logging::MsvcSink()); + DebugSink->SetLevel(logging::Debug); Sinks.push_back(DebugSink); } #endif - spdlog::set_error_handler([](const std::string& msg) { - if (msg == std::bad_alloc().what()) - { - // Don't report out of memory in spdlog as we usually log in response to errors which will cause another OOM crashing the - // program - return; - } - // Bypass zen logging wrapping to reduce potential other error sources - if (auto ErrLogger = zen::logging::ErrorLog()) + bool IsAsync = LogOptions.AllowAsync && !LogOptions.IsDebug && !LogOptions.IsTest; + + if (IsAsync) + { + std::vector AsyncSinks; + AsyncSinks.emplace_back(new logging::AsyncSink(std::move(Sinks))); + DefaultLogger->SetSinks(std::move(AsyncSinks)); + } + else + { + DefaultLogger->SetSinks(std::move(Sinks)); + } + + static struct : logging::ErrorHandler + { + void HandleError(const std::string_view& ErrorMsg) override { + if (ErrorMsg == std::bad_alloc().what()) + { + return; + } + static constinit logging::LogPoint ErrorPoint{{}, logging::Err, "{}"}; + if (auto ErrLogger = zen::logging::ErrorLog()) + { + try + { + ErrLogger->Log(ErrorPoint, fmt::make_format_args(ErrorMsg)); + } + catch (const std::exception&) + { + } + } try { - ErrLogger.SpdLogger->log(spdlog::level::err, msg); + Log()->Log(ErrorPoint, fmt::make_format_args(ErrorMsg)); } catch (const std::exception&) { - // Just ignore any errors when in error handler } } - try - { - Log().SpdLogger->error(msg); - } - catch (const std::exception&) - { - // Just ignore any errors when in error handler - } - }); + } s_ErrorHandler; + logging::Registry::Instance().SetErrorHandler(&s_ErrorHandler); g_FileSink = std::move(FileSink); } @@ -212,24 +199,24 @@ FinishInitializeLogging(const LoggingOptions& LogOptions) { ZEN_MEMSCOPE(ELLMTag::Logging); - logging::level::LogLevel LogLevel = logging::level::Info; + logging::LogLevel LogLevel = logging::Info; if (LogOptions.IsDebug) { - LogLevel = logging::level::Debug; + LogLevel = logging::Debug; } if (LogOptions.IsTest || LogOptions.IsVerbose) { - LogLevel = logging::level::Trace; + LogLevel = logging::Trace; } // Configure all registered loggers according to settings logging::RefreshLogLevels(LogLevel); - spdlog::flush_on(spdlog::level::err); - spdlog::flush_every(std::chrono::seconds{2}); - spdlog::set_formatter(std::make_unique( + logging::Registry::Instance().FlushOn(logging::Err); + logging::Registry::Instance().FlushEvery(std::chrono::seconds{2}); + logging::Registry::Instance().SetFormatter(std::make_unique( LogOptions.LogId, std::chrono::system_clock::now() - std::chrono::milliseconds(GetTimeSinceProcessStart()))); // default to duration prefix @@ -242,16 +229,17 @@ FinishInitializeLogging(const LoggingOptions& LogOptions) { if (LogOptions.AbsLogFile.extension() == ".json") { - g_FileSink->set_formatter(std::make_unique(LogOptions.LogId)); + g_FileSink->SetFormatter(std::make_unique(LogOptions.LogId)); } else { - g_FileSink->set_formatter(std::make_unique(LogOptions.LogId)); // this will have a date prefix + g_FileSink->SetFormatter(std::make_unique(LogOptions.LogId)); // this will have a date prefix } const std::string StartLogTime = zen::DateTime::Now().ToIso8601(); - spdlog::apply_all([&](auto Logger) { Logger->info("log starting at {}", StartLogTime); }); + static constinit logging::LogPoint LogStartPoint{{}, logging::Info, "log starting at {}"}; + logging::Registry::Instance().ApplyAll([&](auto Logger) { Logger->Log(LogStartPoint, fmt::make_format_args(StartLogTime)); }); } g_IsLoggingInitialized = true; @@ -268,7 +256,7 @@ ShutdownLogging() zen::logging::ShutdownLogging(); - g_FileSink.reset(); + g_FileSink = nullptr; } } // namespace zen diff --git a/src/zenutil/xmake.lua b/src/zenutil/xmake.lua index bc33adf9e..1d5be5977 100644 --- a/src/zenutil/xmake.lua +++ b/src/zenutil/xmake.lua @@ -6,7 +6,7 @@ target('zenutil') add_headerfiles("**.h") add_files("**.cpp") add_includedirs("include", {public=true}) - add_deps("zencore", "zenhttp", "spdlog") + add_deps("zencore", "zenhttp") add_deps("cxxopts") add_deps("robin-map") diff --git a/src/zenvfs/xmake.lua b/src/zenvfs/xmake.lua index 7f790c2d4..47665a5d5 100644 --- a/src/zenvfs/xmake.lua +++ b/src/zenvfs/xmake.lua @@ -6,5 +6,5 @@ target('zenvfs') add_headerfiles("**.h") add_files("**.cpp") add_includedirs("include", {public=true}) - add_deps("zencore", "spdlog") + add_deps("zencore") diff --git a/thirdparty/spdlog/.clang-format b/thirdparty/spdlog/.clang-format deleted file mode 100644 index c8c345f6f..000000000 --- a/thirdparty/spdlog/.clang-format +++ /dev/null @@ -1,19 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: Google -AccessModifierOffset: -4 -Standard: c++17 -IndentWidth: 4 -TabWidth: 4 -UseTab: Never -ColumnLimit: 100 -AlignAfterOpenBracket: Align -BinPackParameters: false -AlignEscapedNewlines: Left -AlwaysBreakTemplateDeclarations: Yes -PackConstructorInitializers: Never -BreakConstructorInitializersBeforeComma: false -IndentPPDirectives: BeforeHash -SortIncludes: Never -... - diff --git a/thirdparty/spdlog/.clang-tidy b/thirdparty/spdlog/.clang-tidy deleted file mode 100644 index bc50ee791..000000000 --- a/thirdparty/spdlog/.clang-tidy +++ /dev/null @@ -1,53 +0,0 @@ -Checks: 'cppcoreguidelines-*, -performance-*, -modernize-*, -google-*, -misc-* -cert-*, -readability-*, -clang-analyzer-*, --performance-unnecessary-value-param, --modernize-use-trailing-return-type, --google-runtime-references, --misc-non-private-member-variables-in-classes, --readability-braces-around-statements, --google-readability-braces-around-statements, --cppcoreguidelines-avoid-magic-numbers, --readability-magic-numbers, --readability-magic-numbers, --cppcoreguidelines-pro-type-vararg, --cppcoreguidelines-pro-bounds-pointer-arithmetic, --cppcoreguidelines-avoid-c-arrays, --modernize-avoid-c-arrays, --cppcoreguidelines-pro-bounds-array-to-pointer-decay, --readability-named-parameter, --cert-env33-c -' - - -WarningsAsErrors: '' -HeaderFilterRegex: '*spdlog/[^f].*' -FormatStyle: none - -CheckOptions: - - key: google-readability-braces-around-statements.ShortStatementLines - value: '1' - - key: google-readability-function-size.StatementThreshold - value: '800' - - key: google-readability-namespace-comments.ShortNamespaceLines - value: '10' - - key: google-readability-namespace-comments.SpacesBeforeComments - value: '2' - - key: modernize-loop-convert.MaxCopySize - value: '16' - - key: modernize-loop-convert.MinConfidence - value: reasonable - - key: modernize-loop-convert.NamingStyle - value: CamelCase - - key: modernize-pass-by-value.IncludeStyle - value: llvm - - key: modernize-replace-auto-ptr.IncludeStyle - value: llvm - - key: modernize-use-nullptr.NullMacros - value: 'NULL' - diff --git a/thirdparty/spdlog/.gitattributes b/thirdparty/spdlog/.gitattributes deleted file mode 100644 index fe505b275..000000000 --- a/thirdparty/spdlog/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -* text=false diff --git a/thirdparty/spdlog/.gitignore b/thirdparty/spdlog/.gitignore deleted file mode 100644 index 487fc3847..000000000 --- a/thirdparty/spdlog/.gitignore +++ /dev/null @@ -1,98 +0,0 @@ -# Auto generated files -[Dd]ebug/ -[Rr]elease/ -build/* -*.slo -*.lo -*.o -*.obj -*.suo -*.tlog -*.ilk -*.log -*.pdb -*.idb -*.iobj -*.ipdb -*.opensdf -*.sdf - -# Compiled Dynamic libraries -*.so -*.dylib -*.dll - -# Compiled Static libraries -*.lai -*.la -*.a -*.lib - -# Executables -*.exe -*.out -*.app - -# Codelite -.codelite - -# KDevelop -*.kdev4 - -# .orig files -*.orig - -# example files -example/* -!example/example.cpp -!example/bench.cpp -!example/utils.h -!example/Makefile* -!example/example.sln -!example/example.vcxproj -!example/CMakeLists.txt -!example/meson.build -!example/multisink.cpp -!example/jni - -# generated files -generated -version.rc - -# Cmake -CMakeCache.txt -CMakeFiles -CMakeScripts -Makefile -cmake_install.cmake -install_manifest.txt -/tests/tests.VC.VC.opendb -/tests/tests.VC.db -/tests/tests -/tests/logs/* -spdlogConfig.cmake -spdlogConfigVersion.cmake -compile_commands.json - -# idea -.idea/ -.cache/ -.vscode/ -cmake-build-*/ -*.db -*.ipch -*.filters -*.db-wal -*.opendb -*.db-shm -*.vcxproj -*.tcl -*.user -*.sln - -# macos -*.DS_store -*.xcodeproj/ -/.vs -/out/build -/CMakeSettings.json diff --git a/thirdparty/spdlog/LICENSE b/thirdparty/spdlog/LICENSE deleted file mode 100644 index 6546da0b9..000000000 --- a/thirdparty/spdlog/LICENSE +++ /dev/null @@ -1,26 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Gabi Melman. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - --- NOTE: Third party dependency used by this software -- -This software depends on the fmt lib (MIT License), -and users must comply to its license: https://raw.githubusercontent.com/fmtlib/fmt/master/LICENSE - diff --git a/thirdparty/spdlog/README.md b/thirdparty/spdlog/README.md deleted file mode 100644 index 262f867e5..000000000 --- a/thirdparty/spdlog/README.md +++ /dev/null @@ -1,553 +0,0 @@ -# spdlog - - -[![ci](https://github.com/gabime/spdlog/actions/workflows/linux.yml/badge.svg)](https://github.com/gabime/spdlog/actions/workflows/linux.yml)  -[![ci](https://github.com/gabime/spdlog/actions/workflows/windows.yml/badge.svg)](https://github.com/gabime/spdlog/actions/workflows/windows.yml)  -[![ci](https://github.com/gabime/spdlog/actions/workflows/macos.yml/badge.svg)](https://github.com/gabime/spdlog/actions/workflows/macos.yml)  -[![Build status](https://ci.appveyor.com/api/projects/status/d2jnxclg20vd0o50?svg=true&branch=v1.x)](https://ci.appveyor.com/project/gabime/spdlog) [![Release](https://img.shields.io/github/release/gabime/spdlog.svg)](https://github.com/gabime/spdlog/releases/latest) - -Fast C++ logging library - - -## Install -#### Header-only version -Copy the include [folder](include/spdlog) to your build tree and use a C++11 compiler. - -#### Compiled version (recommended - much faster compile times) -```console -$ git clone https://github.com/gabime/spdlog.git -$ cd spdlog && mkdir build && cd build -$ cmake .. && cmake --build . -``` -see example [CMakeLists.txt](example/CMakeLists.txt) on how to use. - -## Platforms -* Linux, FreeBSD, OpenBSD, Solaris, AIX -* Windows (msvc 2013+, cygwin) -* macOS (clang 3.5+) -* Android - -## Package managers: -* Debian: `sudo apt install libspdlog-dev` -* Homebrew: `brew install spdlog` -* MacPorts: `sudo port install spdlog` -* FreeBSD: `pkg install spdlog` -* Fedora: `dnf install spdlog` -* Gentoo: `emerge dev-libs/spdlog` -* Arch Linux: `pacman -S spdlog` -* openSUSE: `sudo zypper in spdlog-devel` -* ALT Linux: `apt-get install libspdlog-devel` -* vcpkg: `vcpkg install spdlog` -* conan: `conan install --requires=spdlog/[*]` -* conda: `conda install -c conda-forge spdlog` -* build2: ```depends: spdlog ^1.8.2``` - - -## Features -* Very fast (see [benchmarks](#benchmarks) below). -* Headers only or compiled -* Feature-rich formatting, using the excellent [fmt](https://github.com/fmtlib/fmt) library. -* Asynchronous mode (optional) -* [Custom](https://github.com/gabime/spdlog/wiki/Custom-formatting) formatting. -* Multi/Single threaded loggers. -* Various log targets: - * Rotating log files. - * Daily log files. - * Console logging (colors supported). - * syslog. - * Windows event log. - * Windows debugger (```OutputDebugString(..)```). - * Log to Qt widgets ([example](#log-to-qt-with-nice-colors)). - * Easily [extendable](https://github.com/gabime/spdlog/wiki/Sinks#implementing-your-own-sink) with custom log targets. -* Log filtering - log levels can be modified at runtime as well as compile time. -* Support for loading log levels from argv or environment var. -* [Backtrace](#backtrace-support) support - store debug messages in a ring buffer and display them later on demand. - -## Usage samples - -#### Basic usage -```c++ -#include "spdlog/spdlog.h" - -int main() -{ - spdlog::info("Welcome to spdlog!"); - spdlog::error("Some error message with arg: {}", 1); - - spdlog::warn("Easy padding in numbers like {:08d}", 12); - spdlog::critical("Support for int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}", 42); - spdlog::info("Support for floats {:03.2f}", 1.23456); - spdlog::info("Positional args are {1} {0}..", "too", "supported"); - spdlog::info("{:<30}", "left aligned"); - - spdlog::set_level(spdlog::level::debug); // Set *global* log level to debug - spdlog::debug("This message should be displayed.."); - - // change log pattern - spdlog::set_pattern("[%H:%M:%S %z] [%n] [%^---%L---%$] [thread %t] %v"); - - // Compile time log levels - // Note that this does not change the current log level, it will only - // remove (depending on SPDLOG_ACTIVE_LEVEL) the call on the release code. - SPDLOG_TRACE("Some trace message with param {}", 42); - SPDLOG_DEBUG("Some debug message"); -} - -``` ---- -#### Create stdout/stderr logger object -```c++ -#include "spdlog/spdlog.h" -#include "spdlog/sinks/stdout_color_sinks.h" -void stdout_example() -{ - // create a color multi-threaded logger - auto console = spdlog::stdout_color_mt("console"); - auto err_logger = spdlog::stderr_color_mt("stderr"); - spdlog::get("console")->info("loggers can be retrieved from a global registry using the spdlog::get(logger_name)"); -} -``` - ---- -#### Basic file logger -```c++ -#include "spdlog/sinks/basic_file_sink.h" -void basic_logfile_example() -{ - try - { - auto logger = spdlog::basic_logger_mt("basic_logger", "logs/basic-log.txt"); - } - catch (const spdlog::spdlog_ex &ex) - { - std::cout << "Log init failed: " << ex.what() << std::endl; - } -} -``` ---- -#### Rotating files -```c++ -#include "spdlog/sinks/rotating_file_sink.h" -void rotating_example() -{ - // Create a file rotating logger with 5 MB size max and 3 rotated files - auto max_size = 1048576 * 5; - auto max_files = 3; - auto logger = spdlog::rotating_logger_mt("some_logger_name", "logs/rotating.txt", max_size, max_files); -} -``` - ---- -#### Daily files -```c++ - -#include "spdlog/sinks/daily_file_sink.h" -void daily_example() -{ - // Create a daily logger - a new file is created every day at 2:30 am - auto logger = spdlog::daily_logger_mt("daily_logger", "logs/daily.txt", 2, 30); -} - -``` - ---- -#### Backtrace support -```c++ -// Debug messages can be stored in a ring buffer instead of being logged immediately. -// This is useful to display debug logs only when needed (e.g. when an error happens). -// When needed, call dump_backtrace() to dump them to your log. - -spdlog::enable_backtrace(32); // Store the latest 32 messages in a buffer. -// or my_logger->enable_backtrace(32).. -for(int i = 0; i < 100; i++) -{ - spdlog::debug("Backtrace message {}", i); // not logged yet.. -} -// e.g. if some error happened: -spdlog::dump_backtrace(); // log them now! show the last 32 messages -// or my_logger->dump_backtrace(32).. -``` - ---- -#### Periodic flush -```c++ -// periodically flush all *registered* loggers every 3 seconds: -// warning: only use if all your loggers are thread-safe ("_mt" loggers) -spdlog::flush_every(std::chrono::seconds(3)); - -``` - ---- -#### Stopwatch -```c++ -// Stopwatch support for spdlog -#include "spdlog/stopwatch.h" -void stopwatch_example() -{ - spdlog::stopwatch sw; - spdlog::debug("Elapsed {}", sw); - spdlog::debug("Elapsed {:.3}", sw); -} - -``` - ---- -#### Log binary data in hex -```c++ -// many types of std::container types can be used. -// ranges are supported too. -// format flags: -// {:X} - print in uppercase. -// {:s} - don't separate each byte with space. -// {:p} - don't print the position on each line start. -// {:n} - don't split the output into lines. -// {:a} - show ASCII if :n is not set. - -#include "spdlog/fmt/bin_to_hex.h" - -void binary_example() -{ - auto console = spdlog::get("console"); - std::array buf; - console->info("Binary example: {}", spdlog::to_hex(buf)); - console->info("Another binary example:{:n}", spdlog::to_hex(std::begin(buf), std::begin(buf) + 10)); - // more examples: - // logger->info("uppercase: {:X}", spdlog::to_hex(buf)); - // logger->info("uppercase, no delimiters: {:Xs}", spdlog::to_hex(buf)); - // logger->info("uppercase, no delimiters, no position info: {:Xsp}", spdlog::to_hex(buf)); -} - -``` - ---- -#### Logger with multi sinks - each with a different format and log level -```c++ - -// create a logger with 2 targets, with different log levels and formats. -// The console will show only warnings or errors, while the file will log all. -void multi_sink_example() -{ - auto console_sink = std::make_shared(); - console_sink->set_level(spdlog::level::warn); - console_sink->set_pattern("[multi_sink_example] [%^%l%$] %v"); - - auto file_sink = std::make_shared("logs/multisink.txt", true); - file_sink->set_level(spdlog::level::trace); - - spdlog::logger logger("multi_sink", {console_sink, file_sink}); - logger.set_level(spdlog::level::debug); - logger.warn("this should appear in both console and file"); - logger.info("this message should not appear in the console, only in the file"); -} -``` - ---- -#### Register several loggers - change global level -```c++ - -// Creation of loggers. Set levels to all registered loggers. -void set_level_example() -{ - auto logger1 = spdlog::basic_logger_mt("logger1", "logs/logger1.txt"); - auto logger2 = spdlog::basic_logger_mt("logger2", "logs/logger2.txt"); - - spdlog::set_default_logger(logger2); - spdlog::default_logger()->set_level(spdlog::level::trace); // set level for the default logger (logger2) to trace - - spdlog::trace("trace message to the logger2 (specified as default)"); - - spdlog::set_level(spdlog::level::off) // (sic!) set level for *all* registered loggers to off (disable) - - logger1.warn("warn message will not appear because the level set to off"); - logger2.warn("warn message will not appear because the level set to off"); - spdlog::warn("warn message will not appear because the level set to off"); -} -``` - ---- -#### User-defined callbacks about log events -```c++ - -// create a logger with a lambda function callback, the callback will be called -// each time something is logged to the logger -void callback_example() -{ - auto callback_sink = std::make_shared([](const spdlog::details::log_msg &msg) { - // for example you can be notified by sending an email to yourself - }); - callback_sink->set_level(spdlog::level::err); - - auto console_sink = std::make_shared(); - spdlog::logger logger("custom_callback_logger", {console_sink, callback_sink}); - - logger.info("some info log"); - logger.error("critical issue"); // will notify you -} -``` - ---- -#### Asynchronous logging -```c++ -#include "spdlog/async.h" -#include "spdlog/sinks/basic_file_sink.h" -void async_example() -{ - // default thread pool settings can be modified *before* creating the async logger: - // spdlog::init_thread_pool(8192, 1); // queue with 8k items and 1 backing thread. - auto async_file = spdlog::basic_logger_mt("async_file_logger", "logs/async_log.txt"); - // alternatively: - // auto async_file = spdlog::create_async("async_file_logger", "logs/async_log.txt"); -} - -``` - ---- -#### Asynchronous logger with multi sinks -```c++ -#include "spdlog/async.h" -#include "spdlog/sinks/stdout_color_sinks.h" -#include "spdlog/sinks/rotating_file_sink.h" - -void multi_sink_example2() -{ - spdlog::init_thread_pool(8192, 1); - auto stdout_sink = std::make_shared(); - auto rotating_sink = std::make_shared("mylog.txt", 1024*1024*10, 3); - std::vector sinks {stdout_sink, rotating_sink}; - auto logger = std::make_shared("loggername", sinks.begin(), sinks.end(), spdlog::thread_pool(), spdlog::async_overflow_policy::block); - spdlog::register_logger(logger); -} -``` - ---- -#### User-defined types -```c++ -template<> -struct fmt::formatter : fmt::formatter -{ - auto format(my_type my, format_context &ctx) const -> decltype(ctx.out()) - { - return fmt::format_to(ctx.out(), "[my_type i={}]", my.i); - } -}; - -void user_defined_example() -{ - spdlog::info("user defined type: {}", my_type(14)); -} - -``` - ---- -#### User-defined flags in the log pattern -```c++ -// Log patterns can contain custom flags. -// the following example will add new flag '%*' - which will be bound to a instance. -#include "spdlog/pattern_formatter.h" -class my_formatter_flag : public spdlog::custom_flag_formatter -{ -public: - void format(const spdlog::details::log_msg &, const std::tm &, spdlog::memory_buf_t &dest) override - { - std::string some_txt = "custom-flag"; - dest.append(some_txt.data(), some_txt.data() + some_txt.size()); - } - - std::unique_ptr clone() const override - { - return spdlog::details::make_unique(); - } -}; - -void custom_flags_example() -{ - auto formatter = std::make_unique(); - formatter->add_flag('*').set_pattern("[%n] [%*] [%^%l%$] %v"); - spdlog::set_formatter(std::move(formatter)); -} - -``` - ---- -#### Custom error handler -```c++ -void err_handler_example() -{ - // can be set globally or per logger(logger->set_error_handler(..)) - spdlog::set_error_handler([](const std::string &msg) { spdlog::get("console")->error("*** LOGGER ERROR ***: {}", msg); }); - spdlog::get("console")->info("some invalid message to trigger an error {}{}{}{}", 3); -} - -``` - ---- -#### syslog -```c++ -#include "spdlog/sinks/syslog_sink.h" -void syslog_example() -{ - std::string ident = "spdlog-example"; - auto syslog_logger = spdlog::syslog_logger_mt("syslog", ident, LOG_PID); - syslog_logger->warn("This is warning that will end up in syslog."); -} -``` ---- -#### Android example -```c++ -#include "spdlog/sinks/android_sink.h" -void android_example() -{ - std::string tag = "spdlog-android"; - auto android_logger = spdlog::android_logger_mt("android", tag); - android_logger->critical("Use \"adb shell logcat\" to view this message."); -} -``` - ---- -#### Load log levels from the env variable or argv - -```c++ -#include "spdlog/cfg/env.h" -int main (int argc, char *argv[]) -{ - spdlog::cfg::load_env_levels(); - // or specify the env variable name: - // MYAPP_LEVEL=info,mylogger=trace && ./example - // spdlog::cfg::load_env_levels("MYAPP_LEVEL"); - // or from the command line: - // ./example SPDLOG_LEVEL=info,mylogger=trace - // #include "spdlog/cfg/argv.h" // for loading levels from argv - // spdlog::cfg::load_argv_levels(argc, argv); -} -``` -So then you can: - -```console -$ export SPDLOG_LEVEL=info,mylogger=trace -$ ./example -``` - - ---- -#### Log file open/close event handlers -```c++ -// You can get callbacks from spdlog before/after a log file has been opened or closed. -// This is useful for cleanup procedures or for adding something to the start/end of the log file. -void file_events_example() -{ - // pass the spdlog::file_event_handlers to file sinks for open/close log file notifications - spdlog::file_event_handlers handlers; - handlers.before_open = [](spdlog::filename_t filename) { spdlog::info("Before opening {}", filename); }; - handlers.after_open = [](spdlog::filename_t filename, std::FILE *fstream) { fputs("After opening\n", fstream); }; - handlers.before_close = [](spdlog::filename_t filename, std::FILE *fstream) { fputs("Before closing\n", fstream); }; - handlers.after_close = [](spdlog::filename_t filename) { spdlog::info("After closing {}", filename); }; - auto my_logger = spdlog::basic_logger_st("some_logger", "logs/events-sample.txt", true, handlers); -} -``` - ---- -#### Replace the Default Logger -```c++ -void replace_default_logger_example() -{ - auto new_logger = spdlog::basic_logger_mt("new_default_logger", "logs/new-default-log.txt", true); - spdlog::set_default_logger(new_logger); - spdlog::info("new logger log message"); -} -``` - ---- -#### Log to Qt with nice colors -```c++ -#include "spdlog/spdlog.h" -#include "spdlog/sinks/qt_sinks.h" -MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent) -{ - setMinimumSize(640, 480); - auto log_widget = new QTextEdit(this); - setCentralWidget(log_widget); - int max_lines = 500; // keep the text widget to max 500 lines. remove old lines if needed. - auto logger = spdlog::qt_color_logger_mt("qt_logger", log_widget, max_lines); - logger->info("Some info message"); -} -``` ---- - -#### Mapped Diagnostic Context -```c++ -// Mapped Diagnostic Context (MDC) is a map that stores key-value pairs (string values) in thread local storage. -// Each thread maintains its own MDC, which loggers use to append diagnostic information to log outputs. -// Note: it is not supported in asynchronous mode due to its reliance on thread-local storage. -#include "spdlog/mdc.h" -void mdc_example() -{ - spdlog::mdc::put("key1", "value1"); - spdlog::mdc::put("key2", "value2"); - // if not using the default format, use the %& formatter to print mdc data - // spdlog::set_pattern("[%H:%M:%S %z] [%^%L%$] [%&] %v"); -} -``` ---- -## Benchmarks - -Below are some [benchmarks](bench/bench.cpp) done in Ubuntu 64 bit, Intel i7-4770 CPU @ 3.40GHz - -#### Synchronous mode -``` -[info] ************************************************************** -[info] Single thread, 1,000,000 iterations -[info] ************************************************************** -[info] basic_st Elapsed: 0.17 secs 5,777,626/sec -[info] rotating_st Elapsed: 0.18 secs 5,475,894/sec -[info] daily_st Elapsed: 0.20 secs 5,062,659/sec -[info] empty_logger Elapsed: 0.07 secs 14,127,300/sec -[info] ************************************************************** -[info] C-string (400 bytes). Single thread, 1,000,000 iterations -[info] ************************************************************** -[info] basic_st Elapsed: 0.41 secs 2,412,483/sec -[info] rotating_st Elapsed: 0.72 secs 1,389,196/sec -[info] daily_st Elapsed: 0.42 secs 2,393,298/sec -[info] null_st Elapsed: 0.04 secs 27,446,957/sec -[info] ************************************************************** -[info] 10 threads, competing over the same logger object, 1,000,000 iterations -[info] ************************************************************** -[info] basic_mt Elapsed: 0.60 secs 1,659,613/sec -[info] rotating_mt Elapsed: 0.62 secs 1,612,493/sec -[info] daily_mt Elapsed: 0.61 secs 1,638,305/sec -[info] null_mt Elapsed: 0.16 secs 6,272,758/sec -``` -#### Asynchronous mode -``` -[info] ------------------------------------------------- -[info] Messages : 1,000,000 -[info] Threads : 10 -[info] Queue : 8,192 slots -[info] Queue memory : 8,192 x 272 = 2,176 KB -[info] ------------------------------------------------- -[info] -[info] ********************************* -[info] Queue Overflow Policy: block -[info] ********************************* -[info] Elapsed: 1.70784 secs 585,535/sec -[info] Elapsed: 1.69805 secs 588,910/sec -[info] Elapsed: 1.7026 secs 587,337/sec -[info] -[info] ********************************* -[info] Queue Overflow Policy: overrun -[info] ********************************* -[info] Elapsed: 0.372816 secs 2,682,285/sec -[info] Elapsed: 0.379758 secs 2,633,255/sec -[info] Elapsed: 0.373532 secs 2,677,147/sec - -``` - -## Documentation - -Documentation can be found in the [wiki](https://github.com/gabime/spdlog/wiki) pages. - ---- - -### Powered by - - JetBrains logo - diff --git a/thirdparty/spdlog/include/spdlog/async.h b/thirdparty/spdlog/include/spdlog/async.h deleted file mode 100644 index 92fcd9a7d..000000000 --- a/thirdparty/spdlog/include/spdlog/async.h +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -// -// Async logging using global thread pool -// All loggers created here share same global thread pool. -// Each log message is pushed to a queue along with a shared pointer to the -// logger. -// If a logger deleted while having pending messages in the queue, it's actual -// destruction will defer -// until all its messages are processed by the thread pool. -// This is because each message in the queue holds a shared_ptr to the -// originating logger. - -#include -#include -#include - -#include -#include -#include - -namespace spdlog { - -namespace details { -static const size_t default_async_q_size = 8192; -} - -// async logger factory - creates async loggers backed with thread pool. -// if a global thread pool doesn't already exist, create it with default queue -// size of 8192 items and single thread. -template -struct async_factory_impl { - template - static std::shared_ptr create(std::string logger_name, SinkArgs &&...args) { - auto ®istry_inst = details::registry::instance(); - - // create global thread pool if not already exists.. - - auto &mutex = registry_inst.tp_mutex(); - std::lock_guard tp_lock(mutex); - auto tp = registry_inst.get_tp(); - if (tp == nullptr) { - tp = std::make_shared(details::default_async_q_size, 1U); - registry_inst.set_tp(tp); - } - - auto sink = std::make_shared(std::forward(args)...); - auto new_logger = std::make_shared(std::move(logger_name), std::move(sink), - std::move(tp), OverflowPolicy); - registry_inst.initialize_logger(new_logger); - return new_logger; - } -}; - -using async_factory = async_factory_impl; -using async_factory_nonblock = async_factory_impl; - -template -inline std::shared_ptr create_async(std::string logger_name, - SinkArgs &&...sink_args) { - return async_factory::create(std::move(logger_name), - std::forward(sink_args)...); -} - -template -inline std::shared_ptr create_async_nb(std::string logger_name, - SinkArgs &&...sink_args) { - return async_factory_nonblock::create(std::move(logger_name), - std::forward(sink_args)...); -} - -// set global thread pool. -inline void init_thread_pool(size_t q_size, - size_t thread_count, - std::function on_thread_start, - std::function on_thread_stop) { - auto tp = std::make_shared(q_size, thread_count, on_thread_start, - on_thread_stop); - details::registry::instance().set_tp(std::move(tp)); -} - -inline void init_thread_pool(size_t q_size, - size_t thread_count, - std::function on_thread_start) { - init_thread_pool(q_size, thread_count, on_thread_start, [] {}); -} - -inline void init_thread_pool(size_t q_size, size_t thread_count) { - init_thread_pool(q_size, thread_count, [] {}, [] {}); -} - -// get the global thread pool. -inline std::shared_ptr thread_pool() { - return details::registry::instance().get_tp(); -} -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/async_logger-inl.h b/thirdparty/spdlog/include/spdlog/async_logger-inl.h deleted file mode 100644 index a681d97c6..000000000 --- a/thirdparty/spdlog/include/spdlog/async_logger-inl.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -#include -#include - -SPDLOG_INLINE spdlog::async_logger::async_logger(std::string logger_name, - sinks_init_list sinks_list, - std::weak_ptr tp, - async_overflow_policy overflow_policy) - : async_logger(std::move(logger_name), - sinks_list.begin(), - sinks_list.end(), - std::move(tp), - overflow_policy) {} - -SPDLOG_INLINE spdlog::async_logger::async_logger(std::string logger_name, - sink_ptr single_sink, - std::weak_ptr tp, - async_overflow_policy overflow_policy) - : async_logger( - std::move(logger_name), {std::move(single_sink)}, std::move(tp), overflow_policy) {} - -// send the log message to the thread pool -SPDLOG_INLINE void spdlog::async_logger::sink_it_(const details::log_msg &msg){ - SPDLOG_TRY{if (auto pool_ptr = thread_pool_.lock()){ - pool_ptr -> post_log(shared_from_this(), msg, overflow_policy_); -} -else { - throw_spdlog_ex("async log: thread pool doesn't exist anymore"); -} -} -SPDLOG_LOGGER_CATCH(msg.source) -} - -// send flush request to the thread pool -SPDLOG_INLINE void spdlog::async_logger::flush_(){ - SPDLOG_TRY{if (auto pool_ptr = thread_pool_.lock()){ - pool_ptr -> post_flush(shared_from_this(), overflow_policy_); -} -else { - throw_spdlog_ex("async flush: thread pool doesn't exist anymore"); -} -} -SPDLOG_LOGGER_CATCH(source_loc()) -} - -// -// backend functions - called from the thread pool to do the actual job -// -SPDLOG_INLINE void spdlog::async_logger::backend_sink_it_(const details::log_msg &msg) { - for (auto &sink : sinks_) { - if (sink->should_log(msg.level)) { - SPDLOG_TRY { sink->log(msg); } - SPDLOG_LOGGER_CATCH(msg.source) - } - } - - if (should_flush_(msg)) { - backend_flush_(); - } -} - -SPDLOG_INLINE void spdlog::async_logger::backend_flush_() { - for (auto &sink : sinks_) { - SPDLOG_TRY { sink->flush(); } - SPDLOG_LOGGER_CATCH(source_loc()) - } -} - -SPDLOG_INLINE std::shared_ptr spdlog::async_logger::clone(std::string new_name) { - auto cloned = std::make_shared(*this); - cloned->name_ = std::move(new_name); - return cloned; -} diff --git a/thirdparty/spdlog/include/spdlog/async_logger.h b/thirdparty/spdlog/include/spdlog/async_logger.h deleted file mode 100644 index 846c4c6f0..000000000 --- a/thirdparty/spdlog/include/spdlog/async_logger.h +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -// Fast asynchronous logger. -// Uses pre allocated queue. -// Creates a single back thread to pop messages from the queue and log them. -// -// Upon each log write the logger: -// 1. Checks if its log level is enough to log the message -// 2. Push a new copy of the message to a queue (or block the caller until -// space is available in the queue) -// Upon destruction, logs all remaining messages in the queue before -// destructing.. - -#include - -namespace spdlog { - -// Async overflow policy - block by default. -enum class async_overflow_policy { - block, // Block until message can be enqueued - overrun_oldest, // Discard oldest message in the queue if full when trying to - // add new item. - discard_new // Discard new message if the queue is full when trying to add new item. -}; - -namespace details { -class thread_pool; -} - -class SPDLOG_API async_logger final : public std::enable_shared_from_this, - public logger { - friend class details::thread_pool; - -public: - template - async_logger(std::string logger_name, - It begin, - It end, - std::weak_ptr tp, - async_overflow_policy overflow_policy = async_overflow_policy::block) - : logger(std::move(logger_name), begin, end), - thread_pool_(std::move(tp)), - overflow_policy_(overflow_policy) {} - - async_logger(std::string logger_name, - sinks_init_list sinks_list, - std::weak_ptr tp, - async_overflow_policy overflow_policy = async_overflow_policy::block); - - async_logger(std::string logger_name, - sink_ptr single_sink, - std::weak_ptr tp, - async_overflow_policy overflow_policy = async_overflow_policy::block); - - std::shared_ptr clone(std::string new_name) override; - -protected: - void sink_it_(const details::log_msg &msg) override; - void flush_() override; - void backend_sink_it_(const details::log_msg &incoming_log_msg); - void backend_flush_(); - -private: - std::weak_ptr thread_pool_; - async_overflow_policy overflow_policy_; -}; -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "async_logger-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/cfg/argv.h b/thirdparty/spdlog/include/spdlog/cfg/argv.h deleted file mode 100644 index 7de2f83e7..000000000 --- a/thirdparty/spdlog/include/spdlog/cfg/argv.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once -#include -#include - -// -// Init log levels using each argv entry that starts with "SPDLOG_LEVEL=" -// -// set all loggers to debug level: -// example.exe "SPDLOG_LEVEL=debug" - -// set logger1 to trace level -// example.exe "SPDLOG_LEVEL=logger1=trace" - -// turn off all logging except for logger1 and logger2: -// example.exe "SPDLOG_LEVEL=off,logger1=debug,logger2=info" - -namespace spdlog { -namespace cfg { - -// search for SPDLOG_LEVEL= in the args and use it to init the levels -inline void load_argv_levels(int argc, const char **argv) { - const std::string spdlog_level_prefix = "SPDLOG_LEVEL="; - for (int i = 1; i < argc; i++) { - std::string arg = argv[i]; - if (arg.find(spdlog_level_prefix) == 0) { - auto levels_string = arg.substr(spdlog_level_prefix.size()); - helpers::load_levels(levels_string); - } - } -} - -inline void load_argv_levels(int argc, char **argv) { - load_argv_levels(argc, const_cast(argv)); -} - -} // namespace cfg -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/cfg/env.h b/thirdparty/spdlog/include/spdlog/cfg/env.h deleted file mode 100644 index 47bf61c72..000000000 --- a/thirdparty/spdlog/include/spdlog/cfg/env.h +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once -#include -#include -#include - -// -// Init levels and patterns from env variables SPDLOG_LEVEL -// Inspired from Rust's "env_logger" crate (https://crates.io/crates/env_logger). -// Note - fallback to "info" level on unrecognized levels -// -// Examples: -// -// set global level to debug: -// export SPDLOG_LEVEL=debug -// -// turn off all logging except for logger1: -// export SPDLOG_LEVEL="*=off,logger1=debug" -// - -// turn off all logging except for logger1 and logger2: -// export SPDLOG_LEVEL="off,logger1=debug,logger2=info" - -namespace spdlog { -namespace cfg { -inline void load_env_levels(const char* var = "SPDLOG_LEVEL") { - auto env_val = details::os::getenv(var); - if (!env_val.empty()) { - helpers::load_levels(env_val); - } -} - -} // namespace cfg -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/cfg/helpers-inl.h b/thirdparty/spdlog/include/spdlog/cfg/helpers-inl.h deleted file mode 100644 index 61b9b9f64..000000000 --- a/thirdparty/spdlog/include/spdlog/cfg/helpers-inl.h +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -#include -#include -#include -#include - -namespace spdlog { -namespace cfg { -namespace helpers { - -// inplace convert to lowercase -inline std::string &to_lower_(std::string &str) { - std::transform(str.begin(), str.end(), str.begin(), [](char ch) { - return static_cast((ch >= 'A' && ch <= 'Z') ? ch + ('a' - 'A') : ch); - }); - return str; -} - -// inplace trim spaces -inline std::string &trim_(std::string &str) { - const char *spaces = " \n\r\t"; - str.erase(str.find_last_not_of(spaces) + 1); - str.erase(0, str.find_first_not_of(spaces)); - return str; -} - -// return (name,value) trimmed pair from the given "name = value" string. -// return empty string on missing parts -// "key=val" => ("key", "val") -// " key = val " => ("key", "val") -// "key=" => ("key", "") -// "val" => ("", "val") - -inline std::pair extract_kv_(char sep, const std::string &str) { - auto n = str.find(sep); - std::string k, v; - if (n == std::string::npos) { - v = str; - } else { - k = str.substr(0, n); - v = str.substr(n + 1); - } - return std::make_pair(trim_(k), trim_(v)); -} - -// return vector of key/value pairs from a sequence of "K1=V1,K2=V2,.." -// "a=AAA,b=BBB,c=CCC,.." => {("a","AAA"),("b","BBB"),("c", "CCC"),...} -inline std::unordered_map extract_key_vals_(const std::string &str) { - std::string token; - std::istringstream token_stream(str); - std::unordered_map rv{}; - while (std::getline(token_stream, token, ',')) { - if (token.empty()) { - continue; - } - auto kv = extract_kv_('=', token); - rv[kv.first] = kv.second; - } - return rv; -} - -SPDLOG_INLINE void load_levels(const std::string &input) { - if (input.empty() || input.size() >= 32768) { - return; - } - - auto key_vals = extract_key_vals_(input); - std::unordered_map levels; - level::level_enum global_level = level::info; - bool global_level_found = false; - - for (auto &name_level : key_vals) { - const auto &logger_name = name_level.first; - const auto &level_name = to_lower_(name_level.second); - auto level = level::from_str(level_name); - // ignore unrecognized level names - if (level == level::off && level_name != "off") { - continue; - } - if (logger_name.empty()) // no logger name indicates global level - { - global_level_found = true; - global_level = level; - } else { - levels[logger_name] = level; - } - } - - details::registry::instance().set_levels(std::move(levels), - global_level_found ? &global_level : nullptr); -} - -} // namespace helpers -} // namespace cfg -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/cfg/helpers.h b/thirdparty/spdlog/include/spdlog/cfg/helpers.h deleted file mode 100644 index c02381891..000000000 --- a/thirdparty/spdlog/include/spdlog/cfg/helpers.h +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include - -namespace spdlog { -namespace cfg { -namespace helpers { -// -// Init levels from given string -// -// Examples: -// -// set global level to debug: "debug" -// turn off all logging except for logger1: "off,logger1=debug" -// turn off all logging except for logger1 and logger2: "off,logger1=debug,logger2=info" -// -SPDLOG_API void load_levels(const std::string &txt); -} // namespace helpers - -} // namespace cfg -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "helpers-inl.h" -#endif // SPDLOG_HEADER_ONLY diff --git a/thirdparty/spdlog/include/spdlog/common-inl.h b/thirdparty/spdlog/include/spdlog/common-inl.h deleted file mode 100644 index a8a0453c1..000000000 --- a/thirdparty/spdlog/include/spdlog/common-inl.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -namespace spdlog { -namespace level { - -#if __cplusplus >= 201703L -constexpr -#endif - static string_view_t level_string_views[] SPDLOG_LEVEL_NAMES; - -static const char *short_level_names[] SPDLOG_SHORT_LEVEL_NAMES; - -SPDLOG_INLINE const string_view_t &to_string_view(spdlog::level::level_enum l) SPDLOG_NOEXCEPT { - return level_string_views[l]; -} - -SPDLOG_INLINE const char *to_short_c_str(spdlog::level::level_enum l) SPDLOG_NOEXCEPT { - return short_level_names[l]; -} - -SPDLOG_INLINE spdlog::level::level_enum from_str(const std::string &name) SPDLOG_NOEXCEPT { - auto it = std::find(std::begin(level_string_views), std::end(level_string_views), name); - if (it != std::end(level_string_views)) - return static_cast(std::distance(std::begin(level_string_views), it)); - - // check also for "warn" and "err" before giving up.. - if (name == "warn") { - return level::warn; - } - if (name == "err") { - return level::err; - } - return level::off; -} -} // namespace level - -SPDLOG_INLINE spdlog_ex::spdlog_ex(std::string msg) - : msg_(std::move(msg)) {} - -SPDLOG_INLINE spdlog_ex::spdlog_ex(const std::string &msg, int last_errno) { -#ifdef SPDLOG_USE_STD_FORMAT - msg_ = std::system_error(std::error_code(last_errno, std::generic_category()), msg).what(); -#else - memory_buf_t outbuf; - fmt::format_system_error(outbuf, last_errno, msg.c_str()); - msg_ = fmt::to_string(outbuf); -#endif -} - -SPDLOG_INLINE const char *spdlog_ex::what() const SPDLOG_NOEXCEPT { return msg_.c_str(); } - -SPDLOG_INLINE void throw_spdlog_ex(const std::string &msg, int last_errno) { - SPDLOG_THROW(spdlog_ex(msg, last_errno)); -} - -SPDLOG_INLINE void throw_spdlog_ex(std::string msg) { SPDLOG_THROW(spdlog_ex(std::move(msg))); } - -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/common.h b/thirdparty/spdlog/include/spdlog/common.h deleted file mode 100644 index ba9a2e75f..000000000 --- a/thirdparty/spdlog/include/spdlog/common.h +++ /dev/null @@ -1,406 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef SPDLOG_USE_STD_FORMAT - #include - #if __cpp_lib_format >= 202207L - #include - #else - #include - #endif -#endif - -#ifdef SPDLOG_COMPILED_LIB - #undef SPDLOG_HEADER_ONLY - #if defined(SPDLOG_SHARED_LIB) - #if defined(_WIN32) - #ifdef spdlog_EXPORTS - #define SPDLOG_API __declspec(dllexport) - #else // !spdlog_EXPORTS - #define SPDLOG_API __declspec(dllimport) - #endif - #else // !defined(_WIN32) - #define SPDLOG_API __attribute__((visibility("default"))) - #endif - #else // !defined(SPDLOG_SHARED_LIB) - #define SPDLOG_API - #endif - #define SPDLOG_INLINE -#else // !defined(SPDLOG_COMPILED_LIB) - #define SPDLOG_API - #define SPDLOG_HEADER_ONLY - #define SPDLOG_INLINE inline -#endif // #ifdef SPDLOG_COMPILED_LIB - -#include - -#if !defined(SPDLOG_USE_STD_FORMAT) && \ - FMT_VERSION >= 80000 // backward compatibility with fmt versions older than 8 - #define SPDLOG_FMT_RUNTIME(format_string) fmt::runtime(format_string) - #define SPDLOG_FMT_STRING(format_string) FMT_STRING(format_string) - #if defined(SPDLOG_WCHAR_FILENAMES) || defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) - #include - #endif -#else - #define SPDLOG_FMT_RUNTIME(format_string) format_string - #define SPDLOG_FMT_STRING(format_string) format_string -#endif - -// visual studio up to 2013 does not support noexcept nor constexpr -#if defined(_MSC_VER) && (_MSC_VER < 1900) - #define SPDLOG_NOEXCEPT _NOEXCEPT - #define SPDLOG_CONSTEXPR -#else - #define SPDLOG_NOEXCEPT noexcept - #define SPDLOG_CONSTEXPR constexpr -#endif - -// If building with std::format, can just use constexpr, otherwise if building with fmt -// SPDLOG_CONSTEXPR_FUNC needs to be set the same as FMT_CONSTEXPR to avoid situations where -// a constexpr function in spdlog could end up calling a non-constexpr function in fmt -// depending on the compiler -// If fmt determines it can't use constexpr, we should inline the function instead -#ifdef SPDLOG_USE_STD_FORMAT - #define SPDLOG_CONSTEXPR_FUNC constexpr -#else // Being built with fmt - #if FMT_USE_CONSTEXPR - #define SPDLOG_CONSTEXPR_FUNC FMT_CONSTEXPR - #else - #define SPDLOG_CONSTEXPR_FUNC inline - #endif -#endif - -#if defined(__GNUC__) || defined(__clang__) - #define SPDLOG_DEPRECATED __attribute__((deprecated)) -#elif defined(_MSC_VER) - #define SPDLOG_DEPRECATED __declspec(deprecated) -#else - #define SPDLOG_DEPRECATED -#endif - -// disable thread local on msvc 2013 -#ifndef SPDLOG_NO_TLS - #if (defined(_MSC_VER) && (_MSC_VER < 1900)) || defined(__cplusplus_winrt) - #define SPDLOG_NO_TLS 1 - #endif -#endif - -#ifndef SPDLOG_FUNCTION - #define SPDLOG_FUNCTION static_cast(__FUNCTION__) -#endif - -#ifdef SPDLOG_NO_EXCEPTIONS - #define SPDLOG_TRY - #define SPDLOG_THROW(ex) \ - do { \ - printf("spdlog fatal error: %s\n", ex.what()); \ - std::abort(); \ - } while (0) - #define SPDLOG_CATCH_STD -#else - #define SPDLOG_TRY try - #define SPDLOG_THROW(ex) throw(ex) - #define SPDLOG_CATCH_STD \ - catch (const std::exception &) { \ - } -#endif - -namespace spdlog { - -class formatter; - -namespace sinks { -class sink; -} - -#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES) -using filename_t = std::wstring; - // allow macro expansion to occur in SPDLOG_FILENAME_T - #define SPDLOG_FILENAME_T_INNER(s) L##s - #define SPDLOG_FILENAME_T(s) SPDLOG_FILENAME_T_INNER(s) -#else -using filename_t = std::string; - #define SPDLOG_FILENAME_T(s) s -#endif - -using log_clock = std::chrono::system_clock; -using sink_ptr = std::shared_ptr; -using sinks_init_list = std::initializer_list; -using err_handler = std::function; -#ifdef SPDLOG_USE_STD_FORMAT -namespace fmt_lib = std; - -using string_view_t = std::string_view; -using memory_buf_t = std::string; - -template - #if __cpp_lib_format >= 202207L -using format_string_t = std::format_string; - #else -using format_string_t = std::string_view; - #endif - -template -struct is_convertible_to_basic_format_string - : std::integral_constant>::value> {}; - - #if defined(SPDLOG_WCHAR_FILENAMES) || defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) -using wstring_view_t = std::wstring_view; -using wmemory_buf_t = std::wstring; - -template - #if __cpp_lib_format >= 202207L -using wformat_string_t = std::wformat_string; - #else -using wformat_string_t = std::wstring_view; - #endif - #endif - #define SPDLOG_BUF_TO_STRING(x) x -#else // use fmt lib instead of std::format -namespace fmt_lib = fmt; - -using string_view_t = fmt::basic_string_view; -using memory_buf_t = fmt::basic_memory_buffer; - -template -using format_string_t = fmt::format_string; - -template -using remove_cvref_t = typename std::remove_cv::type>::type; - -template - #if FMT_VERSION >= 90101 -using fmt_runtime_string = fmt::runtime_format_string; - #else -using fmt_runtime_string = fmt::basic_runtime; - #endif - -// clang doesn't like SFINAE disabled constructor in std::is_convertible<> so have to repeat the -// condition from basic_format_string here, in addition, fmt::basic_runtime is only -// convertible to basic_format_string but not basic_string_view -template -struct is_convertible_to_basic_format_string - : std::integral_constant>::value || - std::is_same, fmt_runtime_string>::value> { -}; - - #if defined(SPDLOG_WCHAR_FILENAMES) || defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) -using wstring_view_t = fmt::basic_string_view; -using wmemory_buf_t = fmt::basic_memory_buffer; - -template -using wformat_string_t = fmt::wformat_string; - #endif - #define SPDLOG_BUF_TO_STRING(x) fmt::to_string(x) -#endif - -#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT - #ifndef _WIN32 - #error SPDLOG_WCHAR_TO_UTF8_SUPPORT only supported on windows - #endif // _WIN32 -#endif // SPDLOG_WCHAR_TO_UTF8_SUPPORT - -template -struct is_convertible_to_any_format_string - : std::integral_constant::value || - is_convertible_to_basic_format_string::value> {}; - -#if defined(SPDLOG_NO_ATOMIC_LEVELS) -using level_t = details::null_atomic_int; -#else -using level_t = std::atomic; -#endif - -#define SPDLOG_LEVEL_TRACE 0 -#define SPDLOG_LEVEL_DEBUG 1 -#define SPDLOG_LEVEL_INFO 2 -#define SPDLOG_LEVEL_WARN 3 -#define SPDLOG_LEVEL_ERROR 4 -#define SPDLOG_LEVEL_CRITICAL 5 -#define SPDLOG_LEVEL_OFF 6 - -#if !defined(SPDLOG_ACTIVE_LEVEL) - #define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_INFO -#endif - -// Log level enum -namespace level { -enum level_enum : int { - trace = SPDLOG_LEVEL_TRACE, - debug = SPDLOG_LEVEL_DEBUG, - info = SPDLOG_LEVEL_INFO, - warn = SPDLOG_LEVEL_WARN, - err = SPDLOG_LEVEL_ERROR, - critical = SPDLOG_LEVEL_CRITICAL, - off = SPDLOG_LEVEL_OFF, - n_levels -}; - -#define SPDLOG_LEVEL_NAME_TRACE spdlog::string_view_t("trace", 5) -#define SPDLOG_LEVEL_NAME_DEBUG spdlog::string_view_t("debug", 5) -#define SPDLOG_LEVEL_NAME_INFO spdlog::string_view_t("info", 4) -#define SPDLOG_LEVEL_NAME_WARNING spdlog::string_view_t("warning", 7) -#define SPDLOG_LEVEL_NAME_ERROR spdlog::string_view_t("error", 5) -#define SPDLOG_LEVEL_NAME_CRITICAL spdlog::string_view_t("critical", 8) -#define SPDLOG_LEVEL_NAME_OFF spdlog::string_view_t("off", 3) - -#if !defined(SPDLOG_LEVEL_NAMES) - #define SPDLOG_LEVEL_NAMES \ - { \ - SPDLOG_LEVEL_NAME_TRACE, SPDLOG_LEVEL_NAME_DEBUG, SPDLOG_LEVEL_NAME_INFO, \ - SPDLOG_LEVEL_NAME_WARNING, SPDLOG_LEVEL_NAME_ERROR, SPDLOG_LEVEL_NAME_CRITICAL, \ - SPDLOG_LEVEL_NAME_OFF \ - } -#endif - -#if !defined(SPDLOG_SHORT_LEVEL_NAMES) - - #define SPDLOG_SHORT_LEVEL_NAMES \ - { "T", "D", "I", "W", "E", "C", "O" } -#endif - -SPDLOG_API const string_view_t &to_string_view(spdlog::level::level_enum l) SPDLOG_NOEXCEPT; -SPDLOG_API const char *to_short_c_str(spdlog::level::level_enum l) SPDLOG_NOEXCEPT; -SPDLOG_API spdlog::level::level_enum from_str(const std::string &name) SPDLOG_NOEXCEPT; - -} // namespace level - -// -// Color mode used by sinks with color support. -// -enum class color_mode { always, automatic, never }; - -// -// Pattern time - specific time getting to use for pattern_formatter. -// local time by default -// -enum class pattern_time_type { - local, // log localtime - utc // log utc -}; - -// -// Log exception -// -class SPDLOG_API spdlog_ex : public std::exception { -public: - explicit spdlog_ex(std::string msg); - spdlog_ex(const std::string &msg, int last_errno); - const char *what() const SPDLOG_NOEXCEPT override; - -private: - std::string msg_; -}; - -[[noreturn]] SPDLOG_API void throw_spdlog_ex(const std::string &msg, int last_errno); -[[noreturn]] SPDLOG_API void throw_spdlog_ex(std::string msg); - -struct source_loc { - SPDLOG_CONSTEXPR source_loc() = default; - SPDLOG_CONSTEXPR source_loc(const char *filename_in, int line_in, const char *funcname_in) - : filename{filename_in}, - line{line_in}, - funcname{funcname_in} {} - - SPDLOG_CONSTEXPR bool empty() const SPDLOG_NOEXCEPT { return line <= 0; } - const char *filename{nullptr}; - int line{0}; - const char *funcname{nullptr}; -}; - -struct file_event_handlers { - file_event_handlers() - : before_open(nullptr), - after_open(nullptr), - before_close(nullptr), - after_close(nullptr) {} - - std::function before_open; - std::function after_open; - std::function before_close; - std::function after_close; -}; - -namespace details { - -// to_string_view - -SPDLOG_CONSTEXPR_FUNC spdlog::string_view_t to_string_view(const memory_buf_t &buf) - SPDLOG_NOEXCEPT { - return spdlog::string_view_t{buf.data(), buf.size()}; -} - -SPDLOG_CONSTEXPR_FUNC spdlog::string_view_t to_string_view(spdlog::string_view_t str) - SPDLOG_NOEXCEPT { - return str; -} - -#if defined(SPDLOG_WCHAR_FILENAMES) || defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) -SPDLOG_CONSTEXPR_FUNC spdlog::wstring_view_t to_string_view(const wmemory_buf_t &buf) - SPDLOG_NOEXCEPT { - return spdlog::wstring_view_t{buf.data(), buf.size()}; -} - -SPDLOG_CONSTEXPR_FUNC spdlog::wstring_view_t to_string_view(spdlog::wstring_view_t str) - SPDLOG_NOEXCEPT { - return str; -} -#endif - -#if defined(SPDLOG_USE_STD_FORMAT) && __cpp_lib_format >= 202207L -template -SPDLOG_CONSTEXPR_FUNC std::basic_string_view to_string_view( - std::basic_format_string fmt) SPDLOG_NOEXCEPT { - return fmt.get(); -} -#endif - -// make_unique support for pre c++14 -#if __cplusplus >= 201402L // C++14 and beyond -using std::enable_if_t; -using std::make_unique; -#else -template -using enable_if_t = typename std::enable_if::type; - -template -std::unique_ptr make_unique(Args &&...args) { - static_assert(!std::is_array::value, "arrays not supported"); - return std::unique_ptr(new T(std::forward(args)...)); -} -#endif - -// to avoid useless casts (see https://github.com/nlohmann/json/issues/2893#issuecomment-889152324) -template ::value, int> = 0> -constexpr T conditional_static_cast(U value) { - return static_cast(value); -} - -template ::value, int> = 0> -constexpr T conditional_static_cast(U value) { - return value; -} - -} // namespace details -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "common-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/details/backtracer-inl.h b/thirdparty/spdlog/include/spdlog/details/backtracer-inl.h deleted file mode 100644 index 43d100247..000000000 --- a/thirdparty/spdlog/include/spdlog/details/backtracer-inl.h +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif -namespace spdlog { -namespace details { -SPDLOG_INLINE backtracer::backtracer(const backtracer &other) { - std::lock_guard lock(other.mutex_); - enabled_ = other.enabled(); - messages_ = other.messages_; -} - -SPDLOG_INLINE backtracer::backtracer(backtracer &&other) SPDLOG_NOEXCEPT { - std::lock_guard lock(other.mutex_); - enabled_ = other.enabled(); - messages_ = std::move(other.messages_); -} - -SPDLOG_INLINE backtracer &backtracer::operator=(backtracer other) { - std::lock_guard lock(mutex_); - enabled_ = other.enabled(); - messages_ = std::move(other.messages_); - return *this; -} - -SPDLOG_INLINE void backtracer::enable(size_t size) { - std::lock_guard lock{mutex_}; - enabled_.store(true, std::memory_order_relaxed); - messages_ = circular_q{size}; -} - -SPDLOG_INLINE void backtracer::disable() { - std::lock_guard lock{mutex_}; - enabled_.store(false, std::memory_order_relaxed); -} - -SPDLOG_INLINE bool backtracer::enabled() const { return enabled_.load(std::memory_order_relaxed); } - -SPDLOG_INLINE void backtracer::push_back(const log_msg &msg) { - std::lock_guard lock{mutex_}; - messages_.push_back(log_msg_buffer{msg}); -} - -SPDLOG_INLINE bool backtracer::empty() const { - std::lock_guard lock{mutex_}; - return messages_.empty(); -} - -// pop all items in the q and apply the given fun on each of them. -SPDLOG_INLINE void backtracer::foreach_pop(std::function fun) { - std::lock_guard lock{mutex_}; - while (!messages_.empty()) { - auto &front_msg = messages_.front(); - fun(front_msg); - messages_.pop_front(); - } -} -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/backtracer.h b/thirdparty/spdlog/include/spdlog/details/backtracer.h deleted file mode 100644 index 541339cdc..000000000 --- a/thirdparty/spdlog/include/spdlog/details/backtracer.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include - -#include -#include -#include - -// Store log messages in circular buffer. -// Useful for storing debug data in case of error/warning happens. - -namespace spdlog { -namespace details { -class SPDLOG_API backtracer { - mutable std::mutex mutex_; - std::atomic enabled_{false}; - circular_q messages_; - -public: - backtracer() = default; - backtracer(const backtracer &other); - - backtracer(backtracer &&other) SPDLOG_NOEXCEPT; - backtracer &operator=(backtracer other); - - void enable(size_t size); - void disable(); - bool enabled() const; - void push_back(const log_msg &msg); - bool empty() const; - - // pop all items in the q and apply the given fun on each of them. - void foreach_pop(std::function fun); -}; - -} // namespace details -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "backtracer-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/details/circular_q.h b/thirdparty/spdlog/include/spdlog/details/circular_q.h deleted file mode 100644 index 29e9d255f..000000000 --- a/thirdparty/spdlog/include/spdlog/details/circular_q.h +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -// circular q view of std::vector. -#pragma once - -#include -#include - -#include "spdlog/common.h" - -namespace spdlog { -namespace details { -template -class circular_q { - size_t max_items_ = 0; - typename std::vector::size_type head_ = 0; - typename std::vector::size_type tail_ = 0; - size_t overrun_counter_ = 0; - std::vector v_; - -public: - using value_type = T; - - // empty ctor - create a disabled queue with no elements allocated at all - circular_q() = default; - - explicit circular_q(size_t max_items) - : max_items_(max_items + 1) // one item is reserved as marker for full q - , - v_(max_items_) {} - - circular_q(const circular_q &) = default; - circular_q &operator=(const circular_q &) = default; - - // move cannot be default, - // since we need to reset head_, tail_, etc to zero in the moved object - circular_q(circular_q &&other) SPDLOG_NOEXCEPT { copy_moveable(std::move(other)); } - - circular_q &operator=(circular_q &&other) SPDLOG_NOEXCEPT { - copy_moveable(std::move(other)); - return *this; - } - - // push back, overrun (oldest) item if no room left - void push_back(T &&item) { - if (max_items_ > 0) { - v_[tail_] = std::move(item); - tail_ = (tail_ + 1) % max_items_; - - if (tail_ == head_) // overrun last item if full - { - head_ = (head_ + 1) % max_items_; - ++overrun_counter_; - } - } - } - - // Return reference to the front item. - // If there are no elements in the container, the behavior is undefined. - const T &front() const { return v_[head_]; } - - T &front() { return v_[head_]; } - - // Return number of elements actually stored - size_t size() const { - if (tail_ >= head_) { - return tail_ - head_; - } else { - return max_items_ - (head_ - tail_); - } - } - - // Return const reference to item by index. - // If index is out of range 0…size()-1, the behavior is undefined. - const T &at(size_t i) const { - assert(i < size()); - return v_[(head_ + i) % max_items_]; - } - - // Pop item from front. - // If there are no elements in the container, the behavior is undefined. - void pop_front() { head_ = (head_ + 1) % max_items_; } - - bool empty() const { return tail_ == head_; } - - bool full() const { - // head is ahead of the tail by 1 - if (max_items_ > 0) { - return ((tail_ + 1) % max_items_) == head_; - } - return false; - } - - size_t overrun_counter() const { return overrun_counter_; } - - void reset_overrun_counter() { overrun_counter_ = 0; } - -private: - // copy from other&& and reset it to disabled state - void copy_moveable(circular_q &&other) SPDLOG_NOEXCEPT { - max_items_ = other.max_items_; - head_ = other.head_; - tail_ = other.tail_; - overrun_counter_ = other.overrun_counter_; - v_ = std::move(other.v_); - - // put &&other in disabled, but valid state - other.max_items_ = 0; - other.head_ = other.tail_ = 0; - other.overrun_counter_ = 0; - } -}; -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/console_globals.h b/thirdparty/spdlog/include/spdlog/details/console_globals.h deleted file mode 100644 index 9c552106a..000000000 --- a/thirdparty/spdlog/include/spdlog/details/console_globals.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include - -namespace spdlog { -namespace details { - -struct console_mutex { - using mutex_t = std::mutex; - static mutex_t &mutex() { - static mutex_t s_mutex; - return s_mutex; - } -}; - -struct console_nullmutex { - using mutex_t = null_mutex; - static mutex_t &mutex() { - static mutex_t s_mutex; - return s_mutex; - } -}; -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/file_helper-inl.h b/thirdparty/spdlog/include/spdlog/details/file_helper-inl.h deleted file mode 100644 index 0c514ef2a..000000000 --- a/thirdparty/spdlog/include/spdlog/details/file_helper-inl.h +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -#include -#include -#include -#include - -namespace spdlog { -namespace details { - -SPDLOG_INLINE file_helper::file_helper(const file_event_handlers &event_handlers) - : event_handlers_(event_handlers) {} - -SPDLOG_INLINE file_helper::~file_helper() { close(); } - -SPDLOG_INLINE void file_helper::open(const filename_t &fname, bool truncate) { - close(); - filename_ = fname; - - auto *mode = SPDLOG_FILENAME_T("ab"); - auto *trunc_mode = SPDLOG_FILENAME_T("wb"); - - if (event_handlers_.before_open) { - event_handlers_.before_open(filename_); - } - for (int tries = 0; tries < open_tries_; ++tries) { - // create containing folder if not exists already. - os::create_dir(os::dir_name(fname)); - if (truncate) { - // Truncate by opening-and-closing a tmp file in "wb" mode, always - // opening the actual log-we-write-to in "ab" mode, since that - // interacts more politely with eternal processes that might - // rotate/truncate the file underneath us. - std::FILE *tmp; - if (os::fopen_s(&tmp, fname, trunc_mode)) { - continue; - } - std::fclose(tmp); - } - if (!os::fopen_s(&fd_, fname, mode)) { - if (event_handlers_.after_open) { - event_handlers_.after_open(filename_, fd_); - } - return; - } - - details::os::sleep_for_millis(open_interval_); - } - - throw_spdlog_ex("Failed opening file " + os::filename_to_str(filename_) + " for writing", - errno); -} - -SPDLOG_INLINE void file_helper::reopen(bool truncate) { - if (filename_.empty()) { - throw_spdlog_ex("Failed re opening file - was not opened before"); - } - this->open(filename_, truncate); -} - -SPDLOG_INLINE void file_helper::flush() { - if (std::fflush(fd_) != 0) { - throw_spdlog_ex("Failed flush to file " + os::filename_to_str(filename_), errno); - } -} - -SPDLOG_INLINE void file_helper::sync() { - if (!os::fsync(fd_)) { - throw_spdlog_ex("Failed to fsync file " + os::filename_to_str(filename_), errno); - } -} - -SPDLOG_INLINE void file_helper::close() { - if (fd_ != nullptr) { - if (event_handlers_.before_close) { - event_handlers_.before_close(filename_, fd_); - } - - std::fclose(fd_); - fd_ = nullptr; - - if (event_handlers_.after_close) { - event_handlers_.after_close(filename_); - } - } -} - -SPDLOG_INLINE void file_helper::write(const memory_buf_t &buf) { - if (fd_ == nullptr) return; - size_t msg_size = buf.size(); - auto data = buf.data(); - - if (!details::os::fwrite_bytes(data, msg_size, fd_)) { - throw_spdlog_ex("Failed writing to file " + os::filename_to_str(filename_), errno); - } -} - -SPDLOG_INLINE size_t file_helper::size() const { - if (fd_ == nullptr) { - throw_spdlog_ex("Cannot use size() on closed file " + os::filename_to_str(filename_)); - } - return os::filesize(fd_); -} - -SPDLOG_INLINE const filename_t &file_helper::filename() const { return filename_; } - -// -// return file path and its extension: -// -// "mylog.txt" => ("mylog", ".txt") -// "mylog" => ("mylog", "") -// "mylog." => ("mylog.", "") -// "/dir1/dir2/mylog.txt" => ("/dir1/dir2/mylog", ".txt") -// -// the starting dot in filenames is ignored (hidden files): -// -// ".mylog" => (".mylog". "") -// "my_folder/.mylog" => ("my_folder/.mylog", "") -// "my_folder/.mylog.txt" => ("my_folder/.mylog", ".txt") -SPDLOG_INLINE std::tuple file_helper::split_by_extension( - const filename_t &fname) { - auto ext_index = fname.rfind('.'); - - // no valid extension found - return whole path and empty string as - // extension - if (ext_index == filename_t::npos || ext_index == 0 || ext_index == fname.size() - 1) { - return std::make_tuple(fname, filename_t()); - } - - // treat cases like "/etc/rc.d/somelogfile or "/abc/.hiddenfile" - auto folder_index = fname.find_last_of(details::os::folder_seps_filename); - if (folder_index != filename_t::npos && folder_index >= ext_index - 1) { - return std::make_tuple(fname, filename_t()); - } - - // finally - return a valid base and extension tuple - return std::make_tuple(fname.substr(0, ext_index), fname.substr(ext_index)); -} - -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/file_helper.h b/thirdparty/spdlog/include/spdlog/details/file_helper.h deleted file mode 100644 index f0e5d180e..000000000 --- a/thirdparty/spdlog/include/spdlog/details/file_helper.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include - -namespace spdlog { -namespace details { - -// Helper class for file sinks. -// When failing to open a file, retry several times(5) with a delay interval(10 ms). -// Throw spdlog_ex exception on errors. - -class SPDLOG_API file_helper { -public: - file_helper() = default; - explicit file_helper(const file_event_handlers &event_handlers); - - file_helper(const file_helper &) = delete; - file_helper &operator=(const file_helper &) = delete; - ~file_helper(); - - void open(const filename_t &fname, bool truncate = false); - void reopen(bool truncate); - void flush(); - void sync(); - void close(); - void write(const memory_buf_t &buf); - size_t size() const; - const filename_t &filename() const; - - // - // return file path and its extension: - // - // "mylog.txt" => ("mylog", ".txt") - // "mylog" => ("mylog", "") - // "mylog." => ("mylog.", "") - // "/dir1/dir2/mylog.txt" => ("/dir1/dir2/mylog", ".txt") - // - // the starting dot in filenames is ignored (hidden files): - // - // ".mylog" => (".mylog". "") - // "my_folder/.mylog" => ("my_folder/.mylog", "") - // "my_folder/.mylog.txt" => ("my_folder/.mylog", ".txt") - static std::tuple split_by_extension(const filename_t &fname); - -private: - const int open_tries_ = 5; - const unsigned int open_interval_ = 10; - std::FILE *fd_{nullptr}; - filename_t filename_; - file_event_handlers event_handlers_; -}; -} // namespace details -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "file_helper-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/details/fmt_helper.h b/thirdparty/spdlog/include/spdlog/details/fmt_helper.h deleted file mode 100644 index 61306003b..000000000 --- a/thirdparty/spdlog/include/spdlog/details/fmt_helper.h +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -#pragma once - -#include -#include -#include -#include -#include - -#ifdef SPDLOG_USE_STD_FORMAT - #include - #include -#endif - -// Some fmt helpers to efficiently format and pad ints and strings -namespace spdlog { -namespace details { -namespace fmt_helper { - -inline void append_string_view(spdlog::string_view_t view, memory_buf_t &dest) { - auto *buf_ptr = view.data(); - dest.append(buf_ptr, buf_ptr + view.size()); -} - -#ifdef SPDLOG_USE_STD_FORMAT -template -inline void append_int(T n, memory_buf_t &dest) { - // Buffer should be large enough to hold all digits (digits10 + 1) and a sign - SPDLOG_CONSTEXPR const auto BUF_SIZE = std::numeric_limits::digits10 + 2; - char buf[BUF_SIZE]; - - auto [ptr, ec] = std::to_chars(buf, buf + BUF_SIZE, n, 10); - if (ec == std::errc()) { - dest.append(buf, ptr); - } else { - throw_spdlog_ex("Failed to format int", static_cast(ec)); - } -} -#else -template -inline void append_int(T n, memory_buf_t &dest) { - fmt::format_int i(n); - dest.append(i.data(), i.data() + i.size()); -} -#endif - -template -SPDLOG_CONSTEXPR_FUNC unsigned int count_digits_fallback(T n) { - // taken from fmt: https://github.com/fmtlib/fmt/blob/8.0.1/include/fmt/format.h#L899-L912 - unsigned int count = 1; - for (;;) { - // Integer division is slow so do it for a group of four digits instead - // of for every digit. The idea comes from the talk by Alexandrescu - // "Three Optimization Tips for C++". See speed-test for a comparison. - if (n < 10) return count; - if (n < 100) return count + 1; - if (n < 1000) return count + 2; - if (n < 10000) return count + 3; - n /= 10000u; - count += 4; - } -} - -template -inline unsigned int count_digits(T n) { - using count_type = - typename std::conditional<(sizeof(T) > sizeof(uint32_t)), uint64_t, uint32_t>::type; -#ifdef SPDLOG_USE_STD_FORMAT - return count_digits_fallback(static_cast(n)); -#else - return static_cast(fmt:: - // fmt 7.0.0 renamed the internal namespace to detail. - // See: https://github.com/fmtlib/fmt/issues/1538 - #if FMT_VERSION < 70000 - internal - #else - detail - #endif - ::count_digits(static_cast(n))); -#endif -} - -inline void pad2(int n, memory_buf_t &dest) { - if (n >= 0 && n < 100) // 0-99 - { - dest.push_back(static_cast('0' + n / 10)); - dest.push_back(static_cast('0' + n % 10)); - } else // unlikely, but just in case, let fmt deal with it - { - fmt_lib::format_to(std::back_inserter(dest), SPDLOG_FMT_STRING("{:02}"), n); - } -} - -template -inline void pad_uint(T n, unsigned int width, memory_buf_t &dest) { - static_assert(std::is_unsigned::value, "pad_uint must get unsigned T"); - for (auto digits = count_digits(n); digits < width; digits++) { - dest.push_back('0'); - } - append_int(n, dest); -} - -template -inline void pad3(T n, memory_buf_t &dest) { - static_assert(std::is_unsigned::value, "pad3 must get unsigned T"); - if (n < 1000) { - dest.push_back(static_cast(n / 100 + '0')); - n = n % 100; - dest.push_back(static_cast((n / 10) + '0')); - dest.push_back(static_cast((n % 10) + '0')); - } else { - append_int(n, dest); - } -} - -template -inline void pad6(T n, memory_buf_t &dest) { - pad_uint(n, 6, dest); -} - -template -inline void pad9(T n, memory_buf_t &dest) { - pad_uint(n, 9, dest); -} - -// return fraction of a second of the given time_point. -// e.g. -// fraction(tp) -> will return the millis part of the second -template -inline ToDuration time_fraction(log_clock::time_point tp) { - using std::chrono::duration_cast; - using std::chrono::seconds; - auto duration = tp.time_since_epoch(); - auto secs = duration_cast(duration); - return duration_cast(duration) - duration_cast(secs); -} - -} // namespace fmt_helper -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/log_msg-inl.h b/thirdparty/spdlog/include/spdlog/details/log_msg-inl.h deleted file mode 100644 index aa3a95768..000000000 --- a/thirdparty/spdlog/include/spdlog/details/log_msg-inl.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include - -namespace spdlog { -namespace details { - -SPDLOG_INLINE log_msg::log_msg(spdlog::log_clock::time_point log_time, - spdlog::source_loc loc, - string_view_t a_logger_name, - spdlog::level::level_enum lvl, - spdlog::string_view_t msg) - : logger_name(a_logger_name), - level(lvl), - time(log_time) -#ifndef SPDLOG_NO_THREAD_ID - , - thread_id(os::thread_id()) -#endif - , - source(loc), - payload(msg) { -} - -SPDLOG_INLINE log_msg::log_msg(spdlog::source_loc loc, - string_view_t a_logger_name, - spdlog::level::level_enum lvl, - spdlog::string_view_t msg) - : log_msg(os::now(), loc, a_logger_name, lvl, msg) {} - -SPDLOG_INLINE log_msg::log_msg(string_view_t a_logger_name, - spdlog::level::level_enum lvl, - spdlog::string_view_t msg) - : log_msg(os::now(), source_loc{}, a_logger_name, lvl, msg) {} - -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/log_msg.h b/thirdparty/spdlog/include/spdlog/details/log_msg.h deleted file mode 100644 index 87df1e833..000000000 --- a/thirdparty/spdlog/include/spdlog/details/log_msg.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include - -namespace spdlog { -namespace details { -struct SPDLOG_API log_msg { - log_msg() = default; - log_msg(log_clock::time_point log_time, - source_loc loc, - string_view_t logger_name, - level::level_enum lvl, - string_view_t msg); - log_msg(source_loc loc, string_view_t logger_name, level::level_enum lvl, string_view_t msg); - log_msg(string_view_t logger_name, level::level_enum lvl, string_view_t msg); - log_msg(const log_msg &other) = default; - log_msg &operator=(const log_msg &other) = default; - - string_view_t logger_name; - level::level_enum level{level::off}; - log_clock::time_point time; - size_t thread_id{0}; - - // wrapping the formatted text with color (updated by pattern_formatter). - mutable size_t color_range_start{0}; - mutable size_t color_range_end{0}; - - source_loc source; - string_view_t payload; -}; -} // namespace details -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "log_msg-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/details/log_msg_buffer-inl.h b/thirdparty/spdlog/include/spdlog/details/log_msg_buffer-inl.h deleted file mode 100644 index 2eb242859..000000000 --- a/thirdparty/spdlog/include/spdlog/details/log_msg_buffer-inl.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -namespace spdlog { -namespace details { - -SPDLOG_INLINE log_msg_buffer::log_msg_buffer(const log_msg &orig_msg) - : log_msg{orig_msg} { - buffer.append(logger_name.begin(), logger_name.end()); - buffer.append(payload.begin(), payload.end()); - update_string_views(); -} - -SPDLOG_INLINE log_msg_buffer::log_msg_buffer(const log_msg_buffer &other) - : log_msg{other} { - buffer.append(logger_name.begin(), logger_name.end()); - buffer.append(payload.begin(), payload.end()); - update_string_views(); -} - -SPDLOG_INLINE log_msg_buffer::log_msg_buffer(log_msg_buffer &&other) SPDLOG_NOEXCEPT - : log_msg{other}, - buffer{std::move(other.buffer)} { - update_string_views(); -} - -SPDLOG_INLINE log_msg_buffer &log_msg_buffer::operator=(const log_msg_buffer &other) { - log_msg::operator=(other); - buffer.clear(); - buffer.append(other.buffer.data(), other.buffer.data() + other.buffer.size()); - update_string_views(); - return *this; -} - -SPDLOG_INLINE log_msg_buffer &log_msg_buffer::operator=(log_msg_buffer &&other) SPDLOG_NOEXCEPT { - log_msg::operator=(other); - buffer = std::move(other.buffer); - update_string_views(); - return *this; -} - -SPDLOG_INLINE void log_msg_buffer::update_string_views() { - logger_name = string_view_t{buffer.data(), logger_name.size()}; - payload = string_view_t{buffer.data() + logger_name.size(), payload.size()}; -} - -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/log_msg_buffer.h b/thirdparty/spdlog/include/spdlog/details/log_msg_buffer.h deleted file mode 100644 index 1143b3ba4..000000000 --- a/thirdparty/spdlog/include/spdlog/details/log_msg_buffer.h +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include - -namespace spdlog { -namespace details { - -// Extend log_msg with internal buffer to store its payload. -// This is needed since log_msg holds string_views that points to stack data. - -class SPDLOG_API log_msg_buffer : public log_msg { - memory_buf_t buffer; - void update_string_views(); - -public: - log_msg_buffer() = default; - explicit log_msg_buffer(const log_msg &orig_msg); - log_msg_buffer(const log_msg_buffer &other); - log_msg_buffer(log_msg_buffer &&other) SPDLOG_NOEXCEPT; - log_msg_buffer &operator=(const log_msg_buffer &other); - log_msg_buffer &operator=(log_msg_buffer &&other) SPDLOG_NOEXCEPT; -}; - -} // namespace details -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "log_msg_buffer-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/details/mpmc_blocking_q.h b/thirdparty/spdlog/include/spdlog/details/mpmc_blocking_q.h deleted file mode 100644 index 5848cca83..000000000 --- a/thirdparty/spdlog/include/spdlog/details/mpmc_blocking_q.h +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -// multi producer-multi consumer blocking queue. -// enqueue(..) - will block until room found to put the new message. -// enqueue_nowait(..) - will return immediately with false if no room left in -// the queue. -// dequeue_for(..) - will block until the queue is not empty or timeout have -// passed. - -#include - -#include -#include -#include - -namespace spdlog { -namespace details { - -template -class mpmc_blocking_queue { -public: - using item_type = T; - explicit mpmc_blocking_queue(size_t max_items) - : q_(max_items) {} - -#ifndef __MINGW32__ - // try to enqueue and block if no room left - void enqueue(T &&item) { - { - std::unique_lock lock(queue_mutex_); - pop_cv_.wait(lock, [this] { return !this->q_.full(); }); - q_.push_back(std::move(item)); - } - push_cv_.notify_one(); - } - - // enqueue immediately. overrun oldest message in the queue if no room left. - void enqueue_nowait(T &&item) { - { - std::unique_lock lock(queue_mutex_); - q_.push_back(std::move(item)); - } - push_cv_.notify_one(); - } - - void enqueue_if_have_room(T &&item) { - bool pushed = false; - { - std::unique_lock lock(queue_mutex_); - if (!q_.full()) { - q_.push_back(std::move(item)); - pushed = true; - } - } - - if (pushed) { - push_cv_.notify_one(); - } else { - ++discard_counter_; - } - } - - // dequeue with a timeout. - // Return true, if succeeded dequeue item, false otherwise - bool dequeue_for(T &popped_item, std::chrono::milliseconds wait_duration) { - { - std::unique_lock lock(queue_mutex_); - if (!push_cv_.wait_for(lock, wait_duration, [this] { return !this->q_.empty(); })) { - return false; - } - popped_item = std::move(q_.front()); - q_.pop_front(); - } - pop_cv_.notify_one(); - return true; - } - - // blocking dequeue without a timeout. - void dequeue(T &popped_item) { - { - std::unique_lock lock(queue_mutex_); - push_cv_.wait(lock, [this] { return !this->q_.empty(); }); - popped_item = std::move(q_.front()); - q_.pop_front(); - } - pop_cv_.notify_one(); - } - -#else - // apparently mingw deadlocks if the mutex is released before cv.notify_one(), - // so release the mutex at the very end each function. - - // try to enqueue and block if no room left - void enqueue(T &&item) { - std::unique_lock lock(queue_mutex_); - pop_cv_.wait(lock, [this] { return !this->q_.full(); }); - q_.push_back(std::move(item)); - push_cv_.notify_one(); - } - - // enqueue immediately. overrun oldest message in the queue if no room left. - void enqueue_nowait(T &&item) { - std::unique_lock lock(queue_mutex_); - q_.push_back(std::move(item)); - push_cv_.notify_one(); - } - - void enqueue_if_have_room(T &&item) { - bool pushed = false; - std::unique_lock lock(queue_mutex_); - if (!q_.full()) { - q_.push_back(std::move(item)); - pushed = true; - } - - if (pushed) { - push_cv_.notify_one(); - } else { - ++discard_counter_; - } - } - - // dequeue with a timeout. - // Return true, if succeeded dequeue item, false otherwise - bool dequeue_for(T &popped_item, std::chrono::milliseconds wait_duration) { - std::unique_lock lock(queue_mutex_); - if (!push_cv_.wait_for(lock, wait_duration, [this] { return !this->q_.empty(); })) { - return false; - } - popped_item = std::move(q_.front()); - q_.pop_front(); - pop_cv_.notify_one(); - return true; - } - - // blocking dequeue without a timeout. - void dequeue(T &popped_item) { - std::unique_lock lock(queue_mutex_); - push_cv_.wait(lock, [this] { return !this->q_.empty(); }); - popped_item = std::move(q_.front()); - q_.pop_front(); - pop_cv_.notify_one(); - } - -#endif - - size_t overrun_counter() { - std::lock_guard lock(queue_mutex_); - return q_.overrun_counter(); - } - - size_t discard_counter() { return discard_counter_.load(std::memory_order_relaxed); } - - size_t size() { - std::lock_guard lock(queue_mutex_); - return q_.size(); - } - - void reset_overrun_counter() { - std::lock_guard lock(queue_mutex_); - q_.reset_overrun_counter(); - } - - void reset_discard_counter() { discard_counter_.store(0, std::memory_order_relaxed); } - -private: - std::mutex queue_mutex_; - std::condition_variable push_cv_; - std::condition_variable pop_cv_; - spdlog::details::circular_q q_; - std::atomic discard_counter_{0}; -}; -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/null_mutex.h b/thirdparty/spdlog/include/spdlog/details/null_mutex.h deleted file mode 100644 index e3b322041..000000000 --- a/thirdparty/spdlog/include/spdlog/details/null_mutex.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -// null, no cost dummy "mutex" and dummy "atomic" int - -namespace spdlog { -namespace details { -struct null_mutex { - void lock() const {} - void unlock() const {} -}; - -struct null_atomic_int { - int value; - null_atomic_int() = default; - - explicit null_atomic_int(int new_value) - : value(new_value) {} - - int load(std::memory_order = std::memory_order_relaxed) const { return value; } - - void store(int new_value, std::memory_order = std::memory_order_relaxed) { value = new_value; } - - int exchange(int new_value, std::memory_order = std::memory_order_relaxed) { - std::swap(new_value, value); - return new_value; // return value before the call - } -}; - -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/os-inl.h b/thirdparty/spdlog/include/spdlog/details/os-inl.h deleted file mode 100644 index edbbd5c25..000000000 --- a/thirdparty/spdlog/include/spdlog/details/os-inl.h +++ /dev/null @@ -1,605 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 - #include - #include // for _get_osfhandle, _isatty, _fileno - #include // for _get_pid - - #ifdef __MINGW32__ - #include - #endif - - #if defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES) - #include - #include - #endif - - #include // for _mkdir/_wmkdir - -#else // unix - - #include - #include - - #ifdef __linux__ - #include //Use gettid() syscall under linux to get thread id - - #elif defined(_AIX) - #include // for pthread_getthrds_np - - #elif defined(__DragonFly__) || defined(__FreeBSD__) - #include // for pthread_getthreadid_np - - #elif defined(__NetBSD__) - #include // for _lwp_self - - #elif defined(__sun) - #include // for thr_self - #endif - -#endif // unix - -#if defined __APPLE__ - #include -#endif - -#ifndef __has_feature // Clang - feature checking macros. - #define __has_feature(x) 0 // Compatibility with non-clang compilers. -#endif - -namespace spdlog { -namespace details { -namespace os { - -SPDLOG_INLINE spdlog::log_clock::time_point now() SPDLOG_NOEXCEPT { -#if defined __linux__ && defined SPDLOG_CLOCK_COARSE - timespec ts; - ::clock_gettime(CLOCK_REALTIME_COARSE, &ts); - return std::chrono::time_point( - std::chrono::duration_cast( - std::chrono::seconds(ts.tv_sec) + std::chrono::nanoseconds(ts.tv_nsec))); - -#else - return log_clock::now(); -#endif -} -SPDLOG_INLINE std::tm localtime(const std::time_t &time_tt) SPDLOG_NOEXCEPT { -#ifdef _WIN32 - std::tm tm; - ::localtime_s(&tm, &time_tt); -#else - std::tm tm; - ::localtime_r(&time_tt, &tm); -#endif - return tm; -} - -SPDLOG_INLINE std::tm localtime() SPDLOG_NOEXCEPT { - std::time_t now_t = ::time(nullptr); - return localtime(now_t); -} - -SPDLOG_INLINE std::tm gmtime(const std::time_t &time_tt) SPDLOG_NOEXCEPT { -#ifdef _WIN32 - std::tm tm; - ::gmtime_s(&tm, &time_tt); -#else - std::tm tm; - ::gmtime_r(&time_tt, &tm); -#endif - return tm; -} - -SPDLOG_INLINE std::tm gmtime() SPDLOG_NOEXCEPT { - std::time_t now_t = ::time(nullptr); - return gmtime(now_t); -} - -// fopen_s on non windows for writing -SPDLOG_INLINE bool fopen_s(FILE **fp, const filename_t &filename, const filename_t &mode) { -#ifdef _WIN32 - #ifdef SPDLOG_WCHAR_FILENAMES - *fp = ::_wfsopen((filename.c_str()), mode.c_str(), _SH_DENYNO); - #else - *fp = ::_fsopen((filename.c_str()), mode.c_str(), _SH_DENYNO); - #endif - #if defined(SPDLOG_PREVENT_CHILD_FD) - if (*fp != nullptr) { - auto file_handle = reinterpret_cast(_get_osfhandle(::_fileno(*fp))); - if (!::SetHandleInformation(file_handle, HANDLE_FLAG_INHERIT, 0)) { - ::fclose(*fp); - *fp = nullptr; - } - } - #endif -#else // unix - #if defined(SPDLOG_PREVENT_CHILD_FD) - const int mode_flag = mode == SPDLOG_FILENAME_T("ab") ? O_APPEND : O_TRUNC; - const int fd = - ::open((filename.c_str()), O_CREAT | O_WRONLY | O_CLOEXEC | mode_flag, mode_t(0644)); - if (fd == -1) { - return true; - } - *fp = ::fdopen(fd, mode.c_str()); - if (*fp == nullptr) { - ::close(fd); - } - #else - *fp = ::fopen((filename.c_str()), mode.c_str()); - #endif -#endif - - return *fp == nullptr; -} - -SPDLOG_INLINE int remove(const filename_t &filename) SPDLOG_NOEXCEPT { -#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES) - return ::_wremove(filename.c_str()); -#else - return std::remove(filename.c_str()); -#endif -} - -SPDLOG_INLINE int remove_if_exists(const filename_t &filename) SPDLOG_NOEXCEPT { - return path_exists(filename) ? remove(filename) : 0; -} - -SPDLOG_INLINE int rename(const filename_t &filename1, const filename_t &filename2) SPDLOG_NOEXCEPT { -#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES) - return ::_wrename(filename1.c_str(), filename2.c_str()); -#else - return std::rename(filename1.c_str(), filename2.c_str()); -#endif -} - -// Return true if path exists (file or directory) -SPDLOG_INLINE bool path_exists(const filename_t &filename) SPDLOG_NOEXCEPT { -#ifdef _WIN32 - struct _stat buffer; - #ifdef SPDLOG_WCHAR_FILENAMES - return (::_wstat(filename.c_str(), &buffer) == 0); - #else - return (::_stat(filename.c_str(), &buffer) == 0); - #endif -#else // common linux/unix all have the stat system call - struct stat buffer; - return (::stat(filename.c_str(), &buffer) == 0); -#endif -} - -#ifdef _MSC_VER - // avoid warning about unreachable statement at the end of filesize() - #pragma warning(push) - #pragma warning(disable : 4702) -#endif - -// Return file size according to open FILE* object -SPDLOG_INLINE size_t filesize(FILE *f) { - if (f == nullptr) { - throw_spdlog_ex("Failed getting file size. fd is null"); - } -#if defined(_WIN32) && !defined(__CYGWIN__) - int fd = ::_fileno(f); - #if defined(_WIN64) // 64 bits - __int64 ret = ::_filelengthi64(fd); - if (ret >= 0) { - return static_cast(ret); - } - - #else // windows 32 bits - long ret = ::_filelength(fd); - if (ret >= 0) { - return static_cast(ret); - } - #endif - -#else // unix - // OpenBSD and AIX doesn't compile with :: before the fileno(..) - #if defined(__OpenBSD__) || defined(_AIX) - int fd = fileno(f); - #else - int fd = ::fileno(f); - #endif - // 64 bits(but not in osx, linux/musl or cygwin, where fstat64 is deprecated) - #if ((defined(__linux__) && defined(__GLIBC__)) || defined(__sun) || defined(_AIX)) && \ - (defined(__LP64__) || defined(_LP64)) - struct stat64 st; - if (::fstat64(fd, &st) == 0) { - return static_cast(st.st_size); - } - #else // other unix or linux 32 bits or cygwin - struct stat st; - if (::fstat(fd, &st) == 0) { - return static_cast(st.st_size); - } - #endif -#endif - throw_spdlog_ex("Failed getting file size from fd", errno); - return 0; // will not be reached. -} - -#ifdef _MSC_VER - #pragma warning(pop) -#endif - -// Return utc offset in minutes or throw spdlog_ex on failure -SPDLOG_INLINE int utc_minutes_offset(const std::tm &tm) { -#ifdef _WIN32 - #if _WIN32_WINNT < _WIN32_WINNT_WS08 - TIME_ZONE_INFORMATION tzinfo; - auto rv = ::GetTimeZoneInformation(&tzinfo); - #else - DYNAMIC_TIME_ZONE_INFORMATION tzinfo; - auto rv = ::GetDynamicTimeZoneInformation(&tzinfo); - #endif - if (rv == TIME_ZONE_ID_INVALID) throw_spdlog_ex("Failed getting timezone info. ", errno); - - int offset = -tzinfo.Bias; - if (tm.tm_isdst) { - offset -= tzinfo.DaylightBias; - } else { - offset -= tzinfo.StandardBias; - } - return offset; -#else - - #if defined(sun) || defined(__sun) || defined(_AIX) || \ - (defined(__NEWLIB__) && !defined(__TM_GMTOFF)) || \ - (!defined(__APPLE__) && !defined(_BSD_SOURCE) && !defined(_GNU_SOURCE) && \ - (!defined(_POSIX_VERSION) || (_POSIX_VERSION < 202405L))) - // 'tm_gmtoff' field is BSD extension and it's missing on SunOS/Solaris - struct helper { - static long int calculate_gmt_offset(const std::tm &localtm = details::os::localtime(), - const std::tm &gmtm = details::os::gmtime()) { - int local_year = localtm.tm_year + (1900 - 1); - int gmt_year = gmtm.tm_year + (1900 - 1); - - long int days = ( - // difference in day of year - localtm.tm_yday - - gmtm.tm_yday - - // + intervening leap days - + ((local_year >> 2) - (gmt_year >> 2)) - (local_year / 100 - gmt_year / 100) + - ((local_year / 100 >> 2) - (gmt_year / 100 >> 2)) - - // + difference in years * 365 */ - + static_cast(local_year - gmt_year) * 365); - - long int hours = (24 * days) + (localtm.tm_hour - gmtm.tm_hour); - long int mins = (60 * hours) + (localtm.tm_min - gmtm.tm_min); - long int secs = (60 * mins) + (localtm.tm_sec - gmtm.tm_sec); - - return secs; - } - }; - - auto offset_seconds = helper::calculate_gmt_offset(tm); - #else - auto offset_seconds = tm.tm_gmtoff; - #endif - - return static_cast(offset_seconds / 60); -#endif -} - -// Return current thread id as size_t -// It exists because the std::this_thread::get_id() is much slower(especially -// under VS 2013) -SPDLOG_INLINE size_t _thread_id() SPDLOG_NOEXCEPT { -#ifdef _WIN32 - return static_cast(::GetCurrentThreadId()); -#elif defined(__linux__) - #if defined(__ANDROID__) && defined(__ANDROID_API__) && (__ANDROID_API__ < 21) - #define SYS_gettid __NR_gettid - #endif - return static_cast(::syscall(SYS_gettid)); -#elif defined(_AIX) - struct __pthrdsinfo buf; - int reg_size = 0; - pthread_t pt = pthread_self(); - int retval = pthread_getthrds_np(&pt, PTHRDSINFO_QUERY_TID, &buf, sizeof(buf), NULL, ®_size); - int tid = (!retval) ? buf.__pi_tid : 0; - return static_cast(tid); -#elif defined(__DragonFly__) || defined(__FreeBSD__) - return static_cast(::pthread_getthreadid_np()); -#elif defined(__NetBSD__) - return static_cast(::_lwp_self()); -#elif defined(__OpenBSD__) - return static_cast(::getthrid()); -#elif defined(__sun) - return static_cast(::thr_self()); -#elif __APPLE__ - uint64_t tid; - // There is no pthread_threadid_np prior to Mac OS X 10.6, and it is not supported on any PPC, - // including 10.6.8 Rosetta. __POWERPC__ is Apple-specific define encompassing ppc and ppc64. - #ifdef MAC_OS_X_VERSION_MAX_ALLOWED - { - #if (MAC_OS_X_VERSION_MAX_ALLOWED < 1060) || defined(__POWERPC__) - tid = pthread_mach_thread_np(pthread_self()); - #elif MAC_OS_X_VERSION_MIN_REQUIRED < 1060 - if (&pthread_threadid_np) { - pthread_threadid_np(nullptr, &tid); - } else { - tid = pthread_mach_thread_np(pthread_self()); - } - #else - pthread_threadid_np(nullptr, &tid); - #endif - } - #else - pthread_threadid_np(nullptr, &tid); - #endif - return static_cast(tid); -#else // Default to standard C++11 (other Unix) - return static_cast(std::hash()(std::this_thread::get_id())); -#endif -} - -// Return current thread id as size_t (from thread local storage) -SPDLOG_INLINE size_t thread_id() SPDLOG_NOEXCEPT { -#if defined(SPDLOG_NO_TLS) - return _thread_id(); -#else // cache thread id in tls - static thread_local const size_t tid = _thread_id(); - return tid; -#endif -} - -// This is avoid msvc issue in sleep_for that happens if the clock changes. -// See https://github.com/gabime/spdlog/issues/609 -SPDLOG_INLINE void sleep_for_millis(unsigned int milliseconds) SPDLOG_NOEXCEPT { -#if defined(_WIN32) - ::Sleep(milliseconds); -#else - std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds)); -#endif -} - -// wchar support for windows file names (SPDLOG_WCHAR_FILENAMES must be defined) -#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES) -SPDLOG_INLINE std::string filename_to_str(const filename_t &filename) { - memory_buf_t buf; - wstr_to_utf8buf(filename, buf); - return SPDLOG_BUF_TO_STRING(buf); -} -#else -SPDLOG_INLINE std::string filename_to_str(const filename_t &filename) { return filename; } -#endif - -SPDLOG_INLINE int pid() SPDLOG_NOEXCEPT { -#ifdef _WIN32 - return conditional_static_cast(::GetCurrentProcessId()); -#else - return conditional_static_cast(::getpid()); -#endif -} - -// Determine if the terminal supports colors -// Based on: https://github.com/agauniyal/rang/ -SPDLOG_INLINE bool is_color_terminal() SPDLOG_NOEXCEPT { -#ifdef _WIN32 - return true; -#else - - static const bool result = []() { - const char *env_colorterm_p = std::getenv("COLORTERM"); - if (env_colorterm_p != nullptr) { - return true; - } - - static constexpr std::array terms = { - {"ansi", "color", "console", "cygwin", "gnome", "konsole", "kterm", "linux", "msys", - "putty", "rxvt", "screen", "vt100", "xterm", "alacritty", "vt102"}}; - - const char *env_term_p = std::getenv("TERM"); - if (env_term_p == nullptr) { - return false; - } - - return std::any_of(terms.begin(), terms.end(), [&](const char *term) { - return std::strstr(env_term_p, term) != nullptr; - }); - }(); - - return result; -#endif -} - -// Determine if the terminal attached -// Source: https://github.com/agauniyal/rang/ -SPDLOG_INLINE bool in_terminal(FILE *file) SPDLOG_NOEXCEPT { -#ifdef _WIN32 - return ::_isatty(_fileno(file)) != 0; -#else - return ::isatty(fileno(file)) != 0; -#endif -} - -#if (defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES)) && defined(_WIN32) -SPDLOG_INLINE void wstr_to_utf8buf(wstring_view_t wstr, memory_buf_t &target) { - if (wstr.size() > static_cast((std::numeric_limits::max)()) / 4 - 1) { - throw_spdlog_ex("UTF-16 string is too big to be converted to UTF-8"); - } - - int wstr_size = static_cast(wstr.size()); - if (wstr_size == 0) { - target.resize(0); - return; - } - - int result_size = static_cast(target.capacity()); - if ((wstr_size + 1) * 4 > result_size) { - result_size = - ::WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr_size, NULL, 0, NULL, NULL); - } - - if (result_size > 0) { - target.resize(result_size); - result_size = ::WideCharToMultiByte(CP_UTF8, 0, wstr.data(), wstr_size, target.data(), - result_size, NULL, NULL); - - if (result_size > 0) { - target.resize(result_size); - return; - } - } - - throw_spdlog_ex( - fmt_lib::format("WideCharToMultiByte failed. Last error: {}", ::GetLastError())); -} - -SPDLOG_INLINE void utf8_to_wstrbuf(string_view_t str, wmemory_buf_t &target) { - if (str.size() > static_cast((std::numeric_limits::max)()) - 1) { - throw_spdlog_ex("UTF-8 string is too big to be converted to UTF-16"); - } - - int str_size = static_cast(str.size()); - if (str_size == 0) { - target.resize(0); - return; - } - - // find the size to allocate for the result buffer - int result_size = ::MultiByteToWideChar(CP_UTF8, 0, str.data(), str_size, NULL, 0); - - if (result_size > 0) { - target.resize(result_size); - result_size = - ::MultiByteToWideChar(CP_UTF8, 0, str.data(), str_size, target.data(), result_size); - if (result_size > 0) { - assert(result_size == target.size()); - return; - } - } - - throw_spdlog_ex( - fmt_lib::format("MultiByteToWideChar failed. Last error: {}", ::GetLastError())); -} -#endif // (defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES)) && - // defined(_WIN32) - -// return true on success -static SPDLOG_INLINE bool mkdir_(const filename_t &path) { -#ifdef _WIN32 - #ifdef SPDLOG_WCHAR_FILENAMES - return ::_wmkdir(path.c_str()) == 0; - #else - return ::_mkdir(path.c_str()) == 0; - #endif -#else - return ::mkdir(path.c_str(), mode_t(0755)) == 0; -#endif -} - -// create the given directory - and all directories leading to it -// return true on success or if the directory already exists -SPDLOG_INLINE bool create_dir(const filename_t &path) { - if (path_exists(path)) { - return true; - } - - if (path.empty()) { - return false; - } - - size_t search_offset = 0; - do { - auto token_pos = path.find_first_of(folder_seps_filename, search_offset); - // treat the entire path as a folder if no folder separator not found - if (token_pos == filename_t::npos) { - token_pos = path.size(); - } - - auto subdir = path.substr(0, token_pos); -#ifdef _WIN32 - // if subdir is just a drive letter, add a slash e.g. "c:"=>"c:\", - // otherwise path_exists(subdir) returns false (issue #3079) - const bool is_drive = subdir.length() == 2 && subdir[1] == ':'; - if (is_drive) { - subdir += '\\'; - token_pos++; - } -#endif - - if (!subdir.empty() && !path_exists(subdir) && !mkdir_(subdir)) { - return false; // return error if failed creating dir - } - search_offset = token_pos + 1; - } while (search_offset < path.size()); - - return true; -} - -// Return directory name from given path or empty string -// "abc/file" => "abc" -// "abc/" => "abc" -// "abc" => "" -// "abc///" => "abc//" -SPDLOG_INLINE filename_t dir_name(const filename_t &path) { - auto pos = path.find_last_of(folder_seps_filename); - return pos != filename_t::npos ? path.substr(0, pos) : filename_t{}; -} - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4996) -#endif // _MSC_VER -std::string SPDLOG_INLINE getenv(const char *field) { -#if defined(_MSC_VER) && defined(__cplusplus_winrt) - return std::string{}; // not supported under uwp -#else - char *buf = std::getenv(field); - return buf ? buf : std::string{}; -#endif -} -#ifdef _MSC_VER - #pragma warning(pop) -#endif // _MSC_VER - -// Do fsync by FILE handlerpointer -// Return true on success -SPDLOG_INLINE bool fsync(FILE *fp) { -#ifdef _WIN32 - return FlushFileBuffers(reinterpret_cast(_get_osfhandle(_fileno(fp)))) != 0; -#else - return ::fsync(fileno(fp)) == 0; -#endif -} - -// Do non-locking fwrite if possible by the os or use the regular locking fwrite -// Return true on success. -SPDLOG_INLINE bool fwrite_bytes(const void *ptr, const size_t n_bytes, FILE *fp) { -#if defined(_WIN32) && defined(SPDLOG_FWRITE_UNLOCKED) - return _fwrite_nolock(ptr, 1, n_bytes, fp) == n_bytes; -#elif defined(SPDLOG_FWRITE_UNLOCKED) - return ::fwrite_unlocked(ptr, 1, n_bytes, fp) == n_bytes; -#else - return std::fwrite(ptr, 1, n_bytes, fp) == n_bytes; -#endif -} - -} // namespace os -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/os.h b/thirdparty/spdlog/include/spdlog/details/os.h deleted file mode 100644 index 5fd12bac1..000000000 --- a/thirdparty/spdlog/include/spdlog/details/os.h +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include // std::time_t -#include - -namespace spdlog { -namespace details { -namespace os { - -SPDLOG_API spdlog::log_clock::time_point now() SPDLOG_NOEXCEPT; - -SPDLOG_API std::tm localtime(const std::time_t &time_tt) SPDLOG_NOEXCEPT; - -SPDLOG_API std::tm localtime() SPDLOG_NOEXCEPT; - -SPDLOG_API std::tm gmtime(const std::time_t &time_tt) SPDLOG_NOEXCEPT; - -SPDLOG_API std::tm gmtime() SPDLOG_NOEXCEPT; - -// eol definition -#if !defined(SPDLOG_EOL) - #ifdef _WIN32 - #define SPDLOG_EOL "\r\n" - #else - #define SPDLOG_EOL "\n" - #endif -#endif - -SPDLOG_CONSTEXPR static const char *default_eol = SPDLOG_EOL; - -// folder separator -#if !defined(SPDLOG_FOLDER_SEPS) - #ifdef _WIN32 - #define SPDLOG_FOLDER_SEPS "\\/" - #else - #define SPDLOG_FOLDER_SEPS "/" - #endif -#endif - -SPDLOG_CONSTEXPR static const char folder_seps[] = SPDLOG_FOLDER_SEPS; -SPDLOG_CONSTEXPR static const filename_t::value_type folder_seps_filename[] = - SPDLOG_FILENAME_T(SPDLOG_FOLDER_SEPS); - -// fopen_s on non windows for writing -SPDLOG_API bool fopen_s(FILE **fp, const filename_t &filename, const filename_t &mode); - -// Remove filename. return 0 on success -SPDLOG_API int remove(const filename_t &filename) SPDLOG_NOEXCEPT; - -// Remove file if exists. return 0 on success -// Note: Non atomic (might return failure to delete if concurrently deleted by other process/thread) -SPDLOG_API int remove_if_exists(const filename_t &filename) SPDLOG_NOEXCEPT; - -SPDLOG_API int rename(const filename_t &filename1, const filename_t &filename2) SPDLOG_NOEXCEPT; - -// Return if file exists. -SPDLOG_API bool path_exists(const filename_t &filename) SPDLOG_NOEXCEPT; - -// Return file size according to open FILE* object -SPDLOG_API size_t filesize(FILE *f); - -// Return utc offset in minutes or throw spdlog_ex on failure -SPDLOG_API int utc_minutes_offset(const std::tm &tm = details::os::localtime()); - -// Return current thread id as size_t -// It exists because the std::this_thread::get_id() is much slower(especially -// under VS 2013) -SPDLOG_API size_t _thread_id() SPDLOG_NOEXCEPT; - -// Return current thread id as size_t (from thread local storage) -SPDLOG_API size_t thread_id() SPDLOG_NOEXCEPT; - -// This is avoid msvc issue in sleep_for that happens if the clock changes. -// See https://github.com/gabime/spdlog/issues/609 -SPDLOG_API void sleep_for_millis(unsigned int milliseconds) SPDLOG_NOEXCEPT; - -SPDLOG_API std::string filename_to_str(const filename_t &filename); - -SPDLOG_API int pid() SPDLOG_NOEXCEPT; - -// Determine if the terminal supports colors -// Source: https://github.com/agauniyal/rang/ -SPDLOG_API bool is_color_terminal() SPDLOG_NOEXCEPT; - -// Determine if the terminal attached -// Source: https://github.com/agauniyal/rang/ -SPDLOG_API bool in_terminal(FILE *file) SPDLOG_NOEXCEPT; - -#if (defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES)) && defined(_WIN32) -SPDLOG_API void wstr_to_utf8buf(wstring_view_t wstr, memory_buf_t &target); - -SPDLOG_API void utf8_to_wstrbuf(string_view_t str, wmemory_buf_t &target); -#endif - -// Return directory name from given path or empty string -// "abc/file" => "abc" -// "abc/" => "abc" -// "abc" => "" -// "abc///" => "abc//" -SPDLOG_API filename_t dir_name(const filename_t &path); - -// Create a dir from the given path. -// Return true if succeeded or if this dir already exists. -SPDLOG_API bool create_dir(const filename_t &path); - -// non thread safe, cross platform getenv/getenv_s -// return empty string if field not found -SPDLOG_API std::string getenv(const char *field); - -// Do fsync by FILE objectpointer. -// Return true on success. -SPDLOG_API bool fsync(FILE *fp); - -// Do non-locking fwrite if possible by the os or use the regular locking fwrite -// Return true on success. -SPDLOG_API bool fwrite_bytes(const void *ptr, const size_t n_bytes, FILE *fp); - -} // namespace os -} // namespace details -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "os-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/details/periodic_worker-inl.h b/thirdparty/spdlog/include/spdlog/details/periodic_worker-inl.h deleted file mode 100644 index 18f11fbec..000000000 --- a/thirdparty/spdlog/include/spdlog/details/periodic_worker-inl.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -namespace spdlog { -namespace details { - -// stop the worker thread and join it -SPDLOG_INLINE periodic_worker::~periodic_worker() { - if (worker_thread_.joinable()) { - { - std::lock_guard lock(mutex_); - active_ = false; - } - cv_.notify_one(); - worker_thread_.join(); - } -} - -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/periodic_worker.h b/thirdparty/spdlog/include/spdlog/details/periodic_worker.h deleted file mode 100644 index d647b66ee..000000000 --- a/thirdparty/spdlog/include/spdlog/details/periodic_worker.h +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -// periodic worker thread - periodically executes the given callback function. -// -// RAII over the owned thread: -// creates the thread on construction. -// stops and joins the thread on destruction (if the thread is executing a callback, wait for it -// to finish first). - -#include -#include -#include -#include -#include -namespace spdlog { -namespace details { - -class SPDLOG_API periodic_worker { -public: - template - periodic_worker(const std::function &callback_fun, - std::chrono::duration interval) { - active_ = (interval > std::chrono::duration::zero()); - if (!active_) { - return; - } - - worker_thread_ = std::thread([this, callback_fun, interval]() { - for (;;) { - std::unique_lock lock(this->mutex_); - if (this->cv_.wait_for(lock, interval, [this] { return !this->active_; })) { - return; // active_ == false, so exit this thread - } - callback_fun(); - } - }); - } - std::thread &get_thread() { return worker_thread_; } - periodic_worker(const periodic_worker &) = delete; - periodic_worker &operator=(const periodic_worker &) = delete; - // stop the worker thread and join it - ~periodic_worker(); - -private: - bool active_; - std::thread worker_thread_; - std::mutex mutex_; - std::condition_variable cv_; -}; -} // namespace details -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "periodic_worker-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/details/registry-inl.h b/thirdparty/spdlog/include/spdlog/details/registry-inl.h deleted file mode 100644 index 272bebbd3..000000000 --- a/thirdparty/spdlog/include/spdlog/details/registry-inl.h +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include -#include -#include - -#ifndef SPDLOG_DISABLE_DEFAULT_LOGGER - // support for the default stdout color logger - #ifdef _WIN32 - #include - #else - #include - #endif -#endif // SPDLOG_DISABLE_DEFAULT_LOGGER - -#include -#include -#include -#include -#include - -namespace spdlog { -namespace details { - -SPDLOG_INLINE registry::registry() - : formatter_(new pattern_formatter()) { -#ifndef SPDLOG_DISABLE_DEFAULT_LOGGER - // create default logger (ansicolor_stdout_sink_mt or wincolor_stdout_sink_mt in windows). - #ifdef _WIN32 - auto color_sink = std::make_shared(); - #else - auto color_sink = std::make_shared(); - #endif - - const char *default_logger_name = ""; - default_logger_ = std::make_shared(default_logger_name, std::move(color_sink)); - loggers_[default_logger_name] = default_logger_; - -#endif // SPDLOG_DISABLE_DEFAULT_LOGGER -} - -SPDLOG_INLINE registry::~registry() = default; - -SPDLOG_INLINE void registry::register_logger(std::shared_ptr new_logger) { - std::lock_guard lock(logger_map_mutex_); - register_logger_(std::move(new_logger)); -} - -SPDLOG_INLINE void registry::register_or_replace(std::shared_ptr new_logger) { - std::lock_guard lock(logger_map_mutex_); - register_or_replace_(std::move(new_logger)); -} - -SPDLOG_INLINE void registry::initialize_logger(std::shared_ptr new_logger) { - std::lock_guard lock(logger_map_mutex_); - new_logger->set_formatter(formatter_->clone()); - - if (err_handler_) { - new_logger->set_error_handler(err_handler_); - } - - // set new level according to previously configured level or default level - auto it = log_levels_.find(new_logger->name()); - auto new_level = it != log_levels_.end() ? it->second : global_log_level_; - new_logger->set_level(new_level); - - new_logger->flush_on(flush_level_); - - if (backtrace_n_messages_ > 0) { - new_logger->enable_backtrace(backtrace_n_messages_); - } - - if (automatic_registration_) { - register_logger_(std::move(new_logger)); - } -} - -SPDLOG_INLINE std::shared_ptr registry::get(const std::string &logger_name) { - std::lock_guard lock(logger_map_mutex_); - auto found = loggers_.find(logger_name); - return found == loggers_.end() ? nullptr : found->second; -} - -SPDLOG_INLINE std::shared_ptr registry::default_logger() { - std::lock_guard lock(logger_map_mutex_); - return default_logger_; -} - -// Return raw ptr to the default logger. -// To be used directly by the spdlog default api (e.g. spdlog::info) -// This make the default API faster, but cannot be used concurrently with set_default_logger(). -// e.g do not call set_default_logger() from one thread while calling spdlog::info() from another. -SPDLOG_INLINE logger *registry::get_default_raw() { return default_logger_.get(); } - -// set default logger. -// the default logger is stored in default_logger_ (for faster retrieval) and in the loggers_ map. -SPDLOG_INLINE void registry::set_default_logger(std::shared_ptr new_default_logger) { - std::lock_guard lock(logger_map_mutex_); - if (new_default_logger != nullptr) { - loggers_[new_default_logger->name()] = new_default_logger; - } - default_logger_ = std::move(new_default_logger); -} - -SPDLOG_INLINE void registry::set_tp(std::shared_ptr tp) { - std::lock_guard lock(tp_mutex_); - tp_ = std::move(tp); -} - -SPDLOG_INLINE std::shared_ptr registry::get_tp() { - std::lock_guard lock(tp_mutex_); - return tp_; -} - -// Set global formatter. Each sink in each logger will get a clone of this object -SPDLOG_INLINE void registry::set_formatter(std::unique_ptr formatter) { - std::lock_guard lock(logger_map_mutex_); - formatter_ = std::move(formatter); - for (auto &l : loggers_) { - l.second->set_formatter(formatter_->clone()); - } -} - -SPDLOG_INLINE void registry::enable_backtrace(size_t n_messages) { - std::lock_guard lock(logger_map_mutex_); - backtrace_n_messages_ = n_messages; - - for (auto &l : loggers_) { - l.second->enable_backtrace(n_messages); - } -} - -SPDLOG_INLINE void registry::disable_backtrace() { - std::lock_guard lock(logger_map_mutex_); - backtrace_n_messages_ = 0; - for (auto &l : loggers_) { - l.second->disable_backtrace(); - } -} - -SPDLOG_INLINE void registry::set_level(level::level_enum log_level) { - std::lock_guard lock(logger_map_mutex_); - for (auto &l : loggers_) { - l.second->set_level(log_level); - } - global_log_level_ = log_level; -} - -SPDLOG_INLINE void registry::flush_on(level::level_enum log_level) { - std::lock_guard lock(logger_map_mutex_); - for (auto &l : loggers_) { - l.second->flush_on(log_level); - } - flush_level_ = log_level; -} - -SPDLOG_INLINE void registry::set_error_handler(err_handler handler) { - std::lock_guard lock(logger_map_mutex_); - for (auto &l : loggers_) { - l.second->set_error_handler(handler); - } - err_handler_ = std::move(handler); -} - -SPDLOG_INLINE void registry::apply_all( - const std::function)> &fun) { - std::lock_guard lock(logger_map_mutex_); - for (auto &l : loggers_) { - fun(l.second); - } -} - -SPDLOG_INLINE void registry::flush_all() { - std::lock_guard lock(logger_map_mutex_); - for (auto &l : loggers_) { - l.second->flush(); - } -} - -SPDLOG_INLINE void registry::drop(const std::string &logger_name) { - std::lock_guard lock(logger_map_mutex_); - auto is_default_logger = default_logger_ && default_logger_->name() == logger_name; - loggers_.erase(logger_name); - if (is_default_logger) { - default_logger_.reset(); - } -} - -SPDLOG_INLINE void registry::drop_all() { - std::lock_guard lock(logger_map_mutex_); - loggers_.clear(); - default_logger_.reset(); -} - -// clean all resources and threads started by the registry -SPDLOG_INLINE void registry::shutdown() { - { - std::lock_guard lock(flusher_mutex_); - periodic_flusher_.reset(); - } - - drop_all(); - - { - std::lock_guard lock(tp_mutex_); - tp_.reset(); - } -} - -SPDLOG_INLINE std::recursive_mutex ®istry::tp_mutex() { return tp_mutex_; } - -SPDLOG_INLINE void registry::set_automatic_registration(bool automatic_registration) { - std::lock_guard lock(logger_map_mutex_); - automatic_registration_ = automatic_registration; -} - -SPDLOG_INLINE void registry::set_levels(log_levels levels, level::level_enum *global_level) { - std::lock_guard lock(logger_map_mutex_); - log_levels_ = std::move(levels); - auto global_level_requested = global_level != nullptr; - global_log_level_ = global_level_requested ? *global_level : global_log_level_; - - for (auto &logger : loggers_) { - auto logger_entry = log_levels_.find(logger.first); - if (logger_entry != log_levels_.end()) { - logger.second->set_level(logger_entry->second); - } else if (global_level_requested) { - logger.second->set_level(*global_level); - } - } -} - -SPDLOG_INLINE registry ®istry::instance() { - static registry s_instance; - return s_instance; -} - -SPDLOG_INLINE void registry::apply_logger_env_levels(std::shared_ptr new_logger) { - std::lock_guard lock(logger_map_mutex_); - auto it = log_levels_.find(new_logger->name()); - auto new_level = it != log_levels_.end() ? it->second : global_log_level_; - new_logger->set_level(new_level); -} - -SPDLOG_INLINE void registry::throw_if_exists_(const std::string &logger_name) { - if (loggers_.find(logger_name) != loggers_.end()) { - throw_spdlog_ex("logger with name '" + logger_name + "' already exists"); - } -} - -SPDLOG_INLINE void registry::register_logger_(std::shared_ptr new_logger) { - auto &logger_name = new_logger->name(); - throw_if_exists_(logger_name); - loggers_[logger_name] = std::move(new_logger); -} - -SPDLOG_INLINE void registry::register_or_replace_(std::shared_ptr new_logger) { - loggers_[new_logger->name()] = std::move(new_logger); -} - -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/registry.h b/thirdparty/spdlog/include/spdlog/details/registry.h deleted file mode 100644 index 72c70b82b..000000000 --- a/thirdparty/spdlog/include/spdlog/details/registry.h +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -// Loggers registry of unique name->logger pointer -// An attempt to create a logger with an already existing name will result with spdlog_ex exception. -// If user requests a non existing logger, nullptr will be returned -// This class is thread safe - -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace spdlog { -class logger; - -namespace details { -class thread_pool; - -class SPDLOG_API registry { -public: - using log_levels = std::unordered_map; - registry(const registry &) = delete; - registry &operator=(const registry &) = delete; - - void register_logger(std::shared_ptr new_logger); - void register_or_replace(std::shared_ptr new_logger); - void initialize_logger(std::shared_ptr new_logger); - std::shared_ptr get(const std::string &logger_name); - std::shared_ptr default_logger(); - - // Return raw ptr to the default logger. - // To be used directly by the spdlog default api (e.g. spdlog::info) - // This make the default API faster, but cannot be used concurrently with set_default_logger(). - // e.g do not call set_default_logger() from one thread while calling spdlog::info() from - // another. - logger *get_default_raw(); - - // set default logger and add it to the registry if not registered already. - // default logger is stored in default_logger_ (for faster retrieval) and in the loggers_ map. - // Note: Make sure to unregister it when no longer needed or before calling again with a new - // logger. - void set_default_logger(std::shared_ptr new_default_logger); - - void set_tp(std::shared_ptr tp); - - std::shared_ptr get_tp(); - - // Set global formatter. Each sink in each logger will get a clone of this object - void set_formatter(std::unique_ptr formatter); - - void enable_backtrace(size_t n_messages); - - void disable_backtrace(); - - void set_level(level::level_enum log_level); - - void flush_on(level::level_enum log_level); - - template - void flush_every(std::chrono::duration interval) { - std::lock_guard lock(flusher_mutex_); - auto clbk = [this]() { this->flush_all(); }; - periodic_flusher_ = details::make_unique(clbk, interval); - } - - std::unique_ptr &get_flusher() { - std::lock_guard lock(flusher_mutex_); - return periodic_flusher_; - } - - void set_error_handler(err_handler handler); - - void apply_all(const std::function)> &fun); - - void flush_all(); - - void drop(const std::string &logger_name); - - void drop_all(); - - // clean all resources and threads started by the registry - void shutdown(); - - std::recursive_mutex &tp_mutex(); - - void set_automatic_registration(bool automatic_registration); - - // set levels for all existing/future loggers. global_level can be null if should not set. - void set_levels(log_levels levels, level::level_enum *global_level); - - static registry &instance(); - - void apply_logger_env_levels(std::shared_ptr new_logger); - -private: - registry(); - ~registry(); - - void throw_if_exists_(const std::string &logger_name); - void register_logger_(std::shared_ptr new_logger); - void register_or_replace_(std::shared_ptr new_logger); - bool set_level_from_cfg_(logger *logger); - std::mutex logger_map_mutex_, flusher_mutex_; - std::recursive_mutex tp_mutex_; - std::unordered_map> loggers_; - log_levels log_levels_; - std::unique_ptr formatter_; - spdlog::level::level_enum global_log_level_ = level::info; - level::level_enum flush_level_ = level::off; - err_handler err_handler_; - std::shared_ptr tp_; - std::unique_ptr periodic_flusher_; - std::shared_ptr default_logger_; - bool automatic_registration_ = true; - size_t backtrace_n_messages_ = 0; -}; - -} // namespace details -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "registry-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/details/synchronous_factory.h b/thirdparty/spdlog/include/spdlog/details/synchronous_factory.h deleted file mode 100644 index 4bd5a51c8..000000000 --- a/thirdparty/spdlog/include/spdlog/details/synchronous_factory.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include "registry.h" - -namespace spdlog { - -// Default logger factory- creates synchronous loggers -class logger; - -struct synchronous_factory { - template - static std::shared_ptr create(std::string logger_name, SinkArgs &&...args) { - auto sink = std::make_shared(std::forward(args)...); - auto new_logger = std::make_shared(std::move(logger_name), std::move(sink)); - details::registry::instance().initialize_logger(new_logger); - return new_logger; - } -}; -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/thread_pool-inl.h b/thirdparty/spdlog/include/spdlog/details/thread_pool-inl.h deleted file mode 100644 index b172b2801..000000000 --- a/thirdparty/spdlog/include/spdlog/details/thread_pool-inl.h +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -namespace spdlog { -namespace details { - -SPDLOG_INLINE thread_pool::thread_pool(size_t q_max_items, - size_t threads_n, - std::function on_thread_start, - std::function on_thread_stop) - : q_(q_max_items) { - if (threads_n == 0 || threads_n > 1000) { - throw_spdlog_ex( - "spdlog::thread_pool(): invalid threads_n param (valid " - "range is 1-1000)"); - } - for (size_t i = 0; i < threads_n; i++) { - threads_.emplace_back([this, on_thread_start, on_thread_stop] { - on_thread_start(); - this->thread_pool::worker_loop_(); - on_thread_stop(); - }); - } -} - -SPDLOG_INLINE thread_pool::thread_pool(size_t q_max_items, - size_t threads_n, - std::function on_thread_start) - : thread_pool(q_max_items, threads_n, std::move(on_thread_start), [] {}) {} - -SPDLOG_INLINE thread_pool::thread_pool(size_t q_max_items, size_t threads_n) - : thread_pool(q_max_items, threads_n, [] {}, [] {}) {} - -// message all threads to terminate gracefully join them -SPDLOG_INLINE thread_pool::~thread_pool() { - SPDLOG_TRY { - for (size_t i = 0; i < threads_.size(); i++) { - post_async_msg_(async_msg(async_msg_type::terminate), async_overflow_policy::block); - } - - for (auto &t : threads_) { - t.join(); - } - } - SPDLOG_CATCH_STD -} - -void SPDLOG_INLINE thread_pool::post_log(async_logger_ptr &&worker_ptr, - const details::log_msg &msg, - async_overflow_policy overflow_policy) { - async_msg async_m(std::move(worker_ptr), async_msg_type::log, msg); - post_async_msg_(std::move(async_m), overflow_policy); -} - -void SPDLOG_INLINE thread_pool::post_flush(async_logger_ptr &&worker_ptr, - async_overflow_policy overflow_policy) { - post_async_msg_(async_msg(std::move(worker_ptr), async_msg_type::flush), overflow_policy); -} - -size_t SPDLOG_INLINE thread_pool::overrun_counter() { return q_.overrun_counter(); } - -void SPDLOG_INLINE thread_pool::reset_overrun_counter() { q_.reset_overrun_counter(); } - -size_t SPDLOG_INLINE thread_pool::discard_counter() { return q_.discard_counter(); } - -void SPDLOG_INLINE thread_pool::reset_discard_counter() { q_.reset_discard_counter(); } - -size_t SPDLOG_INLINE thread_pool::queue_size() { return q_.size(); } - -void SPDLOG_INLINE thread_pool::post_async_msg_(async_msg &&new_msg, - async_overflow_policy overflow_policy) { - if (overflow_policy == async_overflow_policy::block) { - q_.enqueue(std::move(new_msg)); - } else if (overflow_policy == async_overflow_policy::overrun_oldest) { - q_.enqueue_nowait(std::move(new_msg)); - } else { - assert(overflow_policy == async_overflow_policy::discard_new); - q_.enqueue_if_have_room(std::move(new_msg)); - } -} - -void SPDLOG_INLINE thread_pool::worker_loop_() { - while (process_next_msg_()) { - } -} - -// process next message in the queue -// returns true if this thread should still be active (while no terminated msg was received) -bool SPDLOG_INLINE thread_pool::process_next_msg_() { - async_msg incoming_async_msg; - q_.dequeue(incoming_async_msg); - - switch (incoming_async_msg.msg_type) { - case async_msg_type::log: { - incoming_async_msg.worker_ptr->backend_sink_it_(incoming_async_msg); - return true; - } - case async_msg_type::flush: { - incoming_async_msg.worker_ptr->backend_flush_(); - return true; - } - - case async_msg_type::terminate: { - return false; - } - - default: { - assert(false); - } - } - - return true; -} - -} // namespace details -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/details/thread_pool.h b/thirdparty/spdlog/include/spdlog/details/thread_pool.h deleted file mode 100644 index f22b07821..000000000 --- a/thirdparty/spdlog/include/spdlog/details/thread_pool.h +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace spdlog { -class async_logger; - -namespace details { - -using async_logger_ptr = std::shared_ptr; - -enum class async_msg_type { log, flush, terminate }; - -// Async msg to move to/from the queue -// Movable only. should never be copied -struct async_msg : log_msg_buffer { - async_msg_type msg_type{async_msg_type::log}; - async_logger_ptr worker_ptr; - - async_msg() = default; - ~async_msg() = default; - - // should only be moved in or out of the queue.. - async_msg(const async_msg &) = delete; - -// support for vs2013 move -#if defined(_MSC_VER) && _MSC_VER <= 1800 - async_msg(async_msg &&other) - : log_msg_buffer(std::move(other)), - msg_type(other.msg_type), - worker_ptr(std::move(other.worker_ptr)) {} - - async_msg &operator=(async_msg &&other) { - *static_cast(this) = std::move(other); - msg_type = other.msg_type; - worker_ptr = std::move(other.worker_ptr); - return *this; - } -#else // (_MSC_VER) && _MSC_VER <= 1800 - async_msg(async_msg &&) = default; - async_msg &operator=(async_msg &&) = default; -#endif - - // construct from log_msg with given type - async_msg(async_logger_ptr &&worker, async_msg_type the_type, const details::log_msg &m) - : log_msg_buffer{m}, - msg_type{the_type}, - worker_ptr{std::move(worker)} {} - - async_msg(async_logger_ptr &&worker, async_msg_type the_type) - : log_msg_buffer{}, - msg_type{the_type}, - worker_ptr{std::move(worker)} {} - - explicit async_msg(async_msg_type the_type) - : async_msg{nullptr, the_type} {} -}; - -class SPDLOG_API thread_pool { -public: - using item_type = async_msg; - using q_type = details::mpmc_blocking_queue; - - thread_pool(size_t q_max_items, - size_t threads_n, - std::function on_thread_start, - std::function on_thread_stop); - thread_pool(size_t q_max_items, size_t threads_n, std::function on_thread_start); - thread_pool(size_t q_max_items, size_t threads_n); - - // message all threads to terminate gracefully and join them - ~thread_pool(); - - thread_pool(const thread_pool &) = delete; - thread_pool &operator=(thread_pool &&) = delete; - - void post_log(async_logger_ptr &&worker_ptr, - const details::log_msg &msg, - async_overflow_policy overflow_policy); - void post_flush(async_logger_ptr &&worker_ptr, async_overflow_policy overflow_policy); - size_t overrun_counter(); - void reset_overrun_counter(); - size_t discard_counter(); - void reset_discard_counter(); - size_t queue_size(); - -private: - q_type q_; - - std::vector threads_; - - void post_async_msg_(async_msg &&new_msg, async_overflow_policy overflow_policy); - void worker_loop_(); - - // process next message in the queue - // return true if this thread should still be active (while no terminate msg - // was received) - bool process_next_msg_(); -}; - -} // namespace details -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "thread_pool-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/details/windows_include.h b/thirdparty/spdlog/include/spdlog/details/windows_include.h deleted file mode 100644 index bbab59b1c..000000000 --- a/thirdparty/spdlog/include/spdlog/details/windows_include.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#ifndef NOMINMAX - #define NOMINMAX // prevent windows redefining min/max -#endif - -#ifndef WIN32_LEAN_AND_MEAN - #define WIN32_LEAN_AND_MEAN -#endif - -#include diff --git a/thirdparty/spdlog/include/spdlog/fmt/bin_to_hex.h b/thirdparty/spdlog/include/spdlog/fmt/bin_to_hex.h deleted file mode 100644 index 6ed68e427..000000000 --- a/thirdparty/spdlog/include/spdlog/fmt/bin_to_hex.h +++ /dev/null @@ -1,224 +0,0 @@ -// -// Copyright(c) 2015 Gabi Melman. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -// - -#pragma once - -#include -#include - -#if defined(__has_include) - #if __has_include() - #include - #endif -#endif - -#if __cpp_lib_span >= 202002L - #include -#endif - -// -// Support for logging binary data as hex -// format flags, any combination of the following: -// {:X} - print in uppercase. -// {:s} - don't separate each byte with space. -// {:p} - don't print the position on each line start. -// {:n} - don't split the output to lines. -// {:a} - show ASCII if :n is not set - -// -// Examples: -// -// std::vector v(200, 0x0b); -// logger->info("Some buffer {}", spdlog::to_hex(v)); -// char buf[128]; -// logger->info("Some buffer {:X}", spdlog::to_hex(std::begin(buf), std::end(buf))); -// logger->info("Some buffer {:X}", spdlog::to_hex(std::begin(buf), std::end(buf), 16)); - -namespace spdlog { -namespace details { - -template -class dump_info { -public: - dump_info(It range_begin, It range_end, size_t size_per_line) - : begin_(range_begin), - end_(range_end), - size_per_line_(size_per_line) {} - - // do not use begin() and end() to avoid collision with fmt/ranges - It get_begin() const { return begin_; } - It get_end() const { return end_; } - size_t size_per_line() const { return size_per_line_; } - -private: - It begin_, end_; - size_t size_per_line_; -}; -} // namespace details - -// create a dump_info that wraps the given container -template -inline details::dump_info to_hex(const Container &container, - size_t size_per_line = 32) { - static_assert(sizeof(typename Container::value_type) == 1, - "sizeof(Container::value_type) != 1"); - using Iter = typename Container::const_iterator; - return details::dump_info(std::begin(container), std::end(container), size_per_line); -} - -#if __cpp_lib_span >= 202002L - -template -inline details::dump_info::iterator> to_hex( - const std::span &container, size_t size_per_line = 32) { - using Container = std::span; - static_assert(sizeof(typename Container::value_type) == 1, - "sizeof(Container::value_type) != 1"); - using Iter = typename Container::iterator; - return details::dump_info(std::begin(container), std::end(container), size_per_line); -} - -#endif - -// create dump_info from ranges -template -inline details::dump_info to_hex(const It range_begin, - const It range_end, - size_t size_per_line = 32) { - return details::dump_info(range_begin, range_end, size_per_line); -} - -} // namespace spdlog - -namespace -#ifdef SPDLOG_USE_STD_FORMAT - std -#else - fmt -#endif -{ - -template -struct formatter, char> { - char delimiter = ' '; - bool put_newlines = true; - bool put_delimiters = true; - bool use_uppercase = false; - bool put_positions = true; // position on start of each line - bool show_ascii = false; - - // parse the format string flags - template - SPDLOG_CONSTEXPR_FUNC auto parse(ParseContext &ctx) -> decltype(ctx.begin()) { - auto it = ctx.begin(); - while (it != ctx.end() && *it != '}') { - switch (*it) { - case 'X': - use_uppercase = true; - break; - case 's': - put_delimiters = false; - break; - case 'p': - put_positions = false; - break; - case 'n': - put_newlines = false; - show_ascii = false; - break; - case 'a': - if (put_newlines) { - show_ascii = true; - } - break; - } - - ++it; - } - return it; - } - - // format the given bytes range as hex - template - auto format(const spdlog::details::dump_info &the_range, - FormatContext &ctx) const -> decltype(ctx.out()) { - SPDLOG_CONSTEXPR const char *hex_upper = "0123456789ABCDEF"; - SPDLOG_CONSTEXPR const char *hex_lower = "0123456789abcdef"; - const char *hex_chars = use_uppercase ? hex_upper : hex_lower; - -#if !defined(SPDLOG_USE_STD_FORMAT) && FMT_VERSION < 60000 - auto inserter = ctx.begin(); -#else - auto inserter = ctx.out(); -#endif - - int size_per_line = static_cast(the_range.size_per_line()); - auto start_of_line = the_range.get_begin(); - for (auto i = the_range.get_begin(); i != the_range.get_end(); i++) { - auto ch = static_cast(*i); - - if (put_newlines && - (i == the_range.get_begin() || i - start_of_line >= size_per_line)) { - if (show_ascii && i != the_range.get_begin()) { - *inserter++ = delimiter; - *inserter++ = delimiter; - for (auto j = start_of_line; j < i; j++) { - auto pc = static_cast(*j); - *inserter++ = std::isprint(pc) ? static_cast(*j) : '.'; - } - } - - put_newline(inserter, static_cast(i - the_range.get_begin())); - - // put first byte without delimiter in front of it - *inserter++ = hex_chars[(ch >> 4) & 0x0f]; - *inserter++ = hex_chars[ch & 0x0f]; - start_of_line = i; - continue; - } - - if (put_delimiters && i != the_range.get_begin()) { - *inserter++ = delimiter; - } - - *inserter++ = hex_chars[(ch >> 4) & 0x0f]; - *inserter++ = hex_chars[ch & 0x0f]; - } - if (show_ascii) // add ascii to last line - { - if (the_range.get_end() - the_range.get_begin() > size_per_line) { - auto blank_num = size_per_line - (the_range.get_end() - start_of_line); - while (blank_num-- > 0) { - *inserter++ = delimiter; - *inserter++ = delimiter; - if (put_delimiters) { - *inserter++ = delimiter; - } - } - } - *inserter++ = delimiter; - *inserter++ = delimiter; - for (auto j = start_of_line; j != the_range.get_end(); j++) { - auto pc = static_cast(*j); - *inserter++ = std::isprint(pc) ? static_cast(*j) : '.'; - } - } - return inserter; - } - - // put newline(and position header) - template - void put_newline(It inserter, std::size_t pos) const { -#ifdef _WIN32 - *inserter++ = '\r'; -#endif - *inserter++ = '\n'; - - if (put_positions) { - spdlog::fmt_lib::format_to(inserter, SPDLOG_FMT_STRING("{:04X}: "), pos); - } - } -}; -} // namespace std diff --git a/thirdparty/spdlog/include/spdlog/fmt/chrono.h b/thirdparty/spdlog/include/spdlog/fmt/chrono.h deleted file mode 100644 index a72a5bd64..000000000 --- a/thirdparty/spdlog/include/spdlog/fmt/chrono.h +++ /dev/null @@ -1,23 +0,0 @@ -// -// Copyright(c) 2016 Gabi Melman. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -// - -#pragma once -// -// include bundled or external copy of fmtlib's chrono support -// -#include - -#if !defined(SPDLOG_USE_STD_FORMAT) - #if !defined(SPDLOG_FMT_EXTERNAL) - #ifdef SPDLOG_HEADER_ONLY - #ifndef FMT_HEADER_ONLY - #define FMT_HEADER_ONLY - #endif - #endif - #include - #else - #include - #endif -#endif diff --git a/thirdparty/spdlog/include/spdlog/fmt/compile.h b/thirdparty/spdlog/include/spdlog/fmt/compile.h deleted file mode 100644 index 3c9c25d8b..000000000 --- a/thirdparty/spdlog/include/spdlog/fmt/compile.h +++ /dev/null @@ -1,23 +0,0 @@ -// -// Copyright(c) 2016 Gabi Melman. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -// - -#pragma once -// -// include bundled or external copy of fmtlib's compile-time support -// -#include - -#if !defined(SPDLOG_USE_STD_FORMAT) - #if !defined(SPDLOG_FMT_EXTERNAL) - #ifdef SPDLOG_HEADER_ONLY - #ifndef FMT_HEADER_ONLY - #define FMT_HEADER_ONLY - #endif - #endif - #include - #else - #include - #endif -#endif diff --git a/thirdparty/spdlog/include/spdlog/fmt/fmt.h b/thirdparty/spdlog/include/spdlog/fmt/fmt.h deleted file mode 100644 index ba94a0c5f..000000000 --- a/thirdparty/spdlog/include/spdlog/fmt/fmt.h +++ /dev/null @@ -1,26 +0,0 @@ -// -// Copyright(c) 2016-2018 Gabi Melman. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -// - -#pragma once - -// -// Include a bundled header-only copy of fmtlib or an external one. -// By default spdlog include its own copy. -// -#include - -#if defined(SPDLOG_USE_STD_FORMAT) // SPDLOG_USE_STD_FORMAT is defined - use std::format - #include -#elif !defined(SPDLOG_FMT_EXTERNAL) - #if !defined(SPDLOG_COMPILED_LIB) && !defined(FMT_HEADER_ONLY) - #define FMT_HEADER_ONLY - #endif - #ifndef FMT_USE_WINDOWS_H - #define FMT_USE_WINDOWS_H 0 - #endif - #include -#else // SPDLOG_FMT_EXTERNAL is defined - use external fmtlib - #include -#endif diff --git a/thirdparty/spdlog/include/spdlog/fmt/ostr.h b/thirdparty/spdlog/include/spdlog/fmt/ostr.h deleted file mode 100644 index 2b901055f..000000000 --- a/thirdparty/spdlog/include/spdlog/fmt/ostr.h +++ /dev/null @@ -1,23 +0,0 @@ -// -// Copyright(c) 2016 Gabi Melman. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -// - -#pragma once -// -// include bundled or external copy of fmtlib's ostream support -// -#include - -#if !defined(SPDLOG_USE_STD_FORMAT) - #if !defined(SPDLOG_FMT_EXTERNAL) - #ifdef SPDLOG_HEADER_ONLY - #ifndef FMT_HEADER_ONLY - #define FMT_HEADER_ONLY - #endif - #endif - #include - #else - #include - #endif -#endif diff --git a/thirdparty/spdlog/include/spdlog/fmt/ranges.h b/thirdparty/spdlog/include/spdlog/fmt/ranges.h deleted file mode 100644 index 5bb91e9ac..000000000 --- a/thirdparty/spdlog/include/spdlog/fmt/ranges.h +++ /dev/null @@ -1,23 +0,0 @@ -// -// Copyright(c) 2016 Gabi Melman. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -// - -#pragma once -// -// include bundled or external copy of fmtlib's ranges support -// -#include - -#if !defined(SPDLOG_USE_STD_FORMAT) - #if !defined(SPDLOG_FMT_EXTERNAL) - #ifdef SPDLOG_HEADER_ONLY - #ifndef FMT_HEADER_ONLY - #define FMT_HEADER_ONLY - #endif - #endif - #include - #else - #include - #endif -#endif diff --git a/thirdparty/spdlog/include/spdlog/fmt/std.h b/thirdparty/spdlog/include/spdlog/fmt/std.h deleted file mode 100644 index dabe6f69d..000000000 --- a/thirdparty/spdlog/include/spdlog/fmt/std.h +++ /dev/null @@ -1,24 +0,0 @@ -// -// Copyright(c) 2016 Gabi Melman. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -// - -#pragma once -// -// include bundled or external copy of fmtlib's std support (for formatting e.g. -// std::filesystem::path, std::thread::id, std::monostate, std::variant, ...) -// -#include - -#if !defined(SPDLOG_USE_STD_FORMAT) - #if !defined(SPDLOG_FMT_EXTERNAL) - #ifdef SPDLOG_HEADER_ONLY - #ifndef FMT_HEADER_ONLY - #define FMT_HEADER_ONLY - #endif - #endif - #include - #else - #include - #endif -#endif diff --git a/thirdparty/spdlog/include/spdlog/fmt/xchar.h b/thirdparty/spdlog/include/spdlog/fmt/xchar.h deleted file mode 100644 index 2525f0586..000000000 --- a/thirdparty/spdlog/include/spdlog/fmt/xchar.h +++ /dev/null @@ -1,23 +0,0 @@ -// -// Copyright(c) 2016 Gabi Melman. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -// - -#pragma once -// -// include bundled or external copy of fmtlib's xchar support -// -#include - -#if !defined(SPDLOG_USE_STD_FORMAT) - #if !defined(SPDLOG_FMT_EXTERNAL) - #ifdef SPDLOG_HEADER_ONLY - #ifndef FMT_HEADER_ONLY - #define FMT_HEADER_ONLY - #endif - #endif - #include - #else - #include - #endif -#endif diff --git a/thirdparty/spdlog/include/spdlog/formatter.h b/thirdparty/spdlog/include/spdlog/formatter.h deleted file mode 100644 index 4d482f827..000000000 --- a/thirdparty/spdlog/include/spdlog/formatter.h +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include - -namespace spdlog { - -class formatter { -public: - virtual ~formatter() = default; - virtual void format(const details::log_msg &msg, memory_buf_t &dest) = 0; - virtual std::unique_ptr clone() const = 0; -}; -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/fwd.h b/thirdparty/spdlog/include/spdlog/fwd.h deleted file mode 100644 index 647b16bf0..000000000 --- a/thirdparty/spdlog/include/spdlog/fwd.h +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -namespace spdlog { -class logger; -class formatter; - -namespace sinks { -class sink; -} - -namespace level { -enum level_enum : int; -} - -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/logger-inl.h b/thirdparty/spdlog/include/spdlog/logger-inl.h deleted file mode 100644 index 6879273c3..000000000 --- a/thirdparty/spdlog/include/spdlog/logger-inl.h +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include -#include - -#include - -namespace spdlog { - -// public methods -SPDLOG_INLINE logger::logger(const logger &other) - : name_(other.name_), - sinks_(other.sinks_), - level_(other.level_.load(std::memory_order_relaxed)), - flush_level_(other.flush_level_.load(std::memory_order_relaxed)), - custom_err_handler_(other.custom_err_handler_), - tracer_(other.tracer_) {} - -SPDLOG_INLINE logger::logger(logger &&other) SPDLOG_NOEXCEPT - : name_(std::move(other.name_)), - sinks_(std::move(other.sinks_)), - level_(other.level_.load(std::memory_order_relaxed)), - flush_level_(other.flush_level_.load(std::memory_order_relaxed)), - custom_err_handler_(std::move(other.custom_err_handler_)), - tracer_(std::move(other.tracer_)) - -{} - -SPDLOG_INLINE logger &logger::operator=(logger other) SPDLOG_NOEXCEPT { - this->swap(other); - return *this; -} - -SPDLOG_INLINE void logger::swap(spdlog::logger &other) SPDLOG_NOEXCEPT { - name_.swap(other.name_); - sinks_.swap(other.sinks_); - - // swap level_ - auto other_level = other.level_.load(); - auto my_level = level_.exchange(other_level); - other.level_.store(my_level); - - // swap flush level_ - other_level = other.flush_level_.load(); - my_level = flush_level_.exchange(other_level); - other.flush_level_.store(my_level); - - custom_err_handler_.swap(other.custom_err_handler_); - std::swap(tracer_, other.tracer_); -} - -SPDLOG_INLINE void swap(logger &a, logger &b) noexcept { a.swap(b); } - -SPDLOG_INLINE void logger::set_level(level::level_enum log_level) { level_.store(log_level); } - -SPDLOG_INLINE level::level_enum logger::level() const { - return static_cast(level_.load(std::memory_order_relaxed)); -} - -SPDLOG_INLINE const std::string &logger::name() const { return name_; } - -// set formatting for the sinks in this logger. -// each sink will get a separate instance of the formatter object. -SPDLOG_INLINE void logger::set_formatter(std::unique_ptr f) { - for (auto it = sinks_.begin(); it != sinks_.end(); ++it) { - if (std::next(it) == sinks_.end()) { - // last element - we can be move it. - (*it)->set_formatter(std::move(f)); - break; // to prevent clang-tidy warning - } else { - (*it)->set_formatter(f->clone()); - } - } -} - -SPDLOG_INLINE void logger::set_pattern(std::string pattern, pattern_time_type time_type) { - auto new_formatter = details::make_unique(std::move(pattern), time_type); - set_formatter(std::move(new_formatter)); -} - -// create new backtrace sink and move to it all our child sinks -SPDLOG_INLINE void logger::enable_backtrace(size_t n_messages) { tracer_.enable(n_messages); } - -// restore orig sinks and level and delete the backtrace sink -SPDLOG_INLINE void logger::disable_backtrace() { tracer_.disable(); } - -SPDLOG_INLINE void logger::dump_backtrace() { dump_backtrace_(); } - -// flush functions -SPDLOG_INLINE void logger::flush() { flush_(); } - -SPDLOG_INLINE void logger::flush_on(level::level_enum log_level) { flush_level_.store(log_level); } - -SPDLOG_INLINE level::level_enum logger::flush_level() const { - return static_cast(flush_level_.load(std::memory_order_relaxed)); -} - -// sinks -SPDLOG_INLINE const std::vector &logger::sinks() const { return sinks_; } - -SPDLOG_INLINE std::vector &logger::sinks() { return sinks_; } - -// error handler -SPDLOG_INLINE void logger::set_error_handler(err_handler handler) { - custom_err_handler_ = std::move(handler); -} - -// create new logger with same sinks and configuration. -SPDLOG_INLINE std::shared_ptr logger::clone(std::string logger_name) { - auto cloned = std::make_shared(*this); - cloned->name_ = std::move(logger_name); - return cloned; -} - -// protected methods -SPDLOG_INLINE void logger::log_it_(const spdlog::details::log_msg &log_msg, - bool log_enabled, - bool traceback_enabled) { - if (log_enabled) { - sink_it_(log_msg); - } - if (traceback_enabled) { - tracer_.push_back(log_msg); - } -} - -SPDLOG_INLINE void logger::sink_it_(const details::log_msg &msg) { - for (auto &sink : sinks_) { - if (sink->should_log(msg.level)) { - SPDLOG_TRY { sink->log(msg); } - SPDLOG_LOGGER_CATCH(msg.source) - } - } - - if (should_flush_(msg)) { - flush_(); - } -} - -SPDLOG_INLINE void logger::flush_() { - for (auto &sink : sinks_) { - SPDLOG_TRY { sink->flush(); } - SPDLOG_LOGGER_CATCH(source_loc()) - } -} - -SPDLOG_INLINE void logger::dump_backtrace_() { - using details::log_msg; - if (tracer_.enabled() && !tracer_.empty()) { - sink_it_( - log_msg{name(), level::info, "****************** Backtrace Start ******************"}); - tracer_.foreach_pop([this](const log_msg &msg) { this->sink_it_(msg); }); - sink_it_( - log_msg{name(), level::info, "****************** Backtrace End ********************"}); - } -} - -SPDLOG_INLINE bool logger::should_flush_(const details::log_msg &msg) const { - auto flush_level = flush_level_.load(std::memory_order_relaxed); - return (msg.level >= flush_level) && (msg.level != level::off); -} - -SPDLOG_INLINE void logger::err_handler_(const std::string &msg) const { - if (custom_err_handler_) { - custom_err_handler_(msg); - } else { - using std::chrono::system_clock; - static std::mutex mutex; - static std::chrono::system_clock::time_point last_report_time; - static size_t err_counter = 0; - std::lock_guard lk{mutex}; - auto now = system_clock::now(); - err_counter++; - if (now - last_report_time < std::chrono::seconds(1)) { - return; - } - last_report_time = now; - auto tm_time = details::os::localtime(system_clock::to_time_t(now)); - char date_buf[64]; - std::strftime(date_buf, sizeof(date_buf), "%Y-%m-%d %H:%M:%S", &tm_time); -#if defined(USING_R) && defined(R_R_H) // if in R environment - REprintf("[*** LOG ERROR #%04zu ***] [%s] [%s] %s\n", err_counter, date_buf, name().c_str(), - msg.c_str()); -#else - std::fprintf(stderr, "[*** LOG ERROR #%04zu ***] [%s] [%s] %s\n", err_counter, date_buf, - name().c_str(), msg.c_str()); -#endif - } -} -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/logger.h b/thirdparty/spdlog/include/spdlog/logger.h deleted file mode 100644 index 8c3cd91f5..000000000 --- a/thirdparty/spdlog/include/spdlog/logger.h +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -// Thread safe logger (except for set_error_handler()) -// Has name, log level, vector of std::shared sink pointers and formatter -// Upon each log write the logger: -// 1. Checks if its log level is enough to log the message and if yes: -// 2. Call the underlying sinks to do the job. -// 3. Each sink use its own private copy of a formatter to format the message -// and send to its destination. -// -// The use of private formatter per sink provides the opportunity to cache some -// formatted data, and support for different format per sink. - -#include -#include -#include - -#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT - #ifndef _WIN32 - #error SPDLOG_WCHAR_TO_UTF8_SUPPORT only supported on windows - #endif - #include -#endif - -#include - -#ifndef SPDLOG_NO_EXCEPTIONS - #define SPDLOG_LOGGER_CATCH(location) \ - catch (const std::exception &ex) { \ - if (location.filename) { \ - err_handler_(fmt_lib::format(SPDLOG_FMT_STRING("{} [{}({})]"), ex.what(), \ - location.filename, location.line)); \ - } else { \ - err_handler_(ex.what()); \ - } \ - } \ - catch (...) { \ - err_handler_("Rethrowing unknown exception in logger"); \ - throw; \ - } -#else - #define SPDLOG_LOGGER_CATCH(location) -#endif - -namespace spdlog { - -class SPDLOG_API logger { -public: - // Empty logger - explicit logger(std::string name) - : name_(std::move(name)), - sinks_() {} - - // Logger with range on sinks - template - logger(std::string name, It begin, It end) - : name_(std::move(name)), - sinks_(begin, end) {} - - // Logger with single sink - logger(std::string name, sink_ptr single_sink) - : logger(std::move(name), {std::move(single_sink)}) {} - - // Logger with sinks init list - logger(std::string name, sinks_init_list sinks) - : logger(std::move(name), sinks.begin(), sinks.end()) {} - - virtual ~logger() = default; - - logger(const logger &other); - logger(logger &&other) SPDLOG_NOEXCEPT; - logger &operator=(logger other) SPDLOG_NOEXCEPT; - void swap(spdlog::logger &other) SPDLOG_NOEXCEPT; - - template - void log(source_loc loc, level::level_enum lvl, format_string_t fmt, Args &&...args) { - log_(loc, lvl, details::to_string_view(fmt), std::forward(args)...); - } - - template - void log(level::level_enum lvl, format_string_t fmt, Args &&...args) { - log(source_loc{}, lvl, fmt, std::forward(args)...); - } - - template - void log(level::level_enum lvl, const T &msg) { - log(source_loc{}, lvl, msg); - } - - // T cannot be statically converted to format string (including string_view/wstring_view) - template ::value, - int>::type = 0> - void log(source_loc loc, level::level_enum lvl, const T &msg) { - log(loc, lvl, "{}", msg); - } - - void log(log_clock::time_point log_time, - source_loc loc, - level::level_enum lvl, - string_view_t msg) { - bool log_enabled = should_log(lvl); - bool traceback_enabled = tracer_.enabled(); - if (!log_enabled && !traceback_enabled) { - return; - } - - details::log_msg log_msg(log_time, loc, name_, lvl, msg); - log_it_(log_msg, log_enabled, traceback_enabled); - } - - void log(source_loc loc, level::level_enum lvl, string_view_t msg) { - bool log_enabled = should_log(lvl); - bool traceback_enabled = tracer_.enabled(); - if (!log_enabled && !traceback_enabled) { - return; - } - - details::log_msg log_msg(loc, name_, lvl, msg); - log_it_(log_msg, log_enabled, traceback_enabled); - } - - void log(level::level_enum lvl, string_view_t msg) { log(source_loc{}, lvl, msg); } - - template - void trace(format_string_t fmt, Args &&...args) { - log(level::trace, fmt, std::forward(args)...); - } - - template - void debug(format_string_t fmt, Args &&...args) { - log(level::debug, fmt, std::forward(args)...); - } - - template - void info(format_string_t fmt, Args &&...args) { - log(level::info, fmt, std::forward(args)...); - } - - template - void warn(format_string_t fmt, Args &&...args) { - log(level::warn, fmt, std::forward(args)...); - } - - template - void error(format_string_t fmt, Args &&...args) { - log(level::err, fmt, std::forward(args)...); - } - - template - void critical(format_string_t fmt, Args &&...args) { - log(level::critical, fmt, std::forward(args)...); - } - -#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT - template - void log(source_loc loc, level::level_enum lvl, wformat_string_t fmt, Args &&...args) { - log_(loc, lvl, details::to_string_view(fmt), std::forward(args)...); - } - - template - void log(level::level_enum lvl, wformat_string_t fmt, Args &&...args) { - log(source_loc{}, lvl, fmt, std::forward(args)...); - } - - void log(log_clock::time_point log_time, - source_loc loc, - level::level_enum lvl, - wstring_view_t msg) { - bool log_enabled = should_log(lvl); - bool traceback_enabled = tracer_.enabled(); - if (!log_enabled && !traceback_enabled) { - return; - } - - memory_buf_t buf; - details::os::wstr_to_utf8buf(wstring_view_t(msg.data(), msg.size()), buf); - details::log_msg log_msg(log_time, loc, name_, lvl, string_view_t(buf.data(), buf.size())); - log_it_(log_msg, log_enabled, traceback_enabled); - } - - void log(source_loc loc, level::level_enum lvl, wstring_view_t msg) { - bool log_enabled = should_log(lvl); - bool traceback_enabled = tracer_.enabled(); - if (!log_enabled && !traceback_enabled) { - return; - } - - memory_buf_t buf; - details::os::wstr_to_utf8buf(wstring_view_t(msg.data(), msg.size()), buf); - details::log_msg log_msg(loc, name_, lvl, string_view_t(buf.data(), buf.size())); - log_it_(log_msg, log_enabled, traceback_enabled); - } - - void log(level::level_enum lvl, wstring_view_t msg) { log(source_loc{}, lvl, msg); } - - template - void trace(wformat_string_t fmt, Args &&...args) { - log(level::trace, fmt, std::forward(args)...); - } - - template - void debug(wformat_string_t fmt, Args &&...args) { - log(level::debug, fmt, std::forward(args)...); - } - - template - void info(wformat_string_t fmt, Args &&...args) { - log(level::info, fmt, std::forward(args)...); - } - - template - void warn(wformat_string_t fmt, Args &&...args) { - log(level::warn, fmt, std::forward(args)...); - } - - template - void error(wformat_string_t fmt, Args &&...args) { - log(level::err, fmt, std::forward(args)...); - } - - template - void critical(wformat_string_t fmt, Args &&...args) { - log(level::critical, fmt, std::forward(args)...); - } -#endif - - template - void trace(const T &msg) { - log(level::trace, msg); - } - - template - void debug(const T &msg) { - log(level::debug, msg); - } - - template - void info(const T &msg) { - log(level::info, msg); - } - - template - void warn(const T &msg) { - log(level::warn, msg); - } - - template - void error(const T &msg) { - log(level::err, msg); - } - - template - void critical(const T &msg) { - log(level::critical, msg); - } - - // return true logging is enabled for the given level. - bool should_log(level::level_enum msg_level) const { - return msg_level >= level_.load(std::memory_order_relaxed); - } - - // return true if backtrace logging is enabled. - bool should_backtrace() const { return tracer_.enabled(); } - - void set_level(level::level_enum log_level); - - level::level_enum level() const; - - const std::string &name() const; - - // set formatting for the sinks in this logger. - // each sink will get a separate instance of the formatter object. - void set_formatter(std::unique_ptr f); - - // set formatting for the sinks in this logger. - // equivalent to - // set_formatter(make_unique(pattern, time_type)) - // Note: each sink will get a new instance of a formatter object, replacing the old one. - void set_pattern(std::string pattern, pattern_time_type time_type = pattern_time_type::local); - - // backtrace support. - // efficiently store all debug/trace messages in a circular buffer until needed for debugging. - void enable_backtrace(size_t n_messages); - void disable_backtrace(); - void dump_backtrace(); - - // flush functions - void flush(); - void flush_on(level::level_enum log_level); - level::level_enum flush_level() const; - - // sinks - const std::vector &sinks() const; - - std::vector &sinks(); - - // error handler - void set_error_handler(err_handler); - - // create new logger with same sinks and configuration. - virtual std::shared_ptr clone(std::string logger_name); - -protected: - std::string name_; - std::vector sinks_; - spdlog::level_t level_{level::info}; - spdlog::level_t flush_level_{level::off}; - err_handler custom_err_handler_{nullptr}; - details::backtracer tracer_; - - // common implementation for after templated public api has been resolved - template - void log_(source_loc loc, level::level_enum lvl, string_view_t fmt, Args &&...args) { - bool log_enabled = should_log(lvl); - bool traceback_enabled = tracer_.enabled(); - if (!log_enabled && !traceback_enabled) { - return; - } - SPDLOG_TRY { - memory_buf_t buf; -#ifdef SPDLOG_USE_STD_FORMAT - fmt_lib::vformat_to(std::back_inserter(buf), fmt, fmt_lib::make_format_args(args...)); -#else - fmt::vformat_to(fmt::appender(buf), fmt, fmt::make_format_args(args...)); -#endif - - details::log_msg log_msg(loc, name_, lvl, string_view_t(buf.data(), buf.size())); - log_it_(log_msg, log_enabled, traceback_enabled); - } - SPDLOG_LOGGER_CATCH(loc) - } - -#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT - template - void log_(source_loc loc, level::level_enum lvl, wstring_view_t fmt, Args &&...args) { - bool log_enabled = should_log(lvl); - bool traceback_enabled = tracer_.enabled(); - if (!log_enabled && !traceback_enabled) { - return; - } - SPDLOG_TRY { - // format to wmemory_buffer and convert to utf8 - wmemory_buf_t wbuf; - fmt_lib::vformat_to(std::back_inserter(wbuf), fmt, - fmt_lib::make_format_args(args...)); - - memory_buf_t buf; - details::os::wstr_to_utf8buf(wstring_view_t(wbuf.data(), wbuf.size()), buf); - details::log_msg log_msg(loc, name_, lvl, string_view_t(buf.data(), buf.size())); - log_it_(log_msg, log_enabled, traceback_enabled); - } - SPDLOG_LOGGER_CATCH(loc) - } -#endif // SPDLOG_WCHAR_TO_UTF8_SUPPORT - - // log the given message (if the given log level is high enough), - // and save backtrace (if backtrace is enabled). - void log_it_(const details::log_msg &log_msg, bool log_enabled, bool traceback_enabled); - virtual void sink_it_(const details::log_msg &msg); - virtual void flush_(); - void dump_backtrace_(); - bool should_flush_(const details::log_msg &msg) const; - - // handle errors during logging. - // default handler prints the error to stderr at max rate of 1 message/sec. - void err_handler_(const std::string &msg) const; -}; - -void swap(logger &a, logger &b) noexcept; - -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "logger-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/mdc.h b/thirdparty/spdlog/include/spdlog/mdc.h deleted file mode 100644 index bc131746e..000000000 --- a/thirdparty/spdlog/include/spdlog/mdc.h +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#if defined(SPDLOG_NO_TLS) - #error "This header requires thread local storage support, but SPDLOG_NO_TLS is defined." -#endif - -#include -#include - -#include - -// MDC is a simple map of key->string values stored in thread local storage whose content will be -// printed by the loggers. Note: Not supported in async mode (thread local storage - so the async -// thread pool have different copy). -// -// Usage example: -// spdlog::mdc::put("mdc_key_1", "mdc_value_1"); -// spdlog::info("Hello, {}", "World!"); // => [2024-04-26 02:08:05.040] [info] -// [mdc_key_1:mdc_value_1] Hello, World! - -namespace spdlog { -class SPDLOG_API mdc { -public: - using mdc_map_t = std::map; - - static void put(const std::string &key, const std::string &value) { - get_context()[key] = value; - } - - static std::string get(const std::string &key) { - auto &context = get_context(); - auto it = context.find(key); - if (it != context.end()) { - return it->second; - } - return ""; - } - - static void remove(const std::string &key) { get_context().erase(key); } - - static void clear() { get_context().clear(); } - - static mdc_map_t &get_context() { - static thread_local mdc_map_t context; - return context; - } -}; - -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/pattern_formatter-inl.h b/thirdparty/spdlog/include/spdlog/pattern_formatter-inl.h deleted file mode 100644 index fd408ed52..000000000 --- a/thirdparty/spdlog/include/spdlog/pattern_formatter-inl.h +++ /dev/null @@ -1,1340 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include -#include - -#ifndef SPDLOG_NO_TLS - #include -#endif - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace spdlog { -namespace details { - -/////////////////////////////////////////////////////////////////////// -// name & level pattern appender -/////////////////////////////////////////////////////////////////////// - -class scoped_padder { -public: - scoped_padder(size_t wrapped_size, const padding_info &padinfo, memory_buf_t &dest) - : padinfo_(padinfo), - dest_(dest) { - remaining_pad_ = static_cast(padinfo.width_) - static_cast(wrapped_size); - if (remaining_pad_ <= 0) { - return; - } - - if (padinfo_.side_ == padding_info::pad_side::left) { - pad_it(remaining_pad_); - remaining_pad_ = 0; - } else if (padinfo_.side_ == padding_info::pad_side::center) { - auto half_pad = remaining_pad_ / 2; - auto reminder = remaining_pad_ & 1; - pad_it(half_pad); - remaining_pad_ = half_pad + reminder; // for the right side - } - } - - template - static unsigned int count_digits(T n) { - return fmt_helper::count_digits(n); - } - - ~scoped_padder() { - if (remaining_pad_ >= 0) { - pad_it(remaining_pad_); - } else if (padinfo_.truncate_) { - long new_size = static_cast(dest_.size()) + remaining_pad_; - if (new_size < 0) { - new_size = 0; - } - dest_.resize(static_cast(new_size)); - } - } - -private: - void pad_it(long count) { - fmt_helper::append_string_view(string_view_t(spaces_.data(), static_cast(count)), - dest_); - } - - const padding_info &padinfo_; - memory_buf_t &dest_; - long remaining_pad_; - string_view_t spaces_{" ", 64}; -}; - -struct null_scoped_padder { - null_scoped_padder(size_t /*wrapped_size*/, - const padding_info & /*padinfo*/, - memory_buf_t & /*dest*/) {} - - template - static unsigned int count_digits(T /* number */) { - return 0; - } -}; - -template -class name_formatter final : public flag_formatter { -public: - explicit name_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - ScopedPadder p(msg.logger_name.size(), padinfo_, dest); - fmt_helper::append_string_view(msg.logger_name, dest); - } -}; - -// log level appender -template -class level_formatter final : public flag_formatter { -public: - explicit level_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - const string_view_t &level_name = level::to_string_view(msg.level); - ScopedPadder p(level_name.size(), padinfo_, dest); - fmt_helper::append_string_view(level_name, dest); - } -}; - -// short log level appender -template -class short_level_formatter final : public flag_formatter { -public: - explicit short_level_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - string_view_t level_name{level::to_short_c_str(msg.level)}; - ScopedPadder p(level_name.size(), padinfo_, dest); - fmt_helper::append_string_view(level_name, dest); - } -}; - -/////////////////////////////////////////////////////////////////////// -// Date time pattern appenders -/////////////////////////////////////////////////////////////////////// - -static const char *ampm(const tm &t) { return t.tm_hour >= 12 ? "PM" : "AM"; } - -static int to12h(const tm &t) { return t.tm_hour > 12 ? t.tm_hour - 12 : t.tm_hour; } - -// Abbreviated weekday name -static std::array days{{"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}}; - -template -class a_formatter final : public flag_formatter { -public: - explicit a_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - string_view_t field_value{days[static_cast(tm_time.tm_wday)]}; - ScopedPadder p(field_value.size(), padinfo_, dest); - fmt_helper::append_string_view(field_value, dest); - } -}; - -// Full weekday name -static std::array full_days{ - {"Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"}}; - -template -class A_formatter : public flag_formatter { -public: - explicit A_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - string_view_t field_value{full_days[static_cast(tm_time.tm_wday)]}; - ScopedPadder p(field_value.size(), padinfo_, dest); - fmt_helper::append_string_view(field_value, dest); - } -}; - -// Abbreviated month -static const std::array months{ - {"Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}}; - -template -class b_formatter final : public flag_formatter { -public: - explicit b_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - string_view_t field_value{months[static_cast(tm_time.tm_mon)]}; - ScopedPadder p(field_value.size(), padinfo_, dest); - fmt_helper::append_string_view(field_value, dest); - } -}; - -// Full month name -static const std::array full_months{{"January", "February", "March", "April", - "May", "June", "July", "August", "September", - "October", "November", "December"}}; - -template -class B_formatter final : public flag_formatter { -public: - explicit B_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - string_view_t field_value{full_months[static_cast(tm_time.tm_mon)]}; - ScopedPadder p(field_value.size(), padinfo_, dest); - fmt_helper::append_string_view(field_value, dest); - } -}; - -// Date and time representation (Thu Aug 23 15:35:46 2014) -template -class c_formatter final : public flag_formatter { -public: - explicit c_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 24; - ScopedPadder p(field_size, padinfo_, dest); - - fmt_helper::append_string_view(days[static_cast(tm_time.tm_wday)], dest); - dest.push_back(' '); - fmt_helper::append_string_view(months[static_cast(tm_time.tm_mon)], dest); - dest.push_back(' '); - fmt_helper::append_int(tm_time.tm_mday, dest); - dest.push_back(' '); - // time - - fmt_helper::pad2(tm_time.tm_hour, dest); - dest.push_back(':'); - fmt_helper::pad2(tm_time.tm_min, dest); - dest.push_back(':'); - fmt_helper::pad2(tm_time.tm_sec, dest); - dest.push_back(' '); - fmt_helper::append_int(tm_time.tm_year + 1900, dest); - } -}; - -// year - 2 digit -template -class C_formatter final : public flag_formatter { -public: - explicit C_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 2; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad2(tm_time.tm_year % 100, dest); - } -}; - -// Short MM/DD/YY date, equivalent to %m/%d/%y 08/23/01 -template -class D_formatter final : public flag_formatter { -public: - explicit D_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 8; - ScopedPadder p(field_size, padinfo_, dest); - - fmt_helper::pad2(tm_time.tm_mon + 1, dest); - dest.push_back('/'); - fmt_helper::pad2(tm_time.tm_mday, dest); - dest.push_back('/'); - fmt_helper::pad2(tm_time.tm_year % 100, dest); - } -}; - -// year - 4 digit -template -class Y_formatter final : public flag_formatter { -public: - explicit Y_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 4; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::append_int(tm_time.tm_year + 1900, dest); - } -}; - -// month 1-12 -template -class m_formatter final : public flag_formatter { -public: - explicit m_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 2; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad2(tm_time.tm_mon + 1, dest); - } -}; - -// day of month 1-31 -template -class d_formatter final : public flag_formatter { -public: - explicit d_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 2; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad2(tm_time.tm_mday, dest); - } -}; - -// hours in 24 format 0-23 -template -class H_formatter final : public flag_formatter { -public: - explicit H_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 2; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad2(tm_time.tm_hour, dest); - } -}; - -// hours in 12 format 1-12 -template -class I_formatter final : public flag_formatter { -public: - explicit I_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 2; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad2(to12h(tm_time), dest); - } -}; - -// minutes 0-59 -template -class M_formatter final : public flag_formatter { -public: - explicit M_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 2; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad2(tm_time.tm_min, dest); - } -}; - -// seconds 0-59 -template -class S_formatter final : public flag_formatter { -public: - explicit S_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 2; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad2(tm_time.tm_sec, dest); - } -}; - -// milliseconds -template -class e_formatter final : public flag_formatter { -public: - explicit e_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - auto millis = fmt_helper::time_fraction(msg.time); - const size_t field_size = 3; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad3(static_cast(millis.count()), dest); - } -}; - -// microseconds -template -class f_formatter final : public flag_formatter { -public: - explicit f_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - auto micros = fmt_helper::time_fraction(msg.time); - - const size_t field_size = 6; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad6(static_cast(micros.count()), dest); - } -}; - -// nanoseconds -template -class F_formatter final : public flag_formatter { -public: - explicit F_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - auto ns = fmt_helper::time_fraction(msg.time); - const size_t field_size = 9; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::pad9(static_cast(ns.count()), dest); - } -}; - -// seconds since epoch -template -class E_formatter final : public flag_formatter { -public: - explicit E_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - const size_t field_size = 10; - ScopedPadder p(field_size, padinfo_, dest); - auto duration = msg.time.time_since_epoch(); - auto seconds = std::chrono::duration_cast(duration).count(); - fmt_helper::append_int(seconds, dest); - } -}; - -// AM/PM -template -class p_formatter final : public flag_formatter { -public: - explicit p_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 2; - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::append_string_view(ampm(tm_time), dest); - } -}; - -// 12 hour clock 02:55:02 pm -template -class r_formatter final : public flag_formatter { -public: - explicit r_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 11; - ScopedPadder p(field_size, padinfo_, dest); - - fmt_helper::pad2(to12h(tm_time), dest); - dest.push_back(':'); - fmt_helper::pad2(tm_time.tm_min, dest); - dest.push_back(':'); - fmt_helper::pad2(tm_time.tm_sec, dest); - dest.push_back(' '); - fmt_helper::append_string_view(ampm(tm_time), dest); - } -}; - -// 24-hour HH:MM time, equivalent to %H:%M -template -class R_formatter final : public flag_formatter { -public: - explicit R_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 5; - ScopedPadder p(field_size, padinfo_, dest); - - fmt_helper::pad2(tm_time.tm_hour, dest); - dest.push_back(':'); - fmt_helper::pad2(tm_time.tm_min, dest); - } -}; - -// ISO 8601 time format (HH:MM:SS), equivalent to %H:%M:%S -template -class T_formatter final : public flag_formatter { -public: - explicit T_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 8; - ScopedPadder p(field_size, padinfo_, dest); - - fmt_helper::pad2(tm_time.tm_hour, dest); - dest.push_back(':'); - fmt_helper::pad2(tm_time.tm_min, dest); - dest.push_back(':'); - fmt_helper::pad2(tm_time.tm_sec, dest); - } -}; - -// ISO 8601 offset from UTC in timezone (+-HH:MM) -template -class z_formatter final : public flag_formatter { -public: - explicit z_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - z_formatter() = default; - z_formatter(const z_formatter &) = delete; - z_formatter &operator=(const z_formatter &) = delete; - - void format(const details::log_msg &msg, const std::tm &tm_time, memory_buf_t &dest) override { - const size_t field_size = 6; - ScopedPadder p(field_size, padinfo_, dest); - - auto total_minutes = get_cached_offset(msg, tm_time); - bool is_negative = total_minutes < 0; - if (is_negative) { - total_minutes = -total_minutes; - dest.push_back('-'); - } else { - dest.push_back('+'); - } - - fmt_helper::pad2(total_minutes / 60, dest); // hours - dest.push_back(':'); - fmt_helper::pad2(total_minutes % 60, dest); // minutes - } - -private: - log_clock::time_point last_update_{std::chrono::seconds(0)}; - int offset_minutes_{0}; - - int get_cached_offset(const log_msg &msg, const std::tm &tm_time) { - // refresh every 10 seconds - if (msg.time - last_update_ >= std::chrono::seconds(10)) { - offset_minutes_ = os::utc_minutes_offset(tm_time); - last_update_ = msg.time; - } - return offset_minutes_; - } -}; - -// Thread id -template -class t_formatter final : public flag_formatter { -public: - explicit t_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - const auto field_size = ScopedPadder::count_digits(msg.thread_id); - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::append_int(msg.thread_id, dest); - } -}; - -// Current pid -template -class pid_formatter final : public flag_formatter { -public: - explicit pid_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &, memory_buf_t &dest) override { - const auto pid = static_cast(details::os::pid()); - auto field_size = ScopedPadder::count_digits(pid); - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::append_int(pid, dest); - } -}; - -template -class v_formatter final : public flag_formatter { -public: - explicit v_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - ScopedPadder p(msg.payload.size(), padinfo_, dest); - fmt_helper::append_string_view(msg.payload, dest); - } -}; - -class ch_formatter final : public flag_formatter { -public: - explicit ch_formatter(char ch) - : ch_(ch) {} - - void format(const details::log_msg &, const std::tm &, memory_buf_t &dest) override { - dest.push_back(ch_); - } - -private: - char ch_; -}; - -// aggregate user chars to display as is -class aggregate_formatter final : public flag_formatter { -public: - aggregate_formatter() = default; - - void add_ch(char ch) { str_ += ch; } - void format(const details::log_msg &, const std::tm &, memory_buf_t &dest) override { - fmt_helper::append_string_view(str_, dest); - } - -private: - std::string str_; -}; - -// mark the color range. expect it to be in the form of "%^colored text%$" -class color_start_formatter final : public flag_formatter { -public: - explicit color_start_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - msg.color_range_start = dest.size(); - } -}; - -class color_stop_formatter final : public flag_formatter { -public: - explicit color_stop_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - msg.color_range_end = dest.size(); - } -}; - -// print source location -template -class source_location_formatter final : public flag_formatter { -public: - explicit source_location_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - if (msg.source.empty()) { - ScopedPadder p(0, padinfo_, dest); - return; - } - - size_t text_size; - if (padinfo_.enabled()) { - // calc text size for padding based on "filename:line" - text_size = std::char_traits::length(msg.source.filename) + - ScopedPadder::count_digits(msg.source.line) + 1; - } else { - text_size = 0; - } - - ScopedPadder p(text_size, padinfo_, dest); - fmt_helper::append_string_view(msg.source.filename, dest); - dest.push_back(':'); - fmt_helper::append_int(msg.source.line, dest); - } -}; - -// print source filename -template -class source_filename_formatter final : public flag_formatter { -public: - explicit source_filename_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - if (msg.source.empty()) { - ScopedPadder p(0, padinfo_, dest); - return; - } - size_t text_size = - padinfo_.enabled() ? std::char_traits::length(msg.source.filename) : 0; - ScopedPadder p(text_size, padinfo_, dest); - fmt_helper::append_string_view(msg.source.filename, dest); - } -}; - -template -class short_filename_formatter final : public flag_formatter { -public: - explicit short_filename_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - -#ifdef _MSC_VER - #pragma warning(push) - #pragma warning(disable : 4127) // consider using 'if constexpr' instead -#endif // _MSC_VER - static const char *basename(const char *filename) { - // if the size is 2 (1 character + null terminator) we can use the more efficient strrchr - // the branch will be elided by optimizations - if (sizeof(os::folder_seps) == 2) { - const char *rv = std::strrchr(filename, os::folder_seps[0]); - return rv != nullptr ? rv + 1 : filename; - } else { - const std::reverse_iterator begin(filename + std::strlen(filename)); - const std::reverse_iterator end(filename); - - const auto it = std::find_first_of(begin, end, std::begin(os::folder_seps), - std::end(os::folder_seps) - 1); - return it != end ? it.base() : filename; - } - } -#ifdef _MSC_VER - #pragma warning(pop) -#endif // _MSC_VER - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - if (msg.source.empty()) { - ScopedPadder p(0, padinfo_, dest); - return; - } - auto filename = basename(msg.source.filename); - size_t text_size = padinfo_.enabled() ? std::char_traits::length(filename) : 0; - ScopedPadder p(text_size, padinfo_, dest); - fmt_helper::append_string_view(filename, dest); - } -}; - -template -class source_linenum_formatter final : public flag_formatter { -public: - explicit source_linenum_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - if (msg.source.empty()) { - ScopedPadder p(0, padinfo_, dest); - return; - } - - auto field_size = ScopedPadder::count_digits(msg.source.line); - ScopedPadder p(field_size, padinfo_, dest); - fmt_helper::append_int(msg.source.line, dest); - } -}; - -// print source funcname -template -class source_funcname_formatter final : public flag_formatter { -public: - explicit source_funcname_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - if (msg.source.empty()) { - ScopedPadder p(0, padinfo_, dest); - return; - } - size_t text_size = - padinfo_.enabled() ? std::char_traits::length(msg.source.funcname) : 0; - ScopedPadder p(text_size, padinfo_, dest); - fmt_helper::append_string_view(msg.source.funcname, dest); - } -}; - -// print elapsed time since last message -template -class elapsed_formatter final : public flag_formatter { -public: - using DurationUnits = Units; - - explicit elapsed_formatter(padding_info padinfo) - : flag_formatter(padinfo), - last_message_time_(log_clock::now()) {} - - void format(const details::log_msg &msg, const std::tm &, memory_buf_t &dest) override { - auto delta = (std::max)(msg.time - last_message_time_, log_clock::duration::zero()); - auto delta_units = std::chrono::duration_cast(delta); - last_message_time_ = msg.time; - auto delta_count = static_cast(delta_units.count()); - auto n_digits = static_cast(ScopedPadder::count_digits(delta_count)); - ScopedPadder p(n_digits, padinfo_, dest); - fmt_helper::append_int(delta_count, dest); - } - -private: - log_clock::time_point last_message_time_; -}; - -// Class for formatting Mapped Diagnostic Context (MDC) in log messages. -// Example: [logger-name] [info] [mdc_key_1:mdc_value_1 mdc_key_2:mdc_value_2] some message -#ifndef SPDLOG_NO_TLS -template -class mdc_formatter : public flag_formatter { -public: - explicit mdc_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &, const std::tm &, memory_buf_t &dest) override { - auto &mdc_map = mdc::get_context(); - if (mdc_map.empty()) { - ScopedPadder p(0, padinfo_, dest); - return; - } else { - format_mdc(mdc_map, dest); - } - } - - void format_mdc(const mdc::mdc_map_t &mdc_map, memory_buf_t &dest) { - auto last_element = --mdc_map.end(); - for (auto it = mdc_map.begin(); it != mdc_map.end(); ++it) { - auto &pair = *it; - const auto &key = pair.first; - const auto &value = pair.second; - size_t content_size = key.size() + value.size() + 1; // 1 for ':' - - if (it != last_element) { - content_size++; // 1 for ' ' - } - - ScopedPadder p(content_size, padinfo_, dest); - fmt_helper::append_string_view(key, dest); - fmt_helper::append_string_view(":", dest); - fmt_helper::append_string_view(value, dest); - if (it != last_element) { - fmt_helper::append_string_view(" ", dest); - } - } - } -}; -#endif - -// Full info formatter -// pattern: [%Y-%m-%d %H:%M:%S.%e] [%n] [%l] [%s:%#] %v -class full_formatter final : public flag_formatter { -public: - explicit full_formatter(padding_info padinfo) - : flag_formatter(padinfo) {} - - void format(const details::log_msg &msg, const std::tm &tm_time, memory_buf_t &dest) override { - using std::chrono::duration_cast; - using std::chrono::milliseconds; - using std::chrono::seconds; - - // cache the date/time part for the next second. - auto duration = msg.time.time_since_epoch(); - auto secs = duration_cast(duration); - - if (cache_timestamp_ != secs || cached_datetime_.size() == 0) { - cached_datetime_.clear(); - cached_datetime_.push_back('['); - fmt_helper::append_int(tm_time.tm_year + 1900, cached_datetime_); - cached_datetime_.push_back('-'); - - fmt_helper::pad2(tm_time.tm_mon + 1, cached_datetime_); - cached_datetime_.push_back('-'); - - fmt_helper::pad2(tm_time.tm_mday, cached_datetime_); - cached_datetime_.push_back(' '); - - fmt_helper::pad2(tm_time.tm_hour, cached_datetime_); - cached_datetime_.push_back(':'); - - fmt_helper::pad2(tm_time.tm_min, cached_datetime_); - cached_datetime_.push_back(':'); - - fmt_helper::pad2(tm_time.tm_sec, cached_datetime_); - cached_datetime_.push_back('.'); - - cache_timestamp_ = secs; - } - dest.append(cached_datetime_.begin(), cached_datetime_.end()); - - auto millis = fmt_helper::time_fraction(msg.time); - fmt_helper::pad3(static_cast(millis.count()), dest); - dest.push_back(']'); - dest.push_back(' '); - - // append logger name if exists - if (msg.logger_name.size() > 0) { - dest.push_back('['); - fmt_helper::append_string_view(msg.logger_name, dest); - dest.push_back(']'); - dest.push_back(' '); - } - - dest.push_back('['); - // wrap the level name with color - msg.color_range_start = dest.size(); - // fmt_helper::append_string_view(level::to_c_str(msg.level), dest); - fmt_helper::append_string_view(level::to_string_view(msg.level), dest); - msg.color_range_end = dest.size(); - dest.push_back(']'); - dest.push_back(' '); - - // add source location if present - if (!msg.source.empty()) { - dest.push_back('['); - const char *filename = - details::short_filename_formatter::basename( - msg.source.filename); - fmt_helper::append_string_view(filename, dest); - dest.push_back(':'); - fmt_helper::append_int(msg.source.line, dest); - dest.push_back(']'); - dest.push_back(' '); - } - -#ifndef SPDLOG_NO_TLS - // add mdc if present - auto &mdc_map = mdc::get_context(); - if (!mdc_map.empty()) { - dest.push_back('['); - mdc_formatter_.format_mdc(mdc_map, dest); - dest.push_back(']'); - dest.push_back(' '); - } -#endif - // fmt_helper::append_string_view(msg.msg(), dest); - fmt_helper::append_string_view(msg.payload, dest); - } - -private: - std::chrono::seconds cache_timestamp_{0}; - memory_buf_t cached_datetime_; - -#ifndef SPDLOG_NO_TLS - mdc_formatter mdc_formatter_{padding_info {}}; -#endif -}; - -} // namespace details - -SPDLOG_INLINE pattern_formatter::pattern_formatter(std::string pattern, - pattern_time_type time_type, - std::string eol, - custom_flags custom_user_flags) - : pattern_(std::move(pattern)), - eol_(std::move(eol)), - pattern_time_type_(time_type), - need_localtime_(false), - last_log_secs_(0), - custom_handlers_(std::move(custom_user_flags)) { - std::memset(&cached_tm_, 0, sizeof(cached_tm_)); - compile_pattern_(pattern_); -} - -// use by default full formatter for if pattern is not given -SPDLOG_INLINE pattern_formatter::pattern_formatter(pattern_time_type time_type, std::string eol) - : pattern_("%+"), - eol_(std::move(eol)), - pattern_time_type_(time_type), - need_localtime_(true), - last_log_secs_(0) { - std::memset(&cached_tm_, 0, sizeof(cached_tm_)); - formatters_.push_back(details::make_unique(details::padding_info{})); -} - -SPDLOG_INLINE std::unique_ptr pattern_formatter::clone() const { - custom_flags cloned_custom_formatters; - for (auto &it : custom_handlers_) { - cloned_custom_formatters[it.first] = it.second->clone(); - } - auto cloned = details::make_unique(pattern_, pattern_time_type_, eol_, - std::move(cloned_custom_formatters)); - cloned->need_localtime(need_localtime_); -#if defined(__GNUC__) && __GNUC__ < 5 - return std::move(cloned); -#else - return cloned; -#endif -} - -SPDLOG_INLINE void pattern_formatter::format(const details::log_msg &msg, memory_buf_t &dest) { - if (need_localtime_) { - const auto secs = - std::chrono::duration_cast(msg.time.time_since_epoch()); - if (secs != last_log_secs_) { - cached_tm_ = get_time_(msg); - last_log_secs_ = secs; - } - } - - for (auto &f : formatters_) { - f->format(msg, cached_tm_, dest); - } - // write eol - details::fmt_helper::append_string_view(eol_, dest); -} - -SPDLOG_INLINE void pattern_formatter::set_pattern(std::string pattern) { - pattern_ = std::move(pattern); - need_localtime_ = false; - compile_pattern_(pattern_); -} - -SPDLOG_INLINE void pattern_formatter::need_localtime(bool need) { need_localtime_ = need; } - -SPDLOG_INLINE std::tm pattern_formatter::get_time_(const details::log_msg &msg) { - if (pattern_time_type_ == pattern_time_type::local) { - return details::os::localtime(log_clock::to_time_t(msg.time)); - } - return details::os::gmtime(log_clock::to_time_t(msg.time)); -} - -template -SPDLOG_INLINE void pattern_formatter::handle_flag_(char flag, details::padding_info padding) { - // process custom flags - auto it = custom_handlers_.find(flag); - if (it != custom_handlers_.end()) { - auto custom_handler = it->second->clone(); - custom_handler->set_padding_info(padding); - formatters_.push_back(std::move(custom_handler)); - return; - } - - // process built-in flags - switch (flag) { - case ('+'): // default formatter - formatters_.push_back(details::make_unique(padding)); - need_localtime_ = true; - break; - - case 'n': // logger name - formatters_.push_back(details::make_unique>(padding)); - break; - - case 'l': // level - formatters_.push_back(details::make_unique>(padding)); - break; - - case 'L': // short level - formatters_.push_back( - details::make_unique>(padding)); - break; - - case ('t'): // thread id - formatters_.push_back(details::make_unique>(padding)); - break; - - case ('v'): // the message text - formatters_.push_back(details::make_unique>(padding)); - break; - - case ('a'): // weekday - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('A'): // short weekday - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('b'): - case ('h'): // month - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('B'): // short month - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('c'): // datetime - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('C'): // year 2 digits - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('Y'): // year 4 digits - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('D'): - case ('x'): // datetime MM/DD/YY - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('m'): // month 1-12 - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('d'): // day of month 1-31 - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('H'): // hours 24 - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('I'): // hours 12 - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('M'): // minutes - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('S'): // seconds - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('e'): // milliseconds - formatters_.push_back(details::make_unique>(padding)); - break; - - case ('f'): // microseconds - formatters_.push_back(details::make_unique>(padding)); - break; - - case ('F'): // nanoseconds - formatters_.push_back(details::make_unique>(padding)); - break; - - case ('E'): // seconds since epoch - formatters_.push_back(details::make_unique>(padding)); - break; - - case ('p'): // am/pm - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('r'): // 12 hour clock 02:55:02 pm - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('R'): // 24-hour HH:MM time - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('T'): - case ('X'): // ISO 8601 time format (HH:MM:SS) - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('z'): // timezone - formatters_.push_back(details::make_unique>(padding)); - need_localtime_ = true; - break; - - case ('P'): // pid - formatters_.push_back(details::make_unique>(padding)); - break; - - case ('^'): // color range start - formatters_.push_back(details::make_unique(padding)); - break; - - case ('$'): // color range end - formatters_.push_back(details::make_unique(padding)); - break; - - case ('@'): // source location (filename:filenumber) - formatters_.push_back( - details::make_unique>(padding)); - break; - - case ('s'): // short source filename - without directory name - formatters_.push_back( - details::make_unique>(padding)); - break; - - case ('g'): // full source filename - formatters_.push_back( - details::make_unique>(padding)); - break; - - case ('#'): // source line number - formatters_.push_back( - details::make_unique>(padding)); - break; - - case ('!'): // source funcname - formatters_.push_back( - details::make_unique>(padding)); - break; - - case ('%'): // % char - formatters_.push_back(details::make_unique('%')); - break; - - case ('u'): // elapsed time since last log message in nanos - formatters_.push_back( - details::make_unique>( - padding)); - break; - - case ('i'): // elapsed time since last log message in micros - formatters_.push_back( - details::make_unique>( - padding)); - break; - - case ('o'): // elapsed time since last log message in millis - formatters_.push_back( - details::make_unique>( - padding)); - break; - - case ('O'): // elapsed time since last log message in seconds - formatters_.push_back( - details::make_unique>( - padding)); - break; - -#ifndef SPDLOG_NO_TLS // mdc formatter requires TLS support - case ('&'): - formatters_.push_back(details::make_unique>(padding)); - break; -#endif - - default: // Unknown flag appears as is - auto unknown_flag = details::make_unique(); - - if (!padding.truncate_) { - unknown_flag->add_ch('%'); - unknown_flag->add_ch(flag); - formatters_.push_back((std::move(unknown_flag))); - } - // fix issue #1617 (prev char was '!' and should have been treated as funcname flag - // instead of truncating flag) spdlog::set_pattern("[%10!] %v") => "[ main] some - // message" spdlog::set_pattern("[%3!!] %v") => "[mai] some message" - else { - padding.truncate_ = false; - formatters_.push_back( - details::make_unique>(padding)); - unknown_flag->add_ch(flag); - formatters_.push_back((std::move(unknown_flag))); - } - - break; - } -} - -// Extract given pad spec (e.g. %8X, %=8X, %-8!X, %8!X, %=8!X, %-8!X, %+8!X) -// Advance the given it pass the end of the padding spec found (if any) -// Return padding. -SPDLOG_INLINE details::padding_info pattern_formatter::handle_padspec_( - std::string::const_iterator &it, std::string::const_iterator end) { - using details::padding_info; - using details::scoped_padder; - const size_t max_width = 64; - if (it == end) { - return padding_info{}; - } - - padding_info::pad_side side; - switch (*it) { - case '-': - side = padding_info::pad_side::right; - ++it; - break; - case '=': - side = padding_info::pad_side::center; - ++it; - break; - default: - side = details::padding_info::pad_side::left; - break; - } - - if (it == end || !std::isdigit(static_cast(*it))) { - return padding_info{}; // no padding if no digit found here - } - - auto width = static_cast(*it) - '0'; - for (++it; it != end && std::isdigit(static_cast(*it)); ++it) { - auto digit = static_cast(*it) - '0'; - width = width * 10 + digit; - } - - // search for the optional truncate marker '!' - bool truncate; - if (it != end && *it == '!') { - truncate = true; - ++it; - } else { - truncate = false; - } - return details::padding_info{std::min(width, max_width), side, truncate}; -} - -SPDLOG_INLINE void pattern_formatter::compile_pattern_(const std::string &pattern) { - auto end = pattern.end(); - std::unique_ptr user_chars; - formatters_.clear(); - for (auto it = pattern.begin(); it != end; ++it) { - if (*it == '%') { - if (user_chars) // append user chars found so far - { - formatters_.push_back(std::move(user_chars)); - } - - auto padding = handle_padspec_(++it, end); - - if (it != end) { - if (padding.enabled()) { - handle_flag_(*it, padding); - } else { - handle_flag_(*it, padding); - } - } else { - break; - } - } else // chars not following the % sign should be displayed as is - { - if (!user_chars) { - user_chars = details::make_unique(); - } - user_chars->add_ch(*it); - } - } - if (user_chars) // append raw chars found so far - { - formatters_.push_back(std::move(user_chars)); - } -} -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/pattern_formatter.h b/thirdparty/spdlog/include/spdlog/pattern_formatter.h deleted file mode 100644 index ececd6732..000000000 --- a/thirdparty/spdlog/include/spdlog/pattern_formatter.h +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include -#include - -#include -#include -#include - -#include -#include -#include - -namespace spdlog { -namespace details { - -// padding information. -struct padding_info { - enum class pad_side { left, right, center }; - - padding_info() = default; - padding_info(size_t width, padding_info::pad_side side, bool truncate) - : width_(width), - side_(side), - truncate_(truncate), - enabled_(true) {} - - bool enabled() const { return enabled_; } - size_t width_ = 0; - pad_side side_ = pad_side::left; - bool truncate_ = false; - bool enabled_ = false; -}; - -class SPDLOG_API flag_formatter { -public: - explicit flag_formatter(padding_info padinfo) - : padinfo_(padinfo) {} - flag_formatter() = default; - virtual ~flag_formatter() = default; - virtual void format(const details::log_msg &msg, - const std::tm &tm_time, - memory_buf_t &dest) = 0; - -protected: - padding_info padinfo_; -}; - -} // namespace details - -class SPDLOG_API custom_flag_formatter : public details::flag_formatter { -public: - virtual std::unique_ptr clone() const = 0; - - void set_padding_info(const details::padding_info &padding) { - flag_formatter::padinfo_ = padding; - } -}; - -class SPDLOG_API pattern_formatter final : public formatter { -public: - using custom_flags = std::unordered_map>; - - explicit pattern_formatter(std::string pattern, - pattern_time_type time_type = pattern_time_type::local, - std::string eol = spdlog::details::os::default_eol, - custom_flags custom_user_flags = custom_flags()); - - // use default pattern is not given - explicit pattern_formatter(pattern_time_type time_type = pattern_time_type::local, - std::string eol = spdlog::details::os::default_eol); - - pattern_formatter(const pattern_formatter &other) = delete; - pattern_formatter &operator=(const pattern_formatter &other) = delete; - - std::unique_ptr clone() const override; - void format(const details::log_msg &msg, memory_buf_t &dest) override; - - template - pattern_formatter &add_flag(char flag, Args &&...args) { - custom_handlers_[flag] = details::make_unique(std::forward(args)...); - return *this; - } - void set_pattern(std::string pattern); - void need_localtime(bool need = true); - -private: - std::string pattern_; - std::string eol_; - pattern_time_type pattern_time_type_; - bool need_localtime_; - std::tm cached_tm_; - std::chrono::seconds last_log_secs_; - std::vector> formatters_; - custom_flags custom_handlers_; - - std::tm get_time_(const details::log_msg &msg); - template - void handle_flag_(char flag, details::padding_info padding); - - // Extract given pad spec (e.g. %8X) - // Advance the given it pass the end of the padding spec found (if any) - // Return padding. - static details::padding_info handle_padspec_(std::string::const_iterator &it, - std::string::const_iterator end); - - void compile_pattern_(const std::string &pattern); -}; -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "pattern_formatter-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/sinks/ansicolor_sink-inl.h b/thirdparty/spdlog/include/spdlog/sinks/ansicolor_sink-inl.h deleted file mode 100644 index 6a23f6c70..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/ansicolor_sink-inl.h +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -namespace spdlog { -namespace sinks { - -template -SPDLOG_INLINE ansicolor_sink::ansicolor_sink(FILE *target_file, color_mode mode) - : target_file_(target_file), - mutex_(ConsoleMutex::mutex()), - formatter_(details::make_unique()) - -{ - set_color_mode_(mode); - colors_.at(level::trace) = to_string_(white); - colors_.at(level::debug) = to_string_(cyan); - colors_.at(level::info) = to_string_(green); - colors_.at(level::warn) = to_string_(yellow_bold); - colors_.at(level::err) = to_string_(red_bold); - colors_.at(level::critical) = to_string_(bold_on_red); - colors_.at(level::off) = to_string_(reset); -} - -template -SPDLOG_INLINE void ansicolor_sink::set_color(level::level_enum color_level, - string_view_t color) { - std::lock_guard lock(mutex_); - colors_.at(static_cast(color_level)) = to_string_(color); -} - -template -SPDLOG_INLINE void ansicolor_sink::log(const details::log_msg &msg) { - // Wrap the originally formatted message in color codes. - // If color is not supported in the terminal, log as is instead. - std::lock_guard lock(mutex_); - msg.color_range_start = 0; - msg.color_range_end = 0; - memory_buf_t formatted; - formatter_->format(msg, formatted); - if (should_do_colors_ && msg.color_range_end > msg.color_range_start) { - // before color range - print_range_(formatted, 0, msg.color_range_start); - // in color range - print_ccode_(colors_.at(static_cast(msg.level))); - print_range_(formatted, msg.color_range_start, msg.color_range_end); - print_ccode_(reset); - // after color range - print_range_(formatted, msg.color_range_end, formatted.size()); - } else // no color - { - print_range_(formatted, 0, formatted.size()); - } - fflush(target_file_); -} - -template -SPDLOG_INLINE void ansicolor_sink::flush() { - std::lock_guard lock(mutex_); - fflush(target_file_); -} - -template -SPDLOG_INLINE void ansicolor_sink::set_pattern(const std::string &pattern) { - std::lock_guard lock(mutex_); - formatter_ = std::unique_ptr(new pattern_formatter(pattern)); -} - -template -SPDLOG_INLINE void ansicolor_sink::set_formatter( - std::unique_ptr sink_formatter) { - std::lock_guard lock(mutex_); - formatter_ = std::move(sink_formatter); -} - -template -SPDLOG_INLINE bool ansicolor_sink::should_color() const { - return should_do_colors_; -} - -template -SPDLOG_INLINE void ansicolor_sink::set_color_mode(color_mode mode) { - std::lock_guard lock(mutex_); - set_color_mode_(mode); -} - -template -SPDLOG_INLINE void ansicolor_sink::set_color_mode_(color_mode mode) { - switch (mode) { - case color_mode::always: - should_do_colors_ = true; - return; - case color_mode::automatic: - should_do_colors_ = - details::os::in_terminal(target_file_) && details::os::is_color_terminal(); - return; - case color_mode::never: - should_do_colors_ = false; - return; - default: - should_do_colors_ = false; - } -} - -template -SPDLOG_INLINE void ansicolor_sink::print_ccode_( - const string_view_t &color_code) const { - details::os::fwrite_bytes(color_code.data(), color_code.size(), target_file_); -} - -template -SPDLOG_INLINE void ansicolor_sink::print_range_(const memory_buf_t &formatted, - size_t start, - size_t end) const { - details::os::fwrite_bytes(formatted.data() + start, end - start, target_file_); -} - -template -SPDLOG_INLINE std::string ansicolor_sink::to_string_(const string_view_t &sv) { - return std::string(sv.data(), sv.size()); -} - -// ansicolor_stdout_sink -template -SPDLOG_INLINE ansicolor_stdout_sink::ansicolor_stdout_sink(color_mode mode) - : ansicolor_sink(stdout, mode) {} - -// ansicolor_stderr_sink -template -SPDLOG_INLINE ansicolor_stderr_sink::ansicolor_stderr_sink(color_mode mode) - : ansicolor_sink(stderr, mode) {} - -} // namespace sinks -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/ansicolor_sink.h b/thirdparty/spdlog/include/spdlog/sinks/ansicolor_sink.h deleted file mode 100644 index 47cea9154..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/ansicolor_sink.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -namespace spdlog { -namespace sinks { - -/** - * This sink prefixes the output with an ANSI escape sequence color code - * depending on the severity - * of the message. - * If no color terminal detected, omit the escape codes. - */ - -template -class ansicolor_sink : public sink { -public: - using mutex_t = typename ConsoleMutex::mutex_t; - ansicolor_sink(FILE *target_file, color_mode mode); - ~ansicolor_sink() override = default; - - ansicolor_sink(const ansicolor_sink &other) = delete; - ansicolor_sink(ansicolor_sink &&other) = delete; - - ansicolor_sink &operator=(const ansicolor_sink &other) = delete; - ansicolor_sink &operator=(ansicolor_sink &&other) = delete; - - void set_color(level::level_enum color_level, string_view_t color); - void set_color_mode(color_mode mode); - bool should_color() const; - - void log(const details::log_msg &msg) override; - void flush() override; - void set_pattern(const std::string &pattern) override; - void set_formatter(std::unique_ptr sink_formatter) override; - - // Formatting codes - const string_view_t reset = "\033[m"; - const string_view_t bold = "\033[1m"; - const string_view_t dark = "\033[2m"; - const string_view_t underline = "\033[4m"; - const string_view_t blink = "\033[5m"; - const string_view_t reverse = "\033[7m"; - const string_view_t concealed = "\033[8m"; - const string_view_t clear_line = "\033[K"; - - // Foreground colors - const string_view_t black = "\033[30m"; - const string_view_t red = "\033[31m"; - const string_view_t green = "\033[32m"; - const string_view_t yellow = "\033[33m"; - const string_view_t blue = "\033[34m"; - const string_view_t magenta = "\033[35m"; - const string_view_t cyan = "\033[36m"; - const string_view_t white = "\033[37m"; - - /// Background colors - const string_view_t on_black = "\033[40m"; - const string_view_t on_red = "\033[41m"; - const string_view_t on_green = "\033[42m"; - const string_view_t on_yellow = "\033[43m"; - const string_view_t on_blue = "\033[44m"; - const string_view_t on_magenta = "\033[45m"; - const string_view_t on_cyan = "\033[46m"; - const string_view_t on_white = "\033[47m"; - - /// Bold colors - const string_view_t yellow_bold = "\033[33m\033[1m"; - const string_view_t red_bold = "\033[31m\033[1m"; - const string_view_t bold_on_red = "\033[1m\033[41m"; - -private: - FILE *target_file_; - mutex_t &mutex_; - bool should_do_colors_; - std::unique_ptr formatter_; - std::array colors_; - void set_color_mode_(color_mode mode); - void print_ccode_(const string_view_t &color_code) const; - void print_range_(const memory_buf_t &formatted, size_t start, size_t end) const; - static std::string to_string_(const string_view_t &sv); -}; - -template -class ansicolor_stdout_sink : public ansicolor_sink { -public: - explicit ansicolor_stdout_sink(color_mode mode = color_mode::automatic); -}; - -template -class ansicolor_stderr_sink : public ansicolor_sink { -public: - explicit ansicolor_stderr_sink(color_mode mode = color_mode::automatic); -}; - -using ansicolor_stdout_sink_mt = ansicolor_stdout_sink; -using ansicolor_stdout_sink_st = ansicolor_stdout_sink; - -using ansicolor_stderr_sink_mt = ansicolor_stderr_sink; -using ansicolor_stderr_sink_st = ansicolor_stderr_sink; - -} // namespace sinks -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "ansicolor_sink-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/sinks/base_sink-inl.h b/thirdparty/spdlog/include/spdlog/sinks/base_sink-inl.h deleted file mode 100644 index ada161bcc..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/base_sink-inl.h +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -#include -#include - -template -SPDLOG_INLINE spdlog::sinks::base_sink::base_sink() - : formatter_{details::make_unique()} {} - -template -SPDLOG_INLINE spdlog::sinks::base_sink::base_sink( - std::unique_ptr formatter) - : formatter_{std::move(formatter)} {} - -template -void SPDLOG_INLINE spdlog::sinks::base_sink::log(const details::log_msg &msg) { - std::lock_guard lock(mutex_); - sink_it_(msg); -} - -template -void SPDLOG_INLINE spdlog::sinks::base_sink::flush() { - std::lock_guard lock(mutex_); - flush_(); -} - -template -void SPDLOG_INLINE spdlog::sinks::base_sink::set_pattern(const std::string &pattern) { - std::lock_guard lock(mutex_); - set_pattern_(pattern); -} - -template -void SPDLOG_INLINE -spdlog::sinks::base_sink::set_formatter(std::unique_ptr sink_formatter) { - std::lock_guard lock(mutex_); - set_formatter_(std::move(sink_formatter)); -} - -template -void SPDLOG_INLINE spdlog::sinks::base_sink::set_pattern_(const std::string &pattern) { - set_formatter_(details::make_unique(pattern)); -} - -template -void SPDLOG_INLINE -spdlog::sinks::base_sink::set_formatter_(std::unique_ptr sink_formatter) { - formatter_ = std::move(sink_formatter); -} diff --git a/thirdparty/spdlog/include/spdlog/sinks/base_sink.h b/thirdparty/spdlog/include/spdlog/sinks/base_sink.h deleted file mode 100644 index 1b4bb0689..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/base_sink.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once -// -// base sink templated over a mutex (either dummy or real) -// concrete implementation should override the sink_it_() and flush_() methods. -// locking is taken care of in this class - no locking needed by the -// implementers.. -// - -#include -#include -#include - -namespace spdlog { -namespace sinks { -template -class SPDLOG_API base_sink : public sink { -public: - base_sink(); - explicit base_sink(std::unique_ptr formatter); - ~base_sink() override = default; - - base_sink(const base_sink &) = delete; - base_sink(base_sink &&) = delete; - - base_sink &operator=(const base_sink &) = delete; - base_sink &operator=(base_sink &&) = delete; - - void log(const details::log_msg &msg) final override; - void flush() final override; - void set_pattern(const std::string &pattern) final override; - void set_formatter(std::unique_ptr sink_formatter) final override; - -protected: - // sink formatter - std::unique_ptr formatter_; - Mutex mutex_; - - virtual void sink_it_(const details::log_msg &msg) = 0; - virtual void flush_() = 0; - virtual void set_pattern_(const std::string &pattern); - virtual void set_formatter_(std::unique_ptr sink_formatter); -}; -} // namespace sinks -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "base_sink-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/sinks/basic_file_sink-inl.h b/thirdparty/spdlog/include/spdlog/sinks/basic_file_sink-inl.h deleted file mode 100644 index ce0ddad00..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/basic_file_sink-inl.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -namespace spdlog { -namespace sinks { - -template -SPDLOG_INLINE basic_file_sink::basic_file_sink(const filename_t &filename, - bool truncate, - const file_event_handlers &event_handlers) - : file_helper_{event_handlers} { - file_helper_.open(filename, truncate); -} - -template -SPDLOG_INLINE const filename_t &basic_file_sink::filename() const { - return file_helper_.filename(); -} - -template -SPDLOG_INLINE void basic_file_sink::truncate() { - std::lock_guard lock(base_sink::mutex_); - file_helper_.reopen(true); -} - -template -SPDLOG_INLINE void basic_file_sink::sink_it_(const details::log_msg &msg) { - memory_buf_t formatted; - base_sink::formatter_->format(msg, formatted); - file_helper_.write(formatted); -} - -template -SPDLOG_INLINE void basic_file_sink::flush_() { - file_helper_.flush(); -} - -} // namespace sinks -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/basic_file_sink.h b/thirdparty/spdlog/include/spdlog/sinks/basic_file_sink.h deleted file mode 100644 index 48c07671a..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/basic_file_sink.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include -#include - -#include -#include - -namespace spdlog { -namespace sinks { -/* - * Trivial file sink with single file as target - */ -template -class basic_file_sink final : public base_sink { -public: - explicit basic_file_sink(const filename_t &filename, - bool truncate = false, - const file_event_handlers &event_handlers = {}); - const filename_t &filename() const; - void truncate(); - -protected: - void sink_it_(const details::log_msg &msg) override; - void flush_() override; - -private: - details::file_helper file_helper_; -}; - -using basic_file_sink_mt = basic_file_sink; -using basic_file_sink_st = basic_file_sink; - -} // namespace sinks - -// -// factory functions -// -template -inline std::shared_ptr basic_logger_mt(const std::string &logger_name, - const filename_t &filename, - bool truncate = false, - const file_event_handlers &event_handlers = {}) { - return Factory::template create(logger_name, filename, truncate, - event_handlers); -} - -template -inline std::shared_ptr basic_logger_st(const std::string &logger_name, - const filename_t &filename, - bool truncate = false, - const file_event_handlers &event_handlers = {}) { - return Factory::template create(logger_name, filename, truncate, - event_handlers); -} - -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "basic_file_sink-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/sinks/callback_sink.h b/thirdparty/spdlog/include/spdlog/sinks/callback_sink.h deleted file mode 100644 index 8f0c8d411..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/callback_sink.h +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include - -#include -#include - -namespace spdlog { - -// callbacks type -typedef std::function custom_log_callback; - -namespace sinks { -/* - * Trivial callback sink, gets a callback function and calls it on each log - */ -template -class callback_sink final : public base_sink { -public: - explicit callback_sink(const custom_log_callback &callback) - : callback_{callback} {} - -protected: - void sink_it_(const details::log_msg &msg) override { callback_(msg); } - void flush_() override {} - -private: - custom_log_callback callback_; -}; - -using callback_sink_mt = callback_sink; -using callback_sink_st = callback_sink; - -} // namespace sinks - -// -// factory functions -// -template -inline std::shared_ptr callback_logger_mt(const std::string &logger_name, - const custom_log_callback &callback) { - return Factory::template create(logger_name, callback); -} - -template -inline std::shared_ptr callback_logger_st(const std::string &logger_name, - const custom_log_callback &callback) { - return Factory::template create(logger_name, callback); -} - -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/daily_file_sink.h b/thirdparty/spdlog/include/spdlog/sinks/daily_file_sink.h deleted file mode 100644 index 615c9f7be..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/daily_file_sink.h +++ /dev/null @@ -1,254 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace spdlog { -namespace sinks { - -/* - * Generator of daily log file names in format basename.YYYY-MM-DD.ext - */ -struct daily_filename_calculator { - // Create filename for the form basename.YYYY-MM-DD - static filename_t calc_filename(const filename_t &filename, const tm &now_tm) { - filename_t basename, ext; - std::tie(basename, ext) = details::file_helper::split_by_extension(filename); - return fmt_lib::format(SPDLOG_FMT_STRING(SPDLOG_FILENAME_T("{}_{:04d}-{:02d}-{:02d}{}")), - basename, now_tm.tm_year + 1900, now_tm.tm_mon + 1, now_tm.tm_mday, - ext); - } -}; - -/* - * Generator of daily log file names with strftime format. - * Usages: - * auto sink = - * std::make_shared("myapp-%Y-%m-%d:%H:%M:%S.log", hour, - * minute);" auto logger = spdlog::daily_logger_format_mt("loggername, "myapp-%Y-%m-%d:%X.log", - * hour, minute)" - * - */ -struct daily_filename_format_calculator { - static filename_t calc_filename(const filename_t &file_path, const tm &now_tm) { -#if defined(_WIN32) && defined(SPDLOG_WCHAR_FILENAMES) - std::wstringstream stream; -#else - std::stringstream stream; -#endif - stream << std::put_time(&now_tm, file_path.c_str()); - return stream.str(); - } -}; - -/* - * Rotating file sink based on date. - * If truncate != false , the created file will be truncated. - * If max_files > 0, retain only the last max_files and delete previous. - * Note that old log files from previous executions will not be deleted by this class, - * rotation and deletion is only applied while the program is running. - */ -template -class daily_file_sink final : public base_sink { -public: - // create daily file sink which rotates on given time - daily_file_sink(filename_t base_filename, - int rotation_hour, - int rotation_minute, - bool truncate = false, - uint16_t max_files = 0, - const file_event_handlers &event_handlers = {}) - : base_filename_(std::move(base_filename)), - rotation_h_(rotation_hour), - rotation_m_(rotation_minute), - file_helper_{event_handlers}, - truncate_(truncate), - max_files_(max_files), - filenames_q_() { - if (rotation_hour < 0 || rotation_hour > 23 || rotation_minute < 0 || - rotation_minute > 59) { - throw_spdlog_ex("daily_file_sink: Invalid rotation time in ctor"); - } - - auto now = log_clock::now(); - auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(now)); - file_helper_.open(filename, truncate_); - rotation_tp_ = next_rotation_tp_(); - - if (max_files_ > 0) { - init_filenames_q_(); - } - } - - filename_t filename() { - std::lock_guard lock(base_sink::mutex_); - return file_helper_.filename(); - } - -protected: - void sink_it_(const details::log_msg &msg) override { - auto time = msg.time; - bool should_rotate = time >= rotation_tp_; - if (should_rotate) { - auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(time)); - file_helper_.open(filename, truncate_); - rotation_tp_ = next_rotation_tp_(); - } - memory_buf_t formatted; - base_sink::formatter_->format(msg, formatted); - file_helper_.write(formatted); - - // Do the cleaning only at the end because it might throw on failure. - if (should_rotate && max_files_ > 0) { - delete_old_(); - } - } - - void flush_() override { file_helper_.flush(); } - -private: - void init_filenames_q_() { - using details::os::path_exists; - - filenames_q_ = details::circular_q(static_cast(max_files_)); - std::vector filenames; - auto now = log_clock::now(); - while (filenames.size() < max_files_) { - auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(now)); - if (!path_exists(filename)) { - break; - } - filenames.emplace_back(filename); - now -= std::chrono::hours(24); - } - for (auto iter = filenames.rbegin(); iter != filenames.rend(); ++iter) { - filenames_q_.push_back(std::move(*iter)); - } - } - - tm now_tm(log_clock::time_point tp) { - time_t tnow = log_clock::to_time_t(tp); - return spdlog::details::os::localtime(tnow); - } - - log_clock::time_point next_rotation_tp_() { - auto now = log_clock::now(); - tm date = now_tm(now); - date.tm_hour = rotation_h_; - date.tm_min = rotation_m_; - date.tm_sec = 0; - auto rotation_time = log_clock::from_time_t(std::mktime(&date)); - if (rotation_time > now) { - return rotation_time; - } - return {rotation_time + std::chrono::hours(24)}; - } - - // Delete the file N rotations ago. - // Throw spdlog_ex on failure to delete the old file. - void delete_old_() { - using details::os::filename_to_str; - using details::os::remove_if_exists; - - filename_t current_file = file_helper_.filename(); - if (filenames_q_.full()) { - auto old_filename = std::move(filenames_q_.front()); - filenames_q_.pop_front(); - bool ok = remove_if_exists(old_filename) == 0; - if (!ok) { - filenames_q_.push_back(std::move(current_file)); - throw_spdlog_ex("Failed removing daily file " + filename_to_str(old_filename), - errno); - } - } - filenames_q_.push_back(std::move(current_file)); - } - - filename_t base_filename_; - int rotation_h_; - int rotation_m_; - log_clock::time_point rotation_tp_; - details::file_helper file_helper_; - bool truncate_; - uint16_t max_files_; - details::circular_q filenames_q_; -}; - -using daily_file_sink_mt = daily_file_sink; -using daily_file_sink_st = daily_file_sink; -using daily_file_format_sink_mt = daily_file_sink; -using daily_file_format_sink_st = - daily_file_sink; - -} // namespace sinks - -// -// factory functions -// -template -inline std::shared_ptr daily_logger_mt(const std::string &logger_name, - const filename_t &filename, - int hour = 0, - int minute = 0, - bool truncate = false, - uint16_t max_files = 0, - const file_event_handlers &event_handlers = {}) { - return Factory::template create(logger_name, filename, hour, minute, - truncate, max_files, event_handlers); -} - -template -inline std::shared_ptr daily_logger_format_mt( - const std::string &logger_name, - const filename_t &filename, - int hour = 0, - int minute = 0, - bool truncate = false, - uint16_t max_files = 0, - const file_event_handlers &event_handlers = {}) { - return Factory::template create( - logger_name, filename, hour, minute, truncate, max_files, event_handlers); -} - -template -inline std::shared_ptr daily_logger_st(const std::string &logger_name, - const filename_t &filename, - int hour = 0, - int minute = 0, - bool truncate = false, - uint16_t max_files = 0, - const file_event_handlers &event_handlers = {}) { - return Factory::template create(logger_name, filename, hour, minute, - truncate, max_files, event_handlers); -} - -template -inline std::shared_ptr daily_logger_format_st( - const std::string &logger_name, - const filename_t &filename, - int hour = 0, - int minute = 0, - bool truncate = false, - uint16_t max_files = 0, - const file_event_handlers &event_handlers = {}) { - return Factory::template create( - logger_name, filename, hour, minute, truncate, max_files, event_handlers); -} -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/dist_sink.h b/thirdparty/spdlog/include/spdlog/sinks/dist_sink.h deleted file mode 100644 index 69c4971c9..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/dist_sink.h +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include "base_sink.h" -#include -#include -#include - -#include -#include -#include -#include - -// Distribution sink (mux). Stores a vector of sinks which get called when log -// is called - -namespace spdlog { -namespace sinks { - -template -class dist_sink : public base_sink { -public: - dist_sink() = default; - explicit dist_sink(std::vector> sinks) - : sinks_(sinks) {} - - dist_sink(const dist_sink &) = delete; - dist_sink &operator=(const dist_sink &) = delete; - - void add_sink(std::shared_ptr sub_sink) { - std::lock_guard lock(base_sink::mutex_); - sinks_.push_back(sub_sink); - } - - void remove_sink(std::shared_ptr sub_sink) { - std::lock_guard lock(base_sink::mutex_); - sinks_.erase(std::remove(sinks_.begin(), sinks_.end(), sub_sink), sinks_.end()); - } - - void set_sinks(std::vector> sinks) { - std::lock_guard lock(base_sink::mutex_); - sinks_ = std::move(sinks); - } - - std::vector> &sinks() { return sinks_; } - -protected: - void sink_it_(const details::log_msg &msg) override { - for (auto &sub_sink : sinks_) { - if (sub_sink->should_log(msg.level)) { - sub_sink->log(msg); - } - } - } - - void flush_() override { - for (auto &sub_sink : sinks_) { - sub_sink->flush(); - } - } - - void set_pattern_(const std::string &pattern) override { - set_formatter_(details::make_unique(pattern)); - } - - void set_formatter_(std::unique_ptr sink_formatter) override { - base_sink::formatter_ = std::move(sink_formatter); - for (auto &sub_sink : sinks_) { - sub_sink->set_formatter(base_sink::formatter_->clone()); - } - } - std::vector> sinks_; -}; - -using dist_sink_mt = dist_sink; -using dist_sink_st = dist_sink; - -} // namespace sinks -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/hourly_file_sink.h b/thirdparty/spdlog/include/spdlog/sinks/hourly_file_sink.h deleted file mode 100644 index 3e618725b..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/hourly_file_sink.h +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace spdlog { -namespace sinks { - -/* - * Generator of Hourly log file names in format basename.YYYY-MM-DD-HH.ext - */ -struct hourly_filename_calculator { - // Create filename for the form basename.YYYY-MM-DD-H - static filename_t calc_filename(const filename_t &filename, const tm &now_tm) { - filename_t basename, ext; - std::tie(basename, ext) = details::file_helper::split_by_extension(filename); - return fmt_lib::format(SPDLOG_FILENAME_T("{}_{:04d}-{:02d}-{:02d}_{:02d}{}"), basename, - now_tm.tm_year + 1900, now_tm.tm_mon + 1, now_tm.tm_mday, - now_tm.tm_hour, ext); - } -}; - -/* - * Rotating file sink based on time. - * If truncate != false , the created file will be truncated. - * If max_files > 0, retain only the last max_files and delete previous. - * Note that old log files from previous executions will not be deleted by this class, - * rotation and deletion is only applied while the program is running. - */ -template -class hourly_file_sink final : public base_sink { -public: - // create hourly file sink which rotates on given time - hourly_file_sink(filename_t base_filename, - bool truncate = false, - uint16_t max_files = 0, - const file_event_handlers &event_handlers = {}) - : base_filename_(std::move(base_filename)), - file_helper_{event_handlers}, - truncate_(truncate), - max_files_(max_files), - filenames_q_() { - auto now = log_clock::now(); - auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(now)); - file_helper_.open(filename, truncate_); - remove_init_file_ = file_helper_.size() == 0; - rotation_tp_ = next_rotation_tp_(); - - if (max_files_ > 0) { - init_filenames_q_(); - } - } - - filename_t filename() { - std::lock_guard lock(base_sink::mutex_); - return file_helper_.filename(); - } - -protected: - void sink_it_(const details::log_msg &msg) override { - auto time = msg.time; - bool should_rotate = time >= rotation_tp_; - if (should_rotate) { - if (remove_init_file_) { - file_helper_.close(); - details::os::remove(file_helper_.filename()); - } - auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(time)); - file_helper_.open(filename, truncate_); - rotation_tp_ = next_rotation_tp_(); - } - remove_init_file_ = false; - memory_buf_t formatted; - base_sink::formatter_->format(msg, formatted); - file_helper_.write(formatted); - - // Do the cleaning only at the end because it might throw on failure. - if (should_rotate && max_files_ > 0) { - delete_old_(); - } - } - - void flush_() override { file_helper_.flush(); } - -private: - void init_filenames_q_() { - using details::os::path_exists; - - filenames_q_ = details::circular_q(static_cast(max_files_)); - std::vector filenames; - auto now = log_clock::now(); - while (filenames.size() < max_files_) { - auto filename = FileNameCalc::calc_filename(base_filename_, now_tm(now)); - if (!path_exists(filename)) { - break; - } - filenames.emplace_back(filename); - now -= std::chrono::hours(1); - } - for (auto iter = filenames.rbegin(); iter != filenames.rend(); ++iter) { - filenames_q_.push_back(std::move(*iter)); - } - } - - tm now_tm(log_clock::time_point tp) { - time_t tnow = log_clock::to_time_t(tp); - return spdlog::details::os::localtime(tnow); - } - - log_clock::time_point next_rotation_tp_() { - auto now = log_clock::now(); - tm date = now_tm(now); - date.tm_min = 0; - date.tm_sec = 0; - auto rotation_time = log_clock::from_time_t(std::mktime(&date)); - if (rotation_time > now) { - return rotation_time; - } - return {rotation_time + std::chrono::hours(1)}; - } - - // Delete the file N rotations ago. - // Throw spdlog_ex on failure to delete the old file. - void delete_old_() { - using details::os::filename_to_str; - using details::os::remove_if_exists; - - filename_t current_file = file_helper_.filename(); - if (filenames_q_.full()) { - auto old_filename = std::move(filenames_q_.front()); - filenames_q_.pop_front(); - bool ok = remove_if_exists(old_filename) == 0; - if (!ok) { - filenames_q_.push_back(std::move(current_file)); - SPDLOG_THROW(spdlog_ex( - "Failed removing hourly file " + filename_to_str(old_filename), errno)); - } - } - filenames_q_.push_back(std::move(current_file)); - } - - filename_t base_filename_; - log_clock::time_point rotation_tp_; - details::file_helper file_helper_; - bool truncate_; - uint16_t max_files_; - details::circular_q filenames_q_; - bool remove_init_file_; -}; - -using hourly_file_sink_mt = hourly_file_sink; -using hourly_file_sink_st = hourly_file_sink; - -} // namespace sinks - -// -// factory functions -// -template -inline std::shared_ptr hourly_logger_mt(const std::string &logger_name, - const filename_t &filename, - bool truncate = false, - uint16_t max_files = 0, - const file_event_handlers &event_handlers = {}) { - return Factory::template create(logger_name, filename, truncate, - max_files, event_handlers); -} - -template -inline std::shared_ptr hourly_logger_st(const std::string &logger_name, - const filename_t &filename, - bool truncate = false, - uint16_t max_files = 0, - const file_event_handlers &event_handlers = {}) { - return Factory::template create(logger_name, filename, truncate, - max_files, event_handlers); -} -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/msvc_sink.h b/thirdparty/spdlog/include/spdlog/sinks/msvc_sink.h deleted file mode 100644 index c28d6ebd7..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/msvc_sink.h +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright(c) 2016 Alexander Dalshov & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#if defined(_WIN32) - - #include - #if defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) - #include - #endif - #include - - #include - #include - - // Avoid including windows.h (https://stackoverflow.com/a/30741042) - #if defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) -extern "C" __declspec(dllimport) void __stdcall OutputDebugStringW(const wchar_t *lpOutputString); - #else -extern "C" __declspec(dllimport) void __stdcall OutputDebugStringA(const char *lpOutputString); - #endif -extern "C" __declspec(dllimport) int __stdcall IsDebuggerPresent(); - -namespace spdlog { -namespace sinks { -/* - * MSVC sink (logging using OutputDebugStringA) - */ -template -class msvc_sink : public base_sink { -public: - msvc_sink() = default; - msvc_sink(bool check_debugger_present) - : check_debugger_present_{check_debugger_present} {} - -protected: - void sink_it_(const details::log_msg &msg) override { - if (check_debugger_present_ && !IsDebuggerPresent()) { - return; - } - memory_buf_t formatted; - base_sink::formatter_->format(msg, formatted); - formatted.push_back('\0'); // add a null terminator for OutputDebugString - #if defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) - wmemory_buf_t wformatted; - details::os::utf8_to_wstrbuf(string_view_t(formatted.data(), formatted.size()), wformatted); - OutputDebugStringW(wformatted.data()); - #else - OutputDebugStringA(formatted.data()); - #endif - } - - void flush_() override {} - - bool check_debugger_present_ = true; -}; - -using msvc_sink_mt = msvc_sink; -using msvc_sink_st = msvc_sink; - -using windebug_sink_mt = msvc_sink_mt; -using windebug_sink_st = msvc_sink_st; - -} // namespace sinks -} // namespace spdlog - -#endif diff --git a/thirdparty/spdlog/include/spdlog/sinks/null_sink.h b/thirdparty/spdlog/include/spdlog/sinks/null_sink.h deleted file mode 100644 index 74530b5b1..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/null_sink.h +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include - -#include - -namespace spdlog { -namespace sinks { - -template -class null_sink final : public base_sink { -protected: - void sink_it_(const details::log_msg &) override {} - void flush_() override {} -}; - -using null_sink_mt = null_sink; -using null_sink_st = null_sink; - -} // namespace sinks - -template -inline std::shared_ptr null_logger_mt(const std::string &logger_name) { - auto null_logger = Factory::template create(logger_name); - null_logger->set_level(level::off); - return null_logger; -} - -template -inline std::shared_ptr null_logger_st(const std::string &logger_name) { - auto null_logger = Factory::template create(logger_name); - null_logger->set_level(level::off); - return null_logger; -} - -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/sink-inl.h b/thirdparty/spdlog/include/spdlog/sinks/sink-inl.h deleted file mode 100644 index e4b271404..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/sink-inl.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include - -SPDLOG_INLINE bool spdlog::sinks::sink::should_log(spdlog::level::level_enum msg_level) const { - return msg_level >= level_.load(std::memory_order_relaxed); -} - -SPDLOG_INLINE void spdlog::sinks::sink::set_level(level::level_enum log_level) { - level_.store(log_level, std::memory_order_relaxed); -} - -SPDLOG_INLINE spdlog::level::level_enum spdlog::sinks::sink::level() const { - return static_cast(level_.load(std::memory_order_relaxed)); -} diff --git a/thirdparty/spdlog/include/spdlog/sinks/sink.h b/thirdparty/spdlog/include/spdlog/sinks/sink.h deleted file mode 100644 index 585068536..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/sink.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include - -namespace spdlog { - -namespace sinks { -class SPDLOG_API sink { -public: - virtual ~sink() = default; - virtual void log(const details::log_msg &msg) = 0; - virtual void flush() = 0; - virtual void set_pattern(const std::string &pattern) = 0; - virtual void set_formatter(std::unique_ptr sink_formatter) = 0; - - void set_level(level::level_enum log_level); - level::level_enum level() const; - bool should_log(level::level_enum msg_level) const; - -protected: - // sink log level - default is all - level_t level_{level::trace}; -}; - -} // namespace sinks -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "sink-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/sinks/stdout_color_sinks-inl.h b/thirdparty/spdlog/include/spdlog/sinks/stdout_color_sinks-inl.h deleted file mode 100644 index 166e38614..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/stdout_color_sinks-inl.h +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -namespace spdlog { - -template -SPDLOG_INLINE std::shared_ptr stdout_color_mt(const std::string &logger_name, - color_mode mode) { - return Factory::template create(logger_name, mode); -} - -template -SPDLOG_INLINE std::shared_ptr stdout_color_st(const std::string &logger_name, - color_mode mode) { - return Factory::template create(logger_name, mode); -} - -template -SPDLOG_INLINE std::shared_ptr stderr_color_mt(const std::string &logger_name, - color_mode mode) { - return Factory::template create(logger_name, mode); -} - -template -SPDLOG_INLINE std::shared_ptr stderr_color_st(const std::string &logger_name, - color_mode mode) { - return Factory::template create(logger_name, mode); -} -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/stdout_color_sinks.h b/thirdparty/spdlog/include/spdlog/sinks/stdout_color_sinks.h deleted file mode 100644 index 72991fe0e..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/stdout_color_sinks.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifdef _WIN32 - #include -#else - #include -#endif - -#include - -namespace spdlog { -namespace sinks { -#ifdef _WIN32 -using stdout_color_sink_mt = wincolor_stdout_sink_mt; -using stdout_color_sink_st = wincolor_stdout_sink_st; -using stderr_color_sink_mt = wincolor_stderr_sink_mt; -using stderr_color_sink_st = wincolor_stderr_sink_st; -#else -using stdout_color_sink_mt = ansicolor_stdout_sink_mt; -using stdout_color_sink_st = ansicolor_stdout_sink_st; -using stderr_color_sink_mt = ansicolor_stderr_sink_mt; -using stderr_color_sink_st = ansicolor_stderr_sink_st; -#endif -} // namespace sinks - -template -std::shared_ptr stdout_color_mt(const std::string &logger_name, - color_mode mode = color_mode::automatic); - -template -std::shared_ptr stdout_color_st(const std::string &logger_name, - color_mode mode = color_mode::automatic); - -template -std::shared_ptr stderr_color_mt(const std::string &logger_name, - color_mode mode = color_mode::automatic); - -template -std::shared_ptr stderr_color_st(const std::string &logger_name, - color_mode mode = color_mode::automatic); - -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "stdout_color_sinks-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/sinks/stdout_sinks-inl.h b/thirdparty/spdlog/include/spdlog/sinks/stdout_sinks-inl.h deleted file mode 100644 index dcb21d84a..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/stdout_sinks-inl.h +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include -#include -#include - -#ifdef _WIN32 - // under windows using fwrite to non-binary stream results in \r\r\n (see issue #1675) - // so instead we use ::FileWrite - #include - - #ifndef _USING_V110_SDK71_ // fileapi.h doesn't exist in winxp - #include // WriteFile (..) - #endif - - #include // _get_osfhandle(..) - #include // _fileno(..) -#endif // _WIN32 - -namespace spdlog { - -namespace sinks { - -template -SPDLOG_INLINE stdout_sink_base::stdout_sink_base(FILE *file) - : mutex_(ConsoleMutex::mutex()), - file_(file), - formatter_(details::make_unique()) { -#ifdef _WIN32 - // get windows handle from the FILE* object - - handle_ = reinterpret_cast(::_get_osfhandle(::_fileno(file_))); - - // don't throw to support cases where no console is attached, - // and let the log method to do nothing if (handle_ == INVALID_HANDLE_VALUE). - // throw only if non stdout/stderr target is requested (probably regular file and not console). - if (handle_ == INVALID_HANDLE_VALUE && file != stdout && file != stderr) { - throw_spdlog_ex("spdlog::stdout_sink_base: _get_osfhandle() failed", errno); - } -#endif // _WIN32 -} - -template -SPDLOG_INLINE void stdout_sink_base::log(const details::log_msg &msg) { -#ifdef _WIN32 - if (handle_ == INVALID_HANDLE_VALUE) { - return; - } - std::lock_guard lock(mutex_); - memory_buf_t formatted; - formatter_->format(msg, formatted); - auto size = static_cast(formatted.size()); - DWORD bytes_written = 0; - bool ok = ::WriteFile(handle_, formatted.data(), size, &bytes_written, nullptr) != 0; - if (!ok) { - throw_spdlog_ex("stdout_sink_base: WriteFile() failed. GetLastError(): " + - std::to_string(::GetLastError())); - } -#else - std::lock_guard lock(mutex_); - memory_buf_t formatted; - formatter_->format(msg, formatted); - details::os::fwrite_bytes(formatted.data(), formatted.size(), file_); -#endif // _WIN32 - ::fflush(file_); // flush every line to terminal -} - -template -SPDLOG_INLINE void stdout_sink_base::flush() { - std::lock_guard lock(mutex_); - fflush(file_); -} - -template -SPDLOG_INLINE void stdout_sink_base::set_pattern(const std::string &pattern) { - std::lock_guard lock(mutex_); - formatter_ = std::unique_ptr(new pattern_formatter(pattern)); -} - -template -SPDLOG_INLINE void stdout_sink_base::set_formatter( - std::unique_ptr sink_formatter) { - std::lock_guard lock(mutex_); - formatter_ = std::move(sink_formatter); -} - -// stdout sink -template -SPDLOG_INLINE stdout_sink::stdout_sink() - : stdout_sink_base(stdout) {} - -// stderr sink -template -SPDLOG_INLINE stderr_sink::stderr_sink() - : stdout_sink_base(stderr) {} - -} // namespace sinks - -// factory methods -template -SPDLOG_INLINE std::shared_ptr stdout_logger_mt(const std::string &logger_name) { - return Factory::template create(logger_name); -} - -template -SPDLOG_INLINE std::shared_ptr stdout_logger_st(const std::string &logger_name) { - return Factory::template create(logger_name); -} - -template -SPDLOG_INLINE std::shared_ptr stderr_logger_mt(const std::string &logger_name) { - return Factory::template create(logger_name); -} - -template -SPDLOG_INLINE std::shared_ptr stderr_logger_st(const std::string &logger_name) { - return Factory::template create(logger_name); -} -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/stdout_sinks.h b/thirdparty/spdlog/include/spdlog/sinks/stdout_sinks.h deleted file mode 100644 index 6ef09968a..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/stdout_sinks.h +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include -#include - -#ifdef _WIN32 - #include -#endif - -namespace spdlog { - -namespace sinks { - -template -class stdout_sink_base : public sink { -public: - using mutex_t = typename ConsoleMutex::mutex_t; - explicit stdout_sink_base(FILE *file); - ~stdout_sink_base() override = default; - - stdout_sink_base(const stdout_sink_base &other) = delete; - stdout_sink_base(stdout_sink_base &&other) = delete; - - stdout_sink_base &operator=(const stdout_sink_base &other) = delete; - stdout_sink_base &operator=(stdout_sink_base &&other) = delete; - - void log(const details::log_msg &msg) override; - void flush() override; - void set_pattern(const std::string &pattern) override; - - void set_formatter(std::unique_ptr sink_formatter) override; - -protected: - mutex_t &mutex_; - FILE *file_; - std::unique_ptr formatter_; -#ifdef _WIN32 - HANDLE handle_; -#endif // WIN32 -}; - -template -class stdout_sink : public stdout_sink_base { -public: - stdout_sink(); -}; - -template -class stderr_sink : public stdout_sink_base { -public: - stderr_sink(); -}; - -using stdout_sink_mt = stdout_sink; -using stdout_sink_st = stdout_sink; - -using stderr_sink_mt = stderr_sink; -using stderr_sink_st = stderr_sink; - -} // namespace sinks - -// factory methods -template -std::shared_ptr stdout_logger_mt(const std::string &logger_name); - -template -std::shared_ptr stdout_logger_st(const std::string &logger_name); - -template -std::shared_ptr stderr_logger_mt(const std::string &logger_name); - -template -std::shared_ptr stderr_logger_st(const std::string &logger_name); - -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "stdout_sinks-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/sinks/syslog_sink.h b/thirdparty/spdlog/include/spdlog/sinks/syslog_sink.h deleted file mode 100644 index 913d41be6..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/syslog_sink.h +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include - -#include -#include -#include - -namespace spdlog { -namespace sinks { -/** - * Sink that write to syslog using the `syscall()` library call. - */ -template -class syslog_sink : public base_sink { -public: - syslog_sink(std::string ident, int syslog_option, int syslog_facility, bool enable_formatting) - : enable_formatting_{enable_formatting}, - syslog_levels_{{/* spdlog::level::trace */ LOG_DEBUG, - /* spdlog::level::debug */ LOG_DEBUG, - /* spdlog::level::info */ LOG_INFO, - /* spdlog::level::warn */ LOG_WARNING, - /* spdlog::level::err */ LOG_ERR, - /* spdlog::level::critical */ LOG_CRIT, - /* spdlog::level::off */ LOG_INFO}}, - ident_{std::move(ident)} { - // set ident to be program name if empty - ::openlog(ident_.empty() ? nullptr : ident_.c_str(), syslog_option, syslog_facility); - } - - ~syslog_sink() override { ::closelog(); } - - syslog_sink(const syslog_sink &) = delete; - syslog_sink &operator=(const syslog_sink &) = delete; - -protected: - void sink_it_(const details::log_msg &msg) override { - string_view_t payload; - memory_buf_t formatted; - if (enable_formatting_) { - base_sink::formatter_->format(msg, formatted); - payload = string_view_t(formatted.data(), formatted.size()); - } else { - payload = msg.payload; - } - - size_t length = payload.size(); - // limit to max int - if (length > static_cast(std::numeric_limits::max())) { - length = static_cast(std::numeric_limits::max()); - } - - ::syslog(syslog_prio_from_level(msg), "%.*s", static_cast(length), payload.data()); - } - - void flush_() override {} - bool enable_formatting_ = false; - - // - // Simply maps spdlog's log level to syslog priority level. - // - virtual int syslog_prio_from_level(const details::log_msg &msg) const { - return syslog_levels_.at(static_cast(msg.level)); - } - - using levels_array = std::array; - levels_array syslog_levels_; - -private: - // must store the ident because the man says openlog might use the pointer as - // is and not a string copy - const std::string ident_; -}; - -using syslog_sink_mt = syslog_sink; -using syslog_sink_st = syslog_sink; -} // namespace sinks - -// Create and register a syslog logger -template -inline std::shared_ptr syslog_logger_mt(const std::string &logger_name, - const std::string &syslog_ident = "", - int syslog_option = 0, - int syslog_facility = LOG_USER, - bool enable_formatting = false) { - return Factory::template create(logger_name, syslog_ident, syslog_option, - syslog_facility, enable_formatting); -} - -template -inline std::shared_ptr syslog_logger_st(const std::string &logger_name, - const std::string &syslog_ident = "", - int syslog_option = 0, - int syslog_facility = LOG_USER, - bool enable_formatting = false) { - return Factory::template create(logger_name, syslog_ident, syslog_option, - syslog_facility, enable_formatting); -} -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/systemd_sink.h b/thirdparty/spdlog/include/spdlog/sinks/systemd_sink.h deleted file mode 100644 index d2cd55f2e..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/systemd_sink.h +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright(c) 2019 ZVYAGIN.Alexander@gmail.com -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include -#include - -#include -#ifndef SD_JOURNAL_SUPPRESS_LOCATION - #define SD_JOURNAL_SUPPRESS_LOCATION -#endif -#include - -namespace spdlog { -namespace sinks { - -/** - * Sink that write to systemd journal using the `sd_journal_send()` library call. - */ -template -class systemd_sink : public base_sink { -public: - systemd_sink(std::string ident = "", bool enable_formatting = false) - : ident_{std::move(ident)}, - enable_formatting_{enable_formatting}, - syslog_levels_{{/* spdlog::level::trace */ LOG_DEBUG, - /* spdlog::level::debug */ LOG_DEBUG, - /* spdlog::level::info */ LOG_INFO, - /* spdlog::level::warn */ LOG_WARNING, - /* spdlog::level::err */ LOG_ERR, - /* spdlog::level::critical */ LOG_CRIT, - /* spdlog::level::off */ LOG_INFO}} {} - - ~systemd_sink() override {} - - systemd_sink(const systemd_sink &) = delete; - systemd_sink &operator=(const systemd_sink &) = delete; - -protected: - const std::string ident_; - bool enable_formatting_ = false; - using levels_array = std::array; - levels_array syslog_levels_; - - void sink_it_(const details::log_msg &msg) override { - int err; - string_view_t payload; - memory_buf_t formatted; - if (enable_formatting_) { - base_sink::formatter_->format(msg, formatted); - payload = string_view_t(formatted.data(), formatted.size()); - } else { - payload = msg.payload; - } - - size_t length = payload.size(); - // limit to max int - if (length > static_cast(std::numeric_limits::max())) { - length = static_cast(std::numeric_limits::max()); - } - - const string_view_t syslog_identifier = ident_.empty() ? msg.logger_name : ident_; - - // Do not send source location if not available - if (msg.source.empty()) { - // Note: function call inside '()' to avoid macro expansion - err = (sd_journal_send)("MESSAGE=%.*s", static_cast(length), payload.data(), - "PRIORITY=%d", syslog_level(msg.level), -#ifndef SPDLOG_NO_THREAD_ID - "TID=%zu", msg.thread_id, -#endif - "SYSLOG_IDENTIFIER=%.*s", - static_cast(syslog_identifier.size()), - syslog_identifier.data(), nullptr); - } else { - err = (sd_journal_send)("MESSAGE=%.*s", static_cast(length), payload.data(), - "PRIORITY=%d", syslog_level(msg.level), -#ifndef SPDLOG_NO_THREAD_ID - "TID=%zu", msg.thread_id, -#endif - "SYSLOG_IDENTIFIER=%.*s", - static_cast(syslog_identifier.size()), - syslog_identifier.data(), "CODE_FILE=%s", msg.source.filename, - "CODE_LINE=%d", msg.source.line, "CODE_FUNC=%s", - msg.source.funcname, nullptr); - } - - if (err) { - throw_spdlog_ex("Failed writing to systemd", errno); - } - } - - int syslog_level(level::level_enum l) { - return syslog_levels_.at(static_cast(l)); - } - - void flush_() override {} -}; - -using systemd_sink_mt = systemd_sink; -using systemd_sink_st = systemd_sink; -} // namespace sinks - -// Create and register a syslog logger -template -inline std::shared_ptr systemd_logger_mt(const std::string &logger_name, - const std::string &ident = "", - bool enable_formatting = false) { - return Factory::template create(logger_name, ident, enable_formatting); -} - -template -inline std::shared_ptr systemd_logger_st(const std::string &logger_name, - const std::string &ident = "", - bool enable_formatting = false) { - return Factory::template create(logger_name, ident, enable_formatting); -} -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/win_eventlog_sink.h b/thirdparty/spdlog/include/spdlog/sinks/win_eventlog_sink.h deleted file mode 100644 index 2c9b582d3..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/win_eventlog_sink.h +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -// Writing to Windows Event Log requires the registry entries below to be present, with the -// following modifications: -// 1. should be replaced with your log name (e.g. your application name) -// 2. should be replaced with the specific source name and the key should be -// duplicated for -// each source used in the application -// -// Since typically modifications of this kind require elevation, it's better to do it as a part of -// setup procedure. The snippet below uses mscoree.dll as the message file as it exists on most of -// the Windows systems anyway and happens to contain the needed resource. -// -// You can also specify a custom message file if needed. -// Please refer to Event Log functions descriptions in MSDN for more details on custom message -// files. - -/*--------------------------------------------------------------------------------------- - -Windows Registry Editor Version 5.00 - -[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\EventLog\] - -[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\EventLog\\] -"TypesSupported"=dword:00000007 -"EventMessageFile"=hex(2):25,00,73,00,79,00,73,00,74,00,65,00,6d,00,72,00,6f,\ - 00,6f,00,74,00,25,00,5c,00,53,00,79,00,73,00,74,00,65,00,6d,00,33,00,32,00,\ - 5c,00,6d,00,73,00,63,00,6f,00,72,00,65,00,65,00,2e,00,64,00,6c,00,6c,00,00,\ - 00 - ------------------------------------------------------------------------------------------*/ - -#pragma once - -#include -#include - -#include -#include - -#include -#include -#include - -namespace spdlog { -namespace sinks { - -namespace win_eventlog { - -namespace internal { - -struct local_alloc_t { - HLOCAL hlocal_; - - SPDLOG_CONSTEXPR local_alloc_t() SPDLOG_NOEXCEPT : hlocal_(nullptr) {} - - local_alloc_t(local_alloc_t const &) = delete; - local_alloc_t &operator=(local_alloc_t const &) = delete; - - ~local_alloc_t() SPDLOG_NOEXCEPT { - if (hlocal_) { - LocalFree(hlocal_); - } - } -}; - -/** Windows error */ -struct win32_error : public spdlog_ex { - /** Formats an error report line: "user-message: error-code (system message)" */ - static std::string format(std::string const &user_message, DWORD error_code = GetLastError()) { - std::string system_message; - - local_alloc_t format_message_result{}; - auto format_message_succeeded = - ::FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - nullptr, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPSTR)&format_message_result.hlocal_, 0, nullptr); - - if (format_message_succeeded && format_message_result.hlocal_) { - system_message = fmt_lib::format(" ({})", (LPSTR)format_message_result.hlocal_); - } - - return fmt_lib::format("{}: {}{}", user_message, error_code, system_message); - } - - explicit win32_error(std::string const &func_name, DWORD error = GetLastError()) - : spdlog_ex(format(func_name, error)) {} -}; - -/** Wrapper for security identifiers (SID) on Windows */ -struct sid_t { - std::vector buffer_; - -public: - sid_t() {} - - /** creates a wrapped SID copy */ - static sid_t duplicate_sid(PSID psid) { - if (!::IsValidSid(psid)) { - throw_spdlog_ex("sid_t::sid_t(): invalid SID received"); - } - - auto const sid_length{::GetLengthSid(psid)}; - - sid_t result; - result.buffer_.resize(sid_length); - if (!::CopySid(sid_length, (PSID)result.as_sid(), psid)) { - SPDLOG_THROW(win32_error("CopySid")); - } - - return result; - } - - /** Retrieves pointer to the internal buffer contents as SID* */ - SID *as_sid() const { return buffer_.empty() ? nullptr : (SID *)buffer_.data(); } - - /** Get SID for the current user */ - static sid_t get_current_user_sid() { - /* create and init RAII holder for process token */ - struct process_token_t { - HANDLE token_handle_ = INVALID_HANDLE_VALUE; - explicit process_token_t(HANDLE process) { - if (!::OpenProcessToken(process, TOKEN_QUERY, &token_handle_)) { - SPDLOG_THROW(win32_error("OpenProcessToken")); - } - } - - ~process_token_t() { ::CloseHandle(token_handle_); } - - } current_process_token( - ::GetCurrentProcess()); // GetCurrentProcess returns pseudohandle, no leak here! - - // Get the required size, this is expected to fail with ERROR_INSUFFICIENT_BUFFER and return - // the token size - DWORD tusize = 0; - if (::GetTokenInformation(current_process_token.token_handle_, TokenUser, NULL, 0, - &tusize)) { - SPDLOG_THROW(win32_error("GetTokenInformation should fail")); - } - - // get user token - std::vector buffer(static_cast(tusize)); - if (!::GetTokenInformation(current_process_token.token_handle_, TokenUser, - (LPVOID)buffer.data(), tusize, &tusize)) { - SPDLOG_THROW(win32_error("GetTokenInformation")); - } - - // create a wrapper of the SID data as stored in the user token - return sid_t::duplicate_sid(((TOKEN_USER *)buffer.data())->User.Sid); - } -}; - -struct eventlog { - static WORD get_event_type(details::log_msg const &msg) { - switch (msg.level) { - case level::trace: - case level::debug: - return EVENTLOG_SUCCESS; - - case level::info: - return EVENTLOG_INFORMATION_TYPE; - - case level::warn: - return EVENTLOG_WARNING_TYPE; - - case level::err: - case level::critical: - case level::off: - return EVENTLOG_ERROR_TYPE; - - default: - return EVENTLOG_INFORMATION_TYPE; - } - } - - static WORD get_event_category(details::log_msg const &msg) { return (WORD)msg.level; } -}; - -} // namespace internal - -/* - * Windows Event Log sink - */ -template -class win_eventlog_sink : public base_sink { -private: - HANDLE hEventLog_{NULL}; - internal::sid_t current_user_sid_; - std::string source_; - DWORD event_id_; - - HANDLE event_log_handle() { - if (!hEventLog_) { - hEventLog_ = ::RegisterEventSourceA(nullptr, source_.c_str()); - if (!hEventLog_ || hEventLog_ == (HANDLE)ERROR_ACCESS_DENIED) { - SPDLOG_THROW(internal::win32_error("RegisterEventSource")); - } - } - - return hEventLog_; - } - -protected: - void sink_it_(const details::log_msg &msg) override { - using namespace internal; - - bool succeeded; - memory_buf_t formatted; - base_sink::formatter_->format(msg, formatted); - formatted.push_back('\0'); - -#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT - wmemory_buf_t buf; - details::os::utf8_to_wstrbuf(string_view_t(formatted.data(), formatted.size()), buf); - - LPCWSTR lp_wstr = buf.data(); - succeeded = static_cast(::ReportEventW( - event_log_handle(), eventlog::get_event_type(msg), eventlog::get_event_category(msg), - event_id_, current_user_sid_.as_sid(), 1, 0, &lp_wstr, nullptr)); -#else - LPCSTR lp_str = formatted.data(); - succeeded = static_cast(::ReportEventA( - event_log_handle(), eventlog::get_event_type(msg), eventlog::get_event_category(msg), - event_id_, current_user_sid_.as_sid(), 1, 0, &lp_str, nullptr)); -#endif - - if (!succeeded) { - SPDLOG_THROW(win32_error("ReportEvent")); - } - } - - void flush_() override {} - -public: - win_eventlog_sink(std::string const &source, - DWORD event_id = 1000 /* according to mscoree.dll */) - : source_(source), - event_id_(event_id) { - try { - current_user_sid_ = internal::sid_t::get_current_user_sid(); - } catch (...) { - // get_current_user_sid() is unlikely to fail and if it does, we can still proceed - // without current_user_sid but in the event log the record will have no user name - } - } - - ~win_eventlog_sink() { - if (hEventLog_) DeregisterEventSource(hEventLog_); - } -}; - -} // namespace win_eventlog - -using win_eventlog_sink_mt = win_eventlog::win_eventlog_sink; -using win_eventlog_sink_st = win_eventlog::win_eventlog_sink; - -} // namespace sinks -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/wincolor_sink-inl.h b/thirdparty/spdlog/include/spdlog/sinks/wincolor_sink-inl.h deleted file mode 100644 index a9c0fa25c..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/wincolor_sink-inl.h +++ /dev/null @@ -1,172 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -#include -#include - -namespace spdlog { -namespace sinks { -template -SPDLOG_INLINE wincolor_sink::wincolor_sink(void *out_handle, color_mode mode) - : out_handle_(out_handle), - mutex_(ConsoleMutex::mutex()), - formatter_(details::make_unique()) { - set_color_mode_impl(mode); - // set level colors - colors_[level::trace] = FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE; // white - colors_[level::debug] = FOREGROUND_GREEN | FOREGROUND_BLUE; // cyan - colors_[level::info] = FOREGROUND_GREEN; // green - colors_[level::warn] = - FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_INTENSITY; // intense yellow - colors_[level::err] = FOREGROUND_RED | FOREGROUND_INTENSITY; // intense red - colors_[level::critical] = BACKGROUND_RED | FOREGROUND_RED | FOREGROUND_GREEN | - FOREGROUND_BLUE | - FOREGROUND_INTENSITY; // intense white on red background - colors_[level::off] = 0; -} - -template -SPDLOG_INLINE wincolor_sink::~wincolor_sink() { - this->flush(); -} - -// change the color for the given level -template -void SPDLOG_INLINE wincolor_sink::set_color(level::level_enum level, - std::uint16_t color) { - std::lock_guard lock(mutex_); - colors_[static_cast(level)] = color; -} - -template -void SPDLOG_INLINE wincolor_sink::log(const details::log_msg &msg) { - if (out_handle_ == nullptr || out_handle_ == INVALID_HANDLE_VALUE) { - return; - } - - std::lock_guard lock(mutex_); - msg.color_range_start = 0; - msg.color_range_end = 0; - memory_buf_t formatted; - formatter_->format(msg, formatted); - if (should_do_colors_ && msg.color_range_end > msg.color_range_start) { - // before color range - print_range_(formatted, 0, msg.color_range_start); - // in color range - auto orig_attribs = - static_cast(set_foreground_color_(colors_[static_cast(msg.level)])); - print_range_(formatted, msg.color_range_start, msg.color_range_end); - // reset to orig colors - ::SetConsoleTextAttribute(static_cast(out_handle_), orig_attribs); - print_range_(formatted, msg.color_range_end, formatted.size()); - } else // print without colors if color range is invalid (or color is disabled) - { - write_to_file_(formatted); - } -} - -template -void SPDLOG_INLINE wincolor_sink::flush() { - // windows console always flushed? -} - -template -void SPDLOG_INLINE wincolor_sink::set_pattern(const std::string &pattern) { - std::lock_guard lock(mutex_); - formatter_ = std::unique_ptr(new pattern_formatter(pattern)); -} - -template -void SPDLOG_INLINE -wincolor_sink::set_formatter(std::unique_ptr sink_formatter) { - std::lock_guard lock(mutex_); - formatter_ = std::move(sink_formatter); -} - -template -void SPDLOG_INLINE wincolor_sink::set_color_mode(color_mode mode) { - std::lock_guard lock(mutex_); - set_color_mode_impl(mode); -} - -template -void SPDLOG_INLINE wincolor_sink::set_color_mode_impl(color_mode mode) { - if (mode == color_mode::automatic) { - // should do colors only if out_handle_ points to actual console. - DWORD console_mode; - bool in_console = ::GetConsoleMode(static_cast(out_handle_), &console_mode) != 0; - should_do_colors_ = in_console; - } else { - should_do_colors_ = mode == color_mode::always ? true : false; - } -} - -// set foreground color and return the orig console attributes (for resetting later) -template -std::uint16_t SPDLOG_INLINE -wincolor_sink::set_foreground_color_(std::uint16_t attribs) { - CONSOLE_SCREEN_BUFFER_INFO orig_buffer_info; - if (!::GetConsoleScreenBufferInfo(static_cast(out_handle_), &orig_buffer_info)) { - // just return white if failed getting console info - return FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE; - } - - // change only the foreground bits (lowest 4 bits) - auto new_attribs = static_cast(attribs) | (orig_buffer_info.wAttributes & 0xfff0); - auto ignored = - ::SetConsoleTextAttribute(static_cast(out_handle_), static_cast(new_attribs)); - (void)(ignored); - return static_cast(orig_buffer_info.wAttributes); // return orig attribs -} - -// print a range of formatted message to console -template -void SPDLOG_INLINE wincolor_sink::print_range_(const memory_buf_t &formatted, - size_t start, - size_t end) { - if (end > start) { -#if defined(SPDLOG_UTF8_TO_WCHAR_CONSOLE) - wmemory_buf_t wformatted; - details::os::utf8_to_wstrbuf(string_view_t(formatted.data() + start, end - start), - wformatted); - auto size = static_cast(wformatted.size()); - auto ignored = ::WriteConsoleW(static_cast(out_handle_), wformatted.data(), size, - nullptr, nullptr); -#else - auto size = static_cast(end - start); - auto ignored = ::WriteConsoleA(static_cast(out_handle_), formatted.data() + start, - size, nullptr, nullptr); -#endif - (void)(ignored); - } -} - -template -void SPDLOG_INLINE wincolor_sink::write_to_file_(const memory_buf_t &formatted) { - auto size = static_cast(formatted.size()); - DWORD bytes_written = 0; - auto ignored = ::WriteFile(static_cast(out_handle_), formatted.data(), size, - &bytes_written, nullptr); - (void)(ignored); -} - -// wincolor_stdout_sink -template -SPDLOG_INLINE wincolor_stdout_sink::wincolor_stdout_sink(color_mode mode) - : wincolor_sink(::GetStdHandle(STD_OUTPUT_HANDLE), mode) {} - -// wincolor_stderr_sink -template -SPDLOG_INLINE wincolor_stderr_sink::wincolor_stderr_sink(color_mode mode) - : wincolor_sink(::GetStdHandle(STD_ERROR_HANDLE), mode) {} -} // namespace sinks -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/sinks/wincolor_sink.h b/thirdparty/spdlog/include/spdlog/sinks/wincolor_sink.h deleted file mode 100644 index e62d14d3f..000000000 --- a/thirdparty/spdlog/include/spdlog/sinks/wincolor_sink.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace spdlog { -namespace sinks { -/* - * Windows color console sink. Uses WriteConsoleA to write to the console with - * colors - */ -template -class wincolor_sink : public sink { -public: - wincolor_sink(void *out_handle, color_mode mode); - ~wincolor_sink() override; - - wincolor_sink(const wincolor_sink &other) = delete; - wincolor_sink &operator=(const wincolor_sink &other) = delete; - - // change the color for the given level - void set_color(level::level_enum level, std::uint16_t color); - void log(const details::log_msg &msg) override; - void flush() override; - void set_pattern(const std::string &pattern) override; - void set_formatter(std::unique_ptr sink_formatter) override; - void set_color_mode(color_mode mode); - -protected: - using mutex_t = typename ConsoleMutex::mutex_t; - void *out_handle_; - mutex_t &mutex_; - bool should_do_colors_; - std::unique_ptr formatter_; - std::array colors_; - - // set foreground color and return the orig console attributes (for resetting later) - std::uint16_t set_foreground_color_(std::uint16_t attribs); - - // print a range of formatted message to console - void print_range_(const memory_buf_t &formatted, size_t start, size_t end); - - // in case we are redirected to file (not in console mode) - void write_to_file_(const memory_buf_t &formatted); - - void set_color_mode_impl(color_mode mode); -}; - -template -class wincolor_stdout_sink : public wincolor_sink { -public: - explicit wincolor_stdout_sink(color_mode mode = color_mode::automatic); -}; - -template -class wincolor_stderr_sink : public wincolor_sink { -public: - explicit wincolor_stderr_sink(color_mode mode = color_mode::automatic); -}; - -using wincolor_stdout_sink_mt = wincolor_stdout_sink; -using wincolor_stdout_sink_st = wincolor_stdout_sink; - -using wincolor_stderr_sink_mt = wincolor_stderr_sink; -using wincolor_stderr_sink_st = wincolor_stderr_sink; -} // namespace sinks -} // namespace spdlog - -#ifdef SPDLOG_HEADER_ONLY - #include "wincolor_sink-inl.h" -#endif diff --git a/thirdparty/spdlog/include/spdlog/spdlog-inl.h b/thirdparty/spdlog/include/spdlog/spdlog-inl.h deleted file mode 100644 index e02081fe3..000000000 --- a/thirdparty/spdlog/include/spdlog/spdlog-inl.h +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#ifndef SPDLOG_HEADER_ONLY - #include -#endif - -#include -#include - -namespace spdlog { - -SPDLOG_INLINE void initialize_logger(std::shared_ptr logger) { - details::registry::instance().initialize_logger(std::move(logger)); -} - -SPDLOG_INLINE std::shared_ptr get(const std::string &name) { - return details::registry::instance().get(name); -} - -SPDLOG_INLINE void set_formatter(std::unique_ptr formatter) { - details::registry::instance().set_formatter(std::move(formatter)); -} - -SPDLOG_INLINE void set_pattern(std::string pattern, pattern_time_type time_type) { - set_formatter( - std::unique_ptr(new pattern_formatter(std::move(pattern), time_type))); -} - -SPDLOG_INLINE void enable_backtrace(size_t n_messages) { - details::registry::instance().enable_backtrace(n_messages); -} - -SPDLOG_INLINE void disable_backtrace() { details::registry::instance().disable_backtrace(); } - -SPDLOG_INLINE void dump_backtrace() { default_logger_raw()->dump_backtrace(); } - -SPDLOG_INLINE level::level_enum get_level() { return default_logger_raw()->level(); } - -SPDLOG_INLINE bool should_log(level::level_enum log_level) { - return default_logger_raw()->should_log(log_level); -} - -SPDLOG_INLINE void set_level(level::level_enum log_level) { - details::registry::instance().set_level(log_level); -} - -SPDLOG_INLINE void flush_on(level::level_enum log_level) { - details::registry::instance().flush_on(log_level); -} - -SPDLOG_INLINE void set_error_handler(void (*handler)(const std::string &msg)) { - details::registry::instance().set_error_handler(handler); -} - -SPDLOG_INLINE void register_logger(std::shared_ptr logger) { - details::registry::instance().register_logger(std::move(logger)); -} - -SPDLOG_INLINE void register_or_replace(std::shared_ptr logger) { - details::registry::instance().register_or_replace(std::move(logger)); -} - -SPDLOG_INLINE void apply_all(const std::function)> &fun) { - details::registry::instance().apply_all(fun); -} - -SPDLOG_INLINE void drop(const std::string &name) { details::registry::instance().drop(name); } - -SPDLOG_INLINE void drop_all() { details::registry::instance().drop_all(); } - -SPDLOG_INLINE void shutdown() { details::registry::instance().shutdown(); } - -SPDLOG_INLINE void set_automatic_registration(bool automatic_registration) { - details::registry::instance().set_automatic_registration(automatic_registration); -} - -SPDLOG_INLINE std::shared_ptr default_logger() { - return details::registry::instance().default_logger(); -} - -SPDLOG_INLINE spdlog::logger *default_logger_raw() { - return details::registry::instance().get_default_raw(); -} - -SPDLOG_INLINE void set_default_logger(std::shared_ptr default_logger) { - details::registry::instance().set_default_logger(std::move(default_logger)); -} - -SPDLOG_INLINE void apply_logger_env_levels(std::shared_ptr logger) { - details::registry::instance().apply_logger_env_levels(std::move(logger)); -} - -} // namespace spdlog diff --git a/thirdparty/spdlog/include/spdlog/spdlog.h b/thirdparty/spdlog/include/spdlog/spdlog.h deleted file mode 100644 index 1a927ffc9..000000000 --- a/thirdparty/spdlog/include/spdlog/spdlog.h +++ /dev/null @@ -1,357 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -// spdlog main header file. -// see example.cpp for usage example - -#ifndef SPDLOG_H -#define SPDLOG_H - -#pragma once - -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace spdlog { - -using default_factory = synchronous_factory; - -// Create and register a logger with a templated sink type -// The logger's level, formatter and flush level will be set according to the -// global settings. -// -// Example: -// spdlog::create("logger_name", "dailylog_filename", 11, 59); -template -inline std::shared_ptr create(std::string logger_name, SinkArgs &&...sink_args) { - return default_factory::create(std::move(logger_name), - std::forward(sink_args)...); -} - -// Initialize and register a logger, -// formatter and flush level will be set according the global settings. -// -// Useful for initializing manually created loggers with the global settings. -// -// Example: -// auto mylogger = std::make_shared("mylogger", ...); -// spdlog::initialize_logger(mylogger); -SPDLOG_API void initialize_logger(std::shared_ptr logger); - -// Return an existing logger or nullptr if a logger with such a name doesn't -// exist. -// example: spdlog::get("my_logger")->info("hello {}", "world"); -SPDLOG_API std::shared_ptr get(const std::string &name); - -// Set global formatter. Each sink in each logger will get a clone of this object -SPDLOG_API void set_formatter(std::unique_ptr formatter); - -// Set global format string. -// example: spdlog::set_pattern("%Y-%m-%d %H:%M:%S.%e %l : %v"); -SPDLOG_API void set_pattern(std::string pattern, - pattern_time_type time_type = pattern_time_type::local); - -// enable global backtrace support -SPDLOG_API void enable_backtrace(size_t n_messages); - -// disable global backtrace support -SPDLOG_API void disable_backtrace(); - -// call dump backtrace on default logger -SPDLOG_API void dump_backtrace(); - -// Get global logging level -SPDLOG_API level::level_enum get_level(); - -// Set the global logging level -SPDLOG_API void set_level(level::level_enum log_level); - -// Determine whether the default logger should log messages with a certain level -SPDLOG_API bool should_log(level::level_enum lvl); - -// Set a global flush level -SPDLOG_API void flush_on(level::level_enum log_level); - -// Start/Restart a periodic flusher thread -// Warning: Use only if all your loggers are thread safe! -template -inline void flush_every(std::chrono::duration interval) { - details::registry::instance().flush_every(interval); -} - -// Set global error handler -SPDLOG_API void set_error_handler(void (*handler)(const std::string &msg)); - -// Register the given logger with the given name -// Will throw if a logger with the same name already exists. -SPDLOG_API void register_logger(std::shared_ptr logger); - -// Register the given logger with the given name -// Will replace any existing logger with the same name. -SPDLOG_API void register_or_replace(std::shared_ptr logger); - -// Apply a user-defined function on all registered loggers -// Example: -// spdlog::apply_all([&](std::shared_ptr l) {l->flush();}); -SPDLOG_API void apply_all(const std::function)> &fun); - -// Drop the reference to the given logger -SPDLOG_API void drop(const std::string &name); - -// Drop all references from the registry -SPDLOG_API void drop_all(); - -// stop any running threads started by spdlog and clean registry loggers -SPDLOG_API void shutdown(); - -// Automatic registration of loggers when using spdlog::create() or spdlog::create_async -SPDLOG_API void set_automatic_registration(bool automatic_registration); - -// API for using default logger (stdout_color_mt), -// e.g.: spdlog::info("Message {}", 1); -// -// The default logger object can be accessed using the spdlog::default_logger(): -// For example, to add another sink to it: -// spdlog::default_logger()->sinks().push_back(some_sink); -// -// The default logger can be replaced using spdlog::set_default_logger(new_logger). -// For example, to replace it with a file logger. -// -// IMPORTANT: -// The default API is thread safe (for _mt loggers), but: -// set_default_logger() *should not* be used concurrently with the default API. -// e.g., do not call set_default_logger() from one thread while calling spdlog::info() from another. - -SPDLOG_API std::shared_ptr default_logger(); - -SPDLOG_API spdlog::logger *default_logger_raw(); - -SPDLOG_API void set_default_logger(std::shared_ptr default_logger); - -// Initialize logger level based on environment configs. -// -// Useful for applying SPDLOG_LEVEL to manually created loggers. -// -// Example: -// auto mylogger = std::make_shared("mylogger", ...); -// spdlog::apply_logger_env_levels(mylogger); -SPDLOG_API void apply_logger_env_levels(std::shared_ptr logger); - -template -inline void log(source_loc source, - level::level_enum lvl, - format_string_t fmt, - Args &&...args) { - default_logger_raw()->log(source, lvl, fmt, std::forward(args)...); -} - -template -inline void log(level::level_enum lvl, format_string_t fmt, Args &&...args) { - default_logger_raw()->log(source_loc{}, lvl, fmt, std::forward(args)...); -} - -template -inline void trace(format_string_t fmt, Args &&...args) { - default_logger_raw()->trace(fmt, std::forward(args)...); -} - -template -inline void debug(format_string_t fmt, Args &&...args) { - default_logger_raw()->debug(fmt, std::forward(args)...); -} - -template -inline void info(format_string_t fmt, Args &&...args) { - default_logger_raw()->info(fmt, std::forward(args)...); -} - -template -inline void warn(format_string_t fmt, Args &&...args) { - default_logger_raw()->warn(fmt, std::forward(args)...); -} - -template -inline void error(format_string_t fmt, Args &&...args) { - default_logger_raw()->error(fmt, std::forward(args)...); -} - -template -inline void critical(format_string_t fmt, Args &&...args) { - default_logger_raw()->critical(fmt, std::forward(args)...); -} - -template -inline void log(source_loc source, level::level_enum lvl, const T &msg) { - default_logger_raw()->log(source, lvl, msg); -} - -template -inline void log(level::level_enum lvl, const T &msg) { - default_logger_raw()->log(lvl, msg); -} - -#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT -template -inline void log(source_loc source, - level::level_enum lvl, - wformat_string_t fmt, - Args &&...args) { - default_logger_raw()->log(source, lvl, fmt, std::forward(args)...); -} - -template -inline void log(level::level_enum lvl, wformat_string_t fmt, Args &&...args) { - default_logger_raw()->log(source_loc{}, lvl, fmt, std::forward(args)...); -} - -template -inline void trace(wformat_string_t fmt, Args &&...args) { - default_logger_raw()->trace(fmt, std::forward(args)...); -} - -template -inline void debug(wformat_string_t fmt, Args &&...args) { - default_logger_raw()->debug(fmt, std::forward(args)...); -} - -template -inline void info(wformat_string_t fmt, Args &&...args) { - default_logger_raw()->info(fmt, std::forward(args)...); -} - -template -inline void warn(wformat_string_t fmt, Args &&...args) { - default_logger_raw()->warn(fmt, std::forward(args)...); -} - -template -inline void error(wformat_string_t fmt, Args &&...args) { - default_logger_raw()->error(fmt, std::forward(args)...); -} - -template -inline void critical(wformat_string_t fmt, Args &&...args) { - default_logger_raw()->critical(fmt, std::forward(args)...); -} -#endif - -template -inline void trace(const T &msg) { - default_logger_raw()->trace(msg); -} - -template -inline void debug(const T &msg) { - default_logger_raw()->debug(msg); -} - -template -inline void info(const T &msg) { - default_logger_raw()->info(msg); -} - -template -inline void warn(const T &msg) { - default_logger_raw()->warn(msg); -} - -template -inline void error(const T &msg) { - default_logger_raw()->error(msg); -} - -template -inline void critical(const T &msg) { - default_logger_raw()->critical(msg); -} - -} // namespace spdlog - -// -// enable/disable log calls at compile time according to global level. -// -// define SPDLOG_ACTIVE_LEVEL to one of those (before including spdlog.h): -// SPDLOG_LEVEL_TRACE, -// SPDLOG_LEVEL_DEBUG, -// SPDLOG_LEVEL_INFO, -// SPDLOG_LEVEL_WARN, -// SPDLOG_LEVEL_ERROR, -// SPDLOG_LEVEL_CRITICAL, -// SPDLOG_LEVEL_OFF -// - -#ifndef SPDLOG_NO_SOURCE_LOC - #define SPDLOG_LOGGER_CALL(logger, level, ...) \ - (logger)->log(spdlog::source_loc{__FILE__, __LINE__, SPDLOG_FUNCTION}, level, __VA_ARGS__) -#else - #define SPDLOG_LOGGER_CALL(logger, level, ...) \ - (logger)->log(spdlog::source_loc{}, level, __VA_ARGS__) -#endif - -#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_TRACE - #define SPDLOG_LOGGER_TRACE(logger, ...) \ - SPDLOG_LOGGER_CALL(logger, spdlog::level::trace, __VA_ARGS__) - #define SPDLOG_TRACE(...) SPDLOG_LOGGER_TRACE(spdlog::default_logger_raw(), __VA_ARGS__) -#else - #define SPDLOG_LOGGER_TRACE(logger, ...) (void)0 - #define SPDLOG_TRACE(...) (void)0 -#endif - -#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_DEBUG - #define SPDLOG_LOGGER_DEBUG(logger, ...) \ - SPDLOG_LOGGER_CALL(logger, spdlog::level::debug, __VA_ARGS__) - #define SPDLOG_DEBUG(...) SPDLOG_LOGGER_DEBUG(spdlog::default_logger_raw(), __VA_ARGS__) -#else - #define SPDLOG_LOGGER_DEBUG(logger, ...) (void)0 - #define SPDLOG_DEBUG(...) (void)0 -#endif - -#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_INFO - #define SPDLOG_LOGGER_INFO(logger, ...) \ - SPDLOG_LOGGER_CALL(logger, spdlog::level::info, __VA_ARGS__) - #define SPDLOG_INFO(...) SPDLOG_LOGGER_INFO(spdlog::default_logger_raw(), __VA_ARGS__) -#else - #define SPDLOG_LOGGER_INFO(logger, ...) (void)0 - #define SPDLOG_INFO(...) (void)0 -#endif - -#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_WARN - #define SPDLOG_LOGGER_WARN(logger, ...) \ - SPDLOG_LOGGER_CALL(logger, spdlog::level::warn, __VA_ARGS__) - #define SPDLOG_WARN(...) SPDLOG_LOGGER_WARN(spdlog::default_logger_raw(), __VA_ARGS__) -#else - #define SPDLOG_LOGGER_WARN(logger, ...) (void)0 - #define SPDLOG_WARN(...) (void)0 -#endif - -#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_ERROR - #define SPDLOG_LOGGER_ERROR(logger, ...) \ - SPDLOG_LOGGER_CALL(logger, spdlog::level::err, __VA_ARGS__) - #define SPDLOG_ERROR(...) SPDLOG_LOGGER_ERROR(spdlog::default_logger_raw(), __VA_ARGS__) -#else - #define SPDLOG_LOGGER_ERROR(logger, ...) (void)0 - #define SPDLOG_ERROR(...) (void)0 -#endif - -#if SPDLOG_ACTIVE_LEVEL <= SPDLOG_LEVEL_CRITICAL - #define SPDLOG_LOGGER_CRITICAL(logger, ...) \ - SPDLOG_LOGGER_CALL(logger, spdlog::level::critical, __VA_ARGS__) - #define SPDLOG_CRITICAL(...) SPDLOG_LOGGER_CRITICAL(spdlog::default_logger_raw(), __VA_ARGS__) -#else - #define SPDLOG_LOGGER_CRITICAL(logger, ...) (void)0 - #define SPDLOG_CRITICAL(...) (void)0 -#endif - -#ifdef SPDLOG_HEADER_ONLY - #include "spdlog-inl.h" -#endif - -#endif // SPDLOG_H diff --git a/thirdparty/spdlog/include/spdlog/stopwatch.h b/thirdparty/spdlog/include/spdlog/stopwatch.h deleted file mode 100644 index 54ab3d3b8..000000000 --- a/thirdparty/spdlog/include/spdlog/stopwatch.h +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#include -#include - -// Stopwatch support for spdlog (using std::chrono::steady_clock). -// Displays elapsed seconds since construction as double. -// -// Usage: -// -// spdlog::stopwatch sw; -// ... -// spdlog::debug("Elapsed: {} seconds", sw); => "Elapsed 0.005116733 seconds" -// spdlog::info("Elapsed: {:.6} seconds", sw); => "Elapsed 0.005163 seconds" -// -// -// If other units are needed (e.g. millis instead of double), include "fmt/chrono.h" and use -// "duration_cast<..>(sw.elapsed())": -// -// #include -//.. -// using std::chrono::duration_cast; -// using std::chrono::milliseconds; -// spdlog::info("Elapsed {}", duration_cast(sw.elapsed())); => "Elapsed 5ms" - -namespace spdlog { -class stopwatch { - using clock = std::chrono::steady_clock; - std::chrono::time_point start_tp_; - -public: - stopwatch() - : start_tp_{clock::now()} {} - - std::chrono::duration elapsed() const { - return std::chrono::duration(clock::now() - start_tp_); - } - - std::chrono::milliseconds elapsed_ms() const { - return std::chrono::duration_cast(clock::now() - start_tp_); - } - - void reset() { start_tp_ = clock::now(); } -}; -} // namespace spdlog - -// Support for fmt formatting (e.g. "{:012.9}" or just "{}") -namespace -#ifdef SPDLOG_USE_STD_FORMAT - std -#else - fmt -#endif -{ - -template <> -struct formatter : formatter { - template - auto format(const spdlog::stopwatch &sw, FormatContext &ctx) const -> decltype(ctx.out()) { - return formatter::format(sw.elapsed().count(), ctx); - } -}; -} // namespace std diff --git a/thirdparty/spdlog/include/spdlog/tweakme.h b/thirdparty/spdlog/include/spdlog/tweakme.h deleted file mode 100644 index 17f7a4bc2..000000000 --- a/thirdparty/spdlog/include/spdlog/tweakme.h +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -/////////////////////////////////////////////////////////////////////////////// -// -// Edit this file to squeeze more performance, and to customize supported -// features -// -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Under Linux, the much faster CLOCK_REALTIME_COARSE clock can be used. -// This clock is less accurate - can be off by dozens of millis - depending on -// the kernel HZ. -// Uncomment to use it instead of the regular clock. -// -// #define SPDLOG_CLOCK_COARSE -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment if source location logging is not needed. -// This will prevent spdlog from using __FILE__, __LINE__ and SPDLOG_FUNCTION -// -// #define SPDLOG_NO_SOURCE_LOC -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment if thread id logging is not needed (i.e. no %t in the log pattern). -// This will prevent spdlog from querying the thread id on each log call. -// -// WARNING: If the log pattern contains thread id (i.e, %t) while this flag is -// on, zero will be logged as thread id. -// -// #define SPDLOG_NO_THREAD_ID -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to prevent spdlog from using thread local storage. -// -// WARNING: if your program forks, UNCOMMENT this flag to prevent undefined -// thread ids in the children logs. -// -#define SPDLOG_NO_TLS -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to avoid spdlog's usage of atomic log levels -// Use only if your code never modifies a logger's log levels concurrently by -// different threads. -// -// #define SPDLOG_NO_ATOMIC_LEVELS -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to enable usage of wchar_t for file names on Windows. -// -// #define SPDLOG_WCHAR_FILENAMES -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to override default eol ("\n" or "\r\n" under Linux/Windows) -// -// #define SPDLOG_EOL ";-)\n" -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to override default folder separators ("/" or "\\/" under -// Linux/Windows). Each character in the string is treated as a different -// separator. -// -// #define SPDLOG_FOLDER_SEPS "\\" -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to use your own copy of the fmt library instead of spdlog's copy. -// In this case spdlog will try to include so set your -I flag -// accordingly. -// -#define SPDLOG_FMT_EXTERNAL -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to use C++20 std::format instead of fmt. -// -// #define SPDLOG_USE_STD_FORMAT -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to enable wchar_t support (convert to utf8) -// -// #define SPDLOG_WCHAR_TO_UTF8_SUPPORT -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to prevent child processes from inheriting log file descriptors -// -// #define SPDLOG_PREVENT_CHILD_FD -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to customize level names (e.g. "MY TRACE") -// -// #define SPDLOG_LEVEL_NAMES { "MY TRACE", "MY DEBUG", "MY INFO", "MY WARNING", "MY ERROR", "MY -// CRITICAL", "OFF" } -// -// For C++17 use string_view_literals: -// -// #include -// using namespace std::string_view_literals; -// #define SPDLOG_LEVEL_NAMES { "MY TRACE"sv, "MY DEBUG"sv, "MY INFO"sv, "MY WARNING"sv, "MY -// ERROR"sv, "MY CRITICAL"sv, "OFF"sv } -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to customize short level names (e.g. "MT") -// These can be longer than one character. -// -// #define SPDLOG_SHORT_LEVEL_NAMES { "T", "D", "I", "W", "E", "C", "O" } -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment to disable default logger creation. -// This might save some (very) small initialization time if no default logger is needed. -// -// #define SPDLOG_DISABLE_DEFAULT_LOGGER -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment and set to compile time level with zero cost (default is INFO). -// Macros like SPDLOG_DEBUG(..), SPDLOG_INFO(..) will expand to empty statements if not enabled -// -// #define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_INFO -/////////////////////////////////////////////////////////////////////////////// - -/////////////////////////////////////////////////////////////////////////////// -// Uncomment (and change if desired) macro to use for function names. -// This is compiler dependent. -// __PRETTY_FUNCTION__ might be nicer in clang/gcc, and __FUNCTION__ in msvc. -// Defaults to __FUNCTION__ (should work on all compilers) if not defined. -// -// #ifdef __PRETTY_FUNCTION__ -// # define SPDLOG_FUNCTION __PRETTY_FUNCTION__ -// #else -// # define SPDLOG_FUNCTION __FUNCTION__ -// #endif -/////////////////////////////////////////////////////////////////////////////// diff --git a/thirdparty/spdlog/include/spdlog/version.h b/thirdparty/spdlog/include/spdlog/version.h deleted file mode 100644 index 69ff2571d..000000000 --- a/thirdparty/spdlog/include/spdlog/version.h +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright(c) 2015-present, Gabi Melman & spdlog contributors. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) - -#pragma once - -#define SPDLOG_VER_MAJOR 1 -#define SPDLOG_VER_MINOR 16 -#define SPDLOG_VER_PATCH 0 - -#define SPDLOG_TO_VERSION(major, minor, patch) (major * 10000 + minor * 100 + patch) -#define SPDLOG_VERSION SPDLOG_TO_VERSION(SPDLOG_VER_MAJOR, SPDLOG_VER_MINOR, SPDLOG_VER_PATCH) diff --git a/thirdparty/spdlog/tests/includes.h b/thirdparty/spdlog/tests/includes.h deleted file mode 100644 index 2e49a5cb7..000000000 --- a/thirdparty/spdlog/tests/includes.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#if defined(__GNUC__) && __GNUC__ == 12 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" // Workaround for GCC 12 -#endif -#include -#if defined(__GNUC__) && __GNUC__ == 12 - #pragma GCC diagnostic pop -#endif - -#include "utils.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_DEBUG - -#include "spdlog/spdlog.h" -#include "spdlog/async.h" -#include "spdlog/details/fmt_helper.h" -#include "spdlog/details/os.h" - -#ifndef SPDLOG_NO_TLS - #include "spdlog/mdc.h" -#endif - -#include "spdlog/sinks/basic_file_sink.h" -#include "spdlog/sinks/daily_file_sink.h" -#include "spdlog/sinks/null_sink.h" -#include "spdlog/sinks/ostream_sink.h" -#include "spdlog/sinks/rotating_file_sink.h" -#include "spdlog/sinks/stdout_color_sinks.h" -#include "spdlog/sinks/msvc_sink.h" -#include "spdlog/pattern_formatter.h" diff --git a/thirdparty/spdlog/tests/main.cpp b/thirdparty/spdlog/tests/main.cpp deleted file mode 100644 index a4a4ff154..000000000 --- a/thirdparty/spdlog/tests/main.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#if defined(__GNUC__) && __GNUC__ == 12 - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" // Workaround for GCC 12 -#endif - -#include - -#if defined(__GNUC__) && __GNUC__ == 12 - #pragma GCC diagnostic pop -#endif diff --git a/thirdparty/spdlog/tests/test_async.cpp b/thirdparty/spdlog/tests/test_async.cpp deleted file mode 100644 index 76fdd7c6b..000000000 --- a/thirdparty/spdlog/tests/test_async.cpp +++ /dev/null @@ -1,200 +0,0 @@ -#include "includes.h" -#include "spdlog/async.h" -#include "spdlog/sinks/basic_file_sink.h" -#include "test_sink.h" - -#define TEST_FILENAME "test_logs/async_test.log" - -TEST_CASE("basic async test ", "[async]") { - auto test_sink = std::make_shared(); - size_t overrun_counter = 0; - size_t queue_size = 128; - size_t messages = 256; - { - auto tp = std::make_shared(queue_size, 1); - auto logger = std::make_shared("as", test_sink, tp, - spdlog::async_overflow_policy::block); - for (size_t i = 0; i < messages; i++) { - logger->info("Hello message #{}", i); - } - logger->flush(); - overrun_counter = tp->overrun_counter(); - } - REQUIRE(test_sink->msg_counter() == messages); - REQUIRE(test_sink->flush_counter() == 1); - REQUIRE(overrun_counter == 0); -} - -TEST_CASE("discard policy ", "[async]") { - auto test_sink = std::make_shared(); - test_sink->set_delay(std::chrono::milliseconds(1)); - size_t queue_size = 4; - size_t messages = 1024; - - auto tp = std::make_shared(queue_size, 1); - auto logger = std::make_shared( - "as", test_sink, tp, spdlog::async_overflow_policy::overrun_oldest); - for (size_t i = 0; i < messages; i++) { - logger->info("Hello message"); - } - REQUIRE(test_sink->msg_counter() < messages); - REQUIRE(tp->overrun_counter() > 0); -} - -TEST_CASE("discard policy discard_new ", "[async]") { - auto test_sink = std::make_shared(); - test_sink->set_delay(std::chrono::milliseconds(1)); - size_t queue_size = 4; - size_t messages = 1024; - - auto tp = std::make_shared(queue_size, 1); - auto logger = std::make_shared( - "as", test_sink, tp, spdlog::async_overflow_policy::discard_new); - for (size_t i = 0; i < messages; i++) { - logger->info("Hello message"); - } - REQUIRE(test_sink->msg_counter() < messages); - REQUIRE(tp->discard_counter() > 0); -} - -TEST_CASE("discard policy using factory ", "[async]") { - size_t queue_size = 4; - size_t messages = 1024; - spdlog::init_thread_pool(queue_size, 1); - - auto logger = spdlog::create_async_nb("as2"); - auto test_sink = std::static_pointer_cast(logger->sinks()[0]); - test_sink->set_delay(std::chrono::milliseconds(3)); - - for (size_t i = 0; i < messages; i++) { - logger->info("Hello message"); - } - - REQUIRE(test_sink->msg_counter() < messages); - spdlog::drop_all(); -} - -TEST_CASE("flush", "[async]") { - auto test_sink = std::make_shared(); - size_t queue_size = 256; - size_t messages = 256; - { - auto tp = std::make_shared(queue_size, 1); - auto logger = std::make_shared("as", test_sink, tp, - spdlog::async_overflow_policy::block); - for (size_t i = 0; i < messages; i++) { - logger->info("Hello message #{}", i); - } - - logger->flush(); - } - // std::this_thread::sleep_for(std::chrono::milliseconds(250)); - REQUIRE(test_sink->msg_counter() == messages); - REQUIRE(test_sink->flush_counter() == 1); -} - -TEST_CASE("async periodic flush", "[async]") { - auto logger = spdlog::create_async("as"); - auto test_sink = std::static_pointer_cast(logger->sinks()[0]); - - spdlog::flush_every(std::chrono::seconds(1)); - std::this_thread::sleep_for(std::chrono::milliseconds(1700)); - REQUIRE(test_sink->flush_counter() == 1); - spdlog::flush_every(std::chrono::seconds(0)); - spdlog::drop_all(); -} - -TEST_CASE("tp->wait_empty() ", "[async]") { - auto test_sink = std::make_shared(); - test_sink->set_delay(std::chrono::milliseconds(5)); - size_t messages = 100; - - auto tp = std::make_shared(messages, 2); - auto logger = std::make_shared("as", test_sink, tp, - spdlog::async_overflow_policy::block); - for (size_t i = 0; i < messages; i++) { - logger->info("Hello message #{}", i); - } - logger->flush(); - tp.reset(); - - REQUIRE(test_sink->msg_counter() == messages); - REQUIRE(test_sink->flush_counter() == 1); -} - -TEST_CASE("multi threads", "[async]") { - auto test_sink = std::make_shared(); - size_t queue_size = 128; - size_t messages = 256; - size_t n_threads = 10; - { - auto tp = std::make_shared(queue_size, 1); - auto logger = std::make_shared("as", test_sink, tp, - spdlog::async_overflow_policy::block); - - std::vector threads; - for (size_t i = 0; i < n_threads; i++) { - threads.emplace_back([logger, messages] { - for (size_t j = 0; j < messages; j++) { - logger->info("Hello message #{}", j); - } - }); - logger->flush(); - } - - for (auto &t : threads) { - t.join(); - } - } - - REQUIRE(test_sink->msg_counter() == messages * n_threads); - REQUIRE(test_sink->flush_counter() == n_threads); -} - -TEST_CASE("to_file", "[async]") { - prepare_logdir(); - size_t messages = 1024; - size_t tp_threads = 1; - spdlog::filename_t filename = SPDLOG_FILENAME_T(TEST_FILENAME); - { - auto file_sink = std::make_shared(filename, true); - auto tp = std::make_shared(messages, tp_threads); - auto logger = - std::make_shared("as", std::move(file_sink), std::move(tp)); - - for (size_t j = 0; j < messages; j++) { - logger->info("Hello message #{}", j); - } - } - - require_message_count(TEST_FILENAME, messages); - auto contents = file_contents(TEST_FILENAME); - using spdlog::details::os::default_eol; - REQUIRE(ends_with(contents, spdlog::fmt_lib::format("Hello message #1023{}", default_eol))); -} - -TEST_CASE("to_file multi-workers", "[async]") { - prepare_logdir(); - size_t messages = 1024 * 10; - size_t tp_threads = 10; - spdlog::filename_t filename = SPDLOG_FILENAME_T(TEST_FILENAME); - { - auto file_sink = std::make_shared(filename, true); - auto tp = std::make_shared(messages, tp_threads); - auto logger = - std::make_shared("as", std::move(file_sink), std::move(tp)); - - for (size_t j = 0; j < messages; j++) { - logger->info("Hello message #{}", j); - } - } - require_message_count(TEST_FILENAME, messages); -} - -TEST_CASE("bad_tp", "[async]") { - auto test_sink = std::make_shared(); - std::shared_ptr const empty_tp; - auto logger = std::make_shared("as", test_sink, empty_tp); - logger->info("Please throw an exception"); - REQUIRE(test_sink->msg_counter() == 0); -} diff --git a/thirdparty/spdlog/tests/test_backtrace.cpp b/thirdparty/spdlog/tests/test_backtrace.cpp deleted file mode 100644 index 4d78c0c21..000000000 --- a/thirdparty/spdlog/tests/test_backtrace.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include "includes.h" -#include "test_sink.h" -#include "spdlog/async.h" - -TEST_CASE("bactrace1", "[bactrace]") { - using spdlog::sinks::test_sink_st; - auto test_sink = std::make_shared(); - size_t backtrace_size = 5; - - spdlog::logger logger("test-backtrace", test_sink); - logger.set_pattern("%v"); - logger.enable_backtrace(backtrace_size); - - logger.info("info message"); - for (int i = 0; i < 100; i++) logger.debug("debug message {}", i); - - REQUIRE(test_sink->lines().size() == 1); - REQUIRE(test_sink->lines()[0] == "info message"); - - logger.dump_backtrace(); - REQUIRE(test_sink->lines().size() == backtrace_size + 3); - REQUIRE(test_sink->lines()[1] == "****************** Backtrace Start ******************"); - REQUIRE(test_sink->lines()[2] == "debug message 95"); - REQUIRE(test_sink->lines()[3] == "debug message 96"); - REQUIRE(test_sink->lines()[4] == "debug message 97"); - REQUIRE(test_sink->lines()[5] == "debug message 98"); - REQUIRE(test_sink->lines()[6] == "debug message 99"); - REQUIRE(test_sink->lines()[7] == "****************** Backtrace End ********************"); -} - -TEST_CASE("bactrace-empty", "[bactrace]") { - using spdlog::sinks::test_sink_st; - auto test_sink = std::make_shared(); - size_t backtrace_size = 5; - - spdlog::logger logger("test-backtrace", test_sink); - logger.set_pattern("%v"); - logger.enable_backtrace(backtrace_size); - logger.dump_backtrace(); - REQUIRE(test_sink->lines().size() == 0); -} - -TEST_CASE("bactrace-async", "[bactrace]") { - using spdlog::sinks::test_sink_mt; - auto test_sink = std::make_shared(); - using spdlog::details::os::sleep_for_millis; - - size_t backtrace_size = 5; - - spdlog::init_thread_pool(120, 1); - auto logger = std::make_shared("test-bactrace-async", test_sink, - spdlog::thread_pool()); - logger->set_pattern("%v"); - logger->enable_backtrace(backtrace_size); - - logger->info("info message"); - for (int i = 0; i < 100; i++) logger->debug("debug message {}", i); - - sleep_for_millis(100); - REQUIRE(test_sink->lines().size() == 1); - REQUIRE(test_sink->lines()[0] == "info message"); - - logger->dump_backtrace(); - sleep_for_millis(100); // give time for the async dump to complete - REQUIRE(test_sink->lines().size() == backtrace_size + 3); - REQUIRE(test_sink->lines()[1] == "****************** Backtrace Start ******************"); - REQUIRE(test_sink->lines()[2] == "debug message 95"); - REQUIRE(test_sink->lines()[3] == "debug message 96"); - REQUIRE(test_sink->lines()[4] == "debug message 97"); - REQUIRE(test_sink->lines()[5] == "debug message 98"); - REQUIRE(test_sink->lines()[6] == "debug message 99"); - REQUIRE(test_sink->lines()[7] == "****************** Backtrace End ********************"); -} diff --git a/thirdparty/spdlog/tests/test_bin_to_hex.cpp b/thirdparty/spdlog/tests/test_bin_to_hex.cpp deleted file mode 100644 index 45fc9fa96..000000000 --- a/thirdparty/spdlog/tests/test_bin_to_hex.cpp +++ /dev/null @@ -1,97 +0,0 @@ -#include "includes.h" -#include "test_sink.h" -#include "spdlog/fmt/bin_to_hex.h" - -TEST_CASE("to_hex", "[to_hex]") { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - spdlog::logger oss_logger("oss", oss_sink); - - std::vector v{9, 0xa, 0xb, 0xc, 0xff, 0xff}; - oss_logger.info("{}", spdlog::to_hex(v)); - - auto output = oss.str(); - REQUIRE(ends_with(output, - "0000: 09 0a 0b 0c ff ff" + std::string(spdlog::details::os::default_eol))); -} - -TEST_CASE("to_hex_upper", "[to_hex]") { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - spdlog::logger oss_logger("oss", oss_sink); - - std::vector v{9, 0xa, 0xb, 0xc, 0xff, 0xff}; - oss_logger.info("{:X}", spdlog::to_hex(v)); - - auto output = oss.str(); - REQUIRE(ends_with(output, - "0000: 09 0A 0B 0C FF FF" + std::string(spdlog::details::os::default_eol))); -} - -TEST_CASE("to_hex_no_delimiter", "[to_hex]") { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - spdlog::logger oss_logger("oss", oss_sink); - - std::vector v{9, 0xa, 0xb, 0xc, 0xff, 0xff}; - oss_logger.info("{:sX}", spdlog::to_hex(v)); - - auto output = oss.str(); - REQUIRE( - ends_with(output, "0000: 090A0B0CFFFF" + std::string(spdlog::details::os::default_eol))); -} - -TEST_CASE("to_hex_show_ascii", "[to_hex]") { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - spdlog::logger oss_logger("oss", oss_sink); - - std::vector v{9, 0xa, 0xb, 0x41, 0xc, 0x4b, 0xff, 0xff}; - oss_logger.info("{:Xsa}", spdlog::to_hex(v, 8)); - - REQUIRE(ends_with(oss.str(), "0000: 090A0B410C4BFFFF ...A.K.." + - std::string(spdlog::details::os::default_eol))); -} - -TEST_CASE("to_hex_different_size_per_line", "[to_hex]") { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - spdlog::logger oss_logger("oss", oss_sink); - - std::vector v{9, 0xa, 0xb, 0x41, 0xc, 0x4b, 0xff, 0xff}; - - oss_logger.info("{:Xsa}", spdlog::to_hex(v, 10)); - REQUIRE(ends_with(oss.str(), "0000: 090A0B410C4BFFFF ...A.K.." + - std::string(spdlog::details::os::default_eol))); - - oss_logger.info("{:Xs}", spdlog::to_hex(v, 10)); - REQUIRE(ends_with(oss.str(), - "0000: 090A0B410C4BFFFF" + std::string(spdlog::details::os::default_eol))); - - oss_logger.info("{:Xsa}", spdlog::to_hex(v, 6)); - REQUIRE(ends_with( - oss.str(), "0000: 090A0B410C4B ...A.K" + std::string(spdlog::details::os::default_eol) + - "0006: FFFF .." + std::string(spdlog::details::os::default_eol))); - - oss_logger.info("{:Xs}", spdlog::to_hex(v, 6)); - REQUIRE(ends_with(oss.str(), "0000: 090A0B410C4B" + - std::string(spdlog::details::os::default_eol) + "0006: FFFF" + - std::string(spdlog::details::os::default_eol))); -} - -TEST_CASE("to_hex_no_ascii", "[to_hex]") { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - spdlog::logger oss_logger("oss", oss_sink); - - std::vector v{9, 0xa, 0xb, 0x41, 0xc, 0x4b, 0xff, 0xff}; - oss_logger.info("{:Xs}", spdlog::to_hex(v, 8)); - - REQUIRE(ends_with(oss.str(), - "0000: 090A0B410C4BFFFF" + std::string(spdlog::details::os::default_eol))); - - oss_logger.info("{:Xsna}", spdlog::to_hex(v, 8)); - - REQUIRE( - ends_with(oss.str(), "090A0B410C4BFFFF" + std::string(spdlog::details::os::default_eol))); -} diff --git a/thirdparty/spdlog/tests/test_cfg.cpp b/thirdparty/spdlog/tests/test_cfg.cpp deleted file mode 100644 index 7dec94b44..000000000 --- a/thirdparty/spdlog/tests/test_cfg.cpp +++ /dev/null @@ -1,178 +0,0 @@ - -#include "includes.h" -#include "test_sink.h" - -#include -#include - -using spdlog::cfg::load_argv_levels; -using spdlog::cfg::load_env_levels; -using spdlog::sinks::test_sink_st; - -TEST_CASE("env", "[cfg]") { - spdlog::drop("l1"); - auto l1 = spdlog::create("l1"); -#ifdef CATCH_PLATFORM_WINDOWS - _putenv_s("SPDLOG_LEVEL", "l1=warn"); -#else - setenv("SPDLOG_LEVEL", "l1=warn", 1); -#endif - load_env_levels(); - REQUIRE(l1->level() == spdlog::level::warn); - -#ifdef CATCH_PLATFORM_WINDOWS - _putenv_s("MYAPP_LEVEL", "l1=trace"); -#else - setenv("MYAPP_LEVEL", "l1=trace", 1); -#endif - load_env_levels("MYAPP_LEVEL"); - REQUIRE(l1->level() == spdlog::level::trace); - - spdlog::set_default_logger(spdlog::create("cfg-default")); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::info); -} - -TEST_CASE("argv1", "[cfg]") { - spdlog::drop("l1"); - const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=warn"}; - load_argv_levels(2, argv); - auto l1 = spdlog::create("l1"); - REQUIRE(l1->level() == spdlog::level::warn); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::info); -} - -TEST_CASE("argv2", "[cfg]") { - spdlog::drop("l1"); - const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=warn,trace"}; - load_argv_levels(2, argv); - auto l1 = spdlog::create("l1"); - REQUIRE(l1->level() == spdlog::level::warn); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::trace); -} - -TEST_CASE("argv3", "[cfg]") { - spdlog::set_level(spdlog::level::trace); - - spdlog::drop("l1"); - const char *argv[] = {"ignore", "SPDLOG_LEVEL=junk_name=warn"}; - load_argv_levels(2, argv); - auto l1 = spdlog::create("l1"); - REQUIRE(l1->level() == spdlog::level::trace); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::trace); -} - -TEST_CASE("argv4", "[cfg]") { - spdlog::set_level(spdlog::level::info); - spdlog::drop("l1"); - const char *argv[] = {"ignore", "SPDLOG_LEVEL=junk"}; - load_argv_levels(2, argv); - auto l1 = spdlog::create("l1"); - REQUIRE(l1->level() == spdlog::level::info); -} - -TEST_CASE("argv5", "[cfg]") { - spdlog::set_level(spdlog::level::info); - spdlog::drop("l1"); - const char *argv[] = {"ignore", "ignore", "SPDLOG_LEVEL=l1=warn,trace"}; - load_argv_levels(3, argv); - auto l1 = spdlog::create("l1"); - REQUIRE(l1->level() == spdlog::level::warn); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::trace); - spdlog::set_level(spdlog::level::info); -} - -TEST_CASE("argv6", "[cfg]") { - spdlog::set_level(spdlog::level::err); - const char *argv[] = {""}; - load_argv_levels(1, argv); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::err); - spdlog::set_level(spdlog::level::info); -} - -TEST_CASE("argv7", "[cfg]") { - spdlog::set_level(spdlog::level::err); - const char *argv[] = {""}; - load_argv_levels(0, argv); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::err); - spdlog::set_level(spdlog::level::info); -} - -TEST_CASE("level-not-set-test1", "[cfg]") { - spdlog::drop("l1"); - const char *argv[] = {"ignore", ""}; - load_argv_levels(2, argv); - auto l1 = spdlog::create("l1"); - l1->set_level(spdlog::level::trace); - REQUIRE(l1->level() == spdlog::level::trace); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::info); -} - -TEST_CASE("level-not-set-test2", "[cfg]") { - spdlog::drop("l1"); - spdlog::drop("l2"); - const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=trace"}; - - auto l1 = spdlog::create("l1"); - l1->set_level(spdlog::level::warn); - auto l2 = spdlog::create("l2"); - l2->set_level(spdlog::level::warn); - - load_argv_levels(2, argv); - - REQUIRE(l1->level() == spdlog::level::trace); - REQUIRE(l2->level() == spdlog::level::warn); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::info); -} - -TEST_CASE("level-not-set-test3", "[cfg]") { - spdlog::drop("l1"); - spdlog::drop("l2"); - const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=trace"}; - - load_argv_levels(2, argv); - - auto l1 = spdlog::create("l1"); - auto l2 = spdlog::create("l2"); - - REQUIRE(l1->level() == spdlog::level::trace); - REQUIRE(l2->level() == spdlog::level::info); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::info); -} - -TEST_CASE("level-not-set-test4", "[cfg]") { - spdlog::drop("l1"); - spdlog::drop("l2"); - const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=trace,warn"}; - - load_argv_levels(2, argv); - - auto l1 = spdlog::create("l1"); - auto l2 = spdlog::create("l2"); - - REQUIRE(l1->level() == spdlog::level::trace); - REQUIRE(l2->level() == spdlog::level::warn); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::warn); -} - -TEST_CASE("level-not-set-test5", "[cfg]") { - spdlog::drop("l1"); - spdlog::drop("l2"); - const char *argv[] = {"ignore", "SPDLOG_LEVEL=l1=junk,warn"}; - - load_argv_levels(2, argv); - - auto l1 = spdlog::create("l1"); - auto l2 = spdlog::create("l2"); - - REQUIRE(l1->level() == spdlog::level::warn); - REQUIRE(l2->level() == spdlog::level::warn); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::warn); -} - -TEST_CASE("restore-to-default", "[cfg]") { - spdlog::drop("l1"); - spdlog::drop("l2"); - const char *argv[] = {"ignore", "SPDLOG_LEVEL=info"}; - load_argv_levels(2, argv); - REQUIRE(spdlog::default_logger()->level() == spdlog::level::info); -} diff --git a/thirdparty/spdlog/tests/test_circular_q.cpp b/thirdparty/spdlog/tests/test_circular_q.cpp deleted file mode 100644 index c8b02d363..000000000 --- a/thirdparty/spdlog/tests/test_circular_q.cpp +++ /dev/null @@ -1,50 +0,0 @@ -#include "includes.h" -#include "spdlog/details/circular_q.h" - -using q_type = spdlog::details::circular_q; -TEST_CASE("test_size", "[circular_q]") { - const size_t q_size = 4; - q_type q(q_size); - REQUIRE(q.size() == 0); - REQUIRE(q.empty() == true); - for (size_t i = 0; i < q_size; i++) { - q.push_back(std::move(i)); - } - REQUIRE(q.size() == q_size); - q.push_back(999); - REQUIRE(q.size() == q_size); -} - -TEST_CASE("test_rolling", "[circular_q]") { - const size_t q_size = 4; - q_type q(q_size); - - for (size_t i = 0; i < q_size + 2; i++) { - q.push_back(std::move(i)); - } - - REQUIRE(q.size() == q_size); - - REQUIRE(q.front() == 2); - q.pop_front(); - - REQUIRE(q.front() == 3); - q.pop_front(); - - REQUIRE(q.front() == 4); - q.pop_front(); - - REQUIRE(q.front() == 5); - q.pop_front(); - - REQUIRE(q.empty()); - - q.push_back(6); - REQUIRE(q.front() == 6); -} - -TEST_CASE("test_empty", "[circular_q]") { - q_type q(0); - q.push_back(1); - REQUIRE(q.empty()); -} \ No newline at end of file diff --git a/thirdparty/spdlog/tests/test_create_dir.cpp b/thirdparty/spdlog/tests/test_create_dir.cpp deleted file mode 100644 index fd040339c..000000000 --- a/thirdparty/spdlog/tests/test_create_dir.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * This content is released under the MIT License as specified in - * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE - */ -#include "includes.h" - -using spdlog::details::os::create_dir; -using spdlog::details::os::path_exists; - -bool try_create_dir(const spdlog::filename_t &path, const spdlog::filename_t &normalized_path) { - auto rv = create_dir(path); - REQUIRE(rv == true); - return path_exists(normalized_path); -} - -TEST_CASE("create_dir", "[create_dir]") { - prepare_logdir(); - - REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs/dir1/dir1"), - SPDLOG_FILENAME_T("test_logs/dir1/dir1"))); - REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs/dir1/dir1"), - SPDLOG_FILENAME_T("test_logs/dir1/dir1"))); // test existing - REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs/dir1///dir2//"), - SPDLOG_FILENAME_T("test_logs/dir1/dir2"))); - REQUIRE(try_create_dir(SPDLOG_FILENAME_T("./test_logs/dir1/dir3"), - SPDLOG_FILENAME_T("test_logs/dir1/dir3"))); - REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs/../test_logs/dir1/dir4"), - SPDLOG_FILENAME_T("test_logs/dir1/dir4"))); - -#ifdef WIN32 - // test backslash folder separator - REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs\\dir1\\dir222"), - SPDLOG_FILENAME_T("test_logs\\dir1\\dir222"))); - REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs\\dir1\\dir223\\"), - SPDLOG_FILENAME_T("test_logs\\dir1\\dir223\\"))); - REQUIRE(try_create_dir(SPDLOG_FILENAME_T(".\\test_logs\\dir1\\dir2\\dir99\\..\\dir23"), - SPDLOG_FILENAME_T("test_logs\\dir1\\dir2\\dir23"))); - REQUIRE(try_create_dir(SPDLOG_FILENAME_T("test_logs\\..\\test_logs\\dir1\\dir5"), - SPDLOG_FILENAME_T("test_logs\\dir1\\dir5"))); -#endif -} - -TEST_CASE("create_invalid_dir", "[create_dir]") { - REQUIRE(create_dir(SPDLOG_FILENAME_T("")) == false); - REQUIRE(create_dir(spdlog::filename_t{}) == false); -#ifdef __linux__ - REQUIRE(create_dir("/proc/spdlog-utest") == false); -#endif -} - -TEST_CASE("dir_name", "[create_dir]") { - using spdlog::details::os::dir_name; - REQUIRE(dir_name(SPDLOG_FILENAME_T("")).empty()); - REQUIRE(dir_name(SPDLOG_FILENAME_T("dir")).empty()); - -#ifdef WIN32 - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\)")) == SPDLOG_FILENAME_T("dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\\\)")) == SPDLOG_FILENAME_T(R"(dir\\)")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\file)")) == SPDLOG_FILENAME_T("dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir/file)")) == SPDLOG_FILENAME_T("dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\file.txt)")) == SPDLOG_FILENAME_T("dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir/file)")) == SPDLOG_FILENAME_T("dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(dir\file.txt\)")) == - SPDLOG_FILENAME_T(R"(dir\file.txt)")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(\dir\file.txt)")) == SPDLOG_FILENAME_T(R"(\dir)")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(\\dir\file.txt)")) == SPDLOG_FILENAME_T(R"(\\dir)")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(..\file.txt)")) == SPDLOG_FILENAME_T("..")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(.\file.txt)")) == SPDLOG_FILENAME_T(".")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(c:\\a\b\c\d\file.txt)")) == - SPDLOG_FILENAME_T(R"(c:\\a\b\c\d)")); - REQUIRE(dir_name(SPDLOG_FILENAME_T(R"(c://a/b/c/d/file.txt)")) == - SPDLOG_FILENAME_T(R"(c://a/b/c/d)")); -#endif - REQUIRE(dir_name(SPDLOG_FILENAME_T("dir/")) == SPDLOG_FILENAME_T("dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T("dir///")) == SPDLOG_FILENAME_T("dir//")); - REQUIRE(dir_name(SPDLOG_FILENAME_T("dir/file")) == SPDLOG_FILENAME_T("dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T("dir/file.txt")) == SPDLOG_FILENAME_T("dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T("dir/file.txt/")) == SPDLOG_FILENAME_T("dir/file.txt")); - REQUIRE(dir_name(SPDLOG_FILENAME_T("/dir/file.txt")) == SPDLOG_FILENAME_T("/dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T("//dir/file.txt")) == SPDLOG_FILENAME_T("//dir")); - REQUIRE(dir_name(SPDLOG_FILENAME_T("../file.txt")) == SPDLOG_FILENAME_T("..")); - REQUIRE(dir_name(SPDLOG_FILENAME_T("./file.txt")) == SPDLOG_FILENAME_T(".")); -} - -#ifdef _WIN32 - - // - // test windows cases when drive letter is given e.g. C:\\some-folder - // - #include - #include - -std::string get_full_path(const std::string &relative_folder_path) { - char full_path[MAX_PATH]; - - DWORD result = ::GetFullPathNameA(relative_folder_path.c_str(), MAX_PATH, full_path, nullptr); - // Return an empty string if failed to get full path - return result > 0 && result < MAX_PATH ? std::string(full_path) : std::string(); -} - -std::wstring get_full_path(const std::wstring &relative_folder_path) { - wchar_t full_path[MAX_PATH]; - DWORD result = ::GetFullPathNameW(relative_folder_path.c_str(), MAX_PATH, full_path, nullptr); - return result > 0 && result < MAX_PATH ? std::wstring(full_path) : std::wstring(); -} - -spdlog::filename_t::value_type find_non_existing_drive() { - for (char drive = 'A'; drive <= 'Z'; ++drive) { - std::string root_path = std::string(1, drive) + ":\\"; - UINT drive_type = GetDriveTypeA(root_path.c_str()); - if (drive_type == DRIVE_NO_ROOT_DIR) { - return static_cast(drive); - } - } - return '\0'; // No available drive found -} - -TEST_CASE("create_abs_path1", "[create_dir]") { - prepare_logdir(); - auto abs_path = get_full_path(SPDLOG_FILENAME_T("test_logs\\logdir1")); - REQUIRE(!abs_path.empty()); - REQUIRE(create_dir(abs_path) == true); -} - -TEST_CASE("create_abs_path2", "[create_dir]") { - prepare_logdir(); - auto abs_path = get_full_path(SPDLOG_FILENAME_T("test_logs/logdir2")); - REQUIRE(!abs_path.empty()); - REQUIRE(create_dir(abs_path) == true); -} - -TEST_CASE("non_existing_drive", "[create_dir]") { - prepare_logdir(); - spdlog::filename_t path; - - auto non_existing_drive = find_non_existing_drive(); - path += non_existing_drive; - path += SPDLOG_FILENAME_T(":\\"); - REQUIRE(create_dir(path) == false); - path += SPDLOG_FILENAME_T("subdir"); - REQUIRE(create_dir(path) == false); -} -// #endif // SPDLOG_WCHAR_FILENAMES -#endif // _WIN32 diff --git a/thirdparty/spdlog/tests/test_custom_callbacks.cpp b/thirdparty/spdlog/tests/test_custom_callbacks.cpp deleted file mode 100644 index f14572115..000000000 --- a/thirdparty/spdlog/tests/test_custom_callbacks.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * This content is released under the MIT License as specified in - * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE - */ -#include "includes.h" -#include "test_sink.h" -#include "spdlog/sinks/callback_sink.h" -#include "spdlog/async.h" -#include "spdlog/common.h" - -TEST_CASE("custom_callback_logger", "[custom_callback_logger]") { - std::vector lines; - spdlog::pattern_formatter formatter; - auto callback_logger = - std::make_shared([&](const spdlog::details::log_msg &msg) { - spdlog::memory_buf_t formatted; - formatter.format(msg, formatted); - auto eol_len = strlen(spdlog::details::os::default_eol); - using diff_t = - typename std::iterator_traits::difference_type; - lines.emplace_back(formatted.begin(), formatted.end() - static_cast(eol_len)); - }); - std::shared_ptr test_sink(new spdlog::sinks::test_sink_st); - - spdlog::logger logger("test-callback", {callback_logger, test_sink}); - - logger.info("test message 1"); - logger.info("test message 2"); - logger.info("test message 3"); - - std::vector ref_lines = test_sink->lines(); - - REQUIRE(lines[0] == ref_lines[0]); - REQUIRE(lines[1] == ref_lines[1]); - REQUIRE(lines[2] == ref_lines[2]); - spdlog::drop_all(); -} diff --git a/thirdparty/spdlog/tests/test_daily_logger.cpp b/thirdparty/spdlog/tests/test_daily_logger.cpp deleted file mode 100644 index 8a00c024a..000000000 --- a/thirdparty/spdlog/tests/test_daily_logger.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - * This content is released under the MIT License as specified in - * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE - */ -#include "includes.h" - -#ifdef SPDLOG_USE_STD_FORMAT -using filename_memory_buf_t = std::basic_string; -#else -using filename_memory_buf_t = fmt::basic_memory_buffer; -#endif - -#ifdef SPDLOG_WCHAR_FILENAMES -std::string filename_buf_to_utf8string(const filename_memory_buf_t &w) { - spdlog::memory_buf_t buf; - spdlog::details::os::wstr_to_utf8buf(spdlog::wstring_view_t(w.data(), w.size()), buf); - return SPDLOG_BUF_TO_STRING(buf); -} -#else -std::string filename_buf_to_utf8string(const filename_memory_buf_t &w) { - return SPDLOG_BUF_TO_STRING(w); -} -#endif - -TEST_CASE("daily_logger with dateonly calculator", "[daily_logger]") { - using sink_type = - spdlog::sinks::daily_file_sink; - - prepare_logdir(); - - // calculate filename (time based) - spdlog::filename_t basename = SPDLOG_FILENAME_T("test_logs/daily_dateonly"); - std::tm tm = spdlog::details::os::localtime(); - filename_memory_buf_t w; - spdlog::fmt_lib::format_to(std::back_inserter(w), SPDLOG_FILENAME_T("{}_{:04d}-{:02d}-{:02d}"), - basename, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday); - - auto logger = spdlog::create("logger", basename, 0, 0); - for (int i = 0; i < 10; ++i) { - logger->info("Test message {}", i); - } - logger->flush(); - - require_message_count(filename_buf_to_utf8string(w), 10); -} - -struct custom_daily_file_name_calculator { - static spdlog::filename_t calc_filename(const spdlog::filename_t &basename, const tm &now_tm) { - return spdlog::fmt_lib::format(SPDLOG_FILENAME_T("{}{:04d}{:02d}{:02d}"), basename, - now_tm.tm_year + 1900, now_tm.tm_mon + 1, now_tm.tm_mday); - } -}; - -TEST_CASE("daily_logger with custom calculator", "[daily_logger]") { - using sink_type = spdlog::sinks::daily_file_sink; - - prepare_logdir(); - - // calculate filename (time based) - spdlog::filename_t basename = SPDLOG_FILENAME_T("test_logs/daily_dateonly"); - std::tm tm = spdlog::details::os::localtime(); - filename_memory_buf_t w; - spdlog::fmt_lib::format_to(std::back_inserter(w), SPDLOG_FILENAME_T("{}{:04d}{:02d}{:02d}"), - basename, tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday); - - auto logger = spdlog::create("logger", basename, 0, 0); - for (int i = 0; i < 10; ++i) { - logger->info("Test message {}", i); - } - - logger->flush(); - - require_message_count(filename_buf_to_utf8string(w), 10); -} - -/* - * File name calculations - */ - -TEST_CASE("rotating_file_sink::calc_filename1", "[rotating_file_sink]") { - auto filename = - spdlog::sinks::rotating_file_sink_st::calc_filename(SPDLOG_FILENAME_T("rotated.txt"), 3); - REQUIRE(filename == SPDLOG_FILENAME_T("rotated.3.txt")); -} - -TEST_CASE("rotating_file_sink::calc_filename2", "[rotating_file_sink]") { - auto filename = - spdlog::sinks::rotating_file_sink_st::calc_filename(SPDLOG_FILENAME_T("rotated"), 3); - REQUIRE(filename == SPDLOG_FILENAME_T("rotated.3")); -} - -TEST_CASE("rotating_file_sink::calc_filename3", "[rotating_file_sink]") { - auto filename = - spdlog::sinks::rotating_file_sink_st::calc_filename(SPDLOG_FILENAME_T("rotated.txt"), 0); - REQUIRE(filename == SPDLOG_FILENAME_T("rotated.txt")); -} - -// regex supported only from gcc 4.9 and above -#if defined(_MSC_VER) || !(__GNUC__ <= 4 && __GNUC_MINOR__ < 9) - - #include - -TEST_CASE("daily_file_sink::daily_filename_calculator", "[daily_file_sink]") { - // daily_YYYY-MM-DD_hh-mm.txt - auto filename = spdlog::sinks::daily_filename_calculator::calc_filename( - SPDLOG_FILENAME_T("daily.txt"), spdlog::details::os::localtime()); - // date regex based on https://www.regular-expressions.info/dates.html - std::basic_regex re( - SPDLOG_FILENAME_T(R"(^daily_(19|20)\d\d-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])\.txt$)")); - std::match_results match; - REQUIRE(std::regex_match(filename, match, re)); -} -#endif - -TEST_CASE("daily_file_sink::daily_filename_format_calculator", "[daily_file_sink]") { - std::tm tm = spdlog::details::os::localtime(); - // example-YYYY-MM-DD.log - auto filename = spdlog::sinks::daily_filename_format_calculator::calc_filename( - SPDLOG_FILENAME_T("example-%Y-%m-%d.log"), tm); - - REQUIRE(filename == - spdlog::fmt_lib::format(SPDLOG_FILENAME_T("example-{:04d}-{:02d}-{:02d}.log"), - tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday)); -} - -/* Test removal of old files */ -static spdlog::details::log_msg create_msg(std::chrono::seconds offset) { - using spdlog::log_clock; - spdlog::details::log_msg msg{"test", spdlog::level::info, "Hello Message"}; - msg.time = log_clock::now() + offset; - return msg; -} - -static void test_rotate(int days_to_run, uint16_t max_days, uint16_t expected_n_files) { - using spdlog::log_clock; - using spdlog::details::log_msg; - using spdlog::sinks::daily_file_sink_st; - - prepare_logdir(); - - spdlog::filename_t basename = SPDLOG_FILENAME_T("test_logs/daily_rotate.txt"); - daily_file_sink_st sink{basename, 2, 30, true, max_days}; - - // simulate messages with 24 intervals - - for (int i = 0; i < days_to_run; i++) { - auto offset = std::chrono::seconds{24 * 3600 * i}; - sink.log(create_msg(offset)); - } - - REQUIRE(count_files("test_logs") == static_cast(expected_n_files)); -} - -TEST_CASE("daily_logger rotate", "[daily_file_sink]") { - int days_to_run = 1; - test_rotate(days_to_run, 0, 1); - test_rotate(days_to_run, 1, 1); - test_rotate(days_to_run, 3, 1); - test_rotate(days_to_run, 10, 1); - - days_to_run = 10; - test_rotate(days_to_run, 0, 10); - test_rotate(days_to_run, 1, 1); - test_rotate(days_to_run, 3, 3); - test_rotate(days_to_run, 9, 9); - test_rotate(days_to_run, 10, 10); - test_rotate(days_to_run, 11, 10); - test_rotate(days_to_run, 20, 10); -} diff --git a/thirdparty/spdlog/tests/test_dup_filter.cpp b/thirdparty/spdlog/tests/test_dup_filter.cpp deleted file mode 100644 index 78e22be3b..000000000 --- a/thirdparty/spdlog/tests/test_dup_filter.cpp +++ /dev/null @@ -1,83 +0,0 @@ -#include "includes.h" -#include "spdlog/sinks/dup_filter_sink.h" -#include "test_sink.h" - -TEST_CASE("dup_filter_test1", "[dup_filter_sink]") { - using spdlog::sinks::dup_filter_sink_st; - using spdlog::sinks::test_sink_mt; - - dup_filter_sink_st dup_sink{std::chrono::seconds{5}}; - auto test_sink = std::make_shared(); - dup_sink.add_sink(test_sink); - - for (int i = 0; i < 10; i++) { - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"}); - } - - REQUIRE(test_sink->msg_counter() == 1); -} - -TEST_CASE("dup_filter_test2", "[dup_filter_sink]") { - using spdlog::sinks::dup_filter_sink_st; - using spdlog::sinks::test_sink_mt; - - dup_filter_sink_st dup_sink{std::chrono::seconds{0}}; - auto test_sink = std::make_shared(); - dup_sink.add_sink(test_sink); - - for (int i = 0; i < 10; i++) { - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"}); - std::this_thread::sleep_for(std::chrono::milliseconds(5)); - } - - REQUIRE(test_sink->msg_counter() == 10); -} - -TEST_CASE("dup_filter_test3", "[dup_filter_sink]") { - using spdlog::sinks::dup_filter_sink_st; - using spdlog::sinks::test_sink_mt; - - dup_filter_sink_st dup_sink{std::chrono::seconds{1}}; - auto test_sink = std::make_shared(); - dup_sink.add_sink(test_sink); - - for (int i = 0; i < 10; i++) { - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"}); - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message2"}); - } - - REQUIRE(test_sink->msg_counter() == 20); -} - -TEST_CASE("dup_filter_test4", "[dup_filter_sink]") { - using spdlog::sinks::dup_filter_sink_mt; - using spdlog::sinks::test_sink_mt; - - dup_filter_sink_mt dup_sink{std::chrono::milliseconds{10}}; - auto test_sink = std::make_shared(); - dup_sink.add_sink(test_sink); - - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message"}); - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message"}); - REQUIRE(test_sink->msg_counter() == 2); -} - -TEST_CASE("dup_filter_test5", "[dup_filter_sink]") { - using spdlog::sinks::dup_filter_sink_mt; - using spdlog::sinks::test_sink_mt; - - dup_filter_sink_mt dup_sink{std::chrono::seconds{5}}; - auto test_sink = std::make_shared(); - test_sink->set_pattern("%v"); - dup_sink.add_sink(test_sink); - - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"}); - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"}); - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message1"}); - dup_sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "message2"}); - - REQUIRE(test_sink->msg_counter() == - 3); // skip 2 messages but log the "skipped.." message before message2 - REQUIRE(test_sink->lines()[1] == "Skipped 2 duplicate messages.."); -} diff --git a/thirdparty/spdlog/tests/test_errors.cpp b/thirdparty/spdlog/tests/test_errors.cpp deleted file mode 100644 index 1c24cabc2..000000000 --- a/thirdparty/spdlog/tests/test_errors.cpp +++ /dev/null @@ -1,112 +0,0 @@ -/* - * This content is released under the MIT License as specified in - * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE - */ -#include "includes.h" - -#include - -#define SIMPLE_LOG "test_logs/simple_log.txt" -#define SIMPLE_ASYNC_LOG "test_logs/simple_async_log.txt" - -class failing_sink : public spdlog::sinks::base_sink { -protected: - void sink_it_(const spdlog::details::log_msg &) final { - throw std::runtime_error("some error happened during log"); - } - - void flush_() final { throw std::runtime_error("some error happened during flush"); } -}; -struct custom_ex {}; - -#if !defined(SPDLOG_USE_STD_FORMAT) // std format doesn't fully support runtime strings -TEST_CASE("default_error_handler", "[errors]") { - prepare_logdir(); - spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG); - - auto logger = spdlog::create("test-error", filename, true); - logger->set_pattern("%v"); - logger->info(SPDLOG_FMT_RUNTIME("Test message {} {}"), 1); - logger->info("Test message {}", 2); - logger->flush(); - using spdlog::details::os::default_eol; - REQUIRE(file_contents(SIMPLE_LOG) == spdlog::fmt_lib::format("Test message 2{}", default_eol)); - REQUIRE(count_lines(SIMPLE_LOG) == 1); -} - -TEST_CASE("custom_error_handler", "[errors]") { - prepare_logdir(); - spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG); - auto logger = spdlog::create("logger", filename, true); - logger->flush_on(spdlog::level::info); - logger->set_error_handler([=](const std::string &) { throw custom_ex(); }); - logger->info("Good message #1"); - - REQUIRE_THROWS_AS(logger->info(SPDLOG_FMT_RUNTIME("Bad format msg {} {}"), "xxx"), custom_ex); - logger->info("Good message #2"); - require_message_count(SIMPLE_LOG, 2); -} -#endif - -TEST_CASE("default_error_handler2", "[errors]") { - spdlog::drop_all(); - auto logger = spdlog::create("failed_logger"); - logger->set_error_handler([=](const std::string &) { throw custom_ex(); }); - REQUIRE_THROWS_AS(logger->info("Some message"), custom_ex); -} - -TEST_CASE("flush_error_handler", "[errors]") { - spdlog::drop_all(); - auto logger = spdlog::create("failed_logger"); - logger->set_error_handler([=](const std::string &) { throw custom_ex(); }); - REQUIRE_THROWS_AS(logger->flush(), custom_ex); -} - -#if !defined(SPDLOG_USE_STD_FORMAT) -TEST_CASE("async_error_handler", "[errors]") { - prepare_logdir(); - std::string err_msg("log failed with some msg"); - - spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_ASYNC_LOG); - { - spdlog::init_thread_pool(128, 1); - auto logger = - spdlog::create_async("logger", filename, true); - logger->set_error_handler([=](const std::string &) { - std::ofstream ofs("test_logs/custom_err.txt"); - if (!ofs) { - throw std::runtime_error("Failed open test_logs/custom_err.txt"); - } - ofs << err_msg; - }); - logger->info("Good message #1"); - logger->info(SPDLOG_FMT_RUNTIME("Bad format msg {} {}"), "xxx"); - logger->info("Good message #2"); - spdlog::drop("logger"); // force logger to drain the queue and shutdown - } - spdlog::init_thread_pool(128, 1); - require_message_count(SIMPLE_ASYNC_LOG, 2); - REQUIRE(file_contents("test_logs/custom_err.txt") == err_msg); -} -#endif - -// Make sure async error handler is executed -TEST_CASE("async_error_handler2", "[errors]") { - prepare_logdir(); - std::string err_msg("This is async handler error message"); - { - spdlog::details::os::create_dir(SPDLOG_FILENAME_T("test_logs")); - spdlog::init_thread_pool(128, 1); - auto logger = spdlog::create_async("failed_logger"); - logger->set_error_handler([=](const std::string &) { - std::ofstream ofs("test_logs/custom_err2.txt"); - if (!ofs) throw std::runtime_error("Failed open test_logs/custom_err2.txt"); - ofs << err_msg; - }); - logger->info("Hello failure"); - spdlog::drop("failed_logger"); // force logger to drain the queue and shutdown - } - - spdlog::init_thread_pool(128, 1); - REQUIRE(file_contents("test_logs/custom_err2.txt") == err_msg); -} diff --git a/thirdparty/spdlog/tests/test_eventlog.cpp b/thirdparty/spdlog/tests/test_eventlog.cpp deleted file mode 100644 index 702eabea1..000000000 --- a/thirdparty/spdlog/tests/test_eventlog.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#if _WIN32 - - #include "includes.h" - #include "test_sink.h" - - #include "spdlog/sinks/win_eventlog_sink.h" - -static const LPCSTR TEST_SOURCE = "spdlog_test"; - -static void test_single_print(std::function do_log, - std::string const &expected_contents, - WORD expected_ev_type) { - using namespace std::chrono; - do_log(expected_contents); - const auto expected_time_generated = - duration_cast(system_clock::now().time_since_epoch()).count(); - - struct handle_t { - HANDLE handle_; - - ~handle_t() { - if (handle_) { - REQUIRE(CloseEventLog(handle_)); - } - } - } event_log{::OpenEventLogA(nullptr, TEST_SOURCE)}; - - REQUIRE(event_log.handle_); - - DWORD read_bytes{}, size_needed{}; - auto ok = ::ReadEventLogA(event_log.handle_, EVENTLOG_SEQUENTIAL_READ | EVENTLOG_BACKWARDS_READ, - 0, &read_bytes, 0, &read_bytes, &size_needed); - REQUIRE(!ok); - REQUIRE(::GetLastError() == ERROR_INSUFFICIENT_BUFFER); - - std::vector record_buffer(size_needed); - PEVENTLOGRECORD record = (PEVENTLOGRECORD)record_buffer.data(); - - ok = ::ReadEventLogA(event_log.handle_, EVENTLOG_SEQUENTIAL_READ | EVENTLOG_BACKWARDS_READ, 0, - record, size_needed, &read_bytes, &size_needed); - REQUIRE(ok); - - REQUIRE(record->NumStrings == 1); - REQUIRE(record->EventType == expected_ev_type); - REQUIRE((expected_time_generated - record->TimeGenerated) <= 3u); - - std::string message_in_log(((char *)record + record->StringOffset)); - REQUIRE(message_in_log == expected_contents + spdlog::details::os::default_eol); -} - -TEST_CASE("eventlog", "[eventlog]") { - using namespace spdlog; - - auto test_sink = std::make_shared(TEST_SOURCE); - - spdlog::logger test_logger("eventlog", test_sink); - test_logger.set_level(level::trace); - - test_sink->set_pattern("%v"); - - test_single_print([&test_logger](std::string const &msg) { test_logger.trace(msg); }, - "my trace message", EVENTLOG_SUCCESS); - test_single_print([&test_logger](std::string const &msg) { test_logger.debug(msg); }, - "my debug message", EVENTLOG_SUCCESS); - test_single_print([&test_logger](std::string const &msg) { test_logger.info(msg); }, - "my info message", EVENTLOG_INFORMATION_TYPE); - test_single_print([&test_logger](std::string const &msg) { test_logger.warn(msg); }, - "my warn message", EVENTLOG_WARNING_TYPE); - test_single_print([&test_logger](std::string const &msg) { test_logger.error(msg); }, - "my error message", EVENTLOG_ERROR_TYPE); - test_single_print([&test_logger](std::string const &msg) { test_logger.critical(msg); }, - "my critical message", EVENTLOG_ERROR_TYPE); -} - -#endif //_WIN32 diff --git a/thirdparty/spdlog/tests/test_file_helper.cpp b/thirdparty/spdlog/tests/test_file_helper.cpp deleted file mode 100644 index 56ee75e3e..000000000 --- a/thirdparty/spdlog/tests/test_file_helper.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - * This content is released under the MIT License as specified in - * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE - */ -#include "includes.h" - -#define TEST_FILENAME "test_logs/file_helper_test.txt" - -using spdlog::details::file_helper; - -static void write_with_helper(file_helper &helper, size_t howmany) { - spdlog::memory_buf_t formatted; - spdlog::fmt_lib::format_to(std::back_inserter(formatted), "{}", std::string(howmany, '1')); - helper.write(formatted); - helper.flush(); -} - -TEST_CASE("file_helper_filename", "[file_helper::filename()]") { - prepare_logdir(); - - file_helper helper; - spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME); - helper.open(target_filename); - REQUIRE(helper.filename() == target_filename); -} - -TEST_CASE("file_helper_size", "[file_helper::size()]") { - prepare_logdir(); - spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME); - size_t expected_size = 123; - { - file_helper helper; - helper.open(target_filename); - write_with_helper(helper, expected_size); - REQUIRE(static_cast(helper.size()) == expected_size); - } - REQUIRE(get_filesize(TEST_FILENAME) == expected_size); -} - -TEST_CASE("file_helper_reopen", "[file_helper::reopen()]") { - prepare_logdir(); - spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME); - file_helper helper; - helper.open(target_filename); - write_with_helper(helper, 12); - REQUIRE(helper.size() == 12); - helper.reopen(true); - REQUIRE(helper.size() == 0); -} - -TEST_CASE("file_helper_reopen2", "[file_helper::reopen(false)]") { - prepare_logdir(); - spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME); - size_t expected_size = 14; - file_helper helper; - helper.open(target_filename); - write_with_helper(helper, expected_size); - REQUIRE(helper.size() == expected_size); - helper.reopen(false); - REQUIRE(helper.size() == expected_size); -} - -static void test_split_ext(const spdlog::filename_t::value_type *fname, - const spdlog::filename_t::value_type *expect_base, - const spdlog::filename_t::value_type *expect_ext) { - spdlog::filename_t filename(fname); - spdlog::filename_t expected_base(expect_base); - spdlog::filename_t expected_ext(expect_ext); - - spdlog::filename_t basename; - spdlog::filename_t ext; - std::tie(basename, ext) = file_helper::split_by_extension(filename); - REQUIRE(basename == expected_base); - REQUIRE(ext == expected_ext); -} - -TEST_CASE("file_helper_split_by_extension", "[file_helper::split_by_extension()]") { - test_split_ext(SPDLOG_FILENAME_T("mylog.txt"), SPDLOG_FILENAME_T("mylog"), - SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T(".mylog.txt"), SPDLOG_FILENAME_T(".mylog"), - SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T(".mylog"), SPDLOG_FILENAME_T(".mylog"), SPDLOG_FILENAME_T("")); - test_split_ext(SPDLOG_FILENAME_T("/aaa/bb.d/mylog"), SPDLOG_FILENAME_T("/aaa/bb.d/mylog"), - SPDLOG_FILENAME_T("")); - test_split_ext(SPDLOG_FILENAME_T("/aaa/bb.d/mylog.txt"), SPDLOG_FILENAME_T("/aaa/bb.d/mylog"), - SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T("aaa/bbb/ccc/mylog.txt"), - SPDLOG_FILENAME_T("aaa/bbb/ccc/mylog"), SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T("aaa/bbb/ccc/mylog."), SPDLOG_FILENAME_T("aaa/bbb/ccc/mylog."), - SPDLOG_FILENAME_T("")); - test_split_ext(SPDLOG_FILENAME_T("aaa/bbb/ccc/.mylog.txt"), - SPDLOG_FILENAME_T("aaa/bbb/ccc/.mylog"), SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T("/aaa/bbb/ccc/mylog.txt"), - SPDLOG_FILENAME_T("/aaa/bbb/ccc/mylog"), SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T("/aaa/bbb/ccc/.mylog"), - SPDLOG_FILENAME_T("/aaa/bbb/ccc/.mylog"), SPDLOG_FILENAME_T("")); - test_split_ext(SPDLOG_FILENAME_T("../mylog.txt"), SPDLOG_FILENAME_T("../mylog"), - SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T(".././mylog.txt"), SPDLOG_FILENAME_T(".././mylog"), - SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T(".././mylog.txt/xxx"), SPDLOG_FILENAME_T(".././mylog.txt/xxx"), - SPDLOG_FILENAME_T("")); - test_split_ext(SPDLOG_FILENAME_T("/mylog.txt"), SPDLOG_FILENAME_T("/mylog"), - SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T("//mylog.txt"), SPDLOG_FILENAME_T("//mylog"), - SPDLOG_FILENAME_T(".txt")); - test_split_ext(SPDLOG_FILENAME_T(""), SPDLOG_FILENAME_T(""), SPDLOG_FILENAME_T("")); - test_split_ext(SPDLOG_FILENAME_T("."), SPDLOG_FILENAME_T("."), SPDLOG_FILENAME_T("")); - test_split_ext(SPDLOG_FILENAME_T("..txt"), SPDLOG_FILENAME_T("."), SPDLOG_FILENAME_T(".txt")); -} - -TEST_CASE("file_event_handlers", "[file_helper]") { - enum class flags { before_open, after_open, before_close, after_close }; - prepare_logdir(); - - spdlog::filename_t test_filename = SPDLOG_FILENAME_T(TEST_FILENAME); - // define event handles that update vector of flags when called - std::vector events; - spdlog::file_event_handlers handlers; - handlers.before_open = [&](spdlog::filename_t filename) { - REQUIRE(filename == test_filename); - events.push_back(flags::before_open); - }; - handlers.after_open = [&](spdlog::filename_t filename, std::FILE *fstream) { - REQUIRE(filename == test_filename); - REQUIRE(fstream); - fputs("after_open\n", fstream); - events.push_back(flags::after_open); - }; - handlers.before_close = [&](spdlog::filename_t filename, std::FILE *fstream) { - REQUIRE(filename == test_filename); - REQUIRE(fstream); - fputs("before_close\n", fstream); - events.push_back(flags::before_close); - }; - handlers.after_close = [&](spdlog::filename_t filename) { - REQUIRE(filename == test_filename); - events.push_back(flags::after_close); - }; - { - spdlog::details::file_helper helper{handlers}; - REQUIRE(events.empty()); - - helper.open(test_filename); - REQUIRE(events == std::vector{flags::before_open, flags::after_open}); - - events.clear(); - helper.close(); - REQUIRE(events == std::vector{flags::before_close, flags::after_close}); - REQUIRE(file_contents(TEST_FILENAME) == "after_open\nbefore_close\n"); - - helper.reopen(true); - events.clear(); - } - // make sure that the file_helper destructor calls the close callbacks if needed - REQUIRE(events == std::vector{flags::before_close, flags::after_close}); - REQUIRE(file_contents(TEST_FILENAME) == "after_open\nbefore_close\n"); -} - -TEST_CASE("file_helper_open", "[file_helper]") { - prepare_logdir(); - spdlog::filename_t target_filename = SPDLOG_FILENAME_T(TEST_FILENAME); - file_helper helper; - helper.open(target_filename); - helper.close(); - - target_filename += SPDLOG_FILENAME_T("/invalid"); - REQUIRE_THROWS_AS(helper.open(target_filename), spdlog::spdlog_ex); -} diff --git a/thirdparty/spdlog/tests/test_file_logging.cpp b/thirdparty/spdlog/tests/test_file_logging.cpp deleted file mode 100644 index e3155effd..000000000 --- a/thirdparty/spdlog/tests/test_file_logging.cpp +++ /dev/null @@ -1,187 +0,0 @@ -/* - * This content is released under the MIT License as specified in - * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE - */ -#include "includes.h" - -#define SIMPLE_LOG "test_logs/simple_log" -#define ROTATING_LOG "test_logs/rotating_log" - -TEST_CASE("simple_file_logger", "[simple_logger]") { - prepare_logdir(); - spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG); - - auto logger = spdlog::create("logger", filename); - logger->set_pattern("%v"); - - logger->info("Test message {}", 1); - logger->info("Test message {}", 2); - - logger->flush(); - require_message_count(SIMPLE_LOG, 2); - using spdlog::details::os::default_eol; - REQUIRE(file_contents(SIMPLE_LOG) == - spdlog::fmt_lib::format("Test message 1{}Test message 2{}", default_eol, default_eol)); -} - -TEST_CASE("flush_on", "[flush_on]") { - prepare_logdir(); - spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG); - - auto logger = spdlog::create("logger", filename); - logger->set_pattern("%v"); - logger->set_level(spdlog::level::trace); - logger->flush_on(spdlog::level::info); - logger->trace("Should not be flushed"); - REQUIRE(count_lines(SIMPLE_LOG) == 0); - - logger->info("Test message {}", 1); - logger->info("Test message {}", 2); - - require_message_count(SIMPLE_LOG, 3); - using spdlog::details::os::default_eol; - REQUIRE(file_contents(SIMPLE_LOG) == - spdlog::fmt_lib::format("Should not be flushed{}Test message 1{}Test message 2{}", - default_eol, default_eol, default_eol)); -} - -TEST_CASE("simple_file_logger", "[truncate]") { - prepare_logdir(); - const spdlog::filename_t filename = SPDLOG_FILENAME_T(SIMPLE_LOG); - const bool truncate = true; - const auto sink = std::make_shared(filename, truncate); - const auto logger = std::make_shared("simple_file_logger", sink); - - logger->info("Test message {}", 3.14); - logger->info("Test message {}", 2.71); - logger->flush(); - REQUIRE(count_lines(SIMPLE_LOG) == 2); - - sink->truncate(); - REQUIRE(count_lines(SIMPLE_LOG) == 0); - - logger->info("Test message {}", 6.28); - logger->flush(); - REQUIRE(count_lines(SIMPLE_LOG) == 1); -} - -TEST_CASE("rotating_file_logger1", "[rotating_logger]") { - prepare_logdir(); - size_t max_size = 1024 * 10; - spdlog::filename_t basename = SPDLOG_FILENAME_T(ROTATING_LOG); - auto logger = spdlog::rotating_logger_mt("logger", basename, max_size, 0); - - for (int i = 0; i < 10; ++i) { - logger->info("Test message {}", i); - } - - logger->flush(); - require_message_count(ROTATING_LOG, 10); -} - -TEST_CASE("rotating_file_logger2", "[rotating_logger]") { - prepare_logdir(); - size_t max_size = 1024 * 10; - spdlog::filename_t basename = SPDLOG_FILENAME_T(ROTATING_LOG); - - { - // make an initial logger to create the first output file - auto logger = spdlog::rotating_logger_mt("logger", basename, max_size, 2, true); - for (int i = 0; i < 10; ++i) { - logger->info("Test message {}", i); - } - // drop causes the logger destructor to be called, which is required so the - // next logger can rename the first output file. - spdlog::drop(logger->name()); - } - auto logger = spdlog::rotating_logger_mt("logger", basename, max_size, 2, true); - for (int i = 0; i < 10; ++i) { - logger->info("Test message {}", i); - } - logger->flush(); - require_message_count(ROTATING_LOG, 10); - - for (int i = 0; i < 1000; i++) { - logger->info("Test message {}", i); - } - - logger->flush(); - REQUIRE(get_filesize(ROTATING_LOG) <= max_size); - REQUIRE(get_filesize(ROTATING_LOG ".1") <= max_size); -} - -// test that passing max_size=0 throws -TEST_CASE("rotating_file_logger3", "[rotating_logger]") { - prepare_logdir(); - size_t max_size = 0; - spdlog::filename_t basename = SPDLOG_FILENAME_T(ROTATING_LOG); - REQUIRE_THROWS_AS(spdlog::rotating_logger_mt("logger", basename, max_size, 0), - spdlog::spdlog_ex); -} - -// test on-demand rotation of logs -TEST_CASE("rotating_file_logger4", "[rotating_logger]") { - prepare_logdir(); - size_t max_size = 1024 * 10; - spdlog::filename_t basename = SPDLOG_FILENAME_T(ROTATING_LOG); - auto sink = std::make_shared(basename, max_size, 2); - auto logger = std::make_shared("rotating_sink_logger", sink); - - logger->info("Test message - pre-rotation"); - logger->flush(); - - sink->rotate_now(); - - logger->info("Test message - post-rotation"); - logger->flush(); - - REQUIRE(get_filesize(ROTATING_LOG) > 0); - REQUIRE(get_filesize(ROTATING_LOG ".1") > 0); -} - -// test changing the max size of the rotating file sink -TEST_CASE("rotating_file_logger5", "[rotating_logger]") { - prepare_logdir(); - size_t max_size = 5 * 1024; - size_t max_files = 2; - spdlog::filename_t basename = SPDLOG_FILENAME_T(ROTATING_LOG); - auto sink = - std::make_shared(basename, max_size, max_files); - auto logger = std::make_shared("rotating_sink_logger", sink); - logger->set_pattern("%v"); - - REQUIRE(sink->get_max_size() == max_size); - REQUIRE(sink->get_max_files() == max_files); - max_size = 7 * 1024; - max_files = 3; - - sink->set_max_size(max_size); - sink->set_max_files(max_files); - REQUIRE(sink->get_max_size() == max_size); - REQUIRE(sink->get_max_files() == max_files); - - const auto message = std::string(200, 'x'); - assert(message.size() < max_size); - const auto n_messages = max_files * max_size / message.size(); - for (size_t i = 0; i < n_messages; ++i) { - logger->info(message); - } - logger.reset(); // force flush and close the file - - // validate that the files were rotated correctly with the new max size and max files - for (size_t i = 0; i <= max_files; i++) { - // calc filenames - // e.g. rotating_log, rotating_log.0 rotating_log.1, rotating_log.2, etc. - std::ostringstream oss; - oss << ROTATING_LOG; - if (i > 0) { - oss << '.' << i; - } - const auto filename = oss.str(); - const auto filesize = get_filesize(filename); - REQUIRE(filesize <= max_size); - if (i > 0) { - REQUIRE(filesize >= max_size - message.size() - 2); - } - } -} diff --git a/thirdparty/spdlog/tests/test_fmt_helper.cpp b/thirdparty/spdlog/tests/test_fmt_helper.cpp deleted file mode 100644 index 31b930672..000000000 --- a/thirdparty/spdlog/tests/test_fmt_helper.cpp +++ /dev/null @@ -1,82 +0,0 @@ - -#include "includes.h" - -using spdlog::memory_buf_t; -using spdlog::details::to_string_view; - -void test_pad2(int n, const char *expected) { - memory_buf_t buf; - spdlog::details::fmt_helper::pad2(n, buf); - - REQUIRE(to_string_view(buf) == expected); -} - -void test_pad3(uint32_t n, const char *expected) { - memory_buf_t buf; - spdlog::details::fmt_helper::pad3(n, buf); - - REQUIRE(to_string_view(buf) == expected); -} - -void test_pad6(std::size_t n, const char *expected) { - memory_buf_t buf; - spdlog::details::fmt_helper::pad6(n, buf); - - REQUIRE(to_string_view(buf) == expected); -} - -void test_pad9(std::size_t n, const char *expected) { - memory_buf_t buf; - spdlog::details::fmt_helper::pad9(n, buf); - - REQUIRE(to_string_view(buf) == expected); -} - -TEST_CASE("pad2", "[fmt_helper]") { - test_pad2(0, "00"); - test_pad2(3, "03"); - test_pad2(10, "10"); - test_pad2(23, "23"); - test_pad2(99, "99"); - test_pad2(100, "100"); - test_pad2(123, "123"); - test_pad2(1234, "1234"); - test_pad2(-5, "-5"); -} - -TEST_CASE("pad3", "[fmt_helper]") { - test_pad3(0, "000"); - test_pad3(3, "003"); - test_pad3(10, "010"); - test_pad3(23, "023"); - test_pad3(99, "099"); - test_pad3(100, "100"); - test_pad3(123, "123"); - test_pad3(999, "999"); - test_pad3(1000, "1000"); - test_pad3(1234, "1234"); -} - -TEST_CASE("pad6", "[fmt_helper]") { - test_pad6(0, "000000"); - test_pad6(3, "000003"); - test_pad6(23, "000023"); - test_pad6(123, "000123"); - test_pad6(1234, "001234"); - test_pad6(12345, "012345"); - test_pad6(123456, "123456"); -} - -TEST_CASE("pad9", "[fmt_helper]") { - test_pad9(0, "000000000"); - test_pad9(3, "000000003"); - test_pad9(23, "000000023"); - test_pad9(123, "000000123"); - test_pad9(1234, "000001234"); - test_pad9(12345, "000012345"); - test_pad9(123456, "000123456"); - test_pad9(1234567, "001234567"); - test_pad9(12345678, "012345678"); - test_pad9(123456789, "123456789"); - test_pad9(1234567891, "1234567891"); -} diff --git a/thirdparty/spdlog/tests/test_macros.cpp b/thirdparty/spdlog/tests/test_macros.cpp deleted file mode 100644 index 132706f13..000000000 --- a/thirdparty/spdlog/tests/test_macros.cpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * This content is released under the MIT License as specified in - * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE - */ - -#include "includes.h" - -#if SPDLOG_ACTIVE_LEVEL != SPDLOG_LEVEL_DEBUG - #error "Invalid SPDLOG_ACTIVE_LEVEL in test. Should be SPDLOG_LEVEL_DEBUG" -#endif - -#define TEST_FILENAME "test_logs/simple_log" - -TEST_CASE("debug and trace w/o format string", "[macros]") { - prepare_logdir(); - spdlog::filename_t filename = SPDLOG_FILENAME_T(TEST_FILENAME); - - auto logger = spdlog::create("logger", filename); - logger->set_pattern("%v"); - logger->set_level(spdlog::level::trace); - - SPDLOG_LOGGER_TRACE(logger, "Test message 1"); - SPDLOG_LOGGER_DEBUG(logger, "Test message 2"); - logger->flush(); - - using spdlog::details::os::default_eol; - REQUIRE(ends_with(file_contents(TEST_FILENAME), - spdlog::fmt_lib::format("Test message 2{}", default_eol))); - REQUIRE(count_lines(TEST_FILENAME) == 1); - - auto orig_default_logger = spdlog::default_logger(); - spdlog::set_default_logger(logger); - - SPDLOG_TRACE("Test message 3"); - SPDLOG_DEBUG("Test message {}", 4); - logger->flush(); - - require_message_count(TEST_FILENAME, 2); - REQUIRE(ends_with(file_contents(TEST_FILENAME), - spdlog::fmt_lib::format("Test message 4{}", default_eol))); - spdlog::set_default_logger(std::move(orig_default_logger)); -} - -TEST_CASE("disable param evaluation", "[macros]") { - SPDLOG_TRACE("Test message {}", throw std::runtime_error("Should not be evaluated")); -} - -TEST_CASE("pass logger pointer", "[macros]") { - auto logger = spdlog::create("refmacro"); - auto &ref = *logger; - SPDLOG_LOGGER_TRACE(&ref, "Test message 1"); - SPDLOG_LOGGER_DEBUG(&ref, "Test message 2"); -} diff --git a/thirdparty/spdlog/tests/test_misc.cpp b/thirdparty/spdlog/tests/test_misc.cpp deleted file mode 100644 index deb18e745..000000000 --- a/thirdparty/spdlog/tests/test_misc.cpp +++ /dev/null @@ -1,224 +0,0 @@ -#ifdef _WIN32 // to prevent fopen warning on windows - #define _CRT_SECURE_NO_WARNINGS -#endif - -#include "includes.h" -#include "test_sink.h" - -template -std::string log_info(const T& what, spdlog::level::level_enum logger_level = spdlog::level::info) { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - - spdlog::logger oss_logger("oss", oss_sink); - oss_logger.set_level(logger_level); - oss_logger.set_pattern("%v"); - oss_logger.info(what); - - return oss.str().substr(0, oss.str().length() - strlen(spdlog::details::os::default_eol)); -} - -TEST_CASE("basic_logging ", "[basic_logging]") { - // const char - REQUIRE(log_info("Hello") == "Hello"); - REQUIRE(log_info("").empty()); - - // std::string - REQUIRE(log_info(std::string("Hello")) == "Hello"); - REQUIRE(log_info(std::string()).empty()); - - // Numbers - REQUIRE(log_info(5) == "5"); - REQUIRE(log_info(5.6) == "5.6"); - - // User defined class - // REQUIRE(log_info(some_logged_class("some_val")) == "some_val"); -} - -TEST_CASE("log_levels", "[log_levels]") { - REQUIRE(log_info("Hello", spdlog::level::err).empty()); - REQUIRE(log_info("Hello", spdlog::level::critical).empty()); - REQUIRE(log_info("Hello", spdlog::level::info) == "Hello"); - REQUIRE(log_info("Hello", spdlog::level::debug) == "Hello"); - REQUIRE(log_info("Hello", spdlog::level::trace) == "Hello"); -} - -TEST_CASE("level_to_string_view", "[convert_to_string_view]") { - REQUIRE(spdlog::level::to_string_view(spdlog::level::trace) == "trace"); - REQUIRE(spdlog::level::to_string_view(spdlog::level::debug) == "debug"); - REQUIRE(spdlog::level::to_string_view(spdlog::level::info) == "info"); - REQUIRE(spdlog::level::to_string_view(spdlog::level::warn) == "warning"); - REQUIRE(spdlog::level::to_string_view(spdlog::level::err) == "error"); - REQUIRE(spdlog::level::to_string_view(spdlog::level::critical) == "critical"); - REQUIRE(spdlog::level::to_string_view(spdlog::level::off) == "off"); -} - -TEST_CASE("to_short_c_str", "[convert_to_short_c_str]") { - REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::trace)) == "T"); - REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::debug)) == "D"); - REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::info)) == "I"); - REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::warn)) == "W"); - REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::err)) == "E"); - REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::critical)) == "C"); - REQUIRE(std::string(spdlog::level::to_short_c_str(spdlog::level::off)) == "O"); -} - -TEST_CASE("to_level_enum", "[convert_to_level_enum]") { - REQUIRE(spdlog::level::from_str("trace") == spdlog::level::trace); - REQUIRE(spdlog::level::from_str("debug") == spdlog::level::debug); - REQUIRE(spdlog::level::from_str("info") == spdlog::level::info); - REQUIRE(spdlog::level::from_str("warning") == spdlog::level::warn); - REQUIRE(spdlog::level::from_str("warn") == spdlog::level::warn); - REQUIRE(spdlog::level::from_str("error") == spdlog::level::err); - REQUIRE(spdlog::level::from_str("critical") == spdlog::level::critical); - REQUIRE(spdlog::level::from_str("off") == spdlog::level::off); - REQUIRE(spdlog::level::from_str("null") == spdlog::level::off); -} - -TEST_CASE("periodic flush", "[periodic_flush]") { - using spdlog::sinks::test_sink_mt; - auto logger = spdlog::create("periodic_flush"); - auto test_sink = std::static_pointer_cast(logger->sinks()[0]); - - spdlog::flush_every(std::chrono::seconds(1)); - std::this_thread::sleep_for(std::chrono::milliseconds(1250)); - REQUIRE(test_sink->flush_counter() == 1); - spdlog::flush_every(std::chrono::seconds(0)); - spdlog::drop_all(); -} - -TEST_CASE("clone-logger", "[clone]") { - using spdlog::sinks::test_sink_mt; - auto test_sink = std::make_shared(); - auto logger = std::make_shared("orig", test_sink); - logger->set_pattern("%v"); - auto cloned = logger->clone("clone"); - - REQUIRE(cloned->name() == "clone"); - REQUIRE(logger->sinks() == cloned->sinks()); - REQUIRE(logger->level() == cloned->level()); - REQUIRE(logger->flush_level() == cloned->flush_level()); - logger->info("Some message 1"); - cloned->info("Some message 2"); - - REQUIRE(test_sink->lines().size() == 2); - REQUIRE(test_sink->lines()[0] == "Some message 1"); - REQUIRE(test_sink->lines()[1] == "Some message 2"); - - spdlog::drop_all(); -} - -TEST_CASE("clone async", "[clone]") { - using spdlog::sinks::test_sink_mt; - spdlog::init_thread_pool(4, 1); - auto test_sink = std::make_shared(); - auto logger = std::make_shared("orig", test_sink, spdlog::thread_pool()); - logger->set_pattern("%v"); - auto cloned = logger->clone("clone"); - - REQUIRE(cloned->name() == "clone"); - REQUIRE(logger->sinks() == cloned->sinks()); - REQUIRE(logger->level() == cloned->level()); - REQUIRE(logger->flush_level() == cloned->flush_level()); - - logger->info("Some message 1"); - cloned->info("Some message 2"); - - spdlog::details::os::sleep_for_millis(100); - - REQUIRE(test_sink->lines().size() == 2); - REQUIRE(test_sink->lines()[0] == "Some message 1"); - REQUIRE(test_sink->lines()[1] == "Some message 2"); - - spdlog::drop_all(); -} - -TEST_CASE("default logger API", "[default logger]") { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - - spdlog::set_default_logger(std::make_shared("oss", oss_sink)); - spdlog::set_pattern("*** %v"); - - spdlog::default_logger()->set_level(spdlog::level::trace); - spdlog::trace("hello trace"); - REQUIRE(oss.str() == "*** hello trace" + std::string(spdlog::details::os::default_eol)); - - oss.str(""); - spdlog::debug("hello debug"); - REQUIRE(oss.str() == "*** hello debug" + std::string(spdlog::details::os::default_eol)); - - oss.str(""); - spdlog::info("Hello"); - REQUIRE(oss.str() == "*** Hello" + std::string(spdlog::details::os::default_eol)); - - oss.str(""); - spdlog::warn("Hello again {}", 2); - REQUIRE(oss.str() == "*** Hello again 2" + std::string(spdlog::details::os::default_eol)); - - oss.str(""); - spdlog::error(123); - REQUIRE(oss.str() == "*** 123" + std::string(spdlog::details::os::default_eol)); - - oss.str(""); - spdlog::critical(std::string("some string")); - REQUIRE(oss.str() == "*** some string" + std::string(spdlog::details::os::default_eol)); - - oss.str(""); - spdlog::set_level(spdlog::level::info); - spdlog::debug("should not be logged"); - REQUIRE(oss.str().empty()); - spdlog::drop_all(); - spdlog::set_pattern("%v"); -} - -#if (defined(SPDLOG_WCHAR_TO_UTF8_SUPPORT) || defined(SPDLOG_WCHAR_FILENAMES)) && defined(_WIN32) -TEST_CASE("utf8 to utf16 conversion using windows api", "[windows utf]") { - spdlog::wmemory_buf_t buffer; - - spdlog::details::os::utf8_to_wstrbuf("", buffer); - REQUIRE(std::wstring(buffer.data(), buffer.size()) == std::wstring(L"")); - - spdlog::details::os::utf8_to_wstrbuf("abc", buffer); - REQUIRE(std::wstring(buffer.data(), buffer.size()) == std::wstring(L"abc")); - - spdlog::details::os::utf8_to_wstrbuf("\xc3\x28", buffer); // Invalid UTF-8 sequence. - REQUIRE(std::wstring(buffer.data(), buffer.size()) == std::wstring(L"\xfffd(")); - - spdlog::details::os::utf8_to_wstrbuf("\xe3\x81\xad\xe3\x81\x93", - buffer); // "Neko" in hiragana. - REQUIRE(std::wstring(buffer.data(), buffer.size()) == std::wstring(L"\x306d\x3053")); -} -#endif - -struct auto_closer { - FILE* fp = nullptr; - explicit auto_closer(FILE* f) - : fp(f) {} - auto_closer(const auto_closer&) = delete; - auto_closer& operator=(const auto_closer&) = delete; - ~auto_closer() { - if (fp != nullptr) (void)std::fclose(fp); - } -}; - -TEST_CASE("os::fwrite_bytes", "[os]") { - using spdlog::details::os::create_dir; - using spdlog::details::os::fwrite_bytes; - const char* filename = "log_tests/test_fwrite_bytes.txt"; - const char* msg = "hello"; - prepare_logdir(); - REQUIRE(create_dir(SPDLOG_FILENAME_T("log_tests")) == true); - { - auto_closer closer(std::fopen(filename, "wb")); - REQUIRE(closer.fp != nullptr); - REQUIRE(fwrite_bytes(msg, std::strlen(msg), closer.fp) == true); - REQUIRE(fwrite_bytes(msg, 0, closer.fp) == true); - std::fflush(closer.fp); - REQUIRE(spdlog::details::os::filesize(closer.fp) == 5); - } - // fwrite_bytes should return false on write failure - auto_closer closer(std::fopen(filename, "r")); - REQUIRE(closer.fp != nullptr); - REQUIRE_FALSE(fwrite_bytes("Hello", 5, closer.fp)); -} diff --git a/thirdparty/spdlog/tests/test_mpmc_q.cpp b/thirdparty/spdlog/tests/test_mpmc_q.cpp deleted file mode 100644 index bc7a37d9c..000000000 --- a/thirdparty/spdlog/tests/test_mpmc_q.cpp +++ /dev/null @@ -1,114 +0,0 @@ -#include "includes.h" - -using std::chrono::milliseconds; -using test_clock = std::chrono::high_resolution_clock; - -static milliseconds millis_from(const test_clock::time_point &tp0) { - return std::chrono::duration_cast(test_clock::now() - tp0); -} -TEST_CASE("dequeue-empty-nowait", "[mpmc_blocking_q]") { - size_t q_size = 100; - milliseconds tolerance_wait(20); - spdlog::details::mpmc_blocking_queue q(q_size); - int popped_item = 0; - - auto start = test_clock::now(); - auto rv = q.dequeue_for(popped_item, milliseconds::zero()); - auto delta_ms = millis_from(start); - - REQUIRE(rv == false); - INFO("Delta " << delta_ms.count() << " millis"); - REQUIRE(delta_ms <= tolerance_wait); -} - -TEST_CASE("dequeue-empty-wait", "[mpmc_blocking_q]") { - size_t q_size = 100; - milliseconds wait_ms(250); - milliseconds tolerance_wait(250); - - spdlog::details::mpmc_blocking_queue q(q_size); - int popped_item = 0; - auto start = test_clock::now(); - auto rv = q.dequeue_for(popped_item, wait_ms); - auto delta_ms = millis_from(start); - - REQUIRE(rv == false); - - INFO("Delta " << delta_ms.count() << " millis"); - REQUIRE(delta_ms >= wait_ms - tolerance_wait); - REQUIRE(delta_ms <= wait_ms + tolerance_wait); -} - -TEST_CASE("dequeue-full-nowait", "[mpmc_blocking_q]") { - spdlog::details::mpmc_blocking_queue q(1); - q.enqueue(42); - - int item = 0; - q.dequeue_for(item, milliseconds::zero()); - REQUIRE(item == 42); -} - -TEST_CASE("dequeue-full-wait", "[mpmc_blocking_q]") { - spdlog::details::mpmc_blocking_queue q(1); - q.enqueue(42); - - int item = 0; - q.dequeue(item); - REQUIRE(item == 42); -} - -TEST_CASE("enqueue_nowait", "[mpmc_blocking_q]") { - size_t q_size = 1; - spdlog::details::mpmc_blocking_queue q(q_size); - milliseconds tolerance_wait(10); - - q.enqueue(1); - REQUIRE(q.overrun_counter() == 0); - - auto start = test_clock::now(); - q.enqueue_nowait(2); - auto delta_ms = millis_from(start); - - INFO("Delta " << delta_ms.count() << " millis"); - REQUIRE(delta_ms <= tolerance_wait); - REQUIRE(q.overrun_counter() == 1); -} - -TEST_CASE("bad_queue", "[mpmc_blocking_q]") { - size_t q_size = 0; - spdlog::details::mpmc_blocking_queue q(q_size); - q.enqueue_nowait(1); - REQUIRE(q.overrun_counter() == 1); - int i = 0; - REQUIRE(q.dequeue_for(i, milliseconds(0)) == false); -} - -TEST_CASE("empty_queue", "[mpmc_blocking_q]") { - size_t q_size = 10; - spdlog::details::mpmc_blocking_queue q(q_size); - int i = 0; - REQUIRE(q.dequeue_for(i, milliseconds(10)) == false); -} - -TEST_CASE("full_queue", "[mpmc_blocking_q]") { - size_t q_size = 100; - spdlog::details::mpmc_blocking_queue q(q_size); - for (int i = 0; i < static_cast(q_size); i++) { - q.enqueue(i + 0); // i+0 to force rvalue and avoid tidy warnings on the same time if we - // std::move(i) instead - } - - q.enqueue_nowait(123456); - REQUIRE(q.overrun_counter() == 1); - - for (int i = 1; i < static_cast(q_size); i++) { - int item = -1; - q.dequeue(item); - REQUIRE(item == i); - } - - // last item pushed has overridden the oldest. - int item = -1; - q.dequeue(item); - REQUIRE(item == 123456); -} diff --git a/thirdparty/spdlog/tests/test_pattern_formatter.cpp b/thirdparty/spdlog/tests/test_pattern_formatter.cpp deleted file mode 100644 index 17a1bbcb5..000000000 --- a/thirdparty/spdlog/tests/test_pattern_formatter.cpp +++ /dev/null @@ -1,660 +0,0 @@ -#include "includes.h" -#include "test_sink.h" - -#include - -using spdlog::memory_buf_t; -using spdlog::details::to_string_view; - -// log to str and return it -template -static std::string log_to_str(const std::string &msg, const Args &...args) { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - spdlog::logger oss_logger("pattern_tester", oss_sink); - oss_logger.set_level(spdlog::level::info); - - oss_logger.set_formatter( - std::unique_ptr(new spdlog::pattern_formatter(args...))); - - oss_logger.info(msg); - return oss.str(); -} - -// log to str and return it with time -template -static std::string log_to_str_with_time(spdlog::log_clock::time_point log_time, - const std::string &msg, - const Args &...args) { - std::ostringstream oss; - auto oss_sink = std::make_shared(oss); - spdlog::logger oss_logger("pattern_tester", oss_sink); - oss_logger.set_level(spdlog::level::info); - - oss_logger.set_formatter( - std::unique_ptr(new spdlog::pattern_formatter(args...))); - - oss_logger.log(log_time, {}, spdlog::level::info, msg); - return oss.str(); -} - -TEST_CASE("custom eol", "[pattern_formatter]") { - std::string msg = "Hello custom eol test"; - std::string eol = ";)"; - REQUIRE(log_to_str(msg, "%v", spdlog::pattern_time_type::local, ";)") == msg + eol); -} - -TEST_CASE("empty format", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "", spdlog::pattern_time_type::local, "").empty()); -} - -TEST_CASE("empty format2", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "", spdlog::pattern_time_type::local, "\n") == "\n"); -} - -TEST_CASE("level", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%l] %v", spdlog::pattern_time_type::local, "\n") == - "[info] Some message\n"); -} - -TEST_CASE("short level", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%L] %v", spdlog::pattern_time_type::local, "\n") == - "[I] Some message\n"); -} - -TEST_CASE("name", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%n] %v", spdlog::pattern_time_type::local, "\n") == - "[pattern_tester] Some message\n"); -} - -TEST_CASE("date MM/DD/YY ", "[pattern_formatter]") { - auto now_tm = spdlog::details::os::localtime(); - std::stringstream oss; - oss << std::setfill('0') << std::setw(2) << now_tm.tm_mon + 1 << "/" << std::setw(2) - << now_tm.tm_mday << "/" << std::setw(2) << (now_tm.tm_year + 1900) % 1000 - << " Some message\n"; - REQUIRE(log_to_str("Some message", "%D %v", spdlog::pattern_time_type::local, "\n") == - oss.str()); -} - -TEST_CASE("GMT offset ", "[pattern_formatter]") { - using namespace std::chrono_literals; - const auto now = std::chrono::system_clock::now(); - const auto yesterday = now - 24h; - - REQUIRE(log_to_str_with_time(yesterday, "Some message", "%z", spdlog::pattern_time_type::utc, - "\n") == "+00:00\n"); -} - -TEST_CASE("color range test1", "[pattern_formatter]") { - auto formatter = std::make_shared( - "%^%v%$", spdlog::pattern_time_type::local, "\n"); - - memory_buf_t buf; - spdlog::fmt_lib::format_to(std::back_inserter(buf), "Hello"); - memory_buf_t formatted; - std::string logger_name = "test"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, - spdlog::string_view_t(buf.data(), buf.size())); - formatter->format(msg, formatted); - REQUIRE(msg.color_range_start == 0); - REQUIRE(msg.color_range_end == 5); - REQUIRE(log_to_str("hello", "%^%v%$", spdlog::pattern_time_type::local, "\n") == "hello\n"); -} - -TEST_CASE("color range test2", "[pattern_formatter]") { - auto formatter = - std::make_shared("%^%$", spdlog::pattern_time_type::local, "\n"); - std::string logger_name = "test"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, ""); - memory_buf_t formatted; - formatter->format(msg, formatted); - REQUIRE(msg.color_range_start == 0); - REQUIRE(msg.color_range_end == 0); - REQUIRE(log_to_str("", "%^%$", spdlog::pattern_time_type::local, "\n") == "\n"); -} - -TEST_CASE("color range test3", "[pattern_formatter]") { - auto formatter = std::make_shared("%^***%$"); - std::string logger_name = "test"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, "ignored"); - memory_buf_t formatted; - formatter->format(msg, formatted); - REQUIRE(msg.color_range_start == 0); - REQUIRE(msg.color_range_end == 3); -} - -TEST_CASE("color range test4", "[pattern_formatter]") { - auto formatter = std::make_shared( - "XX%^YYY%$", spdlog::pattern_time_type::local, "\n"); - std::string logger_name = "test"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, "ignored"); - - memory_buf_t formatted; - formatter->format(msg, formatted); - REQUIRE(msg.color_range_start == 2); - REQUIRE(msg.color_range_end == 5); - REQUIRE(log_to_str("ignored", "XX%^YYY%$", spdlog::pattern_time_type::local, "\n") == - "XXYYY\n"); -} - -TEST_CASE("color range test5", "[pattern_formatter]") { - auto formatter = std::make_shared("**%^"); - std::string logger_name = "test"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, "ignored"); - memory_buf_t formatted; - formatter->format(msg, formatted); - REQUIRE(msg.color_range_start == 2); - REQUIRE(msg.color_range_end == 0); -} - -TEST_CASE("color range test6", "[pattern_formatter]") { - auto formatter = std::make_shared("**%$"); - std::string logger_name = "test"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, "ignored"); - memory_buf_t formatted; - formatter->format(msg, formatted); - REQUIRE(msg.color_range_start == 0); - REQUIRE(msg.color_range_end == 2); -} - -// -// Test padding -// - -TEST_CASE("level_left_padded", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%8l] %v", spdlog::pattern_time_type::local, "\n") == - "[ info] Some message\n"); - REQUIRE(log_to_str("Some message", "[%8!l] %v", spdlog::pattern_time_type::local, "\n") == - "[ info] Some message\n"); -} - -TEST_CASE("level_right_padded", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%-8l] %v", spdlog::pattern_time_type::local, "\n") == - "[info ] Some message\n"); - REQUIRE(log_to_str("Some message", "[%-8!l] %v", spdlog::pattern_time_type::local, "\n") == - "[info ] Some message\n"); -} - -TEST_CASE("level_center_padded", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%=8l] %v", spdlog::pattern_time_type::local, "\n") == - "[ info ] Some message\n"); - REQUIRE(log_to_str("Some message", "[%=8!l] %v", spdlog::pattern_time_type::local, "\n") == - "[ info ] Some message\n"); -} - -TEST_CASE("short level_left_padded", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%3L] %v", spdlog::pattern_time_type::local, "\n") == - "[ I] Some message\n"); - REQUIRE(log_to_str("Some message", "[%3!L] %v", spdlog::pattern_time_type::local, "\n") == - "[ I] Some message\n"); -} - -TEST_CASE("short level_right_padded", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%-3L] %v", spdlog::pattern_time_type::local, "\n") == - "[I ] Some message\n"); - REQUIRE(log_to_str("Some message", "[%-3!L] %v", spdlog::pattern_time_type::local, "\n") == - "[I ] Some message\n"); -} - -TEST_CASE("short level_center_padded", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%=3L] %v", spdlog::pattern_time_type::local, "\n") == - "[ I ] Some message\n"); - REQUIRE(log_to_str("Some message", "[%=3!L] %v", spdlog::pattern_time_type::local, "\n") == - "[ I ] Some message\n"); -} - -TEST_CASE("left_padded_short", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%3n] %v", spdlog::pattern_time_type::local, "\n") == - "[pattern_tester] Some message\n"); - REQUIRE(log_to_str("Some message", "[%3!n] %v", spdlog::pattern_time_type::local, "\n") == - "[pat] Some message\n"); -} - -TEST_CASE("right_padded_short", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%-3n] %v", spdlog::pattern_time_type::local, "\n") == - "[pattern_tester] Some message\n"); - REQUIRE(log_to_str("Some message", "[%-3!n] %v", spdlog::pattern_time_type::local, "\n") == - "[pat] Some message\n"); -} - -TEST_CASE("center_padded_short", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%=3n] %v", spdlog::pattern_time_type::local, "\n") == - "[pattern_tester] Some message\n"); - REQUIRE(log_to_str("Some message", "[%=3!n] %v", spdlog::pattern_time_type::local, "\n") == - "[pat] Some message\n"); -} - -TEST_CASE("left_padded_huge", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%-300n] %v", spdlog::pattern_time_type::local, "\n") == - "[pattern_tester ] Some message\n"); - - REQUIRE(log_to_str("Some message", "[%-300!n] %v", spdlog::pattern_time_type::local, "\n") == - "[pattern_tester ] Some message\n"); -} - -TEST_CASE("left_padded_max", "[pattern_formatter]") { - REQUIRE(log_to_str("Some message", "[%-64n] %v", spdlog::pattern_time_type::local, "\n") == - "[pattern_tester ] Some message\n"); - - REQUIRE(log_to_str("Some message", "[%-64!n] %v", spdlog::pattern_time_type::local, "\n") == - "[pattern_tester ] Some message\n"); -} - -// Test padding + truncate flag - -TEST_CASE("paddinng_truncate", "[pattern_formatter]") { - REQUIRE(log_to_str("123456", "%6!v", spdlog::pattern_time_type::local, "\n") == "123456\n"); - REQUIRE(log_to_str("123456", "%5!v", spdlog::pattern_time_type::local, "\n") == "12345\n"); - REQUIRE(log_to_str("123456", "%7!v", spdlog::pattern_time_type::local, "\n") == " 123456\n"); - - REQUIRE(log_to_str("123456", "%-6!v", spdlog::pattern_time_type::local, "\n") == "123456\n"); - REQUIRE(log_to_str("123456", "%-5!v", spdlog::pattern_time_type::local, "\n") == "12345\n"); - REQUIRE(log_to_str("123456", "%-7!v", spdlog::pattern_time_type::local, "\n") == "123456 \n"); - - REQUIRE(log_to_str("123456", "%=6!v", spdlog::pattern_time_type::local, "\n") == "123456\n"); - REQUIRE(log_to_str("123456", "%=5!v", spdlog::pattern_time_type::local, "\n") == "12345\n"); - REQUIRE(log_to_str("123456", "%=7!v", spdlog::pattern_time_type::local, "\n") == "123456 \n"); - - REQUIRE(log_to_str("123456", "%0!v", spdlog::pattern_time_type::local, "\n") == "\n"); -} - -TEST_CASE("padding_truncate_funcname", "[pattern_formatter]") { - spdlog::sinks::test_sink_st test_sink; - - const char *pattern = "%v [%5!!]"; - auto formatter = std::unique_ptr(new spdlog::pattern_formatter(pattern)); - test_sink.set_formatter(std::move(formatter)); - - spdlog::details::log_msg msg1{spdlog::source_loc{"ignored", 1, "func"}, "test_logger", - spdlog::level::info, "message"}; - test_sink.log(msg1); - REQUIRE(test_sink.lines()[0] == "message [ func]"); - - spdlog::details::log_msg msg2{spdlog::source_loc{"ignored", 1, "function"}, "test_logger", - spdlog::level::info, "message"}; - test_sink.log(msg2); - REQUIRE(test_sink.lines()[1] == "message [funct]"); -} - -TEST_CASE("padding_funcname", "[pattern_formatter]") { - spdlog::sinks::test_sink_st test_sink; - - const char *pattern = "%v [%10!]"; - auto formatter = std::unique_ptr(new spdlog::pattern_formatter(pattern)); - test_sink.set_formatter(std::move(formatter)); - - spdlog::details::log_msg msg1{spdlog::source_loc{"ignored", 1, "func"}, "test_logger", - spdlog::level::info, "message"}; - test_sink.log(msg1); - REQUIRE(test_sink.lines()[0] == "message [ func]"); - - spdlog::details::log_msg msg2{spdlog::source_loc{"ignored", 1, "func567890123"}, "test_logger", - spdlog::level::info, "message"}; - test_sink.log(msg2); - REQUIRE(test_sink.lines()[1] == "message [func567890123]"); -} - -TEST_CASE("clone-default-formatter", "[pattern_formatter]") { - auto formatter_1 = std::make_shared(); - auto formatter_2 = formatter_1->clone(); - std::string logger_name = "test"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message"); - - memory_buf_t formatted_1; - memory_buf_t formatted_2; - formatter_1->format(msg, formatted_1); - formatter_2->format(msg, formatted_2); - - REQUIRE(to_string_view(formatted_1) == to_string_view(formatted_2)); -} - -TEST_CASE("clone-default-formatter2", "[pattern_formatter]") { - auto formatter_1 = std::make_shared("%+"); - auto formatter_2 = formatter_1->clone(); - std::string logger_name = "test"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message"); - - memory_buf_t formatted_1; - memory_buf_t formatted_2; - formatter_1->format(msg, formatted_1); - formatter_2->format(msg, formatted_2); - - REQUIRE(to_string_view(formatted_1) == to_string_view(formatted_2)); -} - -TEST_CASE("clone-formatter", "[pattern_formatter]") { - auto formatter_1 = std::make_shared("%D %X [%] [%n] %v"); - auto formatter_2 = formatter_1->clone(); - std::string logger_name = "test"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message"); - - memory_buf_t formatted_1; - memory_buf_t formatted_2; - formatter_1->format(msg, formatted_1); - formatter_2->format(msg, formatted_2); - - REQUIRE(to_string_view(formatted_1) == to_string_view(formatted_2)); -} - -TEST_CASE("clone-formatter-2", "[pattern_formatter]") { - using spdlog::pattern_time_type; - auto formatter_1 = std::make_shared( - "%D %X [%] [%n] %v", pattern_time_type::utc, "xxxxxx\n"); - auto formatter_2 = formatter_1->clone(); - std::string logger_name = "test2"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message"); - - memory_buf_t formatted_1; - memory_buf_t formatted_2; - formatter_1->format(msg, formatted_1); - formatter_2->format(msg, formatted_2); - - REQUIRE(to_string_view(formatted_1) == to_string_view(formatted_2)); -} - -class custom_test_flag : public spdlog::custom_flag_formatter { -public: - explicit custom_test_flag(std::string txt) - : some_txt{std::move(txt)} {} - - void format(const spdlog::details::log_msg &, - const std::tm &tm, - spdlog::memory_buf_t &dest) override { - if (some_txt == "throw_me") { - throw spdlog::spdlog_ex("custom_flag_exception_test"); - } else if (some_txt == "time") { - auto formatted = spdlog::fmt_lib::format("{:d}:{:02d}{:s}", tm.tm_hour % 12, tm.tm_min, - tm.tm_hour / 12 ? "PM" : "AM"); - dest.append(formatted.data(), formatted.data() + formatted.size()); - return; - } - some_txt = std::string(padinfo_.width_, ' ') + some_txt; - dest.append(some_txt.data(), some_txt.data() + some_txt.size()); - } - spdlog::details::padding_info get_padding_info() { return padinfo_; } - - std::string some_txt; - - std::unique_ptr clone() const override { - return spdlog::details::make_unique(some_txt); - } -}; -// test clone with custom flag formatters -TEST_CASE("clone-custom_formatter", "[pattern_formatter]") { - auto formatter_1 = std::make_shared(); - formatter_1->add_flag('t', "custom_output").set_pattern("[%n] [%t] %v"); - auto formatter_2 = formatter_1->clone(); - std::string logger_name = "logger-name"; - spdlog::details::log_msg msg(logger_name, spdlog::level::info, "some message"); - - memory_buf_t formatted_1; - memory_buf_t formatted_2; - formatter_1->format(msg, formatted_1); - formatter_2->format(msg, formatted_2); - - auto expected = spdlog::fmt_lib::format("[logger-name] [custom_output] some message{}", - spdlog::details::os::default_eol); - - REQUIRE(to_string_view(formatted_1) == expected); - REQUIRE(to_string_view(formatted_2) == expected); -} - -// -// Test source location formatting -// - -#ifdef _WIN32 -static const char *const test_path = "\\a\\b\\c/myfile.cpp"; -#else -static const char *const test_path = "/a/b//myfile.cpp"; -#endif - -TEST_CASE("short filename formatter-1", "[pattern_formatter]") { - spdlog::pattern_formatter formatter("%s", spdlog::pattern_time_type::local, ""); - memory_buf_t formatted; - std::string logger_name = "logger-name"; - spdlog::source_loc source_loc{test_path, 123, "some_func()"}; - spdlog::details::log_msg msg(source_loc, "logger-name", spdlog::level::info, "Hello"); - formatter.format(msg, formatted); - - REQUIRE(to_string_view(formatted) == "myfile.cpp"); -} - -TEST_CASE("short filename formatter-2", "[pattern_formatter]") { - spdlog::pattern_formatter formatter("%s:%#", spdlog::pattern_time_type::local, ""); - memory_buf_t formatted; - std::string logger_name = "logger-name"; - spdlog::source_loc source_loc{"myfile.cpp", 123, "some_func()"}; - spdlog::details::log_msg msg(source_loc, "logger-name", spdlog::level::info, "Hello"); - formatter.format(msg, formatted); - - REQUIRE(to_string_view(formatted) == "myfile.cpp:123"); -} - -TEST_CASE("short filename formatter-3", "[pattern_formatter]") { - spdlog::pattern_formatter formatter("%s %v", spdlog::pattern_time_type::local, ""); - memory_buf_t formatted; - std::string logger_name = "logger-name"; - spdlog::source_loc source_loc{"", 123, "some_func()"}; - spdlog::details::log_msg msg(source_loc, "logger-name", spdlog::level::info, "Hello"); - formatter.format(msg, formatted); - - REQUIRE(to_string_view(formatted) == " Hello"); -} - -TEST_CASE("full filename formatter", "[pattern_formatter]") { - spdlog::pattern_formatter formatter("%g", spdlog::pattern_time_type::local, ""); - memory_buf_t formatted; - std::string logger_name = "logger-name"; - spdlog::source_loc source_loc{test_path, 123, "some_func()"}; - spdlog::details::log_msg msg(source_loc, "logger-name", spdlog::level::info, "Hello"); - formatter.format(msg, formatted); - - REQUIRE(to_string_view(formatted) == test_path); -} - -TEST_CASE("custom flags", "[pattern_formatter]") { - auto formatter = std::make_shared(); - formatter->add_flag('t', "custom1") - .add_flag('u', "custom2") - .set_pattern("[%n] [%t] [%u] %v"); - - memory_buf_t formatted; - - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - formatter->format(msg, formatted); - auto expected = spdlog::fmt_lib::format("[logger-name] [custom1] [custom2] some message{}", - spdlog::details::os::default_eol); - - REQUIRE(to_string_view(formatted) == expected); -} - -TEST_CASE("custom flags-padding", "[pattern_formatter]") { - auto formatter = std::make_shared(); - formatter->add_flag('t', "custom1") - .add_flag('u', "custom2") - .set_pattern("[%n] [%t] [%5u] %v"); - - memory_buf_t formatted; - - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - formatter->format(msg, formatted); - auto expected = spdlog::fmt_lib::format("[logger-name] [custom1] [ custom2] some message{}", - spdlog::details::os::default_eol); - - REQUIRE(to_string_view(formatted) == expected); -} - -TEST_CASE("custom flags-exception", "[pattern_formatter]") { - auto formatter = std::make_shared(); - formatter->add_flag('t', "throw_me") - .add_flag('u', "custom2") - .set_pattern("[%n] [%t] [%u] %v"); - - memory_buf_t formatted; - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - CHECK_THROWS_AS(formatter->format(msg, formatted), spdlog::spdlog_ex); -} - -TEST_CASE("override need_localtime", "[pattern_formatter]") { - auto formatter = - std::make_shared(spdlog::pattern_time_type::local, "\n"); - formatter->add_flag('t', "time").set_pattern("%t> %v"); - - { - memory_buf_t formatted; - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - formatter->format(msg, formatted); - REQUIRE(to_string_view(formatted) == "0:00AM> some message\n"); - } - - { - formatter->need_localtime(); - - auto now_tm = spdlog::details::os::localtime(); - std::stringstream oss; - oss << (now_tm.tm_hour % 12) << ":" << std::setfill('0') << std::setw(2) << now_tm.tm_min - << (now_tm.tm_hour / 12 ? "PM" : "AM") << "> some message\n"; - - memory_buf_t formatted; - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - formatter->format(msg, formatted); - REQUIRE(to_string_view(formatted) == oss.str()); - } -} - -#ifndef SPDLOG_NO_TLS -TEST_CASE("mdc formatter test-1", "[pattern_formatter]") { - spdlog::mdc::put("mdc_key_1", "mdc_value_1"); - spdlog::mdc::put("mdc_key_2", "mdc_value_2"); - - auto formatter = std::make_shared(); - formatter->set_pattern("[%n] [%l] [%&] %v"); - - memory_buf_t formatted; - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - formatter->format(msg, formatted); - - auto expected = spdlog::fmt_lib::format( - "[logger-name] [info] [mdc_key_1:mdc_value_1 mdc_key_2:mdc_value_2] some message{}", - spdlog::details::os::default_eol); - REQUIRE(to_string_view(formatted) == expected); - - SECTION("Tear down") { spdlog::mdc::clear(); } -} - -TEST_CASE("mdc formatter value update", "[pattern_formatter]") { - spdlog::mdc::put("mdc_key_1", "mdc_value_1"); - spdlog::mdc::put("mdc_key_2", "mdc_value_2"); - - auto formatter = std::make_shared(); - formatter->set_pattern("[%n] [%l] [%&] %v"); - - memory_buf_t formatted_1; - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - formatter->format(msg, formatted_1); - - auto expected = spdlog::fmt_lib::format( - "[logger-name] [info] [mdc_key_1:mdc_value_1 mdc_key_2:mdc_value_2] some message{}", - spdlog::details::os::default_eol); - - REQUIRE(to_string_view(formatted_1) == expected); - - spdlog::mdc::put("mdc_key_1", "new_mdc_value_1"); - memory_buf_t formatted_2; - formatter->format(msg, formatted_2); - expected = spdlog::fmt_lib::format( - "[logger-name] [info] [mdc_key_1:new_mdc_value_1 mdc_key_2:mdc_value_2] some message{}", - spdlog::details::os::default_eol); - - REQUIRE(to_string_view(formatted_2) == expected); - - SECTION("Tear down") { spdlog::mdc::clear(); } -} - -TEST_CASE("mdc different threads", "[pattern_formatter]") { - auto formatter = std::make_shared(); - formatter->set_pattern("[%n] [%l] [%&] %v"); - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - - memory_buf_t formatted_2; - - auto lambda_1 = [formatter, msg]() { - spdlog::mdc::put("mdc_key", "thread_1_id"); - memory_buf_t formatted; - formatter->format(msg, formatted); - - auto expected = - spdlog::fmt_lib::format("[logger-name] [info] [mdc_key:thread_1_id] some message{}", - spdlog::details::os::default_eol); - - REQUIRE(to_string_view(formatted) == expected); - }; - - auto lambda_2 = [formatter, msg]() { - spdlog::mdc::put("mdc_key", "thread_2_id"); - memory_buf_t formatted; - formatter->format(msg, formatted); - - auto expected = - spdlog::fmt_lib::format("[logger-name] [info] [mdc_key:thread_2_id] some message{}", - spdlog::details::os::default_eol); - - REQUIRE(to_string_view(formatted) == expected); - }; - - std::thread thread_1(lambda_1); - std::thread thread_2(lambda_2); - - thread_1.join(); - thread_2.join(); - - SECTION("Tear down") { spdlog::mdc::clear(); } -} - -TEST_CASE("mdc remove key", "[pattern_formatter]") { - spdlog::mdc::put("mdc_key_1", "mdc_value_1"); - spdlog::mdc::put("mdc_key_2", "mdc_value_2"); - spdlog::mdc::remove("mdc_key_1"); - - auto formatter = std::make_shared(); - formatter->set_pattern("[%n] [%l] [%&] %v"); - - memory_buf_t formatted; - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - formatter->format(msg, formatted); - - auto expected = - spdlog::fmt_lib::format("[logger-name] [info] [mdc_key_2:mdc_value_2] some message{}", - spdlog::details::os::default_eol); - REQUIRE(to_string_view(formatted) == expected); - - SECTION("Tear down") { spdlog::mdc::clear(); } -} - -TEST_CASE("mdc empty", "[pattern_formatter]") { - auto formatter = std::make_shared(); - formatter->set_pattern("[%n] [%l] [%&] %v"); - - memory_buf_t formatted; - spdlog::details::log_msg msg(spdlog::source_loc{}, "logger-name", spdlog::level::info, - "some message"); - formatter->format(msg, formatted); - - auto expected = spdlog::fmt_lib::format("[logger-name] [info] [] some message{}", - spdlog::details::os::default_eol); - REQUIRE(to_string_view(formatted) == expected); - - SECTION("Tear down") { spdlog::mdc::clear(); } -} -#endif diff --git a/thirdparty/spdlog/tests/test_registry.cpp b/thirdparty/spdlog/tests/test_registry.cpp deleted file mode 100644 index 1805ae7f8..000000000 --- a/thirdparty/spdlog/tests/test_registry.cpp +++ /dev/null @@ -1,125 +0,0 @@ -#include "includes.h" - -static const char *const tested_logger_name = "null_logger"; -static const char *const tested_logger_name2 = "null_logger2"; - -#ifndef SPDLOG_NO_EXCEPTIONS -TEST_CASE("register_drop", "[registry]") { - spdlog::drop_all(); - spdlog::create(tested_logger_name); - REQUIRE(spdlog::get(tested_logger_name) != nullptr); - // Throw if registering existing name - REQUIRE_THROWS_AS(spdlog::create(tested_logger_name), - spdlog::spdlog_ex); -} - -TEST_CASE("explicit register", "[registry]") { - spdlog::drop_all(); - auto logger = std::make_shared(tested_logger_name, - std::make_shared()); - spdlog::register_logger(logger); - REQUIRE(spdlog::get(tested_logger_name) != nullptr); - // Throw if registering existing name - REQUIRE_THROWS_AS(spdlog::create(tested_logger_name), - spdlog::spdlog_ex); -} -#endif - -TEST_CASE("register_or_replace", "[registry]") { - spdlog::drop_all(); - auto logger1 = std::make_shared( - tested_logger_name, std::make_shared()); - spdlog::register_logger(logger1); - REQUIRE(spdlog::get(tested_logger_name) == logger1); - - auto logger2 = std::make_shared( - tested_logger_name, std::make_shared()); - spdlog::register_or_replace(logger2); - REQUIRE(spdlog::get(tested_logger_name) == logger2); -} - -TEST_CASE("apply_all", "[registry]") { - spdlog::drop_all(); - auto logger = std::make_shared(tested_logger_name, - std::make_shared()); - spdlog::register_logger(logger); - auto logger2 = std::make_shared( - tested_logger_name2, std::make_shared()); - spdlog::register_logger(logger2); - - int counter = 0; - spdlog::apply_all([&counter](std::shared_ptr) { counter++; }); - REQUIRE(counter == 2); - - counter = 0; - spdlog::drop(tested_logger_name2); - spdlog::apply_all([&counter](std::shared_ptr l) { - REQUIRE(l->name() == tested_logger_name); - counter++; - }); - REQUIRE(counter == 1); -} - -TEST_CASE("drop", "[registry]") { - spdlog::drop_all(); - spdlog::create(tested_logger_name); - spdlog::drop(tested_logger_name); - REQUIRE_FALSE(spdlog::get(tested_logger_name)); -} - -TEST_CASE("drop-default", "[registry]") { - spdlog::set_default_logger(spdlog::null_logger_st(tested_logger_name)); - spdlog::drop(tested_logger_name); - REQUIRE_FALSE(spdlog::default_logger()); - REQUIRE_FALSE(spdlog::get(tested_logger_name)); -} - -TEST_CASE("drop_all", "[registry]") { - spdlog::drop_all(); - spdlog::create(tested_logger_name); - spdlog::create(tested_logger_name2); - spdlog::drop_all(); - REQUIRE_FALSE(spdlog::get(tested_logger_name)); - REQUIRE_FALSE(spdlog::get(tested_logger_name2)); - REQUIRE_FALSE(spdlog::default_logger()); -} - -TEST_CASE("drop non existing", "[registry]") { - spdlog::drop_all(); - spdlog::create(tested_logger_name); - spdlog::drop("some_name"); - REQUIRE_FALSE(spdlog::get("some_name")); - REQUIRE(spdlog::get(tested_logger_name)); - spdlog::drop_all(); -} - -TEST_CASE("default logger", "[registry]") { - spdlog::drop_all(); - spdlog::set_default_logger(spdlog::null_logger_st(tested_logger_name)); - REQUIRE(spdlog::get(tested_logger_name) == spdlog::default_logger()); - spdlog::drop_all(); -} - -TEST_CASE("set_default_logger(nullptr)", "[registry]") { - spdlog::set_default_logger(nullptr); - REQUIRE_FALSE(spdlog::default_logger()); -} - -TEST_CASE("disable automatic registration", "[registry]") { - // set some global parameters - spdlog::level::level_enum log_level = spdlog::level::level_enum::warn; - spdlog::set_level(log_level); - // but disable automatic registration - spdlog::set_automatic_registration(false); - auto logger1 = spdlog::create( - tested_logger_name, SPDLOG_FILENAME_T("filename"), 11, 59); - auto logger2 = spdlog::create_async(tested_logger_name2); - // loggers should not be part of the registry - REQUIRE_FALSE(spdlog::get(tested_logger_name)); - REQUIRE_FALSE(spdlog::get(tested_logger_name2)); - // but make sure they are still initialized according to global defaults - REQUIRE(logger1->level() == log_level); - REQUIRE(logger2->level() == log_level); - spdlog::set_level(spdlog::level::info); - spdlog::set_automatic_registration(true); -} diff --git a/thirdparty/spdlog/tests/test_ringbuffer.cpp b/thirdparty/spdlog/tests/test_ringbuffer.cpp deleted file mode 100644 index 81d791656..000000000 --- a/thirdparty/spdlog/tests/test_ringbuffer.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "includes.h" -#include "spdlog/sinks/ringbuffer_sink.h" - -TEST_CASE("ringbuffer invalid size", "[ringbuffer]") { - REQUIRE_THROWS_AS(spdlog::sinks::ringbuffer_sink_mt(0), spdlog::spdlog_ex); -} - -TEST_CASE("ringbuffer stores formatted messages", "[ringbuffer]") { - spdlog::sinks::ringbuffer_sink_st sink(3); - sink.set_pattern("%v"); - - sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "msg1"}); - sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "msg2"}); - sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "msg3"}); - - auto formatted = sink.last_formatted(); - REQUIRE(formatted.size() == 3); - using spdlog::details::os::default_eol; - REQUIRE(formatted[0] == spdlog::fmt_lib::format("msg1{}", default_eol)); - REQUIRE(formatted[1] == spdlog::fmt_lib::format("msg2{}", default_eol)); - REQUIRE(formatted[2] == spdlog::fmt_lib::format("msg3{}", default_eol)); -} - -TEST_CASE("ringbuffer overrun keeps last items", "[ringbuffer]") { - spdlog::sinks::ringbuffer_sink_st sink(2); - sink.set_pattern("%v"); - - sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "first"}); - sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "second"}); - sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "third"}); - - auto formatted = sink.last_formatted(); - REQUIRE(formatted.size() == 2); - using spdlog::details::os::default_eol; - REQUIRE(formatted[0] == spdlog::fmt_lib::format("second{}", default_eol)); - REQUIRE(formatted[1] == spdlog::fmt_lib::format("third{}", default_eol)); -} - -TEST_CASE("ringbuffer retrieval limit", "[ringbuffer]") { - spdlog::sinks::ringbuffer_sink_st sink(3); - sink.set_pattern("%v"); - - sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "A"}); - sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "B"}); - sink.log(spdlog::details::log_msg{"test", spdlog::level::info, "C"}); - - auto formatted = sink.last_formatted(2); - REQUIRE(formatted.size() == 2); - using spdlog::details::os::default_eol; - REQUIRE(formatted[0] == spdlog::fmt_lib::format("B{}", default_eol)); - REQUIRE(formatted[1] == spdlog::fmt_lib::format("C{}", default_eol)); -} diff --git a/thirdparty/spdlog/tests/test_sink.h b/thirdparty/spdlog/tests/test_sink.h deleted file mode 100644 index 9c0945232..000000000 --- a/thirdparty/spdlog/tests/test_sink.h +++ /dev/null @@ -1,70 +0,0 @@ -// -// Copyright(c) 2018 Gabi Melman. -// Distributed under the MIT License (http://opensource.org/licenses/MIT) -// - -#pragma once - -#include "spdlog/details/null_mutex.h" -#include "spdlog/sinks/base_sink.h" -#include "spdlog/fmt/fmt.h" -#include -#include -#include - -namespace spdlog { -namespace sinks { - -template -class test_sink : public base_sink { - const size_t lines_to_save = 100; - -public: - size_t msg_counter() { - std::lock_guard lock(base_sink::mutex_); - return msg_counter_; - } - - size_t flush_counter() { - std::lock_guard lock(base_sink::mutex_); - return flush_counter_; - } - - void set_delay(std::chrono::milliseconds delay) { - std::lock_guard lock(base_sink::mutex_); - delay_ = delay; - } - - // return last output without the eol - std::vector lines() { - std::lock_guard lock(base_sink::mutex_); - return lines_; - } - -protected: - void sink_it_(const details::log_msg &msg) override { - memory_buf_t formatted; - base_sink::formatter_->format(msg, formatted); - // save the line without the eol - auto eol_len = strlen(details::os::default_eol); - using diff_t = typename std::iterator_traits::difference_type; - if (lines_.size() < lines_to_save) { - lines_.emplace_back(formatted.begin(), formatted.end() - static_cast(eol_len)); - } - msg_counter_++; - std::this_thread::sleep_for(delay_); - } - - void flush_() override { flush_counter_++; } - - size_t msg_counter_{0}; - size_t flush_counter_{0}; - std::chrono::milliseconds delay_{std::chrono::milliseconds::zero()}; - std::vector lines_; -}; - -using test_sink_mt = test_sink; -using test_sink_st = test_sink; - -} // namespace sinks -} // namespace spdlog diff --git a/thirdparty/spdlog/tests/test_stdout_api.cpp b/thirdparty/spdlog/tests/test_stdout_api.cpp deleted file mode 100644 index 67659b84d..000000000 --- a/thirdparty/spdlog/tests/test_stdout_api.cpp +++ /dev/null @@ -1,90 +0,0 @@ -/* - * This content is released under the MIT License as specified in - * https://raw.githubusercontent.com/gabime/spdlog/master/LICENSE - */ -#include "includes.h" -#include "spdlog/sinks/stdout_sinks.h" -#include "spdlog/sinks/stdout_color_sinks.h" -TEST_CASE("stdout_st", "[stdout]") { - auto l = spdlog::stdout_logger_st("test"); - l->set_pattern("%+"); - l->set_level(spdlog::level::trace); - l->trace("Test stdout_st"); - spdlog::drop_all(); -} - -TEST_CASE("stdout_mt", "[stdout]") { - auto l = spdlog::stdout_logger_mt("test"); - l->set_pattern("%+"); - l->set_level(spdlog::level::debug); - l->debug("Test stdout_mt"); - spdlog::drop_all(); -} - -TEST_CASE("stderr_st", "[stderr]") { - auto l = spdlog::stderr_logger_st("test"); - l->set_pattern("%+"); - l->info("Test stderr_st"); - spdlog::drop_all(); -} - -TEST_CASE("stderr_mt", "[stderr]") { - auto l = spdlog::stderr_logger_mt("test"); - l->set_pattern("%+"); - l->info("Test stderr_mt"); - l->warn("Test stderr_mt"); - l->error("Test stderr_mt"); - l->critical("Test stderr_mt"); - spdlog::drop_all(); -} - -// color loggers -TEST_CASE("stdout_color_st", "[stdout]") { - auto l = spdlog::stdout_color_st("test"); - l->set_pattern("%+"); - l->info("Test stdout_color_st"); - spdlog::drop_all(); -} - -TEST_CASE("stdout_color_mt", "[stdout]") { - auto l = spdlog::stdout_color_mt("test"); - l->set_pattern("%+"); - l->set_level(spdlog::level::trace); - l->trace("Test stdout_color_mt"); - spdlog::drop_all(); -} - -TEST_CASE("stderr_color_st", "[stderr]") { - auto l = spdlog::stderr_color_st("test"); - l->set_pattern("%+"); - l->set_level(spdlog::level::debug); - l->debug("Test stderr_color_st"); - spdlog::drop_all(); -} - -TEST_CASE("stderr_color_mt", "[stderr]") { - auto l = spdlog::stderr_color_mt("test"); - l->set_pattern("%+"); - l->info("Test stderr_color_mt"); - l->warn("Test stderr_color_mt"); - l->error("Test stderr_color_mt"); - l->critical("Test stderr_color_mt"); - spdlog::drop_all(); -} - -#ifdef SPDLOG_WCHAR_TO_UTF8_SUPPORT - -TEST_CASE("wchar_api", "[stdout]") { - auto l = spdlog::stdout_logger_st("wchar_logger"); - l->set_pattern("%+"); - l->set_level(spdlog::level::trace); - l->trace(L"Test wchar_api"); - l->trace(L"Test wchar_api {}", L"param"); - l->trace(L"Test wchar_api {}", 1); - l->trace(L"Test wchar_api {}", std::wstring{L"wstring param"}); - l->trace(std::wstring{L"Test wchar_api wstring"}); - SPDLOG_LOGGER_DEBUG(l, L"Test SPDLOG_LOGGER_DEBUG {}", L"param"); - spdlog::drop_all(); -} - -#endif diff --git a/thirdparty/spdlog/tests/test_stopwatch.cpp b/thirdparty/spdlog/tests/test_stopwatch.cpp deleted file mode 100644 index b1b4b191c..000000000 --- a/thirdparty/spdlog/tests/test_stopwatch.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include "includes.h" -#include "test_sink.h" -#include "spdlog/stopwatch.h" - -TEST_CASE("stopwatch1", "[stopwatch]") { - using std::chrono::milliseconds; - using clock = std::chrono::steady_clock; - milliseconds wait_ms(500); - milliseconds tolerance_ms(250); - auto start = clock::now(); - spdlog::stopwatch sw; - std::this_thread::sleep_for(wait_ms); - auto stop = clock::now(); - auto diff_ms = std::chrono::duration_cast(stop - start); - REQUIRE(sw.elapsed() >= diff_ms); - REQUIRE(sw.elapsed() <= diff_ms + tolerance_ms); -} - -TEST_CASE("stopwatch2", "[stopwatch]") { - using spdlog::sinks::test_sink_st; - using std::chrono::duration_cast; - using std::chrono::milliseconds; - using clock = std::chrono::steady_clock; - - clock::duration wait_duration(milliseconds(500)); - clock::duration tolerance_duration(milliseconds(250)); - - auto test_sink = std::make_shared(); - - auto start = clock::now(); - spdlog::stopwatch sw; - spdlog::logger logger("test-stopwatch", test_sink); - logger.set_pattern("%v"); - std::this_thread::sleep_for(wait_duration); - auto stop = clock::now(); - logger.info("{}", sw); - auto val = std::stod(test_sink->lines()[0]); - auto diff_duration = duration_cast>(stop - start); - - REQUIRE(val >= (diff_duration).count() - 0.001); - REQUIRE(val <= (diff_duration + tolerance_duration).count()); -} diff --git a/thirdparty/spdlog/tests/test_systemd.cpp b/thirdparty/spdlog/tests/test_systemd.cpp deleted file mode 100644 index e36365741..000000000 --- a/thirdparty/spdlog/tests/test_systemd.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include "includes.h" -#include "spdlog/sinks/systemd_sink.h" - -TEST_CASE("systemd", "[all]") { - auto systemd_sink = std::make_shared(); - spdlog::logger logger("spdlog_systemd_test", systemd_sink); - logger.set_level(spdlog::level::trace); - logger.trace("test spdlog trace"); - logger.debug("test spdlog debug"); - SPDLOG_LOGGER_INFO((&logger), "test spdlog info"); - SPDLOG_LOGGER_WARN((&logger), "test spdlog warn"); - SPDLOG_LOGGER_ERROR((&logger), "test spdlog error"); - SPDLOG_LOGGER_CRITICAL((&logger), "test spdlog critical"); -} diff --git a/thirdparty/spdlog/tests/test_time_point.cpp b/thirdparty/spdlog/tests/test_time_point.cpp deleted file mode 100644 index b7b1b2323..000000000 --- a/thirdparty/spdlog/tests/test_time_point.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "includes.h" -#include "test_sink.h" -#include "spdlog/async.h" - -TEST_CASE("time_point1", "[time_point log_msg]") { - std::shared_ptr test_sink(new spdlog::sinks::test_sink_st); - spdlog::logger logger("test-time_point", test_sink); - - spdlog::source_loc source{}; - std::chrono::system_clock::time_point tp{std::chrono::system_clock::now()}; - test_sink->set_pattern("%T.%F"); // interested in the time_point - - // all the following should have the same time - test_sink->set_delay(std::chrono::milliseconds(10)); - for (int i = 0; i < 5; i++) { - spdlog::details::log_msg msg{tp, source, "test_logger", spdlog::level::info, "message"}; - test_sink->log(msg); - } - - logger.log(tp, source, spdlog::level::info, "formatted message"); - logger.log(tp, source, spdlog::level::info, "formatted message"); - logger.log(tp, source, spdlog::level::info, "formatted message"); - logger.log(tp, source, spdlog::level::info, "formatted message"); - logger.log(source, spdlog::level::info, - "formatted message"); // last line has different time_point - - // now the real test... that the times are the same. - std::vector lines = test_sink->lines(); - REQUIRE(lines[0] == lines[1]); - REQUIRE(lines[2] == lines[3]); - REQUIRE(lines[4] == lines[5]); - REQUIRE(lines[6] == lines[7]); - REQUIRE(lines[8] != lines[9]); - spdlog::drop_all(); -} diff --git a/thirdparty/spdlog/tests/utils.cpp b/thirdparty/spdlog/tests/utils.cpp deleted file mode 100644 index 405b5e5a2..000000000 --- a/thirdparty/spdlog/tests/utils.cpp +++ /dev/null @@ -1,102 +0,0 @@ -#include "includes.h" - -#ifdef _WIN32 - #include -#else - #include - #include -#endif - -void prepare_logdir() { - spdlog::drop_all(); -#ifdef _WIN32 - system("rmdir /S /Q test_logs"); -#else - auto rv = system("rm -rf test_logs"); - if (rv != 0) { - throw std::runtime_error("Failed to rm -rf test_logs"); - } -#endif -} - -std::string file_contents(const std::string &filename) { - std::ifstream ifs(filename, std::ios_base::binary); - if (!ifs) { - throw std::runtime_error("Failed open file "); - } - return std::string((std::istreambuf_iterator(ifs)), (std::istreambuf_iterator())); -} - -std::size_t count_lines(const std::string &filename) { - std::ifstream ifs(filename); - if (!ifs) { - throw std::runtime_error("Failed open file "); - } - - std::string line; - size_t counter = 0; - while (std::getline(ifs, line)) counter++; - return counter; -} - -void require_message_count(const std::string &filename, const std::size_t messages) { - if (strlen(spdlog::details::os::default_eol) == 0) { - REQUIRE(count_lines(filename) == 1); - } else { - REQUIRE(count_lines(filename) == messages); - } -} - -std::size_t get_filesize(const std::string &filename) { - std::ifstream ifs(filename, std::ifstream::ate | std::ifstream::binary); - if (!ifs) { - throw std::runtime_error("Failed open file " + filename); - } - return static_cast(ifs.tellg()); -} - -// source: https://stackoverflow.com/a/2072890/192001 -bool ends_with(std::string const &value, std::string const &ending) { - if (ending.size() > value.size()) { - return false; - } - return std::equal(ending.rbegin(), ending.rend(), value.rbegin()); -} - -#ifdef _WIN32 -// Based on: https://stackoverflow.com/a/37416569/192001 -std::size_t count_files(const std::string &folder) { - size_t counter = 0; - WIN32_FIND_DATAA ffd; - - // Start iterating over the files in the folder directory. - HANDLE hFind = ::FindFirstFileA((folder + "\\*").c_str(), &ffd); - if (hFind != INVALID_HANDLE_VALUE) { - do // Managed to locate and create an handle to that folder. - { - if (ffd.cFileName[0] != '.') counter++; - } while (::FindNextFileA(hFind, &ffd) != 0); - ::FindClose(hFind); - } else { - throw std::runtime_error("Failed open folder " + folder); - } - - return counter; -} -#else -// Based on: https://stackoverflow.com/a/2802255/192001 -std::size_t count_files(const std::string &folder) { - size_t counter = 0; - DIR *dp = opendir(folder.c_str()); - if (dp == nullptr) { - throw std::runtime_error("Failed open folder " + folder); - } - - struct dirent *ep = nullptr; - while ((ep = readdir(dp)) != nullptr) { - if (ep->d_name[0] != '.') counter++; - } - (void)closedir(dp); - return counter; -} -#endif diff --git a/thirdparty/spdlog/tests/utils.h b/thirdparty/spdlog/tests/utils.h deleted file mode 100644 index 53c09b469..000000000 --- a/thirdparty/spdlog/tests/utils.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include -#include - -std::size_t count_files(const std::string &folder); - -void prepare_logdir(); - -std::string file_contents(const std::string &filename); - -std::size_t count_lines(const std::string &filename); - -void require_message_count(const std::string &filename, const std::size_t messages); - -std::size_t get_filesize(const std::string &filename); - -bool ends_with(std::string const &value, std::string const &ending); \ No newline at end of file diff --git a/thirdparty/xmake.lua b/thirdparty/xmake.lua index a4a183910..f929656a6 100644 --- a/thirdparty/xmake.lua +++ b/thirdparty/xmake.lua @@ -50,12 +50,6 @@ target('protozero') add_headerfiles("protozero/**.hpp") add_includedirs("protozero/include", {public=true}) -target('spdlog') - set_kind('headeronly') - set_group('thirdparty') - add_headerfiles("spdlog/include/spdlog/**.h") - add_includedirs("spdlog/include", {public=true}) - target('cpr') set_kind('static') set_group('thirdparty') -- cgit v1.2.3 From 07649104761ee910b667adb2b865c4e2fd0979c9 Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 9 Mar 2026 12:03:25 +0100 Subject: added auto-detection logic for console colour output (#817) Add auto-detection of colour support to `AnsicolourStdoutSink`. **New `colorMode` enum** (`On`, `Off`, `Auto`) added to the header, accepted by the `AnsicolorStdoutSink` constructor. Defaults to `Auto`, so all existing call sites are unaffected. **`Auto` mode detection logic** (in `IscolourTerminal()`): 1. **TTY check** -- if stdout is not a terminal, colour is disabled. 2. **`NO_COLOR`** -- respects the no-colour.org convention. If set, colour is disabled. 3. **`COLORTERM`** -- if set (e.g. `truecolour`, `24bit`), colour is enabled. 4. **`TERM`** -- rejects `dumb`; accepts known colour-capable terminals via substring match: `alacritty`, `ansi`, `colour`, `console`, `cygwin`, `gnome`, `konsole`, `kterm`, `linux`, `msys`, `putty`, `rxvt`, `screen`, `tmux`, `vt100`, `vt102`, `xterm`. Substring matching covers variants like `xterm-256color` and `rxvt-unicode`. 5. **Fallback** -- Windows defaults to colour enabled (modern console supports ANSI natively); other platforms default to disabled. When colour is disabled, ANSI escape sequences are omitted entirely from the output. NOTE: this doesn't currently apply to all paths which do logging in zen as they may be determining their colour output mode separately from `AnsicolorStdoutSink`. --- .../include/zencore/logging/ansicolorsink.h | 9 +- src/zencore/logging/ansicolorsink.cpp | 103 ++++++++++++++++++++- 2 files changed, 107 insertions(+), 5 deletions(-) diff --git a/src/zencore/include/zencore/logging/ansicolorsink.h b/src/zencore/include/zencore/logging/ansicolorsink.h index 9f859e8d7..5060a8393 100644 --- a/src/zencore/include/zencore/logging/ansicolorsink.h +++ b/src/zencore/include/zencore/logging/ansicolorsink.h @@ -8,10 +8,17 @@ namespace zen::logging { +enum class ColorMode +{ + On, + Off, + Auto +}; + class AnsiColorStdoutSink : public Sink { public: - AnsiColorStdoutSink(); + explicit AnsiColorStdoutSink(ColorMode Mode = ColorMode::Auto); ~AnsiColorStdoutSink() override; void Log(const LogMessage& Msg) override; diff --git a/src/zencore/logging/ansicolorsink.cpp b/src/zencore/logging/ansicolorsink.cpp index 9b9959862..540d22359 100644 --- a/src/zencore/logging/ansicolorsink.cpp +++ b/src/zencore/logging/ansicolorsink.cpp @@ -5,8 +5,19 @@ #include #include +#include #include +#if defined(_WIN32) +# include +# define ZEN_ISATTY _isatty +# define ZEN_FILENO _fileno +#else +# include +# define ZEN_ISATTY isatty +# define ZEN_FILENO fileno +#endif + namespace zen::logging { // Default formatter replicating spdlog's %+ pattern: @@ -98,7 +109,90 @@ GetColorForLevel(LogLevel InLevel) struct AnsiColorStdoutSink::Impl { - Impl() : m_Formatter(std::make_unique()) {} + explicit Impl(ColorMode Mode) : m_Formatter(std::make_unique()), m_UseColor(ResolveColorMode(Mode)) {} + + static bool IsColorTerminal() + { + // If stdout is not a TTY, no color + if (ZEN_ISATTY(ZEN_FILENO(stdout)) == 0) + { + return false; + } + + // NO_COLOR convention (https://no-color.org/) + if (std::getenv("NO_COLOR") != nullptr) + { + return false; + } + + // COLORTERM is set by terminals that support color (e.g. "truecolor", "24bit") + if (std::getenv("COLORTERM") != nullptr) + { + return true; + } + + // Check TERM for known color-capable values + const char* Term = std::getenv("TERM"); + if (Term != nullptr) + { + std::string_view TermView(Term); + // "dumb" terminals do not support color + if (TermView == "dumb") + { + return false; + } + // Match against known color-capable terminal types. + // TERM often includes suffixes like "-256color", so we use substring matching. + constexpr std::string_view ColorTerms[] = { + "alacritty", + "ansi", + "color", + "console", + "cygwin", + "gnome", + "konsole", + "kterm", + "linux", + "msys", + "putty", + "rxvt", + "screen", + "tmux", + "vt100", + "vt102", + "xterm", + }; + for (std::string_view Candidate : ColorTerms) + { + if (TermView.find(Candidate) != std::string_view::npos) + { + return true; + } + } + } + +#if defined(_WIN32) + // Windows console supports ANSI color by default in modern versions + return true; +#else + // Unknown terminal — be conservative + return false; +#endif + } + + static bool ResolveColorMode(ColorMode Mode) + { + switch (Mode) + { + case ColorMode::On: + return true; + case ColorMode::Off: + return false; + case ColorMode::Auto: + default: + return IsColorTerminal(); + } + } void Log(const LogMessage& Msg) { @@ -107,7 +201,7 @@ struct AnsiColorStdoutSink::Impl MemoryBuffer Formatted; m_Formatter->Format(Msg, Formatted); - if (Msg.ColorRangeEnd > Msg.ColorRangeStart) + if (m_UseColor && Msg.ColorRangeEnd > Msg.ColorRangeStart) { // Print pre-color range fwrite(Formatted.data(), 1, Msg.ColorRangeStart, m_File); @@ -148,10 +242,11 @@ struct AnsiColorStdoutSink::Impl private: std::mutex m_Mutex; std::unique_ptr m_Formatter; - FILE* m_File = stdout; + FILE* m_File = stdout; + bool m_UseColor = true; }; -AnsiColorStdoutSink::AnsiColorStdoutSink() : m_Impl(std::make_unique()) +AnsiColorStdoutSink::AnsiColorStdoutSink(ColorMode Mode) : m_Impl(std::make_unique(Mode)) { } -- cgit v1.2.3 From f9d8cbcb3573b47b639b7bd73d3a4eed17653d71 Mon Sep 17 00:00:00 2001 From: Dan Engelbrecht Date: Mon, 9 Mar 2026 13:08:00 +0100 Subject: add fallback for zencache multirange (#816) * clean up BuildStorageResolveResult to allow capabilities * add check for multirange request capability * add MaxRangeCountPerRequest capabilities * project export tests * add InMemoryBuildStorageCache * progress and logging improvements * fix ElapsedSeconds calculations in fileremoteprojectstore.cpp * oplogs/builds test script --- .../test_scripts/builds-download-upload-test.py | 196 +++ scripts/test_scripts/metadatas/AndroidClient.json | 9 + scripts/test_scripts/metadatas/IOSClient.json | 9 + scripts/test_scripts/metadatas/LinuxServer.json | 9 + scripts/test_scripts/metadatas/PS4Client.json | 9 + scripts/test_scripts/metadatas/Switch2Client.json | 9 + scripts/test_scripts/metadatas/SwitchClient.json | 9 + scripts/test_scripts/metadatas/WindowsClient.json | 9 + scripts/test_scripts/metadatas/XB1Client.json | 9 + scripts/test_scripts/oplog-import-export-test.py | 228 +++ src/zen/cmds/builds_cmd.cpp | 139 +- src/zen/cmds/projectstore_cmd.cpp | 26 +- src/zenhttp/include/zenhttp/formatters.h | 2 +- src/zenremotestore/builds/buildstoragecache.cpp | 209 ++- .../builds/buildstorageoperations.cpp | 391 +++-- src/zenremotestore/builds/buildstorageutil.cpp | 97 +- .../include/zenremotestore/builds/buildstorage.h | 2 - .../zenremotestore/builds/buildstoragecache.h | 10 +- .../zenremotestore/builds/buildstorageutil.h | 36 +- .../include/zenremotestore/chunking/chunkblock.h | 1 + .../include/zenremotestore/jupiter/jupiterhost.h | 3 +- .../projectstore/buildsremoteprojectstore.h | 22 +- .../projectstore/remoteprojectstore.h | 68 +- src/zenremotestore/jupiter/jupiterhost.cpp | 11 +- .../projectstore/buildsremoteprojectstore.cpp | 314 +--- .../projectstore/fileremoteprojectstore.cpp | 301 +++- .../projectstore/jupiterremoteprojectstore.cpp | 81 +- .../projectstore/projectstoreoperations.cpp | 14 +- .../projectstore/remoteprojectstore.cpp | 1694 ++++++++++++++------ .../projectstore/zenremoteprojectstore.cpp | 164 +- .../storage/buildstore/httpbuildstore.cpp | 13 + src/zenserver/storage/buildstore/httpbuildstore.h | 2 + .../storage/projectstore/httpprojectstore.cpp | 263 ++- 33 files changed, 2922 insertions(+), 1437 deletions(-) create mode 100644 scripts/test_scripts/builds-download-upload-test.py create mode 100644 scripts/test_scripts/metadatas/AndroidClient.json create mode 100644 scripts/test_scripts/metadatas/IOSClient.json create mode 100644 scripts/test_scripts/metadatas/LinuxServer.json create mode 100644 scripts/test_scripts/metadatas/PS4Client.json create mode 100644 scripts/test_scripts/metadatas/Switch2Client.json create mode 100644 scripts/test_scripts/metadatas/SwitchClient.json create mode 100644 scripts/test_scripts/metadatas/WindowsClient.json create mode 100644 scripts/test_scripts/metadatas/XB1Client.json create mode 100644 scripts/test_scripts/oplog-import-export-test.py diff --git a/scripts/test_scripts/builds-download-upload-test.py b/scripts/test_scripts/builds-download-upload-test.py new file mode 100644 index 000000000..f03528d98 --- /dev/null +++ b/scripts/test_scripts/builds-download-upload-test.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Test script for builds download/upload operations.""" + +from __future__ import annotations + +import argparse +import platform +import subprocess +import sys +from pathlib import Path +from typing import NamedTuple + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" + + +class Build(NamedTuple): + name: str + bucket: str + id: str + + +BUILDS = [ + Build("XB1Client", "fortnitegame.staged-build.fortnite-main.xb1-client", "09a7616c1a388dfe6056aa57"), + Build("WindowsClient", "fortnitegame.staged-build.fortnite-main.windows-client", "09a762c81e2cf213142d0ce5"), + Build("SwitchClient", "fortnitegame.staged-build.fortnite-main.switch-client", "09a75bf9c3ce75bce09f644f"), + Build("LinuxServer", "fortnitegame.staged-build.fortnite-main.linux-server", "09a750ac155eb3e3b62e87e0"), + Build("Switch2Client", "fortnitegame.staged-build.fortnite-main.switch2-client", "09a78f3df07b289691ec5710"), + Build("PS4Client", "fortnitegame.staged-build.fortnite-main.ps4-client", "09a76ea92ad301d4724fafad"), + Build("IOSClient", "fortnitegame.staged-build.fortnite-main.ios-client", "09a7816fa26c23362fef0c5d"), + Build("AndroidClient", "fortnitegame.staged-build.fortnite-main.android-client", "09a76725f1620d62c6be06e4"), +] + +ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") +ZEN_METADATA_DIR: Path = Path(__file__).resolve().parent / "metadatas" + +ZEN_PORT = 8558 +ZEN_CACHE_PORT = 8559 +ZEN_CACHE = f"http://127.0.0.1:{ZEN_CACHE_PORT}" +ZEN_PARTIAL_REQUEST_MODE = "true" + +SERVER_ARGS: tuple[str, ...] = ( + "--http", "asio", + "--gc-cache-duration-seconds", "1209600", + "--gc-interval-seconds", "21600", + "--gc-low-diskspace-threshold", "2147483648", + "--cache-bucket-limit-overwrites", +) + + +def run(cmd: list[str | Path]) -> None: + try: + subprocess.run(cmd, check=True) + except FileNotFoundError: + sys.exit(f"error: executable not found: {cmd[0]}") + except subprocess.CalledProcessError as e: + sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}") + + +def stop_server(label: str, port: int) -> None: + """Stop a zen server. Tolerates failures so it is safe to call from finally blocks.""" + print(f"--------- stopping {label}") + try: + subprocess.run([ZEN_EXE, "down", "--port", str(port)]) + except OSError as e: + print(f"warning: could not stop {label}: {e}", file=sys.stderr) + print() + + +def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None: + print(f"--------- starting {label} {data_dir}") + run([ + ZEN_EXE, "up", "--port", str(port), "--show-console", "--", + f"--data-dir={data_dir}", + *SERVER_ARGS, + *(extra_args or []), + ]) + print() + + +def wipe_or_create(label: str, path: Path) -> None: + if path.exists(): + print(f"--------- cleaning {label} {path}") + run([ZEN_EXE, "wipe", "-y", path]) + else: + print(f"--------- creating {label} {path}") + path.mkdir(parents=True, exist_ok=True) + print() + + +def check_prerequisites() -> None: + if not ZEN_EXE.is_file(): + sys.exit(f"error: zen executable not found: {ZEN_EXE}") + if not ZEN_METADATA_DIR.is_dir(): + sys.exit(f"error: metadata directory not found: {ZEN_METADATA_DIR}") + for build in BUILDS: + metadata = ZEN_METADATA_DIR / f"{build.name}.json" + if not metadata.is_file(): + sys.exit(f"error: metadata file not found: {metadata}") + + +def main() -> None: + global ZEN_EXE + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "positional_path", + nargs="?", + default=None, + type=Path, + metavar="DATA_PATH", + help="root path for all data directories (positional shorthand for --data-path)", + ) + parser.add_argument( + "zen_exe_positional", + nargs="?", + default=None, + type=Path, + metavar="ZEN_EXE_PATH", + help="path to zen executable (positional shorthand for --zen-exe-path)", + ) + parser.add_argument( + "--data-path", + default=Path(Path(__file__).stem + "_datadir"), + type=Path, + metavar="PATH", + help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + ) + parser.add_argument( + "--zen-exe-path", + default=ZEN_EXE, + type=Path, + metavar="PATH", + help=f"path to zen executable (default: {ZEN_EXE})", + ) + args = parser.parse_args() + + data_path = args.positional_path + if data_path is None: + data_path = args.data_path + + ZEN_EXE = args.zen_exe_positional + if ZEN_EXE is None: + ZEN_EXE = args.zen_exe_path + zen_system_dir = data_path / "system" + zen_download_dir = data_path / "Download" + zen_cache_data_dir = data_path / "ZenBuildsCache" + zen_upload_dir = data_path / "Upload" + zen_chunk_cache_path = data_path / "ChunkCache" + + check_prerequisites() + + start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"]) + try: + wipe_or_create("download folder", zen_download_dir) + wipe_or_create("system folder", zen_system_dir) + + for build in BUILDS: + print(f"--------- importing {build.name} build") + run([ + ZEN_EXE, "builds", "download", + "--host", "https://jupiter.devtools.epicgames.com", + "--namespace", "fortnite.oplog", + "--bucket", build.bucket, + "--build-id", build.id, + "--local-path", zen_download_dir / build.name, + f"--zen-cache-host={ZEN_CACHE}", + f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}", + "--verify", + "--system-dir", zen_system_dir, + ]) + print() + + wipe_or_create("upload folder", zen_upload_dir) + + for build in BUILDS: + print(f"--------- exporting {build.name} build") + run([ + ZEN_EXE, "builds", "upload", + "--storage-path", zen_upload_dir, + "--build-id", build.id, + "--local-path", zen_download_dir / build.name, + "--verify", + "--system-dir", zen_system_dir, + "--metadata-path", str(ZEN_METADATA_DIR / f"{build.name}.json"), + "--create-build", + "--chunking-cache-path", zen_chunk_cache_path, + ]) + print() + finally: + stop_server("cache zenserver", ZEN_CACHE_PORT) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_scripts/metadatas/AndroidClient.json b/scripts/test_scripts/metadatas/AndroidClient.json new file mode 100644 index 000000000..378d0454d --- /dev/null +++ b/scripts/test_scripts/metadatas/AndroidClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 AndroidClient", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Android", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/IOSClient.json b/scripts/test_scripts/metadatas/IOSClient.json new file mode 100644 index 000000000..fb0f9a342 --- /dev/null +++ b/scripts/test_scripts/metadatas/IOSClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 IOSClient", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "IOS", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/LinuxServer.json b/scripts/test_scripts/metadatas/LinuxServer.json new file mode 100644 index 000000000..02ae2d970 --- /dev/null +++ b/scripts/test_scripts/metadatas/LinuxServer.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 LinuxServer", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Linux", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/PS4Client.json b/scripts/test_scripts/metadatas/PS4Client.json new file mode 100644 index 000000000..6e49e3e5e --- /dev/null +++ b/scripts/test_scripts/metadatas/PS4Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 PS4Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "PS4", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/Switch2Client.json b/scripts/test_scripts/metadatas/Switch2Client.json new file mode 100644 index 000000000..41732e7bc --- /dev/null +++ b/scripts/test_scripts/metadatas/Switch2Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 Switch2Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Switch2", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/SwitchClient.json b/scripts/test_scripts/metadatas/SwitchClient.json new file mode 100644 index 000000000..49362f23e --- /dev/null +++ b/scripts/test_scripts/metadatas/SwitchClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 SwitchClient", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Switch", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/WindowsClient.json b/scripts/test_scripts/metadatas/WindowsClient.json new file mode 100644 index 000000000..c7af270c2 --- /dev/null +++ b/scripts/test_scripts/metadatas/WindowsClient.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 Windows Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "Windows", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/metadatas/XB1Client.json b/scripts/test_scripts/metadatas/XB1Client.json new file mode 100644 index 000000000..36fb45801 --- /dev/null +++ b/scripts/test_scripts/metadatas/XB1Client.json @@ -0,0 +1,9 @@ +{ + "name": "++Fortnite+Main-CL-50966326 XB1Client", + "branch": "ZenBuildTest2", + "baselineBranch": "ZenBuildTest2", + "platform": "XB1", + "project": "Fortnite", + "changelist": 50966326, + "buildType": "staged-build" +} diff --git a/scripts/test_scripts/oplog-import-export-test.py b/scripts/test_scripts/oplog-import-export-test.py new file mode 100644 index 000000000..51593d5a9 --- /dev/null +++ b/scripts/test_scripts/oplog-import-export-test.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python3 +"""Test script for oplog import/export operations.""" + +from __future__ import annotations + +import argparse +import platform +import subprocess +import sys +from pathlib import Path +from typing import NamedTuple + +_PLATFORM = "windows" if sys.platform == "win32" else "macosx" if sys.platform == "darwin" else "linux" +_ARCH = "x64" if sys.platform == "win32" else platform.machine().lower() +_EXE_SUFFIX = ".exe" if sys.platform == "win32" else "" + + +class Build(NamedTuple): + name: str + bucket: str + id: str + + +BUILDS = [ + Build("XB1Client", "fortnitegame.oplog.fortnite-main.xb1client", "09a75f7f3b7517653dcdaaa4"), + Build("WindowsClient", "fortnitegame.oplog.fortnite-main.windowsclient", "09a75d977ef944ecfd0eddfd"), + Build("SwitchClient", "fortnitegame.oplog.fortnite-main.switchclient", "09a74d03b3598ec94cfd2644"), + Build("XSXClient", "fortnitegame.oplog.fortnite-main.xsxclient", "09a76c2bbd6cd78f4d40d9ea"), + Build("Switch2Client", "fortnitegame.oplog.fortnite-main.switch2client", "09a7686b3d9faa78fb24a38f"), + Build("PS4Client", "fortnitegame.oplog.fortnite-main.ps4client", "09a75b72d1c260ed26020140"), + Build("LinuxServer", "fortnitegame.oplog.fortnite-main.linuxserver", "09a747f5e0ee83a04be013e6"), + Build("IOSClient", "fortnitegame.oplog.fortnite-main.iosclient", "09a75f677e883325a209148c"), + Build("Android_ASTCClient", "fortnitegame.oplog.fortnite-main.android_astcclient", "09a7422c08c6f37becc7d37f"), +] + +ZEN_EXE: Path = Path(f"./build/{_PLATFORM}/{_ARCH}/release/zen{_EXE_SUFFIX}") + +ZEN_PORT = 8558 +ZEN_CACHE_PORT = 8559 +ZEN_CACHE = f"http://127.0.0.1:{ZEN_CACHE_PORT}" +ZEN_CACHE_POPULATE = "true" +ZEN_PARTIAL_REQUEST_MODE = "true" + +SERVER_ARGS: tuple[str, ...] = ( + "--http", "asio", + "--gc-cache-duration-seconds", "1209600", + "--gc-interval-seconds", "21600", + "--gc-low-diskspace-threshold", "2147483648", + "--cache-bucket-limit-overwrites", +) + + +def run(cmd: list[str | Path]) -> None: + try: + subprocess.run(cmd, check=True) + except FileNotFoundError: + sys.exit(f"error: executable not found: {cmd[0]}") + except subprocess.CalledProcessError as e: + sys.exit(f"error: command failed with exit code {e.returncode}:\n {' '.join(str(x) for x in e.cmd)}") + + +def stop_server(label: str, port: int) -> None: + """Stop a zen server. Tolerates failures so it is safe to call from finally blocks.""" + print(f"--------- stopping {label}") + try: + subprocess.run([ZEN_EXE, "down", "--port", str(port)]) + except OSError as e: + print(f"warning: could not stop {label}: {e}", file=sys.stderr) + print() + + +def start_server(label: str, data_dir: Path, port: int, extra_args: list[str] | None = None) -> None: + print(f"--------- starting {label} {data_dir}") + run([ + ZEN_EXE, "up", "--port", str(port), "--show-console", "--", + f"--data-dir={data_dir}", + *SERVER_ARGS, + *(extra_args or []), + ]) + print() + + +def wipe_or_create(label: str, path: Path) -> None: + if path.exists(): + print(f"--------- cleaning {label} {path}") + run([ZEN_EXE, "wipe", "-y", path]) + else: + print(f"--------- creating {label} {path}") + path.mkdir(parents=True, exist_ok=True) + print() + + +def check_prerequisites() -> None: + if not ZEN_EXE.is_file(): + sys.exit(f"error: zen executable not found: {ZEN_EXE}") + + +def setup_project(port: int) -> None: + """Create the FortniteGame project and all oplogs on the server at the given port.""" + print("--------- creating FortniteGame project") + run([ZEN_EXE, "project-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", "--force-update"]) + print() + + for build in BUILDS: + print(f"--------- creating {build.name} oplog") + run([ZEN_EXE, "oplog-create", f"--hosturl=127.0.0.1:{port}", "FortniteGame", build.name, "--force-update"]) + print() + + +def main() -> None: + global ZEN_EXE + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "positional_path", + nargs="?", + default=None, + type=Path, + metavar="DATA_PATH", + help="root path for all data directories (positional shorthand for --data-path)", + ) + parser.add_argument( + "zen_exe_positional", + nargs="?", + default=None, + type=Path, + metavar="ZEN_EXE_PATH", + help="path to zen executable (positional shorthand for --zen-exe-path)", + ) + parser.add_argument( + "--data-path", + default=Path(Path(__file__).stem + "_datadir"), + type=Path, + metavar="PATH", + help=f"root path for all data directories (default: {Path(__file__).stem}_datadir)", + ) + parser.add_argument( + "--zen-exe-path", + default=ZEN_EXE, + type=Path, + metavar="PATH", + help=f"path to zen executable (default: {ZEN_EXE})", + ) + args = parser.parse_args() + + data_path = args.positional_path + if data_path is None: + data_path = args.data_path + + ZEN_EXE = args.zen_exe_positional + if ZEN_EXE is None: + ZEN_EXE = args.zen_exe_path + zen_data_dir = data_path / "DDC" / "OplogsZen" + zen_cache_data_dir = data_path / "DDC" / "ZenBuildsCache" + zen_import_data_dir = data_path / "DDC" / "OplogsZenImport" + export_dir = data_path / "Export" / "FortniteGame" + + check_prerequisites() + + start_server("cache zenserver", zen_cache_data_dir, ZEN_CACHE_PORT, ["--buildstore-enabled"]) + try: + wipe_or_create("zenserver data", zen_data_dir) + start_server("zenserver", zen_data_dir, ZEN_PORT) + try: + setup_project(ZEN_PORT) + + for build in BUILDS: + print(f"--------- importing {build.name} oplog") + run([ + ZEN_EXE, "oplog-import", + f"--hosturl=127.0.0.1:{ZEN_PORT}", + "FortniteGame", build.name, + "--clean", + "--builds", "https://jupiter.devtools.epicgames.com", + "--namespace", "fortnite.oplog", + "--bucket", build.bucket, + "--builds-id", build.id, + f"--zen-cache-host={ZEN_CACHE}", + f"--zen-cache-upload={ZEN_CACHE_POPULATE}", + f"--allow-partial-block-requests={ZEN_PARTIAL_REQUEST_MODE}", + ]) + print() + + print(f"--------- validating {build.name} oplog") + run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + print() + + wipe_or_create("export folder", export_dir) + + for build in BUILDS: + print(f"--------- exporting {build.name} oplog") + run([ + ZEN_EXE, "oplog-export", + f"--hosturl=127.0.0.1:{ZEN_PORT}", + "FortniteGame", build.name, + "--file", export_dir, + "--forcetempblocks", + ]) + print() + finally: + stop_server("zenserver", ZEN_PORT) + + wipe_or_create("alternate zenserver data", zen_import_data_dir) + start_server("import zenserver", zen_import_data_dir, ZEN_PORT) + try: + setup_project(ZEN_PORT) + + for build in BUILDS: + print(f"--------- importing {build.name} oplog") + run([ + ZEN_EXE, "oplog-import", + f"--hosturl=127.0.0.1:{ZEN_PORT}", + "FortniteGame", build.name, + "--file", export_dir, + ]) + print() + + print(f"--------- validating {build.name} oplog") + run([ZEN_EXE, "oplog-validate", f"--hosturl=127.0.0.1:{ZEN_PORT}", "FortniteGame", build.name]) + print() + finally: + stop_server("alternative zenserver", ZEN_PORT) + finally: + stop_server("cache zenserver", ZEN_CACHE_PORT) + + +if __name__ == "__main__": + main() diff --git a/src/zen/cmds/builds_cmd.cpp b/src/zen/cmds/builds_cmd.cpp index e5cbafbea..b4b4df7c9 100644 --- a/src/zen/cmds/builds_cmd.cpp +++ b/src/zen/cmds/builds_cmd.cpp @@ -1594,7 +1594,7 @@ namespace builds_impl { } } } - if (Storage.BuildCacheStorage) + if (Storage.CacheStorage) { if (SB.Size() > 0) { @@ -1649,9 +1649,9 @@ namespace builds_impl { } if (Options.PrimeCacheOnly) { - if (Storage.BuildCacheStorage) + if (Storage.CacheStorage) { - Storage.BuildCacheStorage->Flush(5000, [](intptr_t Remaining) { + Storage.CacheStorage->Flush(5000, [](intptr_t Remaining) { if (!IsQuiet) { if (Remaining == 0) @@ -2826,47 +2826,47 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) BuildStorageResolveResult ResolveRes = ResolveBuildStorage(*Output, ClientSettings, m_Host, m_OverrideHost, m_ZenCacheHost, ZenCacheResolveMode::All, m_Verbose); - if (!ResolveRes.HostUrl.empty()) + if (!ResolveRes.Cloud.Address.empty()) { - ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2; + ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; Result.BuildStorageHttp = - std::make_unique(ResolveRes.HostUrl, ClientSettings, []() { return AbortFlag.load(); }); + std::make_unique(ResolveRes.Cloud.Address, ClientSettings, []() { return AbortFlag.load(); }); - Result.BuildStorage = CreateJupiterBuildStorage(Log(), + Result.BuildStorage = CreateJupiterBuildStorage(Log(), *Result.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, m_AllowRedirect, TempPath / "storage"); - Result.StorageName = ResolveRes.HostName; + Result.BuildStorageHost = ResolveRes.Cloud; - uint64_t HostLatencyNs = ResolveRes.HostLatencySec >= 0 ? uint64_t(ResolveRes.HostLatencySec * 1000000000.0) : 0; + uint64_t HostLatencyNs = ResolveRes.Cloud.LatencySec >= 0 ? uint64_t(ResolveRes.Cloud.LatencySec * 1000000000.0) : 0; - StorageDescription = fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}", - ResolveRes.HostName, - (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), - Result.BuildStorageHttp->GetSessionId(), - m_Namespace, - m_Bucket, - NiceLatencyNs(HostLatencyNs)); - Result.BuildStorageLatencySec = ResolveRes.HostLatencySec; + StorageDescription = + fmt::format("Cloud {}{}. SessionId: '{}'. Namespace '{}', Bucket '{}'. Latency: {}", + ResolveRes.Cloud.Name, + (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), + Result.BuildStorageHttp->GetSessionId(), + m_Namespace, + m_Bucket, + NiceLatencyNs(HostLatencyNs)); - if (!ResolveRes.CacheUrl.empty()) + if (!ResolveRes.Cache.Address.empty()) { Result.CacheHttp = std::make_unique( - ResolveRes.CacheUrl, + ResolveRes.Cache.Address, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = ResolveRes.CacheAssumeHttp2, + .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, .Verbose = m_VerboseHttp, .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, []() { return AbortFlag.load(); }); - Result.BuildCacheStorage = + Result.CacheStorage = CreateZenBuildStorageCache(*Result.CacheHttp, StorageCacheStats, m_Namespace, @@ -2874,19 +2874,17 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) TempPath / "zencache", BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheName = ResolveRes.CacheName; + Result.CacheHost = ResolveRes.Cache; - uint64_t CacheLatencyNs = ResolveRes.CacheLatencySec >= 0 ? uint64_t(ResolveRes.CacheLatencySec * 1000000000.0) : 0; + uint64_t CacheLatencyNs = ResolveRes.Cache.LatencySec >= 0 ? uint64_t(ResolveRes.Cache.LatencySec * 1000000000.0) : 0; CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'. Latency: {}", - ResolveRes.CacheName, - (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl), + ResolveRes.Cache.Name, + (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address), Result.CacheHttp->GetSessionId(), NiceLatencyNs(CacheLatencyNs)); - Result.CacheLatencySec = ResolveRes.CacheLatencySec; - if (!m_Namespace.empty()) { CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); @@ -2902,41 +2900,56 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) { StorageDescription = fmt::format("folder {}", m_StoragePath); Result.BuildStorage = CreateFileBuildStorage(m_StoragePath, StorageStats, false, DefaultLatency, DefaultDelayPerKBSec); - Result.StorageName = fmt::format("Disk {}", m_StoragePath.stem()); + + Result.BuildStorageHost = BuildStorageResolveResult::Host{.Address = m_StoragePath.generic_string(), + .Name = "Disk", + .LatencySec = 1.0 / 100000, // 1 us + .Caps = {.MaxRangeCountPerRequest = 2048u}}; if (!m_ZenCacheHost.empty()) { - Result.CacheHttp = std::make_unique( - m_ZenCacheHost, - HttpClientSettings{ - .LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{3000}, - .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = m_AssumeHttp2, - .AllowResume = true, - .RetryCount = 0, - .Verbose = m_VerboseHttp, - .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, - []() { return AbortFlag.load(); }); - Result.BuildCacheStorage = - CreateZenBuildStorageCache(*Result.CacheHttp, - StorageCacheStats, - m_Namespace, - m_Bucket, - TempPath / "zencache", - BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) - : GetTinyWorkerPool(EWorkloadType::Background)); - Result.CacheName = m_ZenCacheHost; - - CacheDescription = fmt::format("Zen {}{}. SessionId: '{}'", Result.CacheName, "", Result.CacheHttp->GetSessionId()); - ; - if (!m_Namespace.empty()) - { - CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); - } - if (!m_Bucket.empty()) + ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(m_ZenCacheHost, m_AssumeHttp2, m_VerboseHttp); + + if (TestResult.Success) { - CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + Result.CacheHttp = std::make_unique( + m_ZenCacheHost, + HttpClientSettings{ + .LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = m_AssumeHttp2, + .AllowResume = true, + .RetryCount = 0, + .Verbose = m_VerboseHttp, + .MaximumInMemoryDownloadSize = GetMaxMemoryBufferSize(DefaultMaxChunkBlockSize, m_BoostWorkerMemory)}, + []() { return AbortFlag.load(); }); + + Result.CacheStorage = + CreateZenBuildStorageCache(*Result.CacheHttp, + StorageCacheStats, + m_Namespace, + m_Bucket, + TempPath / "zencache", + BoostCacheBackgroundWorkerPool ? GetSmallWorkerPool(EWorkloadType::Background) + : GetTinyWorkerPool(EWorkloadType::Background)); + Result.CacheHost = + BuildStorageResolveResult::Host{.Address = m_ZenCacheHost, + .Name = m_ZenCacheHost, + .AssumeHttp2 = m_AssumeHttp2, + .LatencySec = TestResult.LatencySeconds, + .Caps = {.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest}}; + + CacheDescription = fmt::format("Zen {}. SessionId: '{}'", Result.CacheHost.Name, Result.CacheHttp->GetSessionId()); + + if (!m_Namespace.empty()) + { + CacheDescription += fmt::format(". Namespace '{}'", m_Namespace); + } + if (!m_Bucket.empty()) + { + CacheDescription += fmt::format(" Bucket '{}'", m_Bucket); + } } } } @@ -2948,7 +2961,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!IsQuiet) { ZEN_CONSOLE("Remote: {}", StorageDescription); - if (!Result.CacheName.empty()) + if (!Result.CacheHost.Name.empty()) { ZEN_CONSOLE("Cache : {}", CacheDescription); } @@ -3489,7 +3502,7 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) "Requests: {}\n" "Avg Request Time: {}\n" "Avg I/O Time: {}", - Storage.StorageName, + Storage.BuildStorageHost.Name, NiceBytes(StorageStats.TotalBytesRead.load()), NiceBytes(StorageStats.TotalBytesWritten.load()), StorageStats.TotalRequestCount.load(), @@ -3810,12 +3823,12 @@ BuildsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) if (!IsQuiet) { - if (Storage.BuildCacheStorage) + if (Storage.CacheStorage) { - ZEN_CONSOLE("Uploaded {} ({}) blobs", + ZEN_CONSOLE("Uploaded {} ({}) blobs to {}", StorageCacheStats.PutBlobCount.load(), NiceBytes(StorageCacheStats.PutBlobByteCount), - Storage.CacheName); + Storage.CacheHost.Name); } } diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp index dfc6c1650..5ff591b54 100644 --- a/src/zen/cmds/projectstore_cmd.cpp +++ b/src/zen/cmds/projectstore_cmd.cpp @@ -2602,38 +2602,37 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a StorageInstance Storage; - ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2; + ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2; ClientSettings.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u; - Storage.BuildStorageHttp = std::make_unique(ResolveRes.HostUrl, ClientSettings); - Storage.BuildStorageLatencySec = ResolveRes.HostLatencySec; + Storage.BuildStorageHttp = std::make_unique(ResolveRes.Cloud.Address, ClientSettings); + Storage.BuildStorageHost = ResolveRes.Cloud; BuildStorageCache::Statistics StorageCacheStats; std::atomic AbortFlag(false); - if (!ResolveRes.CacheUrl.empty()) + if (!ResolveRes.Cache.Address.empty()) { Storage.CacheHttp = std::make_unique( - ResolveRes.CacheUrl, + ResolveRes.Cache.Address, HttpClientSettings{ .LogCategory = "httpcacheclient", .ConnectTimeout = std::chrono::milliseconds{3000}, .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = ResolveRes.CacheAssumeHttp2, + .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2, .AllowResume = true, .RetryCount = 0, .MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u}, [&AbortFlag]() { return AbortFlag.load(); }); - Storage.CacheName = ResolveRes.CacheName; - Storage.CacheLatencySec = ResolveRes.CacheLatencySec; + Storage.CacheHost = ResolveRes.Cache; } if (!m_Quiet) { std::string StorageDescription = fmt::format("Cloud {}{}. SessionId {}. Namespace '{}', Bucket '{}'", - ResolveRes.HostName, - (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl), + ResolveRes.Cloud.Name, + (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address), Storage.BuildStorageHttp->GetSessionId(), m_Namespace, m_Bucket); @@ -2644,8 +2643,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a { std::string CacheDescription = fmt::format("Zen {}{}. SessionId {}. Namespace '{}', Bucket '{}'", - ResolveRes.CacheName, - (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl), + ResolveRes.Cache.Name, + (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address), Storage.CacheHttp->GetSessionId(), m_Namespace, m_Bucket); @@ -2661,11 +2660,10 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a Storage.BuildStorage = CreateJupiterBuildStorage(Log(), *Storage.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, m_AllowRedirect, StorageTempPath); - Storage.StorageName = ResolveRes.HostName; if (Storage.CacheHttp) { - Storage.BuildCacheStorage = CreateZenBuildStorageCache( + Storage.CacheStorage = CreateZenBuildStorageCache( *Storage.CacheHttp, StorageCacheStats, m_Namespace, diff --git a/src/zenhttp/include/zenhttp/formatters.h b/src/zenhttp/include/zenhttp/formatters.h index addb00cb8..57ab01158 100644 --- a/src/zenhttp/include/zenhttp/formatters.h +++ b/src/zenhttp/include/zenhttp/formatters.h @@ -73,7 +73,7 @@ struct fmt::formatter if (Response.IsSuccess()) { return fmt::format_to(Ctx.out(), - "OK: Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}s", + "OK: Status: {}, Bytes: {}/{} (Up/Down), Elapsed: {}", ToString(Response.StatusCode), Response.UploadedBytes, Response.DownloadedBytes, diff --git a/src/zenremotestore/builds/buildstoragecache.cpp b/src/zenremotestore/builds/buildstoragecache.cpp index 53d33bd7e..00765903d 100644 --- a/src/zenremotestore/builds/buildstoragecache.cpp +++ b/src/zenremotestore/builds/buildstoragecache.cpp @@ -528,6 +528,192 @@ CreateZenBuildStorageCache(HttpClient& HttpClient, return std::make_unique(HttpClient, Stats, Namespace, Bucket, TempFolderPath, BackgroundWorkerPool); } +#if ZEN_WITH_TESTS + +class InMemoryBuildStorageCache : public BuildStorageCache +{ +public: + // MaxRangeSupported == 0 : no range requests are accepted, always return full blob + // MaxRangeSupported == 1 : single range is supported, multi range returns full blob + // MaxRangeSupported > 1 : multirange is supported up to MaxRangeSupported, more ranges returns empty blob (bad request) + explicit InMemoryBuildStorageCache(uint64_t MaxRangeSupported, + BuildStorageCache::Statistics& Stats, + double LatencySec = 0.0, + double DelayPerKBSec = 0.0) + : m_MaxRangeSupported(MaxRangeSupported) + , m_Stats(Stats) + , m_LatencySec(LatencySec) + , m_DelayPerKBSec(DelayPerKBSec) + { + } + void PutBuildBlob(const Oid&, const IoHash& RawHash, ZenContentType, const CompositeBuffer& Payload) override + { + IoBuffer Buf = Payload.Flatten().AsIoBuffer(); + Buf.MakeOwned(); + const uint64_t SentBytes = Buf.Size(); + uint64_t ReceivedBytes = 0; + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer.GetElapsedTimeUs(), ReceivedBytes, SentBytes); }); + { + std::lock_guard Lock(m_Mutex); + m_Entries[RawHash] = std::move(Buf); + } + m_Stats.PutBlobCount.fetch_add(1); + m_Stats.PutBlobByteCount.fetch_add(SentBytes); + } + + IoBuffer GetBuildBlob(const Oid&, const IoHash& RawHash, uint64_t RangeOffset = 0, uint64_t RangeBytes = (uint64_t)-1) override + { + uint64_t SentBytes = 0; + uint64_t ReceivedBytes = 0; + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer.GetElapsedTimeUs(), ReceivedBytes, SentBytes); }); + IoBuffer FullPayload; + { + std::lock_guard Lock(m_Mutex); + auto It = m_Entries.find(RawHash); + if (It == m_Entries.end()) + { + return {}; + } + FullPayload = It->second; + } + + if (RangeOffset != 0 || RangeBytes != (uint64_t)-1) + { + if (m_MaxRangeSupported == 0) + { + ReceivedBytes = FullPayload.Size(); + return FullPayload; + } + else + { + ReceivedBytes = (RangeBytes == (uint64_t)-1) ? FullPayload.Size() - RangeOffset : RangeBytes; + return IoBuffer(FullPayload, RangeOffset, RangeBytes); + } + } + else + { + ReceivedBytes = FullPayload.Size(); + return FullPayload; + } + } + + BuildBlobRanges GetBuildBlobRanges(const Oid&, const IoHash& RawHash, std::span> Ranges) override + { + ZEN_ASSERT(!Ranges.empty()); + uint64_t SentBytes = 0; + uint64_t ReceivedBytes = 0; + SimulateLatency(SentBytes, 0); + auto _ = MakeGuard([&]() { SimulateLatency(0, ReceivedBytes); }); + Stopwatch ExecutionTimer; + auto __ = MakeGuard([&]() { AddStatistic(ExecutionTimer.GetElapsedTimeUs(), ReceivedBytes, SentBytes); }); + if (m_MaxRangeSupported > 1 && Ranges.size() > m_MaxRangeSupported) + { + return {}; + } + IoBuffer FullPayload; + { + std::lock_guard Lock(m_Mutex); + auto It = m_Entries.find(RawHash); + if (It == m_Entries.end()) + { + return {}; + } + FullPayload = It->second; + } + + if (Ranges.size() > m_MaxRangeSupported) + { + // An empty Ranges signals to the caller: "full buffer given, use it for all requested ranges". + ReceivedBytes = FullPayload.Size(); + return {.PayloadBuffer = FullPayload}; + } + else + { + uint64_t PayloadStart = Ranges.front().first; + uint64_t PayloadSize = Ranges.back().first + Ranges.back().second - PayloadStart; + IoBuffer RangeBuffer = IoBuffer(FullPayload, PayloadStart, PayloadSize); + std::vector> PayloadRanges; + PayloadRanges.reserve(Ranges.size()); + for (const std::pair& Range : Ranges) + { + PayloadRanges.push_back(std::make_pair(Range.first - PayloadStart, Range.second)); + } + ReceivedBytes = PayloadSize; + return {.PayloadBuffer = RangeBuffer, .Ranges = std::move(PayloadRanges)}; + } + } + + void PutBlobMetadatas(const Oid&, std::span, std::span) override {} + + std::vector GetBlobMetadatas(const Oid&, std::span Hashes) override + { + return std::vector(Hashes.size()); + } + + std::vector BlobsExists(const Oid&, std::span Hashes) override + { + std::lock_guard Lock(m_Mutex); + std::vector Result; + Result.reserve(Hashes.size()); + for (const IoHash& Hash : Hashes) + { + auto It = m_Entries.find(Hash); + Result.push_back({.HasBody = (It != m_Entries.end() && It->second)}); + } + return Result; + } + + void Flush(int32_t, std::function&&) override {} + +private: + void AddStatistic(uint64_t ElapsedTimeUs, uint64_t ReceivedBytes, uint64_t SentBytes) + { + m_Stats.TotalBytesWritten += SentBytes; + m_Stats.TotalBytesRead += ReceivedBytes; + m_Stats.TotalExecutionTimeUs += ElapsedTimeUs; + m_Stats.TotalRequestCount++; + SetAtomicMax(m_Stats.PeakSentBytes, SentBytes); + SetAtomicMax(m_Stats.PeakReceivedBytes, ReceivedBytes); + if (ElapsedTimeUs > 0) + { + SetAtomicMax(m_Stats.PeakBytesPerSec, (ReceivedBytes + SentBytes) * 1000000 / ElapsedTimeUs); + } + } + + void SimulateLatency(uint64_t SendBytes, uint64_t ReceiveBytes) + { + double SleepSec = m_LatencySec; + if (m_DelayPerKBSec > 0.0) + { + SleepSec += m_DelayPerKBSec * (double(SendBytes + ReceiveBytes) / 1024u); + } + if (SleepSec > 0) + { + Sleep(int(SleepSec * 1000)); + } + } + + uint64_t m_MaxRangeSupported = 0; + BuildStorageCache::Statistics& m_Stats; + const double m_LatencySec = 0.0; + const double m_DelayPerKBSec = 0.0; + std::mutex m_Mutex; + std::unordered_map m_Entries; +}; + +std::unique_ptr +CreateInMemoryBuildStorageCache(uint64_t MaxRangeSupported, BuildStorageCache::Statistics& Stats, double LatencySec, double DelayPerKBSec) +{ + return std::make_unique(MaxRangeSupported, Stats, LatencySec, DelayPerKBSec); +} +#endif // ZEN_WITH_TESTS + ZenCacheEndpointTestResult TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose) { @@ -542,15 +728,28 @@ TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const boo HttpClient::Response TestResponse = TestHttpClient.Get("/status/builds"); if (TestResponse.IsSuccess()) { - LatencyTestResult LatencyResult = MeasureLatency(TestHttpClient, "/health"); + uint64_t MaxRangeCountPerRequest = 1; + CbObject StatusResponse = TestResponse.AsObject(); + if (StatusResponse["ok"].AsBool()) + { + MaxRangeCountPerRequest = StatusResponse["capabilities"].AsObjectView()["maxrangecountperrequest"].AsUInt64(1); + + LatencyTestResult LatencyResult = MeasureLatency(TestHttpClient, "/health"); + + if (!LatencyResult.Success) + { + return {.Success = false, .FailureReason = LatencyResult.FailureReason}; + } - if (!LatencyResult.Success) + return {.Success = true, .LatencySeconds = LatencyResult.LatencySeconds, .MaxRangeCountPerRequest = MaxRangeCountPerRequest}; + } + else { - return {.Success = false, .FailureReason = LatencyResult.FailureReason}; + return {.Success = false, + .FailureReason = fmt::format("ZenCache endpoint {}/status/builds did not respond with \"ok\"", BaseUrl)}; } - return {.Success = true, .LatencySeconds = LatencyResult.LatencySeconds}; } return {.Success = false, .FailureReason = TestResponse.ErrorMessage("")}; -}; +} } // namespace zen diff --git a/src/zenremotestore/builds/buildstorageoperations.cpp b/src/zenremotestore/builds/buildstorageoperations.cpp index 43a4937f0..f4b167b73 100644 --- a/src/zenremotestore/builds/buildstorageoperations.cpp +++ b/src/zenremotestore/builds/buildstorageoperations.cpp @@ -887,11 +887,12 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) ChunkBlockAnalyser BlockAnalyser( m_LogOutput, m_BlockDescriptions, - ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet, - .IsVerbose = m_Options.IsVerbose, - .HostLatencySec = m_Storage.BuildStorageLatencySec, - .HostHighSpeedLatencySec = m_Storage.CacheLatencySec, - .HostMaxRangeCountPerRequest = BuildStorageBase::MaxRangeCountPerRequest}); + ChunkBlockAnalyser::Options{.IsQuiet = m_Options.IsQuiet, + .IsVerbose = m_Options.IsVerbose, + .HostLatencySec = m_Storage.BuildStorageHost.LatencySec, + .HostHighSpeedLatencySec = m_Storage.CacheHost.LatencySec, + .HostMaxRangeCountPerRequest = m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest, + .HostHighSpeedMaxRangeCountPerRequest = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest}); std::vector NeededBlocks = BlockAnalyser.GetNeeded( m_RemoteLookup.ChunkHashToChunkIndex, @@ -974,7 +975,7 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) } } - if (m_Storage.BuildCacheStorage) + if (m_Storage.CacheStorage) { ZEN_TRACE_CPU("BlobCacheExistCheck"); Stopwatch Timer; @@ -993,7 +994,7 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) } const std::vector CacheExistsResult = - m_Storage.BuildCacheStorage->BlobsExists(m_BuildId, BlobHashes); + m_Storage.CacheStorage->BlobsExists(m_BuildId, BlobHashes); if (CacheExistsResult.size() == BlobHashes.size()) { @@ -1018,32 +1019,50 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) } std::vector BlockPartialDownloadModes; + if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Off) { BlockPartialDownloadModes.resize(m_BlockDescriptions.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); } else { + ChunkBlockAnalyser::EPartialBlockDownloadMode CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; + ChunkBlockAnalyser::EPartialBlockDownloadMode CachePartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; + + switch (m_Options.PartialBlockRequestMode) + { + case EPartialBlockRequestMode::Off: + break; + case EPartialBlockRequestMode::ZenCacheOnly: + CachePartialDownloadMode = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest > 1 + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange; + CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; + break; + case EPartialBlockRequestMode::Mixed: + CachePartialDownloadMode = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest > 1 + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange; + CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange; + break; + case EPartialBlockRequestMode::All: + CachePartialDownloadMode = m_Storage.CacheHost.Caps.MaxRangeCountPerRequest > 1 + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange; + CloudPartialDownloadMode = m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest > 1 + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange + : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange; + break; + default: + ZEN_ASSERT(false); + break; + } + BlockPartialDownloadModes.reserve(m_BlockDescriptions.size()); for (uint32_t BlockIndex = 0; BlockIndex < m_BlockDescriptions.size(); BlockIndex++) { const bool BlockExistInCache = ExistsResult.ExistingBlobs.contains(m_BlockDescriptions[BlockIndex].BlockHash); - if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::All) - { - BlockPartialDownloadModes.push_back(BlockExistInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::Exact - : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange); - } - else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::ZenCacheOnly) - { - BlockPartialDownloadModes.push_back(BlockExistInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::Exact - : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); - } - else if (m_Options.PartialBlockRequestMode == EPartialBlockRequestMode::Mixed) - { - BlockPartialDownloadModes.push_back(BlockExistInCache - ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed - : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange); - } + BlockPartialDownloadModes.push_back(BlockExistInCache ? CachePartialDownloadMode : CloudPartialDownloadMode); } } @@ -1527,20 +1546,20 @@ BuildsOperationUpdateFolder::Execute(FolderContent& OutLocalFolderState) IoBuffer BlockBuffer; const bool ExistsInCache = - m_Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); + m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); if (ExistsInCache) { - BlockBuffer = m_Storage.BuildCacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash); + BlockBuffer = m_Storage.CacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash); } if (!BlockBuffer) { BlockBuffer = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash); - if (BlockBuffer && m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (BlockBuffer && m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, - BlockDescription.BlockHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(BlockBuffer))); + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + BlockDescription.BlockHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(BlockBuffer))); } } if (!BlockBuffer) @@ -3103,10 +3122,10 @@ BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t RemoteChunkInde const IoHash& ChunkHash = m_RemoteContent.ChunkedContent.ChunkHashes[RemoteChunkIndex]; // FilteredDownloadedBytesPerSecond.Start(); IoBuffer BuildBlob; - const bool ExistsInCache = m_Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); + const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(ChunkHash); if (ExistsInCache) { - BuildBlob = m_Storage.BuildCacheStorage->GetBuildBlob(m_BuildId, ChunkHash); + BuildBlob = m_Storage.CacheStorage->GetBuildBlob(m_BuildId, ChunkHash); } if (BuildBlob) { @@ -3134,12 +3153,12 @@ BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t RemoteChunkInde m_DownloadStats.DownloadedChunkCount++; m_DownloadStats.RequestsCompleteCount++; - if (Payload && m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (Payload && m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, - ChunkHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(Payload))); + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); } OnDownloaded(std::move(Payload)); @@ -3148,12 +3167,12 @@ BuildsOperationUpdateFolder::DownloadBuildBlob(uint32_t RemoteChunkInde else { BuildBlob = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, ChunkHash); - if (BuildBlob && m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (BuildBlob && m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, - ChunkHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(BuildBlob))); + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + ChunkHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(BuildBlob))); } if (!BuildBlob) { @@ -3273,34 +3292,7 @@ BuildsOperationUpdateFolder::DownloadPartialBlock( Ranges.push_back(std::make_pair(BlockRange.RangeStart, BlockRange.RangeLength)); } - if (m_Storage.BuildCacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash)) - { - BuildStorageCache::BuildBlobRanges RangeBuffers = - m_Storage.BuildCacheStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, Ranges); - if (RangeBuffers.PayloadBuffer) - { - if (!m_AbortFlag) - { - if (RangeBuffers.Ranges.size() != Ranges.size()) - { - throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges", - Ranges.size(), - BlockDescription.BlockHash, - RangeBuffers.Ranges.size())); - } - - std::vector> BlockOffsetAndLengths = std::move(RangeBuffers.Ranges); - ProcessDownload(BlockDescription, - std::move(RangeBuffers.PayloadBuffer), - BlockRangeStartIndex, - BlockOffsetAndLengths, - OnDownloaded); - } - return; - } - } - - const size_t MaxRangesPerRequestToJupiter = BuildStorageBase::MaxRangeCountPerRequest; + const bool ExistsInCache = m_Storage.CacheStorage && ExistsResult.ExistingBlobs.contains(BlockDescription.BlockHash); size_t SubBlockRangeCount = BlockRangeCount; size_t SubRangeCountComplete = 0; @@ -3311,30 +3303,101 @@ BuildsOperationUpdateFolder::DownloadPartialBlock( { break; } - size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, MaxRangesPerRequestToJupiter); + + // First try to get subrange from cache. + // If not successful, try to get the ranges from the build store and adapt SubRangeCount... + size_t SubRangeStartIndex = BlockRangeStartIndex + SubRangeCountComplete; + if (ExistsInCache) + { + size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, m_Storage.CacheHost.Caps.MaxRangeCountPerRequest); + + if (SubRangeCount == 1) + { + // Legacy single-range path, prefer that for max compatibility + + const std::pair SubRange = RangesSpan[SubRangeCountComplete]; + IoBuffer PayloadBuffer = + m_Storage.CacheStorage->GetBuildBlob(m_BuildId, BlockDescription.BlockHash, SubRange.first, SubRange.second); + if (m_AbortFlag) + { + break; + } + if (PayloadBuffer) + { + ProcessDownload(BlockDescription, + std::move(PayloadBuffer), + SubRangeStartIndex, + std::vector>{std::make_pair(0u, SubRange.second)}, + OnDownloaded); + SubRangeCountComplete += SubRangeCount; + continue; + } + } + else + { + auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); + + BuildStorageCache::BuildBlobRanges RangeBuffers = + m_Storage.CacheStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, SubRanges); + if (m_AbortFlag) + { + break; + } + if (RangeBuffers.PayloadBuffer) + { + if (RangeBuffers.Ranges.empty()) + { + SubRangeCount = Ranges.size() - SubRangeCountComplete; + ProcessDownload(BlockDescription, + std::move(RangeBuffers.PayloadBuffer), + SubRangeStartIndex, + RangesSpan.subspan(SubRangeCountComplete, SubRangeCount), + OnDownloaded); + SubRangeCountComplete += SubRangeCount; + continue; + } + else if (RangeBuffers.Ranges.size() == SubRangeCount) + { + ProcessDownload(BlockDescription, + std::move(RangeBuffers.PayloadBuffer), + SubRangeStartIndex, + RangeBuffers.Ranges, + OnDownloaded); + SubRangeCountComplete += SubRangeCount; + continue; + } + } + } + } + + size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, m_Storage.BuildStorageHost.Caps.MaxRangeCountPerRequest); auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); BuildStorageBase::BuildBlobRanges RangeBuffers = m_Storage.BuildStorage->GetBuildBlobRanges(m_BuildId, BlockDescription.BlockHash, SubRanges); + if (m_AbortFlag) + { + break; + } if (RangeBuffers.PayloadBuffer) { - if (m_AbortFlag) - { - break; - } if (RangeBuffers.Ranges.empty()) { // Jupiter will ignore the ranges and send the whole payload if it fetches the payload from S3 // Upload to cache (if enabled) and use the whole payload for the remaining ranges - if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, - BlockDescription.BlockHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(std::vector{RangeBuffers.PayloadBuffer})); + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + BlockDescription.BlockHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(std::vector{RangeBuffers.PayloadBuffer})); + if (m_AbortFlag) + { + break; + } } SubRangeCount = Ranges.size() - SubRangeCountComplete; @@ -4932,12 +4995,12 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& const IoHash& BlockHash = OutBlocks.BlockDescriptions[BlockIndex].BlockHash; const uint64_t CompressedBlockSize = Payload.GetCompressedSize(); - if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, - BlockHash, - ZenContentType::kCompressedBinary, - Payload.GetCompressed()); + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + BlockHash, + ZenContentType::kCompressedBinary, + Payload.GetCompressed()); } m_Storage.BuildStorage->PutBuildBlob(m_BuildId, @@ -4955,11 +5018,11 @@ BuildsOperationUploadFolder::GenerateBuildBlocks(const ChunkedFolderContent& OutBlocks.BlockDescriptions[BlockIndex].ChunkRawHashes.size()); } - if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId, - std::vector({BlockHash}), - std::vector({BlockMetaData})); + m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); } bool MetadataSucceeded = @@ -5803,11 +5866,11 @@ BuildsOperationUploadFolder::UploadBuildPart(ChunkingController& ChunkController { const CbObject BlockMetaData = BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId, - std::vector({BlockHash}), - std::vector({BlockMetaData})); + m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); } bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); if (MetadataSucceeded) @@ -6001,9 +6064,9 @@ BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Co const CbObject BlockMetaData = BuildChunkBlockDescription(NewBlocks.BlockDescriptions[BlockIndex], NewBlocks.BlockMetaDatas[BlockIndex]); - if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); } m_Storage.BuildStorage->PutBuildBlob(m_BuildId, BlockHash, ZenContentType::kCompressedBinary, Payload); if (m_Options.IsVerbose) @@ -6017,11 +6080,11 @@ BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Co UploadedBlockSize += PayloadSize; TempUploadStats.BlocksBytes += PayloadSize; - if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBlobMetadatas(m_BuildId, - std::vector({BlockHash}), - std::vector({BlockMetaData})); + m_Storage.CacheStorage->PutBlobMetadatas(m_BuildId, + std::vector({BlockHash}), + std::vector({BlockMetaData})); } bool MetadataSucceeded = m_Storage.BuildStorage->PutBlockMetadata(m_BuildId, BlockHash, BlockMetaData); if (MetadataSucceeded) @@ -6084,9 +6147,9 @@ BuildsOperationUploadFolder::UploadPartBlobs(const ChunkedFolderContent& Co const uint64_t PayloadSize = Payload.GetSize(); - if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, RawHash, ZenContentType::kCompressedBinary, Payload); } if (PayloadSize >= LargeAttachmentSize) @@ -6830,14 +6893,14 @@ BuildsOperationPrimeCache::Execute() std::vector BlobsToDownload; BlobsToDownload.reserve(BuildBlobs.size()); - if (m_Storage.BuildCacheStorage && !BuildBlobs.empty() && !m_Options.ForceUpload) + if (m_Storage.CacheStorage && !BuildBlobs.empty() && !m_Options.ForceUpload) { ZEN_TRACE_CPU("BlobCacheExistCheck"); Stopwatch Timer; const std::vector BlobHashes(BuildBlobs.begin(), BuildBlobs.end()); const std::vector CacheExistsResult = - m_Storage.BuildCacheStorage->BlobsExists(m_BuildId, BlobHashes); + m_Storage.CacheStorage->BlobsExists(m_BuildId, BlobHashes); if (CacheExistsResult.size() == BlobHashes.size()) { @@ -6884,33 +6947,33 @@ BuildsOperationPrimeCache::Execute() for (size_t BlobIndex = 0; BlobIndex < BlobCount; BlobIndex++) { - Work.ScheduleWork( - m_NetworkPool, - [this, - &Work, - &BlobsToDownload, - BlobCount, - &LooseChunkRawSizes, - &CompletedDownloadCount, - &FilteredDownloadedBytesPerSecond, - &MultipartAttachmentCount, - BlobIndex](std::atomic&) { - if (!m_AbortFlag) - { - const IoHash& BlobHash = BlobsToDownload[BlobIndex]; + Work.ScheduleWork(m_NetworkPool, + [this, + &Work, + &BlobsToDownload, + BlobCount, + &LooseChunkRawSizes, + &CompletedDownloadCount, + &FilteredDownloadedBytesPerSecond, + &MultipartAttachmentCount, + BlobIndex](std::atomic&) { + if (!m_AbortFlag) + { + const IoHash& BlobHash = BlobsToDownload[BlobIndex]; - bool IsLargeBlob = false; + bool IsLargeBlob = false; - if (auto It = LooseChunkRawSizes.find(BlobHash); It != LooseChunkRawSizes.end()) - { - IsLargeBlob = It->second >= m_Options.LargeAttachmentSize; - } + if (auto It = LooseChunkRawSizes.find(BlobHash); It != LooseChunkRawSizes.end()) + { + IsLargeBlob = It->second >= m_Options.LargeAttachmentSize; + } - FilteredDownloadedBytesPerSecond.Start(); + FilteredDownloadedBytesPerSecond.Start(); - if (IsLargeBlob) - { - DownloadLargeBlob(*m_Storage.BuildStorage, + if (IsLargeBlob) + { + DownloadLargeBlob( + *m_Storage.BuildStorage, m_TempPath, m_BuildId, BlobHash, @@ -6926,12 +6989,12 @@ BuildsOperationPrimeCache::Execute() if (!m_AbortFlag) { - if (Payload && m_Storage.BuildCacheStorage) + if (Payload && m_Storage.CacheStorage) { - m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, - BlobHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(Payload))); + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + BlobHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(Payload))); } } CompletedDownloadCount++; @@ -6940,32 +7003,32 @@ BuildsOperationPrimeCache::Execute() FilteredDownloadedBytesPerSecond.Stop(); } }); - } - else - { - IoBuffer Payload = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlobHash); - m_DownloadStats.DownloadedBlockCount++; - m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); - m_DownloadStats.RequestsCompleteCount++; + } + else + { + IoBuffer Payload = m_Storage.BuildStorage->GetBuildBlob(m_BuildId, BlobHash); + m_DownloadStats.DownloadedBlockCount++; + m_DownloadStats.DownloadedBlockByteCount += Payload.GetSize(); + m_DownloadStats.RequestsCompleteCount++; - if (!m_AbortFlag) - { - if (Payload && m_Storage.BuildCacheStorage) - { - m_Storage.BuildCacheStorage->PutBuildBlob(m_BuildId, - BlobHash, - ZenContentType::kCompressedBinary, - CompositeBuffer(SharedBuffer(std::move(Payload)))); - } - } - CompletedDownloadCount++; - if (CompletedDownloadCount == BlobCount) - { - FilteredDownloadedBytesPerSecond.Stop(); - } - } - } - }); + if (!m_AbortFlag) + { + if (Payload && m_Storage.CacheStorage) + { + m_Storage.CacheStorage->PutBuildBlob(m_BuildId, + BlobHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(SharedBuffer(std::move(Payload)))); + } + } + CompletedDownloadCount++; + if (CompletedDownloadCount == BlobCount) + { + FilteredDownloadedBytesPerSecond.Stop(); + } + } + } + }); } Work.Wait(m_LogOutput.GetProgressUpdateDelayMS(), [&](bool IsAborted, bool IsPaused, std::ptrdiff_t PendingWork) { @@ -6977,10 +7040,10 @@ BuildsOperationPrimeCache::Execute() std::string DownloadRateString = (CompletedDownloadCount == BlobCount) ? "" : fmt::format(" {}bits/s", NiceNum(FilteredDownloadedBytesPerSecond.GetCurrent() * 8)); - std::string UploadDetails = m_Storage.BuildCacheStorage ? fmt::format(" {} ({}) uploaded.", - m_StorageCacheStats.PutBlobCount.load(), - NiceBytes(m_StorageCacheStats.PutBlobByteCount.load())) - : ""; + std::string UploadDetails = m_Storage.CacheStorage ? fmt::format(" {} ({}) uploaded.", + m_StorageCacheStats.PutBlobCount.load(), + NiceBytes(m_StorageCacheStats.PutBlobByteCount.load())) + : ""; std::string Details = fmt::format("{}/{} ({}{}) downloaded.{}", CompletedDownloadCount.load(), @@ -7005,13 +7068,13 @@ BuildsOperationPrimeCache::Execute() return; } - if (m_Storage.BuildCacheStorage) + if (m_Storage.CacheStorage) { - m_Storage.BuildCacheStorage->Flush(m_LogOutput.GetProgressUpdateDelayMS(), [this](intptr_t Remaining) -> bool { + m_Storage.CacheStorage->Flush(m_LogOutput.GetProgressUpdateDelayMS(), [this](intptr_t Remaining) -> bool { ZEN_UNUSED(Remaining); if (!m_Options.IsQuiet) { - ZEN_OPERATION_LOG_INFO(m_LogOutput, "Waiting for {} blobs to finish upload to '{}'", Remaining, m_Storage.CacheName); + ZEN_OPERATION_LOG_INFO(m_LogOutput, "Waiting for {} blobs to finish upload to '{}'", Remaining, m_Storage.CacheHost.Name); } return !m_AbortFlag; }); @@ -7221,7 +7284,7 @@ GetRemoteContent(OperationLogOutput& Output, bool AttemptFallback = false; OutBlockDescriptions = GetBlockDescriptions(Output, *Storage.BuildStorage, - Storage.BuildCacheStorage.get(), + Storage.CacheStorage.get(), BuildId, BlockRawHashes, AttemptFallback, diff --git a/src/zenremotestore/builds/buildstorageutil.cpp b/src/zenremotestore/builds/buildstorageutil.cpp index d65f18b9a..2ae726e29 100644 --- a/src/zenremotestore/builds/buildstorageutil.cpp +++ b/src/zenremotestore/builds/buildstorageutil.cpp @@ -63,13 +63,15 @@ ResolveBuildStorage(OperationLogOutput& Output, std::string HostUrl; std::string HostName; - double HostLatencySec = -1.0; + double HostLatencySec = -1.0; + uint64_t HostMaxRangeCountPerRequest = 1; std::string CacheUrl; std::string CacheName; - bool HostAssumeHttp2 = ClientSettings.AssumeHttp2; - bool CacheAssumeHttp2 = ClientSettings.AssumeHttp2; - double CacheLatencySec = -1.0; + bool HostAssumeHttp2 = ClientSettings.AssumeHttp2; + bool CacheAssumeHttp2 = ClientSettings.AssumeHttp2; + double CacheLatencySec = -1.0; + uint64_t CacheMaxRangeCountPerRequest = 1; JupiterServerDiscovery DiscoveryResponse; const std::string_view DiscoveryHost = Host.empty() ? OverrideHost : Host; @@ -100,9 +102,10 @@ ResolveBuildStorage(OperationLogOutput& Output, { ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", OverrideHost); } - HostUrl = OverrideHost; - HostName = GetHostNameFromUrl(OverrideHost); - HostLatencySec = TestResult.LatencySeconds; + HostUrl = OverrideHost; + HostName = GetHostNameFromUrl(OverrideHost); + HostLatencySec = TestResult.LatencySeconds; + HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; } else { @@ -137,10 +140,11 @@ ResolveBuildStorage(OperationLogOutput& Output, ZEN_OPERATION_LOG_INFO(Output, "Server endpoint at '{}/api/v1/status/servers' succeeded", ServerEndpoint.BaseUrl); } - HostUrl = ServerEndpoint.BaseUrl; - HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; - HostName = ServerEndpoint.Name; - HostLatencySec = TestResult.LatencySeconds; + HostUrl = ServerEndpoint.BaseUrl; + HostAssumeHttp2 = ServerEndpoint.AssumeHttp2; + HostName = ServerEndpoint.Name; + HostLatencySec = TestResult.LatencySeconds; + HostMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; break; } else @@ -184,10 +188,11 @@ ResolveBuildStorage(OperationLogOutput& Output, ZEN_OPERATION_LOG_INFO(Output, "Cache endpoint at '{}/status/builds' succeeded", CacheEndpoint.BaseUrl); } - CacheUrl = CacheEndpoint.BaseUrl; - CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; - CacheName = CacheEndpoint.Name; - CacheLatencySec = TestResult.LatencySeconds; + CacheUrl = CacheEndpoint.BaseUrl; + CacheAssumeHttp2 = CacheEndpoint.AssumeHttp2; + CacheName = CacheEndpoint.Name; + CacheLatencySec = TestResult.LatencySeconds; + CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; break; } } @@ -225,9 +230,10 @@ ResolveBuildStorage(OperationLogOutput& Output, if (ZenCacheEndpointTestResult TestResult = TestZenCacheEndpoint(ZenCacheHost, /*AssumeHttp2*/ false, ClientSettings.Verbose); TestResult.Success) { - CacheUrl = ZenCacheHost; - CacheName = GetHostNameFromUrl(ZenCacheHost); - CacheLatencySec = TestResult.LatencySeconds; + CacheUrl = ZenCacheHost; + CacheName = GetHostNameFromUrl(ZenCacheHost); + CacheLatencySec = TestResult.LatencySeconds; + CacheMaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; } else { @@ -235,15 +241,34 @@ ResolveBuildStorage(OperationLogOutput& Output, } } - return BuildStorageResolveResult{.HostUrl = HostUrl, - .HostName = HostName, - .HostAssumeHttp2 = HostAssumeHttp2, - .HostLatencySec = HostLatencySec, + return BuildStorageResolveResult{ + .Cloud = {.Address = HostUrl, + .Name = HostName, + .AssumeHttp2 = HostAssumeHttp2, + .LatencySec = HostLatencySec, + .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = HostMaxRangeCountPerRequest}}, + .Cache = {.Address = CacheUrl, + .Name = CacheName, + .AssumeHttp2 = CacheAssumeHttp2, + .LatencySec = CacheLatencySec, + .Caps = BuildStorageResolveResult::Capabilities{.MaxRangeCountPerRequest = CacheMaxRangeCountPerRequest}}}; +} - .CacheUrl = CacheUrl, - .CacheName = CacheName, - .CacheAssumeHttp2 = CacheAssumeHttp2, - .CacheLatencySec = CacheLatencySec}; +std::vector +ParseBlockMetadatas(std::span BlockMetadatas) +{ + std::vector UnorderedList; + UnorderedList.reserve(BlockMetadatas.size()); + for (size_t CacheBlockMetadataIndex = 0; CacheBlockMetadataIndex < BlockMetadatas.size(); CacheBlockMetadataIndex++) + { + const CbObject& CacheBlockMetadata = BlockMetadatas[CacheBlockMetadataIndex]; + ChunkBlockDescription Description = ParseChunkBlockDescription(CacheBlockMetadata); + if (Description.BlockHash != IoHash::Zero) + { + UnorderedList.emplace_back(std::move(Description)); + } + } + return UnorderedList; } std::vector @@ -263,25 +288,15 @@ GetBlockDescriptions(OperationLogOutput& Output, if (OptionalCacheStorage && !BlockRawHashes.empty()) { std::vector CacheBlockMetadatas = OptionalCacheStorage->GetBlobMetadatas(BuildId, BlockRawHashes); - UnorderedList.reserve(CacheBlockMetadatas.size()); - for (size_t CacheBlockMetadataIndex = 0; CacheBlockMetadataIndex < CacheBlockMetadatas.size(); CacheBlockMetadataIndex++) + if (!CacheBlockMetadatas.empty()) { - const CbObject& CacheBlockMetadata = CacheBlockMetadatas[CacheBlockMetadataIndex]; - ChunkBlockDescription Description = ParseChunkBlockDescription(CacheBlockMetadata); - if (Description.BlockHash == IoHash::Zero) - { - ZEN_OPERATION_LOG_WARN(Output, "Unexpected/invalid block metadata received from remote cache, skipping block"); - } - else + UnorderedList = ParseBlockMetadatas(CacheBlockMetadatas); + for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) { - UnorderedList.emplace_back(std::move(Description)); + const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); } } - for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) - { - const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; - BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); - } } if (UnorderedList.size() < BlockRawHashes.size()) diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h index ce3da41c1..da8437a58 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorage.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorage.h @@ -58,8 +58,6 @@ public: uint64_t RangeOffset = 0, uint64_t RangeBytes = (uint64_t)-1) = 0; - static constexpr size_t MaxRangeCountPerRequest = 128u; - struct BuildBlobRanges { IoBuffer PayloadBuffer; diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h index 67c93480b..24702df0f 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstoragecache.h @@ -69,11 +69,19 @@ std::unique_ptr CreateZenBuildStorageCache(HttpClient& H const std::filesystem::path& TempFolderPath, WorkerThreadPool& BackgroundWorkerPool); +#if ZEN_WITH_TESTS +std::unique_ptr CreateInMemoryBuildStorageCache(uint64_t MaxRangeSupported, + BuildStorageCache::Statistics& Stats, + double LatencySec = 0.0, + double DelayPerKBSec = 0.0); +#endif // ZEN_WITH_TESTS + struct ZenCacheEndpointTestResult { bool Success = false; std::string FailureReason; - double LatencySeconds = -1.0; + double LatencySeconds = -1.0; + uint64_t MaxRangeCountPerRequest = 1; }; ZenCacheEndpointTestResult TestZenCacheEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose); diff --git a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h index 764a24e61..7306188ca 100644 --- a/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h +++ b/src/zenremotestore/include/zenremotestore/builds/buildstorageutil.h @@ -14,15 +14,20 @@ class BuildStorageCache; struct BuildStorageResolveResult { - std::string HostUrl; - std::string HostName; - bool HostAssumeHttp2 = false; - double HostLatencySec = -1.0; - - std::string CacheUrl; - std::string CacheName; - bool CacheAssumeHttp2 = false; - double CacheLatencySec = -1.0; + struct Capabilities + { + uint64_t MaxRangeCountPerRequest = 1; + }; + struct Host + { + std::string Address; + std::string Name; + bool AssumeHttp2 = false; + double LatencySec = -1.0; + Capabilities Caps; + }; + Host Cloud; + Host Cache; }; enum class ZenCacheResolveMode @@ -52,14 +57,13 @@ std::vector GetBlockDescriptions(OperationLogOutput& Out struct StorageInstance { - std::unique_ptr BuildStorageHttp; - std::unique_ptr BuildStorage; - std::string StorageName; - double BuildStorageLatencySec = -1.0; + BuildStorageResolveResult::Host BuildStorageHost; + std::unique_ptr BuildStorageHttp; + std::unique_ptr BuildStorage; + + BuildStorageResolveResult::Host CacheHost; std::unique_ptr CacheHttp; - std::unique_ptr BuildCacheStorage; - std::string CacheName; - double CacheLatencySec = -1.0; + std::unique_ptr CacheStorage; }; } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h index 20b6fd371..c085f10e7 100644 --- a/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h +++ b/src/zenremotestore/include/zenremotestore/chunking/chunkblock.h @@ -31,6 +31,7 @@ struct ChunkBlockDescription : public ThinChunkBlockDescription std::vector ParseChunkBlockDescriptionList(const CbObjectView& BlocksObject); ChunkBlockDescription ParseChunkBlockDescription(const CbObjectView& BlockObject); +std::vector ParseBlockMetadatas(std::span BlockMetadatas); CbObject BuildChunkBlockDescription(const ChunkBlockDescription& Block, CbObjectView MetaData); ChunkBlockDescription GetChunkBlockDescription(const SharedBuffer& BlockPayload, const IoHash& RawHash); typedef std::function(const IoHash& RawHash)> FetchChunkFunc; diff --git a/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h b/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h index 7bbf40dfa..bb41f9efc 100644 --- a/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h +++ b/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h @@ -28,7 +28,8 @@ struct JupiterEndpointTestResult { bool Success = false; std::string FailureReason; - double LatencySeconds = -1.0; + double LatencySeconds = -1.0; + uint64_t MaxRangeCountPerRequest = 1; }; JupiterEndpointTestResult TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpVerbose); diff --git a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h index 66dfcc62d..c058e1c1f 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/buildsremoteprojectstore.h @@ -2,6 +2,7 @@ #pragma once +#include #include namespace zen { @@ -10,9 +11,6 @@ class AuthMgr; struct BuildsRemoteStoreOptions : RemoteStoreOptions { - std::string Host; - std::string OverrideHost; - std::string ZenHost; std::string Namespace; std::string Bucket; Oid BuildId; @@ -22,20 +20,16 @@ struct BuildsRemoteStoreOptions : RemoteStoreOptions std::filesystem::path OidcExePath; bool ForceDisableBlocks = false; bool ForceDisableTempBlocks = false; - bool AssumeHttp2 = false; - bool PopulateCache = true; IoBuffer MetaData; size_t MaximumInMemoryDownloadSize = 1024u * 1024u; }; -std::shared_ptr CreateJupiterBuildsRemoteStore(LoggerRef InLog, - const BuildsRemoteStoreOptions& Options, - const std::filesystem::path& TempFilePath, - bool Quiet, - bool Unattended, - bool Hidden, - WorkerThreadPool& CacheBackgroundWorkerPool, - double& OutHostLatencySec, - double& OutCacheLatencySec); +struct BuildStorageResolveResult; + +std::shared_ptr CreateJupiterBuildsRemoteStore(LoggerRef InLog, + const BuildStorageResolveResult& ResolveResult, + std::function&& TokenProvider, + const BuildsRemoteStoreOptions& Options, + const std::filesystem::path& TempFilePath); } // namespace zen diff --git a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h index 42786d0f2..084d975a2 100644 --- a/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h +++ b/src/zenremotestore/include/zenremotestore/projectstore/remoteprojectstore.h @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -79,11 +80,6 @@ public: std::vector Blocks; }; - struct AttachmentExistsInCacheResult : public Result - { - std::vector HasBody; - }; - struct LoadAttachmentRangesResult : public Result { IoBuffer Bytes; @@ -128,28 +124,17 @@ public: virtual FinalizeResult FinalizeContainer(const IoHash& RawHash) = 0; virtual SaveAttachmentsResult SaveAttachments(const std::vector& Payloads) = 0; - virtual LoadContainerResult LoadContainer() = 0; - virtual GetKnownBlocksResult GetKnownBlocks() = 0; - virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) = 0; - virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) = 0; - - enum ESourceMode - { - kAny, - kCacheOnly, - kHostOnly - }; + virtual LoadContainerResult LoadContainer() = 0; + virtual GetKnownBlocksResult GetKnownBlocks() = 0; + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes, + BuildStorageCache* OptionalCache, + const Oid& CacheBuildId) = 0; - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode = ESourceMode::kAny) = 0; - - static constexpr size_t MaxRangeCountPerRequest = 128u; + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) = 0; virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, - std::span> Ranges, - ESourceMode SourceMode = ESourceMode::kAny) = 0; - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode = ESourceMode::kAny) = 0; - - virtual void Flush() = 0; + std::span> Ranges) = 0; + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) = 0; }; struct RemoteStoreOptions @@ -211,18 +196,29 @@ RemoteProjectStore::Result SaveOplog(CidStore& ChunkStore, bool IgnoreMissingAttachments, JobContext* OptionalContext); -RemoteProjectStore::Result LoadOplog(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - ProjectStore::Oplog& Oplog, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, - bool ForceDownload, - bool IgnoreMissingAttachments, - bool CleanOplog, - EPartialBlockRequestMode PartialBlockRequestMode, - double HostLatencySec, - double CacheLatencySec, - JobContext* OptionalContext); +struct LoadOplogContext +{ + CidStore& ChunkStore; + RemoteProjectStore& RemoteStore; + BuildStorageCache* OptionalCache = nullptr; + Oid CacheBuildId = Oid::Zero; + BuildStorageCache::Statistics* OptionalCacheStats = nullptr; + ProjectStore::Oplog& Oplog; + WorkerThreadPool& NetworkWorkerPool; + WorkerThreadPool& WorkerPool; + bool ForceDownload = false; + bool IgnoreMissingAttachments = false; + bool CleanOplog = false; + EPartialBlockRequestMode PartialBlockRequestMode = EPartialBlockRequestMode::All; + bool PopulateCache = false; + double StoreLatencySec = -1.0; + uint64_t StoreMaxRangeCountPerRequest = 1; + double CacheLatencySec = -1.0; + uint64_t CacheMaxRangeCountPerRequest = 1; + JobContext* OptionalJobContext = nullptr; +}; + +RemoteProjectStore::Result LoadOplog(LoadOplogContext&& Context); std::vector GetBlockHashesFromOplog(CbObjectView ContainerObject); std::vector GetBlocksFromOplog(CbObjectView ContainerObject, std::span IncludeBlockHashes); diff --git a/src/zenremotestore/jupiter/jupiterhost.cpp b/src/zenremotestore/jupiter/jupiterhost.cpp index 2583cfc84..4479c8b97 100644 --- a/src/zenremotestore/jupiter/jupiterhost.cpp +++ b/src/zenremotestore/jupiter/jupiterhost.cpp @@ -59,13 +59,22 @@ TestJupiterEndpoint(std::string_view BaseUrl, const bool AssumeHttp2, const bool HttpClient::Response TestResponse = TestHttpClient.Get("/health/live"); if (TestResponse.IsSuccess()) { + // TODO: dan.engelbrecht 20260305 - replace this naive nginx detection with proper capabilites end point once it exists in Jupiter + uint64_t MaxRangeCountPerRequest = 1; + if (auto It = TestResponse.Header.Entries.find("Server"); It != TestResponse.Header.Entries.end()) + { + if (StrCaseCompare(It->second.c_str(), "nginx", 5) == 0) + { + MaxRangeCountPerRequest = 128u; + } + } LatencyTestResult LatencyResult = MeasureLatency(TestHttpClient, "/health/ready"); if (!LatencyResult.Success) { return {.Success = false, .FailureReason = LatencyResult.FailureReason}; } - return {.Success = true, .LatencySeconds = LatencyResult.LatencySeconds}; + return {.Success = true, .LatencySeconds = LatencyResult.LatencySeconds, .MaxRangeCountPerRequest = MaxRangeCountPerRequest}; } return {.Success = false, .FailureReason = TestResponse.ErrorMessage("")}; } diff --git a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp index 3400cdbf5..2282a31dd 100644 --- a/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/buildsremoteprojectstore.cpp @@ -7,8 +7,6 @@ #include #include -#include -#include #include #include #include @@ -26,18 +24,14 @@ class BuildsRemoteStore : public RemoteProjectStore public: BuildsRemoteStore(LoggerRef InLog, const HttpClientSettings& ClientSettings, - HttpClientSettings* OptionalCacheClientSettings, std::string_view HostUrl, - std::string_view CacheUrl, const std::filesystem::path& TempFilePath, - WorkerThreadPool& CacheBackgroundWorkerPool, std::string_view Namespace, std::string_view Bucket, const Oid& BuildId, const IoBuffer& MetaData, bool ForceDisableBlocks, - bool ForceDisableTempBlocks, - bool PopulateCache) + bool ForceDisableTempBlocks) : m_Log(InLog) , m_BuildStorageHttp(HostUrl, ClientSettings) , m_BuildStorage(CreateJupiterBuildStorage(Log(), @@ -53,20 +47,8 @@ public: , m_MetaData(MetaData) , m_EnableBlocks(!ForceDisableBlocks) , m_UseTempBlocks(!ForceDisableTempBlocks) - , m_PopulateCache(PopulateCache) { m_MetaData.MakeOwned(); - if (OptionalCacheClientSettings) - { - ZEN_ASSERT(!CacheUrl.empty()); - m_BuildCacheStorageHttp = std::make_unique(CacheUrl, *OptionalCacheClientSettings); - m_BuildCacheStorage = CreateZenBuildStorageCache(*m_BuildCacheStorageHttp, - m_StorageCacheStats, - Namespace, - Bucket, - TempFilePath, - CacheBackgroundWorkerPool); - } } virtual RemoteStoreInfo GetInfo() const override @@ -75,9 +57,8 @@ public: .UseTempBlockFiles = m_UseTempBlocks, .AllowChunking = true, .ContainerName = fmt::format("{}/{}/{}", m_Namespace, m_Bucket, m_BuildId), - .Description = fmt::format("[cloud] {}{}. SessionId: {}. {}/{}/{}"sv, + .Description = fmt::format("[cloud] {}. SessionId: {}. {}/{}/{}"sv, m_BuildStorageHttp.GetBaseUri(), - m_BuildCacheStorage ? fmt::format(" (Cache: {})", m_BuildCacheStorageHttp->GetBaseUri()) : ""sv, m_BuildStorageHttp.GetSessionId(), m_Namespace, m_Bucket, @@ -86,15 +67,13 @@ public: virtual Stats GetStats() const override { - return { - .m_SentBytes = m_BuildStorageStats.TotalBytesWritten.load() + m_StorageCacheStats.TotalBytesWritten.load(), - .m_ReceivedBytes = m_BuildStorageStats.TotalBytesRead.load() + m_StorageCacheStats.TotalBytesRead.load(), - .m_RequestTimeNS = m_BuildStorageStats.TotalRequestTimeUs.load() * 1000 + m_StorageCacheStats.TotalRequestTimeUs.load() * 1000, - .m_RequestCount = m_BuildStorageStats.TotalRequestCount.load() + m_StorageCacheStats.TotalRequestCount.load(), - .m_PeakSentBytes = Max(m_BuildStorageStats.PeakSentBytes.load(), m_StorageCacheStats.PeakSentBytes.load()), - .m_PeakReceivedBytes = Max(m_BuildStorageStats.PeakReceivedBytes.load(), m_StorageCacheStats.PeakReceivedBytes.load()), - .m_PeakBytesPerSec = Max(m_BuildStorageStats.PeakBytesPerSec.load(), m_StorageCacheStats.PeakBytesPerSec.load()), - }; + return {.m_SentBytes = m_BuildStorageStats.TotalBytesWritten.load(), + .m_ReceivedBytes = m_BuildStorageStats.TotalBytesRead.load(), + .m_RequestTimeNS = m_BuildStorageStats.TotalRequestTimeUs.load() * 1000, + .m_RequestCount = m_BuildStorageStats.TotalRequestCount.load(), + .m_PeakSentBytes = m_BuildStorageStats.PeakSentBytes.load(), + .m_PeakReceivedBytes = m_BuildStorageStats.PeakReceivedBytes.load(), + .m_PeakBytesPerSec = m_BuildStorageStats.PeakBytesPerSec.load()}; } virtual bool GetExtendedStats(ExtendedStats& OutStats) const override @@ -109,11 +88,6 @@ public: } Result = true; } - if (m_BuildCacheStorage) - { - OutStats.m_ReceivedBytesPerSource.insert_or_assign("Cache", m_StorageCacheStats.TotalBytesRead); - Result = true; - } return Result; } @@ -462,11 +436,14 @@ public: return Result; } - virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) override + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes, + BuildStorageCache* OptionalCache, + const Oid& CacheBuildId) override { std::unique_ptr Output(CreateStandardLogOutput(Log())); ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); + ZEN_ASSERT(OptionalCache == nullptr || CacheBuildId == m_BuildId); GetBlockDescriptionsResult Result; Stopwatch Timer; @@ -476,7 +453,7 @@ public: { Result.Blocks = zen::GetBlockDescriptions(*Output, *m_BuildStorage, - m_BuildCacheStorage.get(), + OptionalCache, m_BuildId, BlockHashes, /*AttemptFallback*/ false, @@ -506,49 +483,7 @@ public: return Result; } - virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) override - { - AttachmentExistsInCacheResult Result; - Stopwatch Timer; - auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; }); - try - { - const std::vector CacheExistsResult = - m_BuildCacheStorage->BlobsExists(m_BuildId, RawHashes); - - if (CacheExistsResult.size() == RawHashes.size()) - { - Result.HasBody.reserve(CacheExistsResult.size()); - for (size_t BlobIndex = 0; BlobIndex < CacheExistsResult.size(); BlobIndex++) - { - Result.HasBody.push_back(CacheExistsResult[BlobIndex].HasBody); - } - } - } - catch (const HttpClientError& Ex) - { - Result.ErrorCode = MakeErrorCode(Ex); - Result.Reason = fmt::format("Remote cache: Failed finding known blobs for {}/{}/{}/{}. Reason: '{}'", - m_BuildStorageHttp.GetBaseUri(), - m_Namespace, - m_Bucket, - m_BuildId, - Ex.what()); - } - catch (const std::exception& Ex) - { - Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("Remote cache: Failed finding known blobs for {}/{}/{}/{}. Reason: '{}'", - m_BuildStorageHttp.GetBaseUri(), - m_Namespace, - m_Bucket, - m_BuildId, - Ex.what()); - } - return Result; - } - - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode) override + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override { ZEN_ASSERT(m_OplogBuildPartId != Oid::Zero); @@ -558,25 +493,7 @@ public: try { - if (m_BuildCacheStorage && SourceMode != ESourceMode::kHostOnly) - { - IoBuffer CachedBlob = m_BuildCacheStorage->GetBuildBlob(m_BuildId, RawHash); - if (CachedBlob) - { - Result.Bytes = std::move(CachedBlob); - } - } - if (!Result.Bytes && SourceMode != ESourceMode::kCacheOnly) - { - Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash); - if (m_BuildCacheStorage && Result.Bytes && m_PopulateCache) - { - m_BuildCacheStorage->PutBuildBlob(m_BuildId, - RawHash, - Result.Bytes.GetContentType(), - CompositeBuffer(SharedBuffer(Result.Bytes))); - } - } + Result.Bytes = m_BuildStorage->GetBuildBlob(m_BuildId, RawHash); } catch (const HttpClientError& Ex) { @@ -605,45 +522,20 @@ public: } virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, - std::span> Ranges, - ESourceMode SourceMode) override + std::span> Ranges) override { + ZEN_ASSERT(!Ranges.empty()); LoadAttachmentRangesResult Result; Stopwatch Timer; auto _ = MakeGuard([&Timer, &Result]() { Result.ElapsedSeconds = Timer.GetElapsedTimeUs() / 1000000.0; }); try { - if (m_BuildCacheStorage && SourceMode != ESourceMode::kHostOnly) + BuildStorageBase::BuildBlobRanges BlobRanges = m_BuildStorage->GetBuildBlobRanges(m_BuildId, RawHash, Ranges); + if (BlobRanges.PayloadBuffer) { - BuildStorageCache::BuildBlobRanges BlobRanges = m_BuildCacheStorage->GetBuildBlobRanges(m_BuildId, RawHash, Ranges); - if (BlobRanges.PayloadBuffer) - { - Result.Bytes = std::move(BlobRanges.PayloadBuffer); - Result.Ranges = std::move(BlobRanges.Ranges); - } - } - if (!Result.Bytes && SourceMode != ESourceMode::kCacheOnly) - { - BuildStorageBase::BuildBlobRanges BlobRanges = m_BuildStorage->GetBuildBlobRanges(m_BuildId, RawHash, Ranges); - if (BlobRanges.PayloadBuffer) - { - Result.Bytes = std::move(BlobRanges.PayloadBuffer); - Result.Ranges = std::move(BlobRanges.Ranges); - - if (Result.Ranges.empty()) - { - // Jupiter will ignore the ranges and send the whole payload if it fetches the payload from S3/Replicated - // Upload to cache (if enabled) - if (m_BuildCacheStorage && Result.Bytes && m_PopulateCache) - { - m_BuildCacheStorage->PutBuildBlob(m_BuildId, - RawHash, - Result.Bytes.GetContentType(), - CompositeBuffer(SharedBuffer(Result.Bytes))); - } - } - } + Result.Bytes = std::move(BlobRanges.PayloadBuffer); + Result.Ranges = std::move(BlobRanges.Ranges); } } catch (const HttpClientError& Ex) @@ -674,7 +566,7 @@ public: return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode) override + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) override { LoadAttachmentsResult Result; Stopwatch Timer; @@ -682,67 +574,20 @@ public: std::vector AttachmentsLeftToFind = RawHashes; - if (m_BuildCacheStorage && SourceMode != ESourceMode::kHostOnly) - { - std::vector ExistCheck = m_BuildCacheStorage->BlobsExists(m_BuildId, RawHashes); - if (ExistCheck.size() == RawHashes.size()) - { - AttachmentsLeftToFind.clear(); - for (size_t BlobIndex = 0; BlobIndex < RawHashes.size(); BlobIndex++) - { - const IoHash& Hash = RawHashes[BlobIndex]; - const BuildStorageCache::BlobExistsResult& BlobExists = ExistCheck[BlobIndex]; - if (BlobExists.HasBody) - { - IoBuffer CachedPayload = m_BuildCacheStorage->GetBuildBlob(m_BuildId, Hash); - if (CachedPayload) - { - Result.Chunks.emplace_back( - std::pair{Hash, - CompressedBuffer::FromCompressedNoValidate(std::move(CachedPayload))}); - } - else - { - AttachmentsLeftToFind.push_back(Hash); - } - } - else - { - AttachmentsLeftToFind.push_back(Hash); - } - } - } - } - for (const IoHash& Hash : AttachmentsLeftToFind) { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash, SourceMode); + LoadAttachmentResult ChunkResult = LoadAttachment(Hash); if (ChunkResult.ErrorCode) { return LoadAttachmentsResult{ChunkResult}; } ZEN_DEBUG("Loaded attachment in {}", NiceTimeSpanMs(static_cast(ChunkResult.ElapsedSeconds * 1000))); - if (m_BuildCacheStorage && ChunkResult.Bytes && m_PopulateCache) - { - m_BuildCacheStorage->PutBuildBlob(m_BuildId, - Hash, - ChunkResult.Bytes.GetContentType(), - CompositeBuffer(SharedBuffer(ChunkResult.Bytes))); - } Result.Chunks.emplace_back( std::pair{Hash, CompressedBuffer::FromCompressedNoValidate(std::move(ChunkResult.Bytes))}); } return Result; } - virtual void Flush() override - { - if (m_BuildCacheStorage) - { - m_BuildCacheStorage->Flush(100, [](intptr_t) { return false; }); - } - } - private: static int MakeErrorCode(const HttpClientError& Ex) { @@ -759,10 +604,6 @@ private: HttpClient m_BuildStorageHttp; std::unique_ptr m_BuildStorage; - BuildStorageCache::Statistics m_StorageCacheStats; - std::unique_ptr m_BuildCacheStorageHttp; - std::unique_ptr m_BuildCacheStorage; - const std::string m_Namespace; const std::string m_Bucket; const Oid m_BuildId; @@ -771,125 +612,34 @@ private: const bool m_EnableBlocks = true; const bool m_UseTempBlocks = true; const bool m_AllowRedirect = false; - const bool m_PopulateCache = true; }; std::shared_ptr -CreateJupiterBuildsRemoteStore(LoggerRef InLog, - const BuildsRemoteStoreOptions& Options, - const std::filesystem::path& TempFilePath, - bool Quiet, - bool Unattended, - bool Hidden, - WorkerThreadPool& CacheBackgroundWorkerPool, - double& OutHostLatencySec, - double& OutCacheLatencySec) +CreateJupiterBuildsRemoteStore(LoggerRef InLog, + const BuildStorageResolveResult& ResolveResult, + std::function&& TokenProvider, + const BuildsRemoteStoreOptions& Options, + const std::filesystem::path& TempFilePath) { - std::string Host = Options.Host; - if (!Host.empty() && Host.find("://"sv) == std::string::npos) - { - // Assume https URL - Host = fmt::format("https://{}"sv, Host); - } - std::string OverrideUrl = Options.OverrideHost; - if (!OverrideUrl.empty() && OverrideUrl.find("://"sv) == std::string::npos) - { - // Assume https URL - OverrideUrl = fmt::format("https://{}"sv, OverrideUrl); - } - std::string ZenHost = Options.ZenHost; - if (!ZenHost.empty() && ZenHost.find("://"sv) == std::string::npos) - { - // Assume https URL - ZenHost = fmt::format("https://{}"sv, ZenHost); - } - - // 1) openid-provider if given (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider - // 2) Access token as parameter in request - // 3) Environment variable (different win vs linux/mac) - // 4) Default openid-provider (assumes oidctoken.exe -Zen true has been run with matching Options.OpenIdProvider - - std::function TokenProvider; - if (!Options.OpenIdProvider.empty()) - { - TokenProvider = httpclientauth::CreateFromOpenIdProvider(Options.AuthManager, Options.OpenIdProvider); - } - else if (!Options.AccessToken.empty()) - { - TokenProvider = httpclientauth::CreateFromStaticToken(Options.AccessToken); - } - else if (!Options.OidcExePath.empty()) - { - if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(Options.OidcExePath, - Host.empty() ? OverrideUrl : Host, - Quiet, - Unattended, - Hidden); - TokenProviderMaybe) - { - TokenProvider = TokenProviderMaybe.value(); - } - } - - if (!TokenProvider) - { - TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(Options.AuthManager); - } - - BuildStorageResolveResult ResolveRes; - { - HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", - .AccessTokenProvider = TokenProvider, - .AssumeHttp2 = Options.AssumeHttp2, - .AllowResume = true, - .RetryCount = 2}; - - std::unique_ptr Output(CreateStandardLogOutput(InLog)); - - ResolveRes = - ResolveBuildStorage(*Output, ClientSettings, Host, OverrideUrl, ZenHost, ZenCacheResolveMode::Discovery, /*Verbose*/ false); - } - HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", .ConnectTimeout = std::chrono::milliseconds(3000), .Timeout = std::chrono::milliseconds(1800000), .AccessTokenProvider = std::move(TokenProvider), - .AssumeHttp2 = ResolveRes.HostAssumeHttp2, + .AssumeHttp2 = ResolveResult.Cloud.AssumeHttp2, .AllowResume = true, .RetryCount = 4, .MaximumInMemoryDownloadSize = Options.MaximumInMemoryDownloadSize}; - std::unique_ptr CacheClientSettings; - - if (!ResolveRes.CacheUrl.empty()) - { - CacheClientSettings = - std::make_unique(HttpClientSettings{.LogCategory = "httpcacheclient", - .ConnectTimeout = std::chrono::milliseconds{3000}, - .Timeout = std::chrono::milliseconds{30000}, - .AssumeHttp2 = ResolveRes.CacheAssumeHttp2, - .AllowResume = true, - .RetryCount = 0, - .MaximumInMemoryDownloadSize = Options.MaximumInMemoryDownloadSize}); - } - std::shared_ptr RemoteStore = std::make_shared(InLog, ClientSettings, - CacheClientSettings.get(), - ResolveRes.HostUrl, - ResolveRes.CacheUrl, + ResolveResult.Cloud.Address, TempFilePath, - CacheBackgroundWorkerPool, Options.Namespace, Options.Bucket, Options.BuildId, Options.MetaData, Options.ForceDisableBlocks, - Options.ForceDisableTempBlocks, - Options.PopulateCache); - - OutHostLatencySec = ResolveRes.HostLatencySec; - OutCacheLatencySec = ResolveRes.CacheLatencySec; + Options.ForceDisableTempBlocks); return RemoteStore; } diff --git a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp index f950fd46c..bb21de12c 100644 --- a/src/zenremotestore/projectstore/fileremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/fileremoteprojectstore.cpp @@ -7,8 +7,12 @@ #include #include #include +#include #include #include +#include + +#include namespace zen { @@ -74,9 +78,11 @@ public: virtual SaveResult SaveContainer(const IoBuffer& Payload) override { - Stopwatch Timer; SaveResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Result, &Timer]() { Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; }); + { CbObject ContainerObject = LoadCompactBinaryObject(Payload); @@ -87,6 +93,10 @@ public: { Result.Needs.insert(AttachmentHash); } + else if (std::filesystem::path AttachmentMetaPath = GetAttachmentMetaPath(AttachmentHash); IsFile(AttachmentMetaPath)) + { + BasicFile TouchIt(AttachmentMetaPath, BasicFile::Mode::kWrite); + } }); } @@ -112,14 +122,18 @@ public: Result.Reason = fmt::format("Failed saving oplog container to '{}'. Reason: {}", ContainerPath, Ex.what()); } AddStats(Payload.GetSize(), 0, Timer.GetElapsedTimeUs() * 1000); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; return Result; } - virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, const IoHash& RawHash, ChunkBlockDescription&&) override + virtual SaveAttachmentResult SaveAttachment(const CompositeBuffer& Payload, + const IoHash& RawHash, + ChunkBlockDescription&& BlockDescription) override { - Stopwatch Timer; - SaveAttachmentResult Result; + SaveAttachmentResult Result; + + Stopwatch Timer; + auto _ = MakeGuard([&Result, &Timer]() { Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; }); + std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); if (!IsFile(ChunkPath)) { @@ -142,14 +156,33 @@ public: Result.Reason = fmt::format("Failed saving oplog attachment to '{}'. Reason: {}", ChunkPath, Ex.what()); } } + if (!Result.ErrorCode && BlockDescription.BlockHash != IoHash::Zero) + { + try + { + std::filesystem::path MetaPath = GetAttachmentMetaPath(RawHash); + CbObject MetaData = BuildChunkBlockDescription(BlockDescription, {}); + SharedBuffer MetaBuffer = MetaData.GetBuffer(); + BasicFile MetaFile; + MetaFile.Open(MetaPath, BasicFile::Mode::kTruncate); + MetaFile.Write(MetaBuffer.GetView(), 0); + } + catch (const std::exception& Ex) + { + Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("Failed saving block description to '{}'. Reason: {}", RawHash, Ex.what()); + } + } AddStats(Payload.GetSize(), 0, Timer.GetElapsedTimeUs() * 1000); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; return Result; } virtual SaveAttachmentsResult SaveAttachments(const std::vector& Chunks) override { + SaveAttachmentsResult Result; + Stopwatch Timer; + auto _ = MakeGuard([&Result, &Timer]() { Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; }); for (const SharedBuffer& Chunk : Chunks) { @@ -157,12 +190,10 @@ public: SaveAttachmentResult ChunkResult = SaveAttachment(Compressed.GetCompressed(), Compressed.DecodeRawHash(), {}); if (ChunkResult.ErrorCode) { - ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; - return SaveAttachmentsResult{ChunkResult}; + Result = SaveAttachmentsResult{ChunkResult}; + break; } } - SaveAttachmentsResult Result; - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; return Result; } @@ -172,21 +203,60 @@ public: virtual GetKnownBlocksResult GetKnownBlocks() override { + Stopwatch Timer; if (m_OptionalBaseName.empty()) { - return GetKnownBlocksResult{{.ErrorCode = static_cast(HttpResponseCode::NoContent)}}; + size_t MaxBlockCount = 10000; + + GetKnownBlocksResult Result; + + DirectoryContent Content; + GetDirectoryContent( + m_OutputPath, + DirectoryContentFlags::IncludeFiles | DirectoryContentFlags::Recursive | DirectoryContentFlags::IncludeModificationTick, + Content); + std::vector RecentOrder(Content.Files.size()); + std::iota(RecentOrder.begin(), RecentOrder.end(), 0u); + std::sort(RecentOrder.begin(), RecentOrder.end(), [&Content](size_t Lhs, size_t Rhs) { + return Content.FileModificationTicks[Lhs] > Content.FileModificationTicks[Rhs]; + }); + + for (size_t FileIndex : RecentOrder) + { + std::filesystem::path MetaPath = Content.Files[FileIndex]; + if (MetaPath.extension() == MetaExtension) + { + IoBuffer MetaFile = ReadFile(MetaPath).Flatten(); + CbValidateError Err; + CbObject ValidatedObject = ValidateAndReadCompactBinaryObject(std::move(MetaFile), Err); + if (Err == CbValidateError::None) + { + ChunkBlockDescription Description = ParseChunkBlockDescription(ValidatedObject); + if (Description.BlockHash != IoHash::Zero) + { + Result.Blocks.emplace_back(std::move(Description)); + if (Result.Blocks.size() == MaxBlockCount) + { + break; + } + } + } + } + } + + Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; + return Result; } LoadContainerResult LoadResult = LoadContainer(m_OptionalBaseName); if (LoadResult.ErrorCode) { return GetKnownBlocksResult{LoadResult}; } - Stopwatch Timer; std::vector BlockHashes = GetBlockHashesFromOplog(LoadResult.ContainerObject); if (BlockHashes.empty()) { return GetKnownBlocksResult{{.ErrorCode = static_cast(HttpResponseCode::NoContent), - .ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; + .ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeMs() / 1000.0}}; } std::vector ExistingBlockHashes; for (const IoHash& RawHash : BlockHashes) @@ -200,15 +270,15 @@ public: if (ExistingBlockHashes.empty()) { return GetKnownBlocksResult{{.ErrorCode = static_cast(HttpResponseCode::NoContent), - .ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; + .ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeMs() / 1000.0}}; } std::vector ThinKnownBlocks = GetBlocksFromOplog(LoadResult.ContainerObject, ExistingBlockHashes); - const size_t KnowBlockCount = ThinKnownBlocks.size(); + const size_t KnownBlockCount = ThinKnownBlocks.size(); - GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeUs() * 1000}}; - Result.Blocks.resize(KnowBlockCount); - for (size_t BlockIndex = 0; BlockIndex < KnowBlockCount; BlockIndex++) + GetKnownBlocksResult Result{{.ElapsedSeconds = LoadResult.ElapsedSeconds + Timer.GetElapsedTimeMs() / 1000.0}}; + Result.Blocks.resize(KnownBlockCount); + for (size_t BlockIndex = 0; BlockIndex < KnownBlockCount; BlockIndex++) { Result.Blocks[BlockIndex].BlockHash = ThinKnownBlocks[BlockIndex].BlockHash; Result.Blocks[BlockIndex].ChunkRawHashes = std::move(ThinKnownBlocks[BlockIndex].ChunkRawHashes); @@ -217,87 +287,141 @@ public: return Result; } - virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) override + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes, + BuildStorageCache* OptionalCache, + const Oid& CacheBuildId) override { - ZEN_UNUSED(BlockHashes); - return GetBlockDescriptionsResult{Result{.ErrorCode = int(HttpResponseCode::NotFound)}}; - } + GetBlockDescriptionsResult Result; - virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) override - { - return AttachmentExistsInCacheResult{Result{.ErrorCode = 0}, std::vector(RawHashes.size(), false)}; - } + Stopwatch Timer; + auto _ = MakeGuard([&Result, &Timer]() { Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; }); - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode) override - { - Stopwatch Timer; - LoadAttachmentResult Result; - if (SourceMode != ESourceMode::kCacheOnly) + Result.Blocks.reserve(BlockHashes.size()); + + uint64_t ByteCount = 0; + + std::vector UnorderedList; { - std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!IsFile(ChunkPath)) + if (OptionalCache) { - Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); - Result.Reason = - fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; - return Result; + std::vector CacheBlockMetadatas = OptionalCache->GetBlobMetadatas(CacheBuildId, BlockHashes); + for (const CbObject& BlockObject : CacheBlockMetadatas) + { + ByteCount += BlockObject.GetSize(); + } + UnorderedList = ParseBlockMetadatas(CacheBlockMetadatas); } + + tsl::robin_map BlockDescriptionLookup; + BlockDescriptionLookup.reserve(BlockHashes.size()); + for (size_t DescriptionIndex = 0; DescriptionIndex < UnorderedList.size(); DescriptionIndex++) { - BasicFile ChunkFile; - ChunkFile.Open(ChunkPath, BasicFile::Mode::kRead); - Result.Bytes = ChunkFile.ReadAll(); + const ChunkBlockDescription& Description = UnorderedList[DescriptionIndex]; + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, DescriptionIndex); + } + + if (UnorderedList.size() < BlockHashes.size()) + { + for (const IoHash& RawHash : BlockHashes) + { + if (!BlockDescriptionLookup.contains(RawHash)) + { + std::filesystem::path MetaPath = GetAttachmentMetaPath(RawHash); + IoBuffer MetaFile = ReadFile(MetaPath).Flatten(); + ByteCount += MetaFile.GetSize(); + CbValidateError Err; + CbObject ValidatedObject = ValidateAndReadCompactBinaryObject(std::move(MetaFile), Err); + if (Err == CbValidateError::None) + { + ChunkBlockDescription Description = ParseChunkBlockDescription(ValidatedObject); + if (Description.BlockHash != IoHash::Zero) + { + BlockDescriptionLookup.insert_or_assign(Description.BlockHash, UnorderedList.size()); + UnorderedList.emplace_back(std::move(Description)); + } + } + } + } + } + + Result.Blocks.reserve(UnorderedList.size()); + for (const IoHash& RawHash : BlockHashes) + { + if (auto It = BlockDescriptionLookup.find(RawHash); It != BlockDescriptionLookup.end()) + { + Result.Blocks.emplace_back(std::move(UnorderedList[It->second])); + } } } + AddStats(0, ByteCount, Timer.GetElapsedTimeUs() * 1000); + return Result; + } + + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override + { + LoadAttachmentResult Result; + + Stopwatch Timer; + auto _ = MakeGuard([&Result, &Timer]() { Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; }); + + std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); + if (!IsFile(ChunkPath)) + { + Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); + Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); + return Result; + } + { + BasicFile ChunkFile; + ChunkFile.Open(ChunkPath, BasicFile::Mode::kRead); + Result.Bytes = ChunkFile.ReadAll(); + } AddStats(0, Result.Bytes.GetSize(), Timer.GetElapsedTimeUs() * 1000); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; return Result; } virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, - std::span> Ranges, - ESourceMode SourceMode) override + std::span> Ranges) override { - Stopwatch Timer; + ZEN_ASSERT(!Ranges.empty()); LoadAttachmentRangesResult Result; - if (SourceMode != ESourceMode::kCacheOnly) - { - std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); - if (!IsFile(ChunkPath)) - { - Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); - Result.Reason = - fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; - return Result; - } - { - BasicFile ChunkFile; - ChunkFile.Open(ChunkPath, BasicFile::Mode::kRead); - uint64_t Start = Ranges.front().first; - uint64_t Length = Ranges.back().first + Ranges.back().second - Ranges.front().first; + Stopwatch Timer; + auto _ = MakeGuard([&Result, &Timer]() { Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; }); - Result.Bytes = ChunkFile.ReadRange(Start, Length); - Result.Ranges.reserve(Ranges.size()); - for (const std::pair& Range : Ranges) - { - Result.Ranges.push_back(std::make_pair(Range.first - Start, Range.second)); - } + std::filesystem::path ChunkPath = GetAttachmentPath(RawHash); + if (!IsFile(ChunkPath)) + { + Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); + Result.Reason = fmt::format("Failed loading oplog attachment from '{}'. Reason: 'The file does not exist'", ChunkPath.string()); + return Result; + } + { + uint64_t Start = Ranges.front().first; + uint64_t Length = Ranges.back().first + Ranges.back().second - Ranges.front().first; + Result.Bytes = IoBufferBuilder::MakeFromFile(ChunkPath, Start, Length); + Result.Ranges.reserve(Ranges.size()); + for (const std::pair& Range : Ranges) + { + Result.Ranges.push_back(std::make_pair(Range.first - Start, Range.second)); } } - AddStats(0, Result.Bytes.GetSize(), Timer.GetElapsedTimeUs() * 1000); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; + AddStats(0, + std::accumulate(Result.Ranges.begin(), + Result.Ranges.end(), + uint64_t(0), + [](uint64_t Current, const std::pair& Value) { return Current + Value.second; }), + Timer.GetElapsedTimeUs() * 1000); return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode) override + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) override { Stopwatch Timer; LoadAttachmentsResult Result; for (const IoHash& Hash : RawHashes) { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash, SourceMode); + LoadAttachmentResult ChunkResult = LoadAttachment(Hash); if (ChunkResult.ErrorCode) { ChunkResult.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; @@ -310,20 +434,20 @@ public: return Result; } - virtual void Flush() override {} - private: LoadContainerResult LoadContainer(const std::string& Name) { - Stopwatch Timer; - LoadContainerResult Result; + LoadContainerResult Result; + + Stopwatch Timer; + auto _ = MakeGuard([&Result, &Timer]() { Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; }); + std::filesystem::path SourcePath = m_OutputPath; SourcePath.append(Name); if (!IsFile(SourcePath)) { Result.ErrorCode = gsl::narrow(HttpResponseCode::NotFound); Result.Reason = fmt::format("Failed loading oplog container from '{}'. Reason: 'The file does not exist'", SourcePath.string()); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; return Result; } IoBuffer ContainerPayload; @@ -337,18 +461,16 @@ private: if (Result.ContainerObject = ValidateAndReadCompactBinaryObject(std::move(ContainerPayload), ValidateResult); ValidateResult != CbValidateError::None || !Result.ContainerObject) { - Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); - Result.Reason = fmt::format("The file {} is not formatted as a compact binary object ('{}')", - SourcePath.string(), - ToString(ValidateResult)); - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; + Result.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError); + Result.Reason = fmt::format("The file {} is not formatted as a compact binary object ('{}')", + SourcePath.string(), + ToString(ValidateResult)); return Result; } - Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; return Result; } - std::filesystem::path GetAttachmentPath(const IoHash& RawHash) const + std::filesystem::path GetAttachmentBasePath(const IoHash& RawHash) const { ExtendablePathBuilder<128> ShardedPath; ShardedPath.Append(m_OutputPath.c_str()); @@ -367,6 +489,19 @@ private: return ShardedPath.ToPath(); } + static constexpr std::string_view BlobExtension = ".blob"; + static constexpr std::string_view MetaExtension = ".meta"; + + std::filesystem::path GetAttachmentPath(const IoHash& RawHash) + { + return GetAttachmentBasePath(RawHash).replace_extension(BlobExtension); + } + + std::filesystem::path GetAttachmentMetaPath(const IoHash& RawHash) + { + return GetAttachmentBasePath(RawHash).replace_extension(MetaExtension); + } + void AddStats(uint64_t UploadedBytes, uint64_t DownloadedBytes, uint64_t ElapsedNS) { m_SentBytes.fetch_add(UploadedBytes); diff --git a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp index 514484f30..5b456cb4c 100644 --- a/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/jupiterremoteprojectstore.cpp @@ -212,73 +212,64 @@ public: return Result; } - virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) override + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes, + BuildStorageCache* OptionalCache, + const Oid& CacheBuildId) override { - ZEN_UNUSED(BlockHashes); + ZEN_UNUSED(BlockHashes, OptionalCache, CacheBuildId); return GetBlockDescriptionsResult{Result{.ErrorCode = int(HttpResponseCode::NotFound)}}; } - virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) override - { - return AttachmentExistsInCacheResult{Result{.ErrorCode = 0}, std::vector(RawHashes.size(), false)}; - } - - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode) override + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override { LoadAttachmentResult Result; - if (SourceMode != ESourceMode::kCacheOnly) - { - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); - AddStats(GetResult); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); + JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); + AddStats(GetResult); - Result = {ConvertResult(GetResult), std::move(GetResult.Response)}; - if (GetResult.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - RawHash, - Result.Reason); - } + Result = {ConvertResult(GetResult), std::move(GetResult.Response)}; + if (GetResult.ErrorCode) + { + Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'", + m_JupiterClient->ServiceUrl(), + m_Namespace, + RawHash, + Result.Reason); } return Result; } virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, - std::span> Ranges, - ESourceMode SourceMode) override + std::span> Ranges) override { + ZEN_ASSERT(!Ranges.empty()); LoadAttachmentRangesResult Result; - if (SourceMode != ESourceMode::kCacheOnly) - { - JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); - JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); - AddStats(GetResult); + JupiterSession Session(m_JupiterClient->Logger(), m_JupiterClient->Client(), m_AllowRedirect); + JupiterResult GetResult = Session.GetCompressedBlob(m_Namespace, RawHash, m_TempFilePath); + AddStats(GetResult); - Result = LoadAttachmentRangesResult{ConvertResult(GetResult), std::move(GetResult.Response)}; - if (GetResult.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'", - m_JupiterClient->ServiceUrl(), - m_Namespace, - RawHash, - Result.Reason); - } - else - { - Result.Ranges = std::vector>(Ranges.begin(), Ranges.end()); - } + Result = LoadAttachmentRangesResult{ConvertResult(GetResult), std::move(GetResult.Response)}; + if (GetResult.ErrorCode) + { + Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}. Reason: '{}'", + m_JupiterClient->ServiceUrl(), + m_Namespace, + RawHash, + Result.Reason); + } + else + { + Result.Ranges = std::vector>(Ranges.begin(), Ranges.end()); } return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode) override + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) override { LoadAttachmentsResult Result; for (const IoHash& Hash : RawHashes) { - LoadAttachmentResult ChunkResult = LoadAttachment(Hash, SourceMode); + LoadAttachmentResult ChunkResult = LoadAttachment(Hash); if (ChunkResult.ErrorCode) { return LoadAttachmentsResult{ChunkResult}; @@ -290,8 +281,6 @@ public: return Result; } - virtual void Flush() override {} - private: LoadContainerResult LoadContainer(const IoHash& Key) { diff --git a/src/zenremotestore/projectstore/projectstoreoperations.cpp b/src/zenremotestore/projectstore/projectstoreoperations.cpp index becac3d4c..36dc4d868 100644 --- a/src/zenremotestore/projectstore/projectstoreoperations.cpp +++ b/src/zenremotestore/projectstore/projectstoreoperations.cpp @@ -426,19 +426,19 @@ ProjectStoreOperationDownloadAttachments::Execute() auto GetBuildBlob = [this](const IoHash& RawHash, const std::filesystem::path& OutputPath) { IoBuffer Payload; - if (m_Storage.BuildCacheStorage) + if (m_Storage.CacheStorage) { - Payload = m_Storage.BuildCacheStorage->GetBuildBlob(m_State.GetBuildId(), RawHash); + Payload = m_Storage.CacheStorage->GetBuildBlob(m_State.GetBuildId(), RawHash); } if (!Payload) { Payload = m_Storage.BuildStorage->GetBuildBlob(m_State.GetBuildId(), RawHash); - if (m_Storage.BuildCacheStorage && m_Options.PopulateCache) + if (m_Storage.CacheStorage && m_Options.PopulateCache) { - m_Storage.BuildCacheStorage->PutBuildBlob(m_State.GetBuildId(), - RawHash, - Payload.GetContentType(), - CompositeBuffer(SharedBuffer(Payload))); + m_Storage.CacheStorage->PutBuildBlob(m_State.GetBuildId(), + RawHash, + Payload.GetContentType(), + CompositeBuffer(SharedBuffer(Payload))); } } uint64_t PayloadSize = Payload.GetSize(); diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index c8c5f201d..d5c6286a8 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -124,14 +125,17 @@ namespace remotestore_impl { return OptionalContext->IsCancelled(); } - std::string GetStats(const RemoteProjectStore::Stats& Stats, uint64_t ElapsedWallTimeMS) + std::string GetStats(const RemoteProjectStore::Stats& Stats, + const BuildStorageCache::Statistics* OptionalCacheStats, + uint64_t ElapsedWallTimeMS) { - return fmt::format( - "Sent: {} ({}bits/s) Recv: {} ({}bits/s)", - NiceBytes(Stats.m_SentBytes), - NiceNum(ElapsedWallTimeMS > 0u ? static_cast((Stats.m_SentBytes * 8 * 1000) / ElapsedWallTimeMS) : 0u), - NiceBytes(Stats.m_ReceivedBytes), - NiceNum(ElapsedWallTimeMS > 0u ? static_cast((Stats.m_ReceivedBytes * 8 * 1000) / ElapsedWallTimeMS) : 0u)); + uint64_t SentBytes = Stats.m_SentBytes + (OptionalCacheStats ? OptionalCacheStats->TotalBytesWritten.load() : 0); + uint64_t ReceivedBytes = Stats.m_ReceivedBytes + (OptionalCacheStats ? OptionalCacheStats->TotalBytesRead.load() : 0); + return fmt::format("Sent: {} ({}bits/s) Recv: {} ({}bits/s)", + NiceBytes(SentBytes), + NiceNum(ElapsedWallTimeMS > 0u ? static_cast((SentBytes * 8 * 1000) / ElapsedWallTimeMS) : 0u), + NiceBytes(ReceivedBytes), + NiceNum(ElapsedWallTimeMS > 0u ? static_cast((ReceivedBytes * 8 * 1000) / ElapsedWallTimeMS) : 0u)); } void LogRemoteStoreStatsDetails(const RemoteProjectStore::Stats& Stats) @@ -269,12 +273,7 @@ namespace remotestore_impl { JobContext* m_OptionalContext; }; - void DownloadAndSaveBlockChunks(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - bool IgnoreMissingAttachments, - JobContext* OptionalContext, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, + void DownloadAndSaveBlockChunks(LoadOplogContext& Context, Latch& AttachmentsDownloadLatch, Latch& AttachmentsWriteLatch, AsyncRemoteResult& RemoteResult, @@ -285,10 +284,8 @@ namespace remotestore_impl { std::vector&& NeededChunkIndexes) { AttachmentsDownloadLatch.AddCount(1); - NetworkWorkerPool.ScheduleWork( - [&RemoteStore, - &ChunkStore, - &WorkerPool, + Context.NetworkWorkerPool.ScheduleWork( + [&Context, &AttachmentsDownloadLatch, &AttachmentsWriteLatch, &RemoteResult, @@ -296,9 +293,7 @@ namespace remotestore_impl { NeededChunkIndexes = std::move(NeededChunkIndexes), &Info, &LoadAttachmentsTimer, - &DownloadStartMS, - IgnoreMissingAttachments, - OptionalContext]() { + &DownloadStartMS]() { ZEN_TRACE_CPU("DownloadBlockChunks"); auto _ = MakeGuard([&AttachmentsDownloadLatch] { AttachmentsDownloadLatch.CountDown(); }); @@ -317,16 +312,16 @@ namespace remotestore_impl { uint64_t Unset = (std::uint64_t)-1; DownloadStartMS.compare_exchange_strong(Unset, LoadAttachmentsTimer.GetElapsedTimeMs()); - RemoteProjectStore::LoadAttachmentsResult Result = RemoteStore.LoadAttachments(Chunks); + RemoteProjectStore::LoadAttachmentsResult Result = Context.RemoteStore.LoadAttachments(Chunks); if (Result.ErrorCode) { - ReportMessage(OptionalContext, + ReportMessage(Context.OptionalJobContext, fmt::format("Failed to load attachments with {} chunks ({}): {}", Chunks.size(), RemoteResult.GetError(), RemoteResult.GetErrorReason())); Info.MissingAttachmentCount.fetch_add(1); - if (IgnoreMissingAttachments) + if (Context.IgnoreMissingAttachments) { RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text); } @@ -338,7 +333,7 @@ namespace remotestore_impl { uint64_t ChunkSize = It.second.GetCompressedSize(); Info.AttachmentBytesDownloaded.fetch_add(ChunkSize); } - remotestore_impl::ReportMessage(OptionalContext, + remotestore_impl::ReportMessage(Context.OptionalJobContext, fmt::format("Loaded {} bulk attachments in {}", Chunks.size(), NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000)))); @@ -347,8 +342,8 @@ namespace remotestore_impl { return; } AttachmentsWriteLatch.AddCount(1); - WorkerPool.ScheduleWork( - [&AttachmentsWriteLatch, &RemoteResult, &Info, &ChunkStore, Chunks = std::move(Result.Chunks)]() { + Context.WorkerPool.ScheduleWork( + [&AttachmentsWriteLatch, &RemoteResult, &Info, &Context, Chunks = std::move(Result.Chunks)]() { auto _ = MakeGuard([&AttachmentsWriteLatch] { AttachmentsWriteLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -369,7 +364,9 @@ namespace remotestore_impl { WriteRawHashes.push_back(It.first); } std::vector InsertResults = - ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes, CidStore::InsertMode::kCopyOnly); + Context.ChunkStore.AddChunks(WriteAttachmentBuffers, + WriteRawHashes, + CidStore::InsertMode::kCopyOnly); for (size_t Index = 0; Index < InsertResults.size(); Index++) { @@ -400,12 +397,7 @@ namespace remotestore_impl { WorkerThreadPool::EMode::EnableBacklog); }; - void DownloadAndSaveBlock(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - bool IgnoreMissingAttachments, - JobContext* OptionalContext, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, + void DownloadAndSaveBlock(LoadOplogContext& Context, Latch& AttachmentsDownloadLatch, Latch& AttachmentsWriteLatch, AsyncRemoteResult& RemoteResult, @@ -418,19 +410,14 @@ namespace remotestore_impl { uint32_t RetriesLeft) { AttachmentsDownloadLatch.AddCount(1); - NetworkWorkerPool.ScheduleWork( + Context.NetworkWorkerPool.ScheduleWork( [&AttachmentsDownloadLatch, &AttachmentsWriteLatch, - &ChunkStore, - &RemoteStore, - &NetworkWorkerPool, - &WorkerPool, + &Context, &RemoteResult, &Info, &LoadAttachmentsTimer, &DownloadStartMS, - IgnoreMissingAttachments, - OptionalContext, RetriesLeft, BlockHash = IoHash(BlockHash), &AllNeededPartialChunkHashesLookup, @@ -446,52 +433,65 @@ namespace remotestore_impl { { uint64_t Unset = (std::uint64_t)-1; DownloadStartMS.compare_exchange_strong(Unset, LoadAttachmentsTimer.GetElapsedTimeMs()); - RemoteProjectStore::LoadAttachmentResult BlockResult = RemoteStore.LoadAttachment(BlockHash); - if (BlockResult.ErrorCode) + + IoBuffer BlobBuffer; + if (Context.OptionalCache) { - ReportMessage(OptionalContext, - fmt::format("Failed to download block attachment {} ({}): {}", - BlockHash, - RemoteResult.GetError(), - RemoteResult.GetErrorReason())); - Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) - { - RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text); - } - return; + BlobBuffer = Context.OptionalCache->GetBuildBlob(Context.CacheBuildId, BlockHash); } - if (RemoteResult.IsError()) + + if (!BlobBuffer) { - return; + RemoteProjectStore::LoadAttachmentResult BlockResult = Context.RemoteStore.LoadAttachment(BlockHash); + if (BlockResult.ErrorCode) + { + ReportMessage(Context.OptionalJobContext, + fmt::format("Failed to download block attachment {} ({}): {}", + BlockHash, + BlockResult.Reason, + BlockResult.Text)); + Info.MissingAttachmentCount.fetch_add(1); + if (!Context.IgnoreMissingAttachments) + { + RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text); + } + return; + } + if (RemoteResult.IsError()) + { + return; + } + BlobBuffer = std::move(BlockResult.Bytes); + ZEN_DEBUG("Loaded block attachment '{}' in {} ({})", + BlockHash, + NiceTimeSpanMs(static_cast(BlockResult.ElapsedSeconds * 1000)), + NiceBytes(BlobBuffer.Size())); + if (Context.OptionalCache && Context.PopulateCache) + { + Context.OptionalCache->PutBuildBlob(Context.CacheBuildId, + BlockHash, + BlobBuffer.GetContentType(), + CompositeBuffer(SharedBuffer(BlobBuffer))); + } } - uint64_t BlockSize = BlockResult.Bytes.GetSize(); + uint64_t BlockSize = BlobBuffer.GetSize(); Info.AttachmentBlocksDownloaded.fetch_add(1); - ZEN_DEBUG("Loaded block attachment '{}' in {} ({})", - BlockHash, - NiceTimeSpanMs(static_cast(BlockResult.ElapsedSeconds * 1000)), - NiceBytes(BlockSize)); Info.AttachmentBlockBytesDownloaded.fetch_add(BlockSize); AttachmentsWriteLatch.AddCount(1); - WorkerPool.ScheduleWork( + Context.WorkerPool.ScheduleWork( [&AttachmentsDownloadLatch, &AttachmentsWriteLatch, - &ChunkStore, - &RemoteStore, - &NetworkWorkerPool, - &WorkerPool, + &Context, &RemoteResult, &Info, &LoadAttachmentsTimer, &DownloadStartMS, - IgnoreMissingAttachments, - OptionalContext, RetriesLeft, BlockHash = IoHash(BlockHash), &AllNeededPartialChunkHashesLookup, ChunkDownloadedFlags, - Bytes = std::move(BlockResult.Bytes)]() { + Bytes = std::move(BlobBuffer)]() { auto _ = MakeGuard([&AttachmentsWriteLatch] { AttachmentsWriteLatch.CountDown(); }); if (RemoteResult.IsError()) { @@ -569,7 +569,7 @@ namespace remotestore_impl { if (!WriteAttachmentBuffers.empty()) { std::vector Results = - ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + Context.ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (size_t Index = 0; Index < Results.size(); Index++) { const CidStore::InsertResult& Result = Results[Index]; @@ -598,14 +598,9 @@ namespace remotestore_impl { { if (RetriesLeft > 0) { - ReportMessage(OptionalContext, fmt::format("{}, retrying download", ErrorString)); - - return DownloadAndSaveBlock(ChunkStore, - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, - NetworkWorkerPool, - WorkerPool, + ReportMessage(Context.OptionalJobContext, fmt::format("{}, retrying download", ErrorString)); + + return DownloadAndSaveBlock(Context, AttachmentsDownloadLatch, AttachmentsWriteLatch, RemoteResult, @@ -619,7 +614,7 @@ namespace remotestore_impl { } else { - ReportMessage(OptionalContext, ErrorString); + ReportMessage(Context.OptionalJobContext, ErrorString); RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), ErrorString, {}); return; } @@ -637,28 +632,29 @@ namespace remotestore_impl { catch (const std::exception& Ex) { RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), - fmt::format("Failed to block attachment {}", BlockHash), + fmt::format("Failed to download block attachment {}", BlockHash), Ex.what()); } }, WorkerThreadPool::EMode::EnableBacklog); }; - bool DownloadPartialBlock(RemoteProjectStore& RemoteStore, - bool IgnoreMissingAttachments, - JobContext* OptionalContext, - AsyncRemoteResult& RemoteResult, - DownloadInfo& Info, - double& DownloadTimeSeconds, - const ChunkBlockDescription& BlockDescription, - bool BlockExistsInCache, - std::span BlockRangeDescriptors, - size_t BlockRangeIndexStart, - size_t BlockRangeCount, + void DownloadPartialBlock(LoadOplogContext& Context, + AsyncRemoteResult& RemoteResult, + DownloadInfo& Info, + double& DownloadTimeSeconds, + const ChunkBlockDescription& BlockDescription, + bool BlockExistsInCache, + std::span BlockRangeDescriptors, + size_t BlockRangeIndexStart, + size_t BlockRangeCount, std::function> OffsetAndLengths)>&& OnDownloaded) { + ZEN_ASSERT(Context.StoreMaxRangeCountPerRequest != 0); + ZEN_ASSERT(BlockExistsInCache == false || Context.CacheMaxRangeCountPerRequest != 0); + std::vector> Ranges; Ranges.reserve(BlockRangeDescriptors.size()); for (size_t BlockRangeIndex = BlockRangeIndexStart; BlockRangeIndex < BlockRangeIndexStart + BlockRangeCount; BlockRangeIndex++) @@ -667,65 +663,104 @@ namespace remotestore_impl { Ranges.push_back(std::make_pair(BlockRange.RangeStart, BlockRange.RangeLength)); } - if (BlockExistsInCache) - { - RemoteProjectStore::LoadAttachmentRangesResult BlockResult = - RemoteStore.LoadAttachmentRanges(BlockDescription.BlockHash, Ranges, RemoteProjectStore::ESourceMode::kCacheOnly); - DownloadTimeSeconds += BlockResult.ElapsedSeconds; - if (RemoteResult.IsError()) - { - return false; - } - if (!BlockResult.ErrorCode && BlockResult.Bytes) - { - if (BlockResult.Ranges.size() != Ranges.size()) - { - throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges", - Ranges.size(), - BlockDescription.BlockHash, - BlockResult.Ranges.size())); - } - OnDownloaded(std::move(BlockResult.Bytes), BlockRangeIndexStart, BlockResult.Ranges); - return true; - } - } - - const size_t MaxRangesPerRequestToJupiter = RemoteProjectStore::MaxRangeCountPerRequest; - size_t SubBlockRangeCount = BlockRangeCount; size_t SubRangeCountComplete = 0; std::span> RangesSpan(Ranges); + while (SubRangeCountComplete < SubBlockRangeCount) { if (RemoteResult.IsError()) { break; } - size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, MaxRangesPerRequestToJupiter); + size_t SubRangeStartIndex = BlockRangeIndexStart + SubRangeCountComplete; + if (BlockExistsInCache) + { + ZEN_ASSERT(Context.OptionalCache); + size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, Context.CacheMaxRangeCountPerRequest); + + if (SubRangeCount == 1) + { + // Legacy single-range path, prefer that for max compatibility + + const std::pair SubRange = RangesSpan[SubRangeCountComplete]; + Stopwatch CacheTimer; + IoBuffer PayloadBuffer = Context.OptionalCache->GetBuildBlob(Context.CacheBuildId, + BlockDescription.BlockHash, + SubRange.first, + SubRange.second); + DownloadTimeSeconds += CacheTimer.GetElapsedTimeMs() / 1000.0; + if (RemoteResult.IsError()) + { + break; + } + if (PayloadBuffer) + { + OnDownloaded(std::move(PayloadBuffer), + SubRangeStartIndex, + std::vector>{std::make_pair(0u, SubRange.second)}); + SubRangeCountComplete += SubRangeCount; + continue; + } + } + else + { + auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); + + Stopwatch CacheTimer; + BuildStorageCache::BuildBlobRanges RangeBuffers = + Context.OptionalCache->GetBuildBlobRanges(Context.CacheBuildId, BlockDescription.BlockHash, SubRanges); + DownloadTimeSeconds += CacheTimer.GetElapsedTimeMs() / 1000.0; + if (RemoteResult.IsError()) + { + break; + } + if (RangeBuffers.PayloadBuffer) + { + if (RangeBuffers.Ranges.empty()) + { + SubRangeCount = Ranges.size() - SubRangeCountComplete; + OnDownloaded(std::move(RangeBuffers.PayloadBuffer), + SubRangeStartIndex, + RangesSpan.subspan(SubRangeCountComplete, SubRangeCount)); + SubRangeCountComplete += SubRangeCount; + continue; + } + else if (RangeBuffers.Ranges.size() == SubRangeCount) + { + OnDownloaded(std::move(RangeBuffers.PayloadBuffer), SubRangeStartIndex, RangeBuffers.Ranges); + SubRangeCountComplete += SubRangeCount; + continue; + } + } + } + } + + size_t SubRangeCount = Min(BlockRangeCount - SubRangeCountComplete, Context.StoreMaxRangeCountPerRequest); auto SubRanges = RangesSpan.subspan(SubRangeCountComplete, SubRangeCount); RemoteProjectStore::LoadAttachmentRangesResult BlockResult = - RemoteStore.LoadAttachmentRanges(BlockDescription.BlockHash, SubRanges, RemoteProjectStore::ESourceMode::kHostOnly); + Context.RemoteStore.LoadAttachmentRanges(BlockDescription.BlockHash, SubRanges); DownloadTimeSeconds += BlockResult.ElapsedSeconds; if (RemoteResult.IsError()) { - return false; + break; } if (BlockResult.ErrorCode || !BlockResult.Bytes) { - ReportMessage(OptionalContext, + ReportMessage(Context.OptionalJobContext, fmt::format("Failed to download {} ranges from block attachment '{}' ({}): {}", SubRanges.size(), BlockDescription.BlockHash, BlockResult.ErrorCode, BlockResult.Reason)); Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) + if (!Context.IgnoreMissingAttachments) { RemoteResult.SetError(BlockResult.ErrorCode, BlockResult.Reason, BlockResult.Text); - return false; + break; } } else @@ -734,6 +769,18 @@ namespace remotestore_impl { { // Jupiter will ignore the ranges and send the whole payload if it fetches the payload from S3 // Use the whole payload for the remaining ranges + + if (Context.OptionalCache && Context.PopulateCache) + { + Context.OptionalCache->PutBuildBlob(Context.CacheBuildId, + BlockDescription.BlockHash, + ZenContentType::kCompressedBinary, + CompositeBuffer(std::vector{BlockResult.Bytes})); + if (RemoteResult.IsError()) + { + break; + } + } SubRangeCount = Ranges.size() - SubRangeCountComplete; OnDownloaded(std::move(BlockResult.Bytes), SubRangeStartIndex, @@ -743,10 +790,13 @@ namespace remotestore_impl { { if (BlockResult.Ranges.size() != SubRanges.size()) { - throw std::runtime_error(fmt::format("Fetching {} ranges from {} resulted in {} ranges", - SubRanges.size(), - BlockDescription.BlockHash, - BlockResult.Ranges.size())); + RemoteResult.SetError(gsl::narrow(HttpResponseCode::InternalServerError), + fmt::format("Range response for block {} contains {} ranges, expected {} ranges", + BlockDescription.BlockHash, + BlockResult.Ranges.size(), + SubRanges.size()), + ""); + break; } OnDownloaded(std::move(BlockResult.Bytes), SubRangeStartIndex, BlockResult.Ranges); } @@ -754,15 +804,9 @@ namespace remotestore_impl { SubRangeCountComplete += SubRangeCount; } - return true; } - void DownloadAndSavePartialBlock(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - bool IgnoreMissingAttachments, - JobContext* OptionalContext, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, + void DownloadAndSavePartialBlock(LoadOplogContext& Context, Latch& AttachmentsDownloadLatch, Latch& AttachmentsWriteLatch, AsyncRemoteResult& RemoteResult, @@ -779,19 +823,14 @@ namespace remotestore_impl { uint32_t RetriesLeft) { AttachmentsDownloadLatch.AddCount(1); - NetworkWorkerPool.ScheduleWork( + Context.NetworkWorkerPool.ScheduleWork( [&AttachmentsDownloadLatch, &AttachmentsWriteLatch, - &ChunkStore, - &RemoteStore, - &NetworkWorkerPool, - &WorkerPool, + &Context, &RemoteResult, &Info, &LoadAttachmentsTimer, &DownloadStartMS, - IgnoreMissingAttachments, - OptionalContext, BlockDescription, BlockExistsInCache, BlockRangeDescriptors, @@ -811,10 +850,8 @@ namespace remotestore_impl { double DownloadElapsedSeconds = 0; uint64_t DownloadedBytes = 0; - bool Success = DownloadPartialBlock( - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, + DownloadPartialBlock( + Context, RemoteResult, Info, DownloadElapsedSeconds, @@ -833,19 +870,14 @@ namespace remotestore_impl { Info.AttachmentBlocksRangesDownloaded++; AttachmentsWriteLatch.AddCount(1); - WorkerPool.ScheduleWork( + Context.WorkerPool.ScheduleWork( [&AttachmentsWriteLatch, - &ChunkStore, - &RemoteStore, - &NetworkWorkerPool, - &WorkerPool, + &Context, &AttachmentsDownloadLatch, &RemoteResult, &Info, &LoadAttachmentsTimer, &DownloadStartMS, - IgnoreMissingAttachments, - OptionalContext, BlockDescription, BlockExistsInCache, BlockRangeDescriptors, @@ -945,14 +977,9 @@ namespace remotestore_impl { { if (RetriesLeft > 0) { - ReportMessage(OptionalContext, + ReportMessage(Context.OptionalJobContext, fmt::format("{}, retrying download", ErrorString)); - return DownloadAndSavePartialBlock(ChunkStore, - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, - NetworkWorkerPool, - WorkerPool, + return DownloadAndSavePartialBlock(Context, AttachmentsDownloadLatch, AttachmentsWriteLatch, RemoteResult, @@ -969,9 +996,9 @@ namespace remotestore_impl { RetriesLeft - 1); } - ReportMessage(OptionalContext, ErrorString); + ReportMessage(Context.OptionalJobContext, ErrorString); Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) + if (!Context.IgnoreMissingAttachments) { RemoteResult.SetError(gsl::narrow(HttpResponseCode::NotFound), "Malformed chunk block", @@ -998,7 +1025,7 @@ namespace remotestore_impl { if (!WriteAttachmentBuffers.empty()) { std::vector Results = - ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); + Context.ChunkStore.AddChunks(WriteAttachmentBuffers, WriteRawHashes); for (size_t Index = 0; Index < Results.size(); Index++) { const CidStore::InsertResult& Result = Results[Index]; @@ -1037,7 +1064,7 @@ namespace remotestore_impl { }, WorkerThreadPool::EMode::EnableBacklog); }); - if (Success) + if (!RemoteResult.IsError()) { ZEN_DEBUG("Loaded {} ranges from block attachment '{}' in {} ({})", BlockRangeCount, @@ -1056,12 +1083,7 @@ namespace remotestore_impl { WorkerThreadPool::EMode::EnableBacklog); }; - void DownloadAndSaveAttachment(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - bool IgnoreMissingAttachments, - JobContext* OptionalContext, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, + void DownloadAndSaveAttachment(LoadOplogContext& Context, Latch& AttachmentsDownloadLatch, Latch& AttachmentsWriteLatch, AsyncRemoteResult& RemoteResult, @@ -1071,19 +1093,15 @@ namespace remotestore_impl { const IoHash& RawHash) { AttachmentsDownloadLatch.AddCount(1); - NetworkWorkerPool.ScheduleWork( - [&RemoteStore, - &ChunkStore, - &WorkerPool, + Context.NetworkWorkerPool.ScheduleWork( + [&Context, &RemoteResult, &AttachmentsDownloadLatch, &AttachmentsWriteLatch, RawHash, &LoadAttachmentsTimer, &DownloadStartMS, - &Info, - IgnoreMissingAttachments, - OptionalContext]() { + &Info]() { ZEN_TRACE_CPU("DownloadAttachment"); auto _ = MakeGuard([&AttachmentsDownloadLatch] { AttachmentsDownloadLatch.CountDown(); }); @@ -1095,43 +1113,52 @@ namespace remotestore_impl { { uint64_t Unset = (std::uint64_t)-1; DownloadStartMS.compare_exchange_strong(Unset, LoadAttachmentsTimer.GetElapsedTimeMs()); - RemoteProjectStore::LoadAttachmentResult AttachmentResult = RemoteStore.LoadAttachment(RawHash); - if (AttachmentResult.ErrorCode) + IoBuffer BlobBuffer; + if (Context.OptionalCache) { - ReportMessage(OptionalContext, - fmt::format("Failed to download large attachment {}: '{}', error code : {}", - RawHash, - AttachmentResult.Reason, - AttachmentResult.ErrorCode)); - Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) + BlobBuffer = Context.OptionalCache->GetBuildBlob(Context.CacheBuildId, RawHash); + } + if (!BlobBuffer) + { + RemoteProjectStore::LoadAttachmentResult AttachmentResult = Context.RemoteStore.LoadAttachment(RawHash); + if (AttachmentResult.ErrorCode) { - RemoteResult.SetError(AttachmentResult.ErrorCode, AttachmentResult.Reason, AttachmentResult.Text); + ReportMessage(Context.OptionalJobContext, + fmt::format("Failed to download large attachment {}: '{}', error code : {}", + RawHash, + AttachmentResult.Reason, + AttachmentResult.ErrorCode)); + Info.MissingAttachmentCount.fetch_add(1); + if (!Context.IgnoreMissingAttachments) + { + RemoteResult.SetError(AttachmentResult.ErrorCode, AttachmentResult.Reason, AttachmentResult.Text); + } + return; + } + BlobBuffer = std::move(AttachmentResult.Bytes); + ZEN_DEBUG("Loaded large attachment '{}' in {} ({})", + RawHash, + NiceTimeSpanMs(static_cast(AttachmentResult.ElapsedSeconds * 1000)), + NiceBytes(BlobBuffer.GetSize())); + if (Context.OptionalCache && Context.PopulateCache) + { + Context.OptionalCache->PutBuildBlob(Context.CacheBuildId, + RawHash, + BlobBuffer.GetContentType(), + CompositeBuffer(SharedBuffer(BlobBuffer))); } - return; } - uint64_t AttachmentSize = AttachmentResult.Bytes.GetSize(); - ZEN_DEBUG("Loaded large attachment '{}' in {} ({})", - RawHash, - NiceTimeSpanMs(static_cast(AttachmentResult.ElapsedSeconds * 1000)), - NiceBytes(AttachmentSize)); - Info.AttachmentsDownloaded.fetch_add(1); if (RemoteResult.IsError()) { return; } + uint64_t AttachmentSize = BlobBuffer.GetSize(); + Info.AttachmentsDownloaded.fetch_add(1); Info.AttachmentBytesDownloaded.fetch_add(AttachmentSize); AttachmentsWriteLatch.AddCount(1); - WorkerPool.ScheduleWork( - [&AttachmentsWriteLatch, - &RemoteResult, - &Info, - &ChunkStore, - RawHash, - AttachmentSize, - Bytes = std::move(AttachmentResult.Bytes), - OptionalContext]() { + Context.WorkerPool.ScheduleWork( + [&Context, &AttachmentsWriteLatch, &RemoteResult, &Info, RawHash, AttachmentSize, Bytes = std::move(BlobBuffer)]() { ZEN_TRACE_CPU("WriteAttachment"); auto _ = MakeGuard([&AttachmentsWriteLatch] { AttachmentsWriteLatch.CountDown(); }); @@ -1141,7 +1168,7 @@ namespace remotestore_impl { } try { - CidStore::InsertResult InsertResult = ChunkStore.AddChunk(Bytes, RawHash); + CidStore::InsertResult InsertResult = Context.ChunkStore.AddChunk(Bytes, RawHash); if (InsertResult.New) { Info.AttachmentBytesStored.fetch_add(AttachmentSize); @@ -1557,7 +1584,9 @@ namespace remotestore_impl { uint64_t PartialTransferWallTimeMS = Timer.GetElapsedTimeMs(); ReportProgress(OptionalContext, "Saving attachments"sv, - fmt::format("{} remaining... {}", Remaining, GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)), + fmt::format("{} remaining... {}", + Remaining, + GetStats(RemoteStore.GetStats(), /*OptionalCacheStats*/ nullptr, PartialTransferWallTimeMS)), AttachmentsToSave, Remaining); } @@ -1566,7 +1595,7 @@ namespace remotestore_impl { { ReportProgress(OptionalContext, "Saving attachments"sv, - fmt::format("{}", GetStats(RemoteStore.GetStats(), ElapsedTimeMS)), + fmt::format("{}", GetStats(RemoteStore.GetStats(), /*OptionalCacheStats*/ nullptr, ElapsedTimeMS)), AttachmentsToSave, 0); } @@ -1577,7 +1606,7 @@ namespace remotestore_impl { LargeAttachmentCountToUpload, BulkAttachmentCountToUpload, NiceTimeSpanMs(ElapsedTimeMS), - GetStats(RemoteStore.GetStats(), ElapsedTimeMS))); + GetStats(RemoteStore.GetStats(), /*OptionalCacheStats*/ nullptr, ElapsedTimeMS))); } } // namespace remotestore_impl @@ -2186,31 +2215,36 @@ BuildContainer(CidStore& ChunkStore, } ResolveAttachmentsLatch.CountDown(); - while (!ResolveAttachmentsLatch.Wait(1000)) { - ptrdiff_t Remaining = ResolveAttachmentsLatch.Remaining(); - if (remotestore_impl::IsCancelled(OptionalContext)) + ptrdiff_t AttachmentCountToUseForProgress = ResolveAttachmentsLatch.Remaining(); + while (!ResolveAttachmentsLatch.Wait(1000)) { - RemoteResult.SetError(gsl::narrow(HttpResponseCode::OK), "Operation cancelled", ""); - remotestore_impl::ReportMessage(OptionalContext, - fmt::format("Aborting ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason())); - while (!ResolveAttachmentsLatch.Wait(1000)) + ptrdiff_t Remaining = ResolveAttachmentsLatch.Remaining(); + if (remotestore_impl::IsCancelled(OptionalContext)) { - Remaining = ResolveAttachmentsLatch.Remaining(); - remotestore_impl::ReportProgress(OptionalContext, - "Resolving attachments"sv, - fmt::format("Aborting, {} attachments remaining...", Remaining), - UploadAttachments.size(), - Remaining); + RemoteResult.SetError(gsl::narrow(HttpResponseCode::OK), "Operation cancelled", ""); + remotestore_impl::ReportMessage( + OptionalContext, + fmt::format("Aborting ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason())); + while (!ResolveAttachmentsLatch.Wait(1000)) + { + Remaining = ResolveAttachmentsLatch.Remaining(); + remotestore_impl::ReportProgress(OptionalContext, + "Resolving attachments"sv, + fmt::format("Aborting, {} attachments remaining...", Remaining), + UploadAttachments.size(), + Remaining); + } + remotestore_impl::ReportProgress(OptionalContext, "Resolving attachments"sv, "Aborted"sv, UploadAttachments.size(), 0); + return {}; } - remotestore_impl::ReportProgress(OptionalContext, "Resolving attachments"sv, "Aborted"sv, UploadAttachments.size(), 0); - return {}; + AttachmentCountToUseForProgress = Max(Remaining, AttachmentCountToUseForProgress); + remotestore_impl::ReportProgress(OptionalContext, + "Resolving attachments"sv, + fmt::format("{} remaining...", Remaining), + AttachmentCountToUseForProgress, + Remaining); } - remotestore_impl::ReportProgress(OptionalContext, - "Resolving attachments"sv, - fmt::format("{} remaining...", Remaining), - UploadAttachments.size(), - Remaining); } if (UploadAttachments.size() > 0) { @@ -2598,12 +2632,14 @@ BuildContainer(CidStore& ChunkStore, 0); } - remotestore_impl::ReportMessage(OptionalContext, - fmt::format("Built oplog and collected {} attachments from {} ops into {} blocks and in {}", - ChunkAssembleCount, - TotalOpCount, - GeneratedBlockCount, - NiceTimeSpanMs(static_cast(Timer.GetElapsedTimeMs())))); + remotestore_impl::ReportMessage( + OptionalContext, + fmt::format("Built oplog and collected {} attachments from {} ops into {} blocks and {} loose attachments in {}", + ChunkAssembleCount, + TotalOpCount, + GeneratedBlockCount, + LargeChunkHashes.size(), + NiceTimeSpanMs(static_cast(Timer.GetElapsedTimeMs())))); if (remotestore_impl::IsCancelled(OptionalContext)) { @@ -3155,17 +3191,18 @@ SaveOplog(CidStore& ChunkStore, remotestore_impl::LogRemoteStoreStatsDetails(RemoteStore.GetStats()); - remotestore_impl::ReportMessage(OptionalContext, - fmt::format("Saved oplog '{}' {} in {} ({}), Blocks: {} ({}), Attachments: {} ({}) {}", - RemoteStoreInfo.ContainerName, - RemoteResult.GetError() == 0 ? "SUCCESS" : "FAILURE", - NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000.0)), - NiceBytes(Info.OplogSizeBytes), - Info.AttachmentBlocksUploaded.load(), - NiceBytes(Info.AttachmentBlockBytesUploaded.load()), - Info.AttachmentsUploaded.load(), - NiceBytes(Info.AttachmentBytesUploaded.load()), - remotestore_impl::GetStats(RemoteStore.GetStats(), TransferWallTimeMS))); + remotestore_impl::ReportMessage( + OptionalContext, + fmt::format("Saved oplog '{}' {} in {} ({}), Blocks: {} ({}), Attachments: {} ({}) {}", + RemoteStoreInfo.ContainerName, + RemoteResult.GetError() == 0 ? "SUCCESS" : "FAILURE", + NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000.0)), + NiceBytes(Info.OplogSizeBytes), + Info.AttachmentBlocksUploaded.load(), + NiceBytes(Info.AttachmentBlockBytesUploaded.load()), + Info.AttachmentsUploaded.load(), + NiceBytes(Info.AttachmentBytesUploaded.load()), + remotestore_impl::GetStats(RemoteStore.GetStats(), /*OptionalCacheStats*/ nullptr, TransferWallTimeMS))); return Result; }; @@ -3234,6 +3271,11 @@ ParseOplogContainer( OpCount - OpsCompleteCount); } } + remotestore_impl::ReportProgress(OptionalContext, + "Scanning oplog"sv, + fmt::format("{} attachments found", NeededAttachments.size()), + OpCount, + OpCount - OpsCompleteCount); } { std::vector ReferencedAttachments(NeededAttachments.begin(), NeededAttachments.end()); @@ -3406,22 +3448,11 @@ SaveOplogContainer( } RemoteProjectStore::Result -LoadOplog(CidStore& ChunkStore, - RemoteProjectStore& RemoteStore, - ProjectStore::Oplog& Oplog, - WorkerThreadPool& NetworkWorkerPool, - WorkerThreadPool& WorkerPool, - bool ForceDownload, - bool IgnoreMissingAttachments, - bool CleanOplog, - EPartialBlockRequestMode PartialBlockRequestMode, - double HostLatencySec, - double CacheLatencySec, - JobContext* OptionalContext) +LoadOplog(LoadOplogContext&& Context) { using namespace std::literals; - std::unique_ptr LogOutput(std::make_unique(OptionalContext)); + std::unique_ptr LogOutput(std::make_unique(Context.OptionalJobContext)); remotestore_impl::DownloadInfo Info; @@ -3430,25 +3461,25 @@ LoadOplog(CidStore& ChunkStore, std::unordered_set Attachments; uint64_t BlockCountToDownload = 0; - RemoteProjectStore::RemoteStoreInfo RemoteStoreInfo = RemoteStore.GetInfo(); - remotestore_impl::ReportMessage(OptionalContext, fmt::format("Loading oplog container '{}'", RemoteStoreInfo.ContainerName)); + RemoteProjectStore::RemoteStoreInfo RemoteStoreInfo = Context.RemoteStore.GetInfo(); + remotestore_impl::ReportMessage(Context.OptionalJobContext, fmt::format("Loading oplog container '{}'", RemoteStoreInfo.ContainerName)); uint64_t TransferWallTimeMS = 0; Stopwatch LoadContainerTimer; - RemoteProjectStore::LoadContainerResult LoadContainerResult = RemoteStore.LoadContainer(); + RemoteProjectStore::LoadContainerResult LoadContainerResult = Context.RemoteStore.LoadContainer(); TransferWallTimeMS += LoadContainerTimer.GetElapsedTimeMs(); if (LoadContainerResult.ErrorCode) { remotestore_impl::ReportMessage( - OptionalContext, + Context.OptionalJobContext, fmt::format("Failed to load oplog container: '{}', error code: {}", LoadContainerResult.Reason, LoadContainerResult.ErrorCode)); return RemoteProjectStore::Result{.ErrorCode = LoadContainerResult.ErrorCode, .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0, .Reason = LoadContainerResult.Reason, .Text = LoadContainerResult.Text}; } - remotestore_impl::ReportMessage(OptionalContext, + remotestore_impl::ReportMessage(Context.OptionalJobContext, fmt::format("Loaded container in {} ({})", NiceTimeSpanMs(static_cast(LoadContainerResult.ElapsedSeconds * 1000)), NiceBytes(LoadContainerResult.ContainerObject.GetSize()))); @@ -3462,12 +3493,12 @@ LoadOplog(CidStore& ChunkStore, Stopwatch LoadAttachmentsTimer; std::atomic_uint64_t DownloadStartMS = (std::uint64_t)-1; - auto HasAttachment = [&Oplog, &ChunkStore, ForceDownload](const IoHash& RawHash) { - if (ForceDownload) + auto HasAttachment = [&Context](const IoHash& RawHash) { + if (Context.ForceDownload) { return false; } - if (ChunkStore.ContainsChunk(RawHash)) + if (Context.ChunkStore.ContainsChunk(RawHash)) { return true; } @@ -3482,10 +3513,7 @@ LoadOplog(CidStore& ChunkStore, std::vector NeededBlockDownloads; - auto OnNeedBlock = [&RemoteStore, - &ChunkStore, - &NetworkWorkerPool, - &WorkerPool, + auto OnNeedBlock = [&Context, &AttachmentsDownloadLatch, &AttachmentsWriteLatch, &AttachmentCount, @@ -3494,9 +3522,8 @@ LoadOplog(CidStore& ChunkStore, &Info, &LoadAttachmentsTimer, &DownloadStartMS, - &NeededBlockDownloads, - IgnoreMissingAttachments, - OptionalContext](ThinChunkBlockDescription&& ThinBlockDescription, std::vector&& NeededChunkIndexes) { + &NeededBlockDownloads](ThinChunkBlockDescription&& ThinBlockDescription, + std::vector&& NeededChunkIndexes) { if (RemoteResult.IsError()) { return; @@ -3506,12 +3533,7 @@ LoadOplog(CidStore& ChunkStore, AttachmentCount.fetch_add(1); if (ThinBlockDescription.BlockHash == IoHash::Zero) { - DownloadAndSaveBlockChunks(ChunkStore, - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, - NetworkWorkerPool, - WorkerPool, + DownloadAndSaveBlockChunks(Context, AttachmentsDownloadLatch, AttachmentsWriteLatch, RemoteResult, @@ -3528,11 +3550,7 @@ LoadOplog(CidStore& ChunkStore, } }; - auto OnNeedAttachment = [&RemoteStore, - &Oplog, - &ChunkStore, - &NetworkWorkerPool, - &WorkerPool, + auto OnNeedAttachment = [&Context, &AttachmentsDownloadLatch, &AttachmentsWriteLatch, &RemoteResult, @@ -3540,9 +3558,7 @@ LoadOplog(CidStore& ChunkStore, &AttachmentCount, &LoadAttachmentsTimer, &DownloadStartMS, - &Info, - IgnoreMissingAttachments, - OptionalContext](const IoHash& RawHash) { + &Info](const IoHash& RawHash) { if (!Attachments.insert(RawHash).second) { return; @@ -3552,12 +3568,7 @@ LoadOplog(CidStore& ChunkStore, return; } AttachmentCount.fetch_add(1); - DownloadAndSaveAttachment(ChunkStore, - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, - NetworkWorkerPool, - WorkerPool, + DownloadAndSaveAttachment(Context, AttachmentsDownloadLatch, AttachmentsWriteLatch, RemoteResult, @@ -3570,11 +3581,11 @@ LoadOplog(CidStore& ChunkStore, std::vector FilesToDechunk; auto OnChunkedAttachment = [&FilesToDechunk](const ChunkedInfo& Chunked) { FilesToDechunk.push_back(Chunked); }; - auto OnReferencedAttachments = [&Oplog](std::span RawHashes) { Oplog.CaptureAddedAttachments(RawHashes); }; + auto OnReferencedAttachments = [&Context](std::span RawHashes) { Context.Oplog.CaptureAddedAttachments(RawHashes); }; // Make sure we retain any attachments we download before writing the oplog - Oplog.EnableUpdateCapture(); - auto _ = MakeGuard([&Oplog]() { Oplog.DisableUpdateCapture(); }); + Context.Oplog.EnableUpdateCapture(); + auto _ = MakeGuard([&Context]() { Context.Oplog.DisableUpdateCapture(); }); CbObject OplogSection; RemoteProjectStore::Result Result = ParseOplogContainer(LoadContainerResult.ContainerObject, @@ -3584,12 +3595,12 @@ LoadOplog(CidStore& ChunkStore, OnNeedAttachment, OnChunkedAttachment, OplogSection, - OptionalContext); + Context.OptionalJobContext); if (Result.ErrorCode != 0) { RemoteResult.SetError(Result.ErrorCode, Result.Reason, Result.Text); } - remotestore_impl::ReportMessage(OptionalContext, + remotestore_impl::ReportMessage(Context.OptionalJobContext, fmt::format("Parsed oplog in {}, found {} attachments, {} blocks and {} chunked files to download", NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000.0)), Attachments.size(), @@ -3613,11 +3624,12 @@ LoadOplog(CidStore& ChunkStore, std::vector DownloadedViaLegacyChunkFlag(AllNeededChunkHashes.size(), false); ChunkBlockAnalyser::BlockResult PartialBlocksResult; - remotestore_impl::ReportMessage(OptionalContext, fmt::format("Fetching descriptions for {} blocks", BlockHashes.size())); + remotestore_impl::ReportMessage(Context.OptionalJobContext, fmt::format("Fetching descriptions for {} blocks", BlockHashes.size())); - RemoteProjectStore::GetBlockDescriptionsResult BlockDescriptions = RemoteStore.GetBlockDescriptions(BlockHashes); + RemoteProjectStore::GetBlockDescriptionsResult BlockDescriptions = + Context.RemoteStore.GetBlockDescriptions(BlockHashes, Context.OptionalCache, Context.CacheBuildId); - remotestore_impl::ReportMessage(OptionalContext, + remotestore_impl::ReportMessage(Context.OptionalJobContext, fmt::format("GetBlockDescriptions took {}. Found {} blocks", NiceTimeSpanMs(uint64_t(BlockDescriptions.ElapsedSeconds * 1000)), BlockDescriptions.Blocks.size())); @@ -3636,12 +3648,7 @@ LoadOplog(CidStore& ChunkStore, if (FindIt == BlockDescriptions.Blocks.end()) { // Fall back to full download as we can't get enough information about the block - DownloadAndSaveBlock(ChunkStore, - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, - NetworkWorkerPool, - WorkerPool, + DownloadAndSaveBlock(Context, AttachmentsDownloadLatch, AttachmentsWriteLatch, RemoteResult, @@ -3678,56 +3685,86 @@ LoadOplog(CidStore& ChunkStore, if (!AllNeededChunkHashes.empty()) { std::vector PartialBlockDownloadModes; - std::vector BlockExistsInCache; + std::vector BlockExistsInCache(BlocksWithDescription.size(), false); - if (PartialBlockRequestMode == EPartialBlockRequestMode::Off) + if (Context.PartialBlockRequestMode == EPartialBlockRequestMode::Off) { PartialBlockDownloadModes.resize(BlocksWithDescription.size(), ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); } else { - RemoteProjectStore::AttachmentExistsInCacheResult CacheExistsResult = - RemoteStore.AttachmentExistsInCache(BlocksWithDescription); - if (CacheExistsResult.ErrorCode != 0 || CacheExistsResult.HasBody.size() != BlocksWithDescription.size()) + if (Context.OptionalCache) { - BlockExistsInCache.resize(BlocksWithDescription.size(), false); + std::vector CacheExistsResult = + Context.OptionalCache->BlobsExists(Context.CacheBuildId, BlocksWithDescription); + if (CacheExistsResult.size() == BlocksWithDescription.size()) + { + for (size_t BlobIndex = 0; BlobIndex < CacheExistsResult.size(); BlobIndex++) + { + BlockExistsInCache[BlobIndex] = CacheExistsResult[BlobIndex].HasBody; + } + } + uint64_t FoundBlocks = + std::accumulate(BlockExistsInCache.begin(), + BlockExistsInCache.end(), + uint64_t(0u), + [](uint64_t Current, bool Exists) -> uint64_t { return Current + (Exists ? 1 : 0); }); + if (FoundBlocks > 0) + { + remotestore_impl::ReportMessage( + Context.OptionalJobContext, + fmt::format("Found {} out of {} blocks in cache", FoundBlocks, BlockExistsInCache.size())); + } } - else + + ChunkBlockAnalyser::EPartialBlockDownloadMode CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; + ChunkBlockAnalyser::EPartialBlockDownloadMode CachePartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; + + switch (Context.PartialBlockRequestMode) { - BlockExistsInCache = std::move(CacheExistsResult.HasBody); + case EPartialBlockRequestMode::Off: + break; + case EPartialBlockRequestMode::ZenCacheOnly: + CachePartialDownloadMode = Context.CacheMaxRangeCountPerRequest > 1 + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange; + CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::Off; + break; + case EPartialBlockRequestMode::Mixed: + CachePartialDownloadMode = Context.CacheMaxRangeCountPerRequest > 1 + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange; + CloudPartialDownloadMode = ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange; + break; + case EPartialBlockRequestMode::All: + CachePartialDownloadMode = Context.CacheMaxRangeCountPerRequest > 1 + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed + : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange; + CloudPartialDownloadMode = Context.StoreMaxRangeCountPerRequest > 1 + ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange + : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange; + break; } PartialBlockDownloadModes.reserve(BlocksWithDescription.size()); - - for (bool ExistsInCache : BlockExistsInCache) + for (uint32_t BlockIndex = 0; BlockIndex < BlocksWithDescription.size(); BlockIndex++) { - if (PartialBlockRequestMode == EPartialBlockRequestMode::All) - { - PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::Exact - : ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRange); - } - else if (PartialBlockRequestMode == EPartialBlockRequestMode::ZenCacheOnly) - { - PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::Exact - : ChunkBlockAnalyser::EPartialBlockDownloadMode::Off); - } - else if (PartialBlockRequestMode == EPartialBlockRequestMode::Mixed) - { - PartialBlockDownloadModes.push_back(ExistsInCache ? ChunkBlockAnalyser::EPartialBlockDownloadMode::MultiRangeHighSpeed - : ChunkBlockAnalyser::EPartialBlockDownloadMode::SingleRange); - } + const bool BlockExistInCache = BlockExistsInCache[BlockIndex]; + PartialBlockDownloadModes.push_back(BlockExistInCache ? CachePartialDownloadMode : CloudPartialDownloadMode); } } ZEN_ASSERT(PartialBlockDownloadModes.size() == BlocksWithDescription.size()); + ChunkBlockAnalyser PartialAnalyser( *LogOutput, BlockDescriptions.Blocks, - ChunkBlockAnalyser::Options{.IsQuiet = false, - .IsVerbose = false, - .HostLatencySec = HostLatencySec, - .HostHighSpeedLatencySec = CacheLatencySec, - .HostMaxRangeCountPerRequest = RemoteProjectStore::MaxRangeCountPerRequest}); + ChunkBlockAnalyser::Options{.IsQuiet = false, + .IsVerbose = false, + .HostLatencySec = Context.StoreLatencySec, + .HostHighSpeedLatencySec = Context.CacheLatencySec, + .HostMaxRangeCountPerRequest = Context.StoreMaxRangeCountPerRequest, + .HostHighSpeedMaxRangeCountPerRequest = Context.CacheMaxRangeCountPerRequest}); std::vector NeededBlocks = PartialAnalyser.GetNeeded(AllNeededPartialChunkHashesLookup, @@ -3736,12 +3773,7 @@ LoadOplog(CidStore& ChunkStore, PartialBlocksResult = PartialAnalyser.CalculatePartialBlockDownloads(NeededBlocks, PartialBlockDownloadModes); for (uint32_t FullBlockIndex : PartialBlocksResult.FullBlockIndexes) { - DownloadAndSaveBlock(ChunkStore, - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, - NetworkWorkerPool, - WorkerPool, + DownloadAndSaveBlock(Context, AttachmentsDownloadLatch, AttachmentsWriteLatch, RemoteResult, @@ -3765,12 +3797,7 @@ LoadOplog(CidStore& ChunkStore, RangeCount++; } - DownloadAndSavePartialBlock(ChunkStore, - RemoteStore, - IgnoreMissingAttachments, - OptionalContext, - NetworkWorkerPool, - WorkerPool, + DownloadAndSavePartialBlock(Context, AttachmentsDownloadLatch, AttachmentsWriteLatch, RemoteResult, @@ -3791,38 +3818,44 @@ LoadOplog(CidStore& ChunkStore, } AttachmentsDownloadLatch.CountDown(); - while (!AttachmentsDownloadLatch.Wait(1000)) { - ptrdiff_t Remaining = AttachmentsDownloadLatch.Remaining(); - if (remotestore_impl::IsCancelled(OptionalContext)) + ptrdiff_t AttachmentCountToUseForProgress = AttachmentsDownloadLatch.Remaining(); + while (!AttachmentsDownloadLatch.Wait(1000)) { - if (!RemoteResult.IsError()) + ptrdiff_t Remaining = AttachmentsDownloadLatch.Remaining(); + if (remotestore_impl::IsCancelled(Context.OptionalJobContext)) { - RemoteResult.SetError(gsl::narrow(HttpResponseCode::OK), "Operation cancelled", ""); + if (!RemoteResult.IsError()) + { + RemoteResult.SetError(gsl::narrow(HttpResponseCode::OK), "Operation cancelled", ""); + } + } + uint64_t PartialTransferWallTimeMS = TransferWallTimeMS; + if (DownloadStartMS != (uint64_t)-1) + { + PartialTransferWallTimeMS += LoadAttachmentsTimer.GetElapsedTimeMs() - DownloadStartMS.load(); } - } - uint64_t PartialTransferWallTimeMS = TransferWallTimeMS; - if (DownloadStartMS != (uint64_t)-1) - { - PartialTransferWallTimeMS += LoadAttachmentsTimer.GetElapsedTimeMs() - DownloadStartMS.load(); - } - - uint64_t AttachmentsDownloaded = - Info.AttachmentBlocksDownloaded.load() + Info.AttachmentBlocksRangesDownloaded.load() + Info.AttachmentsDownloaded.load(); - uint64_t AttachmentBytesDownloaded = Info.AttachmentBlockBytesDownloaded.load() + Info.AttachmentBlockRangeBytesDownloaded.load() + - Info.AttachmentBytesDownloaded.load(); - remotestore_impl::ReportProgress(OptionalContext, - "Loading attachments"sv, - fmt::format("{} ({}) downloaded, {} ({}) stored, {} remaining. {}", - AttachmentsDownloaded, - NiceBytes(AttachmentBytesDownloaded), - Info.AttachmentsStored.load(), - NiceBytes(Info.AttachmentBytesStored.load()), - Remaining, - remotestore_impl::GetStats(RemoteStore.GetStats(), PartialTransferWallTimeMS)), - AttachmentCount.load(), - Remaining); + uint64_t AttachmentsDownloaded = + Info.AttachmentBlocksDownloaded.load() + Info.AttachmentBlocksRangesDownloaded.load() + Info.AttachmentsDownloaded.load(); + uint64_t AttachmentBytesDownloaded = Info.AttachmentBlockBytesDownloaded.load() + + Info.AttachmentBlockRangeBytesDownloaded.load() + Info.AttachmentBytesDownloaded.load(); + + AttachmentCountToUseForProgress = Max(Remaining, AttachmentCountToUseForProgress); + remotestore_impl::ReportProgress( + Context.OptionalJobContext, + "Loading attachments"sv, + fmt::format( + "{} ({}) downloaded, {} ({}) stored, {} remaining. {}", + AttachmentsDownloaded, + NiceBytes(AttachmentBytesDownloaded), + Info.AttachmentsStored.load(), + NiceBytes(Info.AttachmentBytesStored.load()), + Remaining, + remotestore_impl::GetStats(Context.RemoteStore.GetStats(), Context.OptionalCacheStats, PartialTransferWallTimeMS)), + AttachmentCountToUseForProgress, + Remaining); + } } if (DownloadStartMS != (uint64_t)-1) { @@ -3831,58 +3864,58 @@ LoadOplog(CidStore& ChunkStore, if (AttachmentCount.load() > 0) { - remotestore_impl::ReportProgress(OptionalContext, - "Loading attachments"sv, - fmt::format("{}", remotestore_impl::GetStats(RemoteStore.GetStats(), TransferWallTimeMS)), - AttachmentCount.load(), - 0); + remotestore_impl::ReportProgress( + Context.OptionalJobContext, + "Loading attachments"sv, + fmt::format("{}", remotestore_impl::GetStats(Context.RemoteStore.GetStats(), Context.OptionalCacheStats, TransferWallTimeMS)), + AttachmentCount.load(), + 0); } AttachmentsWriteLatch.CountDown(); - while (!AttachmentsWriteLatch.Wait(1000)) { - ptrdiff_t Remaining = AttachmentsWriteLatch.Remaining(); - if (remotestore_impl::IsCancelled(OptionalContext)) + ptrdiff_t AttachmentCountToUseForProgress = AttachmentsWriteLatch.Remaining(); + while (!AttachmentsWriteLatch.Wait(1000)) { - if (!RemoteResult.IsError()) + ptrdiff_t Remaining = AttachmentsWriteLatch.Remaining(); + if (remotestore_impl::IsCancelled(Context.OptionalJobContext)) { - RemoteResult.SetError(gsl::narrow(HttpResponseCode::OK), "Operation cancelled", ""); + if (!RemoteResult.IsError()) + { + RemoteResult.SetError(gsl::narrow(HttpResponseCode::OK), "Operation cancelled", ""); + } } + AttachmentCountToUseForProgress = Max(Remaining, AttachmentCountToUseForProgress); + remotestore_impl::ReportProgress(Context.OptionalJobContext, + "Writing attachments"sv, + fmt::format("{} ({}), {} remaining.", + Info.AttachmentsStored.load(), + NiceBytes(Info.AttachmentBytesStored.load()), + Remaining), + AttachmentCountToUseForProgress, + Remaining); } - remotestore_impl::ReportProgress( - OptionalContext, - "Writing attachments"sv, - fmt::format("{} ({}), {} remaining.", Info.AttachmentsStored.load(), NiceBytes(Info.AttachmentBytesStored.load()), Remaining), - AttachmentCount.load(), - Remaining); } if (AttachmentCount.load() > 0) { - remotestore_impl::ReportProgress(OptionalContext, "Writing attachments", ""sv, AttachmentCount.load(), 0); + remotestore_impl::ReportProgress(Context.OptionalJobContext, "Writing attachments", ""sv, AttachmentCount.load(), 0); } if (Result.ErrorCode == 0) { if (!FilesToDechunk.empty()) { - remotestore_impl::ReportMessage(OptionalContext, fmt::format("Dechunking {} attachments", FilesToDechunk.size())); + remotestore_impl::ReportMessage(Context.OptionalJobContext, fmt::format("Dechunking {} attachments", FilesToDechunk.size())); Latch DechunkLatch(1); - std::filesystem::path TempFilePath = Oplog.TempPath(); + std::filesystem::path TempFilePath = Context.Oplog.TempPath(); for (const ChunkedInfo& Chunked : FilesToDechunk) { std::filesystem::path TempFileName = TempFilePath / Chunked.RawHash.ToHexString(); DechunkLatch.AddCount(1); - WorkerPool.ScheduleWork( - [&ChunkStore, - &DechunkLatch, - TempFileName, - &Chunked, - &RemoteResult, - IgnoreMissingAttachments, - &Info, - OptionalContext]() { + Context.WorkerPool.ScheduleWork( + [&Context, &DechunkLatch, TempFileName, &Chunked, &RemoteResult, &Info]() { ZEN_TRACE_CPU("DechunkAttachment"); auto _ = MakeGuard([&DechunkLatch, &TempFileName] { @@ -3916,16 +3949,16 @@ LoadOplog(CidStore& ChunkStore, for (std::uint32_t SequenceIndex : Chunked.ChunkSequence) { const IoHash& ChunkHash = Chunked.ChunkHashes[SequenceIndex]; - IoBuffer Chunk = ChunkStore.FindChunkByCid(ChunkHash); + IoBuffer Chunk = Context.ChunkStore.FindChunkByCid(ChunkHash); if (!Chunk) { remotestore_impl::ReportMessage( - OptionalContext, + Context.OptionalJobContext, fmt::format("Missing chunk {} for chunked attachment {}", ChunkHash, Chunked.RawHash)); // We only add 1 as the resulting missing count will be 1 for the dechunked file Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) + if (!Context.IgnoreMissingAttachments) { RemoteResult.SetError( gsl::narrow(HttpResponseCode::NotFound), @@ -3943,7 +3976,7 @@ LoadOplog(CidStore& ChunkStore, if (RawHash != ChunkHash) { remotestore_impl::ReportMessage( - OptionalContext, + Context.OptionalJobContext, fmt::format("Mismatching raw hash {} for chunk {} for chunked attachment {}", RawHash, ChunkHash, @@ -3951,7 +3984,7 @@ LoadOplog(CidStore& ChunkStore, // We only add 1 as the resulting missing count will be 1 for the dechunked file Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) + if (!Context.IgnoreMissingAttachments) { RemoteResult.SetError( gsl::narrow(HttpResponseCode::NotFound), @@ -3988,14 +4021,14 @@ LoadOplog(CidStore& ChunkStore, })) { remotestore_impl::ReportMessage( - OptionalContext, + Context.OptionalJobContext, fmt::format("Failed to decompress chunk {} for chunked attachment {}", ChunkHash, Chunked.RawHash)); // We only add 1 as the resulting missing count will be 1 for the dechunked file Info.MissingAttachmentCount.fetch_add(1); - if (!IgnoreMissingAttachments) + if (!Context.IgnoreMissingAttachments) { RemoteResult.SetError( gsl::narrow(HttpResponseCode::NotFound), @@ -4019,18 +4052,17 @@ LoadOplog(CidStore& ChunkStore, } uint64_t TmpBufferSize = TmpBuffer.GetSize(); CidStore::InsertResult InsertResult = - ChunkStore.AddChunk(TmpBuffer, Chunked.RawHash, CidStore::InsertMode::kMayBeMovedInPlace); + Context.ChunkStore.AddChunk(TmpBuffer, Chunked.RawHash, CidStore::InsertMode::kMayBeMovedInPlace); if (InsertResult.New) { Info.AttachmentBytesStored.fetch_add(TmpBufferSize); Info.AttachmentsStored.fetch_add(1); } - remotestore_impl::ReportMessage(OptionalContext, - fmt::format("Dechunked attachment {} ({}) in {}", - Chunked.RawHash, - NiceBytes(Chunked.RawSize), - NiceTimeSpanMs(Timer.GetElapsedTimeMs()))); + ZEN_INFO("Dechunked attachment {} ({}) in {}", + Chunked.RawHash, + NiceBytes(Chunked.RawSize), + NiceTimeSpanMs(Timer.GetElapsedTimeMs())); } catch (const std::exception& Ex) { @@ -4046,54 +4078,58 @@ LoadOplog(CidStore& ChunkStore, while (!DechunkLatch.Wait(1000)) { ptrdiff_t Remaining = DechunkLatch.Remaining(); - if (remotestore_impl::IsCancelled(OptionalContext)) + if (remotestore_impl::IsCancelled(Context.OptionalJobContext)) { if (!RemoteResult.IsError()) { RemoteResult.SetError(gsl::narrow(HttpResponseCode::OK), "Operation cancelled", ""); remotestore_impl::ReportMessage( - OptionalContext, + Context.OptionalJobContext, fmt::format("Aborting ({}): {}", RemoteResult.GetError(), RemoteResult.GetErrorReason())); } } - remotestore_impl::ReportProgress(OptionalContext, + remotestore_impl::ReportProgress(Context.OptionalJobContext, "Dechunking attachments"sv, fmt::format("{} remaining...", Remaining), FilesToDechunk.size(), Remaining); } - remotestore_impl::ReportProgress(OptionalContext, "Dechunking attachments"sv, ""sv, FilesToDechunk.size(), 0); + remotestore_impl::ReportProgress(Context.OptionalJobContext, "Dechunking attachments"sv, ""sv, FilesToDechunk.size(), 0); } Result = RemoteResult.ConvertResult(); } if (Result.ErrorCode == 0) { - if (CleanOplog) + if (Context.CleanOplog) { - RemoteStore.Flush(); - if (!Oplog.Reset()) + if (Context.OptionalCache) + { + Context.OptionalCache->Flush(100, [](intptr_t) { return /*DontWaitForPendingOperation*/ false; }); + } + if (!Context.Oplog.Reset()) { Result = RemoteProjectStore::Result{.ErrorCode = gsl::narrow(HttpResponseCode::InternalServerError), .ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0, - .Reason = fmt::format("Failed to clean existing oplog '{}'", Oplog.OplogId())}; - remotestore_impl::ReportMessage(OptionalContext, fmt::format("Aborting ({}): {}", Result.ErrorCode, Result.Reason)); + .Reason = fmt::format("Failed to clean existing oplog '{}'", Context.Oplog.OplogId())}; + remotestore_impl::ReportMessage(Context.OptionalJobContext, + fmt::format("Aborting ({}): {}", Result.ErrorCode, Result.Reason)); } } if (Result.ErrorCode == 0) { - remotestore_impl::WriteOplogSection(Oplog, OplogSection, OptionalContext); + remotestore_impl::WriteOplogSection(Context.Oplog, OplogSection, Context.OptionalJobContext); } } Result.ElapsedSeconds = Timer.GetElapsedTimeMs() / 1000.0; - remotestore_impl::LogRemoteStoreStatsDetails(RemoteStore.GetStats()); + remotestore_impl::LogRemoteStoreStatsDetails(Context.RemoteStore.GetStats()); { std::string DownloadDetails; RemoteProjectStore::ExtendedStats ExtendedStats; - if (RemoteStore.GetExtendedStats(ExtendedStats)) + if (Context.RemoteStore.GetExtendedStats(ExtendedStats)) { if (!ExtendedStats.m_ReceivedBytesPerSource.empty()) { @@ -4112,7 +4148,8 @@ LoadOplog(CidStore& ChunkStore, Total += It.second; } - remotestore_impl::ReportMessage(OptionalContext, fmt::format("Downloaded {} ({})", NiceBytes(Total), SB.ToView())); + remotestore_impl::ReportMessage(Context.OptionalJobContext, + fmt::format("Downloaded {} ({})", NiceBytes(Total), SB.ToView())); } } } @@ -4122,25 +4159,26 @@ LoadOplog(CidStore& ChunkStore, uint64_t TotalBytesDownloaded = Info.OplogSizeBytes + Info.AttachmentBlockBytesDownloaded.load() + Info.AttachmentBlockRangeBytesDownloaded.load() + Info.AttachmentBytesDownloaded.load(); - remotestore_impl::ReportMessage(OptionalContext, - fmt::format("Loaded oplog '{}' {} in {} ({}), Blocks: {} ({}), BlockRanges: {} ({}), Attachments: {} " - "({}), Total: {} ({}), Stored: {} ({}), Missing: {} {}", - RemoteStoreInfo.ContainerName, - Result.ErrorCode == 0 ? "SUCCESS" : "FAILURE", - NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000.0)), - NiceBytes(Info.OplogSizeBytes), - Info.AttachmentBlocksDownloaded.load(), - NiceBytes(Info.AttachmentBlockBytesDownloaded.load()), - Info.AttachmentBlocksRangesDownloaded.load(), - NiceBytes(Info.AttachmentBlockRangeBytesDownloaded.load()), - Info.AttachmentsDownloaded.load(), - NiceBytes(Info.AttachmentBytesDownloaded.load()), - TotalDownloads, - NiceBytes(TotalBytesDownloaded), - Info.AttachmentsStored.load(), - NiceBytes(Info.AttachmentBytesStored.load()), - Info.MissingAttachmentCount.load(), - remotestore_impl::GetStats(RemoteStore.GetStats(), TransferWallTimeMS))); + remotestore_impl::ReportMessage( + Context.OptionalJobContext, + fmt::format("Loaded oplog '{}' {} in {} ({}), Blocks: {} ({}), BlockRanges: {} ({}), Attachments: {} " + "({}), Total: {} ({}), Stored: {} ({}), Missing: {} {}", + RemoteStoreInfo.ContainerName, + Result.ErrorCode == 0 ? "SUCCESS" : "FAILURE", + NiceTimeSpanMs(static_cast(Result.ElapsedSeconds * 1000.0)), + NiceBytes(Info.OplogSizeBytes), + Info.AttachmentBlocksDownloaded.load(), + NiceBytes(Info.AttachmentBlockBytesDownloaded.load()), + Info.AttachmentBlocksRangesDownloaded.load(), + NiceBytes(Info.AttachmentBlockRangeBytesDownloaded.load()), + Info.AttachmentsDownloaded.load(), + NiceBytes(Info.AttachmentBytesDownloaded.load()), + TotalDownloads, + NiceBytes(TotalBytesDownloaded), + Info.AttachmentsStored.load(), + NiceBytes(Info.AttachmentBytesStored.load()), + Info.MissingAttachmentCount.load(), + remotestore_impl::GetStats(Context.RemoteStore.GetStats(), Context.OptionalCacheStats, TransferWallTimeMS))); return Result; } @@ -4237,6 +4275,26 @@ namespace projectstore_testutils { return Result; } + class TestJobContext : public JobContext + { + public: + explicit TestJobContext(int& OpIndex) : m_OpIndex(OpIndex) {} + virtual bool IsCancelled() const { return false; } + virtual void ReportMessage(std::string_view Message) { ZEN_INFO("Job {}: {}", m_OpIndex, Message); } + virtual void ReportProgress(std::string_view CurrentOp, std::string_view Details, ptrdiff_t TotalCount, ptrdiff_t RemainingCount) + { + ZEN_INFO("Job {}: Op '{}'{} {}/{}", + m_OpIndex, + CurrentOp, + Details.empty() ? "" : fmt::format(" {}", Details), + TotalCount - RemainingCount, + TotalCount); + } + + private: + int& m_OpIndex; + }; + } // namespace projectstore_testutils TEST_SUITE_BEGIN("remotestore.projectstore"); @@ -4334,66 +4392,708 @@ TEST_CASE_TEMPLATE("project.store.export", false, nullptr); - CHECK(ExportResult.ErrorCode == 0); + REQUIRE(ExportResult.ErrorCode == 0); Ref OplogImport = Project->NewOplog("oplog2", {}); CHECK(OplogImport); - RemoteProjectStore::Result ImportResult = LoadOplog(CidStore, - *RemoteStore, - *OplogImport, - NetworkPool, - WorkerPool, - /*Force*/ false, - /*IgnoreMissingAttachments*/ false, - /*CleanOplog*/ false, - EPartialBlockRequestMode::Mixed, - /*HostLatencySec*/ -1.0, - /*CacheLatencySec*/ -1.0, - nullptr); + int OpJobIndex = 0; + TestJobContext OpJobContext(OpJobIndex); + + RemoteProjectStore::Result ImportResult = LoadOplog(LoadOplogContext{.ChunkStore = CidStore, + .RemoteStore = *RemoteStore, + .OptionalCache = nullptr, + .CacheBuildId = Oid::Zero, + .Oplog = *OplogImport, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = false, + .IgnoreMissingAttachments = false, + .CleanOplog = false, + .PartialBlockRequestMode = EPartialBlockRequestMode::Mixed, + .OptionalJobContext = &OpJobContext}); CHECK(ImportResult.ErrorCode == 0); - - RemoteProjectStore::Result ImportForceResult = LoadOplog(CidStore, - *RemoteStore, - *OplogImport, - NetworkPool, - WorkerPool, - /*Force*/ true, - /*IgnoreMissingAttachments*/ false, - /*CleanOplog*/ false, - EPartialBlockRequestMode::Mixed, - /*HostLatencySec*/ -1.0, - /*CacheLatencySec*/ -1.0, - nullptr); + OpJobIndex++; + + RemoteProjectStore::Result ImportForceResult = LoadOplog(LoadOplogContext{.ChunkStore = CidStore, + .RemoteStore = *RemoteStore, + .OptionalCache = nullptr, + .CacheBuildId = Oid::Zero, + .Oplog = *OplogImport, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = true, + .IgnoreMissingAttachments = false, + .CleanOplog = false, + .PartialBlockRequestMode = EPartialBlockRequestMode::Mixed, + .OptionalJobContext = &OpJobContext}); CHECK(ImportForceResult.ErrorCode == 0); - - RemoteProjectStore::Result ImportCleanResult = LoadOplog(CidStore, - *RemoteStore, - *OplogImport, - NetworkPool, - WorkerPool, - /*Force*/ false, - /*IgnoreMissingAttachments*/ false, - /*CleanOplog*/ true, - EPartialBlockRequestMode::Mixed, - /*HostLatencySec*/ -1.0, - /*CacheLatencySec*/ -1.0, - nullptr); + OpJobIndex++; + + RemoteProjectStore::Result ImportCleanResult = LoadOplog(LoadOplogContext{.ChunkStore = CidStore, + .RemoteStore = *RemoteStore, + .OptionalCache = nullptr, + .CacheBuildId = Oid::Zero, + .Oplog = *OplogImport, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = false, + .IgnoreMissingAttachments = false, + .CleanOplog = true, + .PartialBlockRequestMode = EPartialBlockRequestMode::Mixed, + .OptionalJobContext = &OpJobContext}); CHECK(ImportCleanResult.ErrorCode == 0); - - RemoteProjectStore::Result ImportForceCleanResult = LoadOplog(CidStore, - *RemoteStore, - *OplogImport, - NetworkPool, - WorkerPool, - /*Force*/ true, - /*IgnoreMissingAttachments*/ false, - /*CleanOplog*/ true, - EPartialBlockRequestMode::Mixed, - /*HostLatencySec*/ -1.0, - /*CacheLatencySec*/ -1.0, - nullptr); + OpJobIndex++; + + RemoteProjectStore::Result ImportForceCleanResult = + LoadOplog(LoadOplogContext{.ChunkStore = CidStore, + .RemoteStore = *RemoteStore, + .OptionalCache = nullptr, + .CacheBuildId = Oid::Zero, + .Oplog = *OplogImport, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = true, + .IgnoreMissingAttachments = false, + .CleanOplog = true, + .PartialBlockRequestMode = EPartialBlockRequestMode::Mixed, + .OptionalJobContext = &OpJobContext}); CHECK(ImportForceCleanResult.ErrorCode == 0); + OpJobIndex++; +} + +// Common oplog setup used by the two tests below. +// Returns a FileRemoteStore backed by ExportDir that has been populated with a SaveOplog call. +// Keeps the test data identical to project.store.export so the two test suites exercise the same blocks/attachments. +static RemoteProjectStore::Result +SetupExportStore(CidStore& CidStore, + ProjectStore::Project& Project, + WorkerThreadPool& NetworkPool, + WorkerThreadPool& WorkerPool, + const std::filesystem::path& ExportDir, + std::shared_ptr& OutRemoteStore) +{ + using namespace projectstore_testutils; + using namespace std::literals; + + Ref Oplog = Project.NewOplog("oplog_export", {}); + if (!Oplog) + { + return RemoteProjectStore::Result{.ErrorCode = -1}; + } + + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), {})); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list{77}))); + Oplog->AppendNewOplogEntry( + CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list{7123, 583, 690, 99}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage(Oid::NewOid(), CreateAttachments(std::initializer_list{55, 122}))); + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage( + Oid::NewOid(), + CreateAttachments(std::initializer_list{256u * 1024u, 92u * 1024u}, OodleCompressionLevel::None))); + + FileRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = 64u * 1024, + .MaxChunksPerBlock = 1000, + .MaxChunkEmbedSize = 32 * 1024u, + .ChunkFileSizeLimit = 64u * 1024u}, + /*.FolderPath =*/ExportDir, + /*.Name =*/std::string("oplog_export"), + /*.OptionalBaseName =*/std::string(), + /*.ForceDisableBlocks =*/false, + /*.ForceEnableTempBlocks =*/false}; + + OutRemoteStore = CreateFileRemoteStore(Log(), Options); + return SaveOplog(CidStore, + *OutRemoteStore, + Project, + *Oplog, + NetworkPool, + WorkerPool, + Options.MaxBlockSize, + Options.MaxChunksPerBlock, + Options.MaxChunkEmbedSize, + Options.ChunkFileSizeLimit, + /*EmbedLooseFiles*/ true, + /*ForceUpload*/ false, + /*IgnoreMissingAttachments*/ false, + /*OptionalContext*/ nullptr); +} + +// Creates an export store with a single oplog entry that packs six 512 KB chunks into one +// ~3 MB block (MaxBlockSize = 8 MB). The resulting block slack (~1.5 MB) far exceeds the +// 512 KB threshold that ChunkBlockAnalyser requires before it will consider partial-block +// downloads instead of full-block downloads. +// +// This function is self-contained: it creates its own GcManager, CidStore, ProjectStore and +// Project internally so that each call is independent of any outer test context. After +// SaveOplog returns, all persistent data lives on disk inside ExportDir and the caller can +// freely query OutRemoteStore without holding any references to the internal context. +static RemoteProjectStore::Result +SetupPartialBlockExportStore(WorkerThreadPool& NetworkPool, + WorkerThreadPool& WorkerPool, + const std::filesystem::path& ExportDir, + std::shared_ptr& OutRemoteStore) +{ + using namespace projectstore_testutils; + using namespace std::literals; + + // Self-contained CAS and project store. Subdirectories of ExportDir keep everything + // together without relying on the outer TEST_CASE's ExportCidStore / ExportProject. + GcManager LocalGc; + CidStore LocalCidStore(LocalGc); + CidStoreConfiguration LocalCidConfig = {.RootDirectory = ExportDir / "cas", .TinyValueThreshold = 1024, .HugeValueThreshold = 4096}; + LocalCidStore.Initialize(LocalCidConfig); + + std::filesystem::path LocalProjectBasePath = ExportDir / "proj"; + ProjectStore LocalProjectStore(LocalCidStore, LocalProjectBasePath, LocalGc, ProjectStore::Configuration{}); + Ref LocalProject(LocalProjectStore.NewProject(LocalProjectBasePath / "p"sv, + "p"sv, + (ExportDir / "root").string(), + (ExportDir / "engine").string(), + (ExportDir / "game").string(), + (ExportDir / "game" / "game.uproject").string())); + + Ref Oplog = LocalProject->NewOplog("oplog_partial_block", {}); + if (!Oplog) + { + return RemoteProjectStore::Result{.ErrorCode = -1}; + } + + // Six 512 KB chunks with OodleCompressionLevel::None so the compressed size stays large + // and the block genuinely exceeds the 512 KB slack threshold. + Oplog->AppendNewOplogEntry(CreateBulkDataOplogPackage( + Oid::NewOid(), + CreateAttachments(std::initializer_list{512u * 1024u, 512u * 1024u, 512u * 1024u, 512u * 1024u, 512u * 1024u, 512u * 1024u}, + OodleCompressionLevel::None))); + + // MaxChunkEmbedSize must be larger than the compressed size of each 512 KB chunk + // (OodleCompressionLevel::None → compressed ≈ raw ≈ 512 KB). With the legacy + // 32 KB limit all six chunks would become loose large attachments and no block would + // be created, so we use the production default of 1.5 MB instead. + FileRemoteStoreOptions Options = {RemoteStoreOptions{.MaxBlockSize = 8u * 1024u * 1024u, + .MaxChunksPerBlock = 1000, + .MaxChunkEmbedSize = RemoteStoreOptions::DefaultMaxChunkEmbedSize, + .ChunkFileSizeLimit = 64u * 1024u * 1024u}, + /*.FolderPath =*/ExportDir, + /*.Name =*/std::string("oplog_partial_block"), + /*.OptionalBaseName =*/std::string(), + /*.ForceDisableBlocks =*/false, + /*.ForceEnableTempBlocks =*/false}; + OutRemoteStore = CreateFileRemoteStore(Log(), Options); + return SaveOplog(LocalCidStore, + *OutRemoteStore, + *LocalProject, + *Oplog, + NetworkPool, + WorkerPool, + Options.MaxBlockSize, + Options.MaxChunksPerBlock, + Options.MaxChunkEmbedSize, + Options.ChunkFileSizeLimit, + /*EmbedLooseFiles*/ true, + /*ForceUpload*/ false, + /*IgnoreMissingAttachments*/ false, + /*OptionalContext*/ nullptr); +} + +// Returns the first block hash that has at least MinChunkCount chunks, or a zero IoHash +// if no qualifying block exists in Store. +static IoHash +FindBlockWithMultipleChunks(RemoteProjectStore& Store, size_t MinChunkCount) +{ + RemoteProjectStore::LoadContainerResult ContainerResult = Store.LoadContainer(); + if (ContainerResult.ErrorCode != 0) + { + return {}; + } + std::vector BlockHashes = GetBlockHashesFromOplog(ContainerResult.ContainerObject); + if (BlockHashes.empty()) + { + return {}; + } + RemoteProjectStore::GetBlockDescriptionsResult Descriptions = Store.GetBlockDescriptions(BlockHashes, nullptr, Oid{}); + if (Descriptions.ErrorCode != 0) + { + return {}; + } + for (const ChunkBlockDescription& Desc : Descriptions.Blocks) + { + if (Desc.ChunkRawHashes.size() >= MinChunkCount) + { + return Desc.BlockHash; + } + } + return {}; +} + +// Loads BlockHash from Source and inserts every even-indexed chunk (0, 2, 4, …) into +// TargetCidStore. Odd-indexed chunks are left absent so that when an import is run +// against the same block, HasAttachment returns false for three non-adjacent positions +// — the minimum needed to exercise the multi-range partial-block download paths. +static void +SeedCidStoreWithAlternateChunks(CidStore& TargetCidStore, RemoteProjectStore& Source, const IoHash& BlockHash) +{ + RemoteProjectStore::LoadAttachmentResult BlockResult = Source.LoadAttachment(BlockHash); + if (BlockResult.ErrorCode != 0 || !BlockResult.Bytes) + { + return; + } + + IoHash RawHash; + uint64_t RawSize; + CompressedBuffer Compressed = CompressedBuffer::FromCompressed(SharedBuffer(BlockResult.Bytes), RawHash, RawSize); + if (!Compressed) + { + return; + } + CompositeBuffer BlockPayload = Compressed.DecompressToComposite(); + if (!BlockPayload) + { + return; + } + + uint32_t ChunkIndex = 0; + uint64_t HeaderSize = 0; + IterateChunkBlock( + BlockPayload.Flatten(), + [&TargetCidStore, &ChunkIndex](CompressedBuffer&& Chunk, const IoHash& AttachmentHash) { + if (ChunkIndex % 2 == 0) + { + IoBuffer ChunkData = Chunk.GetCompressed().Flatten().AsIoBuffer(); + TargetCidStore.AddChunk(ChunkData, AttachmentHash); + } + ++ChunkIndex; + }, + HeaderSize); +} + +TEST_CASE("project.store.import.context_settings") +{ + using namespace std::literals; + using namespace projectstore_testutils; + + ScopedTemporaryDirectory TempDir; + ScopedTemporaryDirectory ExportDir; + + std::filesystem::path RootDir = TempDir.Path() / "root"; + std::filesystem::path EngineRootDir = TempDir.Path() / "engine"; + std::filesystem::path ProjectRootDir = TempDir.Path() / "game"; + std::filesystem::path ProjectFilePath = TempDir.Path() / "game" / "game.uproject"; + + // Export-side CAS and project store: used only by SetupExportStore to build the remote store + // payload. Kept separate from the import side so the two CAS instances are disjoint. + GcManager ExportGc; + CidStore ExportCidStore(ExportGc); + CidStoreConfiguration ExportCidConfig = {.RootDirectory = TempDir.Path() / "export_cas", + .TinyValueThreshold = 1024, + .HugeValueThreshold = 4096}; + ExportCidStore.Initialize(ExportCidConfig); + + std::filesystem::path ExportBasePath = TempDir.Path() / "export_projectstore"; + ProjectStore ExportProjectStore(ExportCidStore, ExportBasePath, ExportGc, ProjectStore::Configuration{}); + Ref ExportProject(ExportProjectStore.NewProject(ExportBasePath / "proj1"sv, + "proj1"sv, + RootDir.string(), + EngineRootDir.string(), + ProjectRootDir.string(), + ProjectFilePath.string())); + + uint32_t NetworkWorkerCount = Max(GetHardwareConcurrency() / 4u, 2u); + uint32_t WorkerCount = (NetworkWorkerCount < GetHardwareConcurrency()) ? Max(GetHardwareConcurrency() - NetworkWorkerCount, 4u) : 4u; + WorkerThreadPool WorkerPool(WorkerCount); + WorkerThreadPool NetworkPool(NetworkWorkerCount); + + std::shared_ptr RemoteStore; + RemoteProjectStore::Result ExportResult = + SetupExportStore(ExportCidStore, *ExportProject, NetworkPool, WorkerPool, ExportDir.Path(), RemoteStore); + REQUIRE(ExportResult.ErrorCode == 0); + + // Import-side CAS and project store: starts empty, mirroring a fresh machine that has never + // downloaded the data. HasAttachment() therefore returns false for every chunk, so the import + // genuinely contacts the remote store without needing ForceDownload on the populate pass. + GcManager ImportGc; + CidStore ImportCidStore(ImportGc); + CidStoreConfiguration ImportCidConfig = {.RootDirectory = TempDir.Path() / "import_cas", + .TinyValueThreshold = 1024, + .HugeValueThreshold = 4096}; + ImportCidStore.Initialize(ImportCidConfig); + + std::filesystem::path ImportBasePath = TempDir.Path() / "import_projectstore"; + ProjectStore ImportProjectStore(ImportCidStore, ImportBasePath, ImportGc, ProjectStore::Configuration{}); + Ref ImportProject(ImportProjectStore.NewProject(ImportBasePath / "proj1"sv, + "proj1"sv, + RootDir.string(), + EngineRootDir.string(), + ProjectRootDir.string(), + ProjectFilePath.string())); + + const Oid CacheBuildId = Oid::NewOid(); + BuildStorageCache::Statistics CacheStats; + std::unique_ptr Cache = CreateInMemoryBuildStorageCache(256u, CacheStats); + auto ResetCacheStats = [&]() { + CacheStats.TotalBytesRead = 0; + CacheStats.TotalBytesWritten = 0; + CacheStats.TotalRequestCount = 0; + CacheStats.TotalRequestTimeUs = 0; + CacheStats.TotalExecutionTimeUs = 0; + CacheStats.PeakSentBytes = 0; + CacheStats.PeakReceivedBytes = 0; + CacheStats.PeakBytesPerSec = 0; + CacheStats.PutBlobCount = 0; + CacheStats.PutBlobByteCount = 0; + }; + + int OpJobIndex = 0; + + TestJobContext OpJobContext(OpJobIndex); + + // Helper: run a LoadOplog against the import-side CAS/project with the given context knobs. + // Each call creates a fresh oplog so repeated calls within one SUBCASE don't short-circuit on + // already-present data. + auto DoImport = [&](BuildStorageCache* OptCache, + EPartialBlockRequestMode Mode, + double StoreLatency, + uint64_t StoreRanges, + double CacheLatency, + uint64_t CacheRanges, + bool PopulateCache, + bool ForceDownload) -> RemoteProjectStore::Result { + Ref ImportOplog = ImportProject->NewOplog(fmt::format("import_{}", OpJobIndex++), {}); + return LoadOplog(LoadOplogContext{.ChunkStore = ImportCidStore, + .RemoteStore = *RemoteStore, + .OptionalCache = OptCache, + .CacheBuildId = CacheBuildId, + .Oplog = *ImportOplog, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = ForceDownload, + .IgnoreMissingAttachments = false, + .CleanOplog = false, + .PartialBlockRequestMode = Mode, + .PopulateCache = PopulateCache, + .StoreLatencySec = StoreLatency, + .StoreMaxRangeCountPerRequest = StoreRanges, + .CacheLatencySec = CacheLatency, + .CacheMaxRangeCountPerRequest = CacheRanges, + .OptionalJobContext = &OpJobContext}); + }; + + // Shorthand: Mode=All, low latency, 128 ranges for both store and cache. + auto ImportAll = [&](BuildStorageCache* OptCache, bool Populate, bool Force) { + return DoImport(OptCache, EPartialBlockRequestMode::All, 0.001, 128u, 0.001, 128u, Populate, Force); + }; + + SUBCASE("mode_off_no_cache") + { + // Baseline: no partial block requests, no cache. + RemoteProjectStore::Result R = + DoImport(nullptr, EPartialBlockRequestMode::Off, -1.0, (uint64_t)-1, -1.0, (uint64_t)-1, false, false); + CHECK(R.ErrorCode == 0); + } + + SUBCASE("mode_all_multirange_cloud_no_cache") + { + // StoreMaxRangeCountPerRequest > 1 → MultiRange cloud path. + RemoteProjectStore::Result R = DoImport(nullptr, EPartialBlockRequestMode::All, 0.001, 128u, -1.0, 0u, false, false); + CHECK(R.ErrorCode == 0); + } + + SUBCASE("mode_all_singlerange_cloud_no_cache") + { + // StoreMaxRangeCountPerRequest == 1 → SingleRange cloud path. + RemoteProjectStore::Result R = DoImport(nullptr, EPartialBlockRequestMode::All, 0.001, 1u, -1.0, 0u, false, false); + CHECK(R.ErrorCode == 0); + } + + SUBCASE("mode_mixed_high_latency_no_cache") + { + // High store latency encourages range merging; Mixed uses SingleRange for cloud, Off for cache. + RemoteProjectStore::Result R = DoImport(nullptr, EPartialBlockRequestMode::Mixed, 0.1, 128u, -1.0, 0u, false, false); + CHECK(R.ErrorCode == 0); + } + + SUBCASE("cache_populate_and_hit") + { + // First import: ImportCidStore is empty so all blocks are downloaded from the remote store + // and written to the cache. + RemoteProjectStore::Result PopulateResult = ImportAll(Cache.get(), /*PopulateCache=*/true, /*Force=*/false); + CHECK(PopulateResult.ErrorCode == 0); + CHECK(CacheStats.PutBlobCount > 0); + + // Re-import with ForceDownload=true: all chunks are now in ImportCidStore but Force overrides + // HasAttachment() so the download logic re-runs and serves blocks from the cache instead of + // the remote store. + ResetCacheStats(); + RemoteProjectStore::Result HitResult = ImportAll(Cache.get(), /*PopulateCache=*/false, /*Force=*/true); + CHECK(HitResult.ErrorCode == 0); + CHECK(CacheStats.PutBlobCount == 0); + // TotalRequestCount covers both full-blob cache hits and partial-range cache hits. + CHECK(CacheStats.TotalRequestCount > 0); + } + + SUBCASE("cache_no_populate_flag") + { + // Cache is provided but PopulateCache=false: blocks are downloaded to ImportCidStore but + // nothing should be written to the cache. + RemoteProjectStore::Result R = ImportAll(Cache.get(), /*PopulateCache=*/false, /*Force=*/false); + CHECK(R.ErrorCode == 0); + CHECK(CacheStats.PutBlobCount == 0); + } + + SUBCASE("mode_zencacheonly_cache_multirange") + { + // Pre-populate the cache via a plain import, then re-import with ZenCacheOnly + + // CacheMaxRangeCountPerRequest=128. With 100% of chunks needed, all blocks go to + // FullBlockIndexes and GetBuildBlob (full blob) is called from the cache. + // CacheMaxRangeCountPerRequest > 1 would route partial downloads through GetBuildBlobRanges + // if the analyser ever emits BlockRanges entries. + RemoteProjectStore::Result Populate = ImportAll(Cache.get(), /*PopulateCache=*/true, /*Force=*/false); + CHECK(Populate.ErrorCode == 0); + ResetCacheStats(); + + RemoteProjectStore::Result R = DoImport(Cache.get(), EPartialBlockRequestMode::ZenCacheOnly, 0.1, 128u, 0.001, 128u, false, true); + CHECK(R.ErrorCode == 0); + CHECK(CacheStats.TotalRequestCount > 0); + } + + SUBCASE("mode_zencacheonly_cache_singlerange") + { + // Pre-populate the cache, then re-import with ZenCacheOnly + CacheMaxRangeCountPerRequest=1. + // With 100% of chunks needed the analyser sends all blocks to FullBlockIndexes (full-block + // download path), which calls GetBuildBlob with no range offset — a full-blob cache hit. + // The single-range vs multi-range distinction only matters for the partial-block (BlockRanges) + // path, which is not reached when all chunks are needed. + RemoteProjectStore::Result Populate = ImportAll(Cache.get(), /*PopulateCache=*/true, /*Force=*/false); + CHECK(Populate.ErrorCode == 0); + ResetCacheStats(); + + RemoteProjectStore::Result R = DoImport(Cache.get(), EPartialBlockRequestMode::ZenCacheOnly, 0.1, 128u, 0.001, 1u, false, true); + CHECK(R.ErrorCode == 0); + CHECK(CacheStats.TotalRequestCount > 0); + } + + SUBCASE("mode_all_cache_and_cloud_multirange") + { + // Pre-populate cache; All mode uses multi-range for both the cache and cloud paths. + RemoteProjectStore::Result Populate = ImportAll(Cache.get(), /*PopulateCache=*/true, /*Force=*/false); + CHECK(Populate.ErrorCode == 0); + ResetCacheStats(); + + RemoteProjectStore::Result R = ImportAll(Cache.get(), /*PopulateCache=*/false, /*Force=*/true); + CHECK(R.ErrorCode == 0); + CHECK(CacheStats.TotalRequestCount > 0); + } + + SUBCASE("partial_block_cloud_multirange") + { + // Export store with 6 × 512 KB chunks packed into one ~3 MB block. + ScopedTemporaryDirectory PartialExportDir; + std::shared_ptr PartialRemoteStore; + RemoteProjectStore::Result ExportR = + SetupPartialBlockExportStore(NetworkPool, WorkerPool, PartialExportDir.Path(), PartialRemoteStore); + REQUIRE(ExportR.ErrorCode == 0); + + // Seeding even-indexed chunks (0, 2, 4) leaves odd ones (1, 3, 5) absent in + // ImportCidStore. Three non-adjacent needed positions → three BlockRangeDescriptors. + IoHash BlockHash = FindBlockWithMultipleChunks(*PartialRemoteStore, 4u); + CHECK(BlockHash != IoHash::Zero); + SeedCidStoreWithAlternateChunks(ImportCidStore, *PartialRemoteStore, BlockHash); + + // StoreMaxRangeCountPerRequest=128 → all three ranges sent in one LoadAttachmentRanges call. + Ref PartialOplog = ImportProject->NewOplog(fmt::format("partial_cloud_multi_{}", OpJobIndex++), {}); + RemoteProjectStore::Result R = LoadOplog(LoadOplogContext{.ChunkStore = ImportCidStore, + .RemoteStore = *PartialRemoteStore, + .OptionalCache = nullptr, + .CacheBuildId = CacheBuildId, + .Oplog = *PartialOplog, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = false, + .IgnoreMissingAttachments = false, + .CleanOplog = false, + .PartialBlockRequestMode = EPartialBlockRequestMode::All, + .PopulateCache = false, + .StoreLatencySec = 0.001, + .StoreMaxRangeCountPerRequest = 128u, + .CacheLatencySec = -1.0, + .CacheMaxRangeCountPerRequest = 0u, + .OptionalJobContext = &OpJobContext}); + CHECK(R.ErrorCode == 0); + } + + SUBCASE("partial_block_cloud_singlerange") + { + // Same block layout as partial_block_cloud_multirange but StoreMaxRangeCountPerRequest=1. + // DownloadPartialBlock issues one LoadAttachmentRanges call per range. + ScopedTemporaryDirectory PartialExportDir; + std::shared_ptr PartialRemoteStore; + RemoteProjectStore::Result ExportR = + SetupPartialBlockExportStore(NetworkPool, WorkerPool, PartialExportDir.Path(), PartialRemoteStore); + REQUIRE(ExportR.ErrorCode == 0); + + IoHash BlockHash = FindBlockWithMultipleChunks(*PartialRemoteStore, 4u); + CHECK(BlockHash != IoHash::Zero); + SeedCidStoreWithAlternateChunks(ImportCidStore, *PartialRemoteStore, BlockHash); + + Ref PartialOplog = ImportProject->NewOplog(fmt::format("partial_cloud_single_{}", OpJobIndex++), {}); + RemoteProjectStore::Result R = LoadOplog(LoadOplogContext{.ChunkStore = ImportCidStore, + .RemoteStore = *PartialRemoteStore, + .OptionalCache = nullptr, + .CacheBuildId = CacheBuildId, + .Oplog = *PartialOplog, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = false, + .IgnoreMissingAttachments = false, + .CleanOplog = false, + .PartialBlockRequestMode = EPartialBlockRequestMode::All, + .PopulateCache = false, + .StoreLatencySec = 0.001, + .StoreMaxRangeCountPerRequest = 1u, + .CacheLatencySec = -1.0, + .CacheMaxRangeCountPerRequest = 0u, + .OptionalJobContext = &OpJobContext}); + CHECK(R.ErrorCode == 0); + } + + SUBCASE("partial_block_cache_multirange") + { + ScopedTemporaryDirectory PartialExportDir; + std::shared_ptr PartialRemoteStore; + RemoteProjectStore::Result ExportR = + SetupPartialBlockExportStore(NetworkPool, WorkerPool, PartialExportDir.Path(), PartialRemoteStore); + REQUIRE(ExportR.ErrorCode == 0); + + IoHash BlockHash = FindBlockWithMultipleChunks(*PartialRemoteStore, 4u); + CHECK(BlockHash != IoHash::Zero); + + // Phase 1: ImportCidStore starts empty → full block download from remote → PutBuildBlob + // populates the cache. + { + Ref Phase1Oplog = ImportProject->NewOplog(fmt::format("partial_cache_multi_p1_{}", OpJobIndex++), {}); + RemoteProjectStore::Result Phase1R = LoadOplog(LoadOplogContext{.ChunkStore = ImportCidStore, + .RemoteStore = *PartialRemoteStore, + .OptionalCache = Cache.get(), + .CacheBuildId = CacheBuildId, + .Oplog = *Phase1Oplog, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = false, + .IgnoreMissingAttachments = false, + .CleanOplog = false, + .PartialBlockRequestMode = EPartialBlockRequestMode::All, + .PopulateCache = true, + .StoreLatencySec = 0.001, + .StoreMaxRangeCountPerRequest = 128u, + .CacheLatencySec = 0.001, + .CacheMaxRangeCountPerRequest = 128u, + .OptionalJobContext = &OpJobContext}); + CHECK(Phase1R.ErrorCode == 0); + CHECK(CacheStats.PutBlobCount > 0); + } + ResetCacheStats(); + + // Phase 2: fresh CidStore with only even-indexed chunks seeded. + // HasAttachment returns false for odd chunks (1, 3, 5) → three BlockRangeDescriptors. + // Block is in cache from Phase 1 → cache partial path. + // CacheMaxRangeCountPerRequest=128 → SubRangeCount=3 > 1 → GetBuildBlobRanges. + GcManager Phase2Gc; + CidStore Phase2CidStore(Phase2Gc); + CidStoreConfiguration Phase2CidConfig = {.RootDirectory = TempDir.Path() / "partial_cas", + .TinyValueThreshold = 1024, + .HugeValueThreshold = 4096}; + Phase2CidStore.Initialize(Phase2CidConfig); + SeedCidStoreWithAlternateChunks(Phase2CidStore, *PartialRemoteStore, BlockHash); + + Ref Phase2Oplog = ImportProject->NewOplog(fmt::format("partial_cache_multi_p2_{}", OpJobIndex++), {}); + RemoteProjectStore::Result Phase2R = LoadOplog(LoadOplogContext{.ChunkStore = Phase2CidStore, + .RemoteStore = *PartialRemoteStore, + .OptionalCache = Cache.get(), + .CacheBuildId = CacheBuildId, + .Oplog = *Phase2Oplog, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = false, + .IgnoreMissingAttachments = false, + .CleanOplog = false, + .PartialBlockRequestMode = EPartialBlockRequestMode::ZenCacheOnly, + .PopulateCache = false, + .StoreLatencySec = 0.001, + .StoreMaxRangeCountPerRequest = 128u, + .CacheLatencySec = 0.001, + .CacheMaxRangeCountPerRequest = 128u, + .OptionalJobContext = &OpJobContext}); + CHECK(Phase2R.ErrorCode == 0); + CHECK(CacheStats.TotalRequestCount > 0); + } + + SUBCASE("partial_block_cache_singlerange") + { + ScopedTemporaryDirectory PartialExportDir; + std::shared_ptr PartialRemoteStore; + RemoteProjectStore::Result ExportR = + SetupPartialBlockExportStore(NetworkPool, WorkerPool, PartialExportDir.Path(), PartialRemoteStore); + REQUIRE(ExportR.ErrorCode == 0); + + IoHash BlockHash = FindBlockWithMultipleChunks(*PartialRemoteStore, 4u); + CHECK(BlockHash != IoHash::Zero); + + // Phase 1: full block download from remote into cache. + { + Ref Phase1Oplog = ImportProject->NewOplog(fmt::format("partial_cache_single_p1_{}", OpJobIndex++), {}); + RemoteProjectStore::Result Phase1R = LoadOplog(LoadOplogContext{.ChunkStore = ImportCidStore, + .RemoteStore = *PartialRemoteStore, + .OptionalCache = Cache.get(), + .CacheBuildId = CacheBuildId, + .Oplog = *Phase1Oplog, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = false, + .IgnoreMissingAttachments = false, + .CleanOplog = false, + .PartialBlockRequestMode = EPartialBlockRequestMode::All, + .PopulateCache = true, + .StoreLatencySec = 0.001, + .StoreMaxRangeCountPerRequest = 128u, + .CacheLatencySec = 0.001, + .CacheMaxRangeCountPerRequest = 128u, + .OptionalJobContext = &OpJobContext}); + CHECK(Phase1R.ErrorCode == 0); + CHECK(CacheStats.PutBlobCount > 0); + } + ResetCacheStats(); + + // Phase 2: fresh CidStore with only even-indexed chunks seeded. + // CacheMaxRangeCountPerRequest=1 → SubRangeCount=Min(3,1)=1 → GetBuildBlob with range + // offset (single-range legacy cache path), called once per needed chunk range. + GcManager Phase2Gc; + CidStore Phase2CidStore(Phase2Gc); + CidStoreConfiguration Phase2CidConfig = {.RootDirectory = TempDir.Path() / "partial_cas_single", + .TinyValueThreshold = 1024, + .HugeValueThreshold = 4096}; + Phase2CidStore.Initialize(Phase2CidConfig); + SeedCidStoreWithAlternateChunks(Phase2CidStore, *PartialRemoteStore, BlockHash); + + Ref Phase2Oplog = ImportProject->NewOplog(fmt::format("partial_cache_single_p2_{}", OpJobIndex++), {}); + RemoteProjectStore::Result Phase2R = LoadOplog(LoadOplogContext{.ChunkStore = Phase2CidStore, + .RemoteStore = *PartialRemoteStore, + .OptionalCache = Cache.get(), + .CacheBuildId = CacheBuildId, + .Oplog = *Phase2Oplog, + .NetworkWorkerPool = NetworkPool, + .WorkerPool = WorkerPool, + .ForceDownload = false, + .IgnoreMissingAttachments = false, + .CleanOplog = false, + .PartialBlockRequestMode = EPartialBlockRequestMode::ZenCacheOnly, + .PopulateCache = false, + .StoreLatencySec = 0.001, + .StoreMaxRangeCountPerRequest = 128u, + .CacheLatencySec = 0.001, + .CacheMaxRangeCountPerRequest = 1u, + .OptionalJobContext = &OpJobContext}); + CHECK(Phase2R.ErrorCode == 0); + CHECK(CacheStats.TotalRequestCount > 0); + } } TEST_SUITE_END(); diff --git a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp index ef82c45e0..115d6438d 100644 --- a/src/zenremotestore/projectstore/zenremoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/zenremoteprojectstore.cpp @@ -157,59 +157,56 @@ public: return Result; } - virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes, ESourceMode SourceMode) override + virtual LoadAttachmentsResult LoadAttachments(const std::vector& RawHashes) override { LoadAttachmentsResult Result; - if (SourceMode != ESourceMode::kCacheOnly) - { - std::string LoadRequest = fmt::format("/{}/oplog/{}/rpc"sv, m_Project, m_Oplog); + std::string LoadRequest = fmt::format("/{}/oplog/{}/rpc"sv, m_Project, m_Oplog); - CbObject Request; + CbObject Request; + { + CbObjectWriter RequestWriter; + RequestWriter.AddString("method"sv, "getchunks"sv); + RequestWriter.BeginObject("Request"sv); { - CbObjectWriter RequestWriter; - RequestWriter.AddString("method"sv, "getchunks"sv); - RequestWriter.BeginObject("Request"sv); + RequestWriter.BeginArray("Chunks"sv); { - RequestWriter.BeginArray("Chunks"sv); + for (const IoHash& RawHash : RawHashes) { - for (const IoHash& RawHash : RawHashes) + RequestWriter.BeginObject(); { - RequestWriter.BeginObject(); - { - RequestWriter.AddHash("RawHash", RawHash); - } - RequestWriter.EndObject(); + RequestWriter.AddHash("RawHash", RawHash); } + RequestWriter.EndObject(); } - RequestWriter.EndArray(); // "chunks" } - RequestWriter.EndObject(); - Request = RequestWriter.Save(); + RequestWriter.EndArray(); // "chunks" } + RequestWriter.EndObject(); + Request = RequestWriter.Save(); + } - HttpClient::Response Response = m_Client.Post(LoadRequest, Request, HttpClient::Accept(ZenContentType::kCbPackage)); - AddStats(Response); + HttpClient::Response Response = m_Client.Post(LoadRequest, Request, HttpClient::Accept(ZenContentType::kCbPackage)); + AddStats(Response); - Result = LoadAttachmentsResult{ConvertResult(Response)}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching {} oplog attachments from {}/{}/{}. Reason: '{}'", - RawHashes.size(), - m_ProjectStoreUrl, - m_Project, - m_Oplog, - Result.Reason); - } - else + Result = LoadAttachmentsResult{ConvertResult(Response)}; + if (Result.ErrorCode) + { + Result.Reason = fmt::format("Failed fetching {} oplog attachments from {}/{}/{}. Reason: '{}'", + RawHashes.size(), + m_ProjectStoreUrl, + m_Project, + m_Oplog, + Result.Reason); + } + else + { + CbPackage Package = Response.AsPackage(); + std::span Attachments = Package.GetAttachments(); + Result.Chunks.reserve(Attachments.size()); + for (const CbAttachment& Attachment : Attachments) { - CbPackage Package = Response.AsPackage(); - std::span Attachments = Package.GetAttachments(); - Result.Chunks.reserve(Attachments.size()); - for (const CbAttachment& Attachment : Attachments) - { - Result.Chunks.emplace_back( - std::pair{Attachment.GetHash(), Attachment.AsCompressedBinary().MakeOwned()}); - } + Result.Chunks.emplace_back( + std::pair{Attachment.GetHash(), Attachment.AsCompressedBinary().MakeOwned()}); } } return Result; @@ -253,75 +250,64 @@ public: return GetKnownBlocksResult{{.ErrorCode = static_cast(HttpResponseCode::NoContent)}}; } - virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes) override + virtual GetBlockDescriptionsResult GetBlockDescriptions(std::span BlockHashes, + BuildStorageCache* OptionalCache, + const Oid& CacheBuildId) override { - ZEN_UNUSED(BlockHashes); + ZEN_UNUSED(BlockHashes, OptionalCache, CacheBuildId); return GetBlockDescriptionsResult{Result{.ErrorCode = int(HttpResponseCode::NotFound)}}; } - virtual AttachmentExistsInCacheResult AttachmentExistsInCache(std::span RawHashes) override - { - return AttachmentExistsInCacheResult{Result{.ErrorCode = 0}, std::vector(RawHashes.size(), false)}; - } - - virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash, ESourceMode SourceMode) override + virtual LoadAttachmentResult LoadAttachment(const IoHash& RawHash) override { LoadAttachmentResult Result; - if (SourceMode != ESourceMode::kCacheOnly) - { - std::string LoadRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash); - HttpClient::Response Response = - m_Client.Download(LoadRequest, m_TempFilePath, HttpClient::Accept(ZenContentType::kCompressedBinary)); - AddStats(Response); + std::string LoadRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash); + HttpClient::Response Response = + m_Client.Download(LoadRequest, m_TempFilePath, HttpClient::Accept(ZenContentType::kCompressedBinary)); + AddStats(Response); - Result = LoadAttachmentResult{ConvertResult(Response)}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'", - m_ProjectStoreUrl, - m_Project, - m_Oplog, - RawHash, - Result.Reason); - } - Result.Bytes = Response.ResponsePayload; - Result.Bytes.MakeOwned(); + Result = LoadAttachmentResult{ConvertResult(Response)}; + if (Result.ErrorCode) + { + Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'", + m_ProjectStoreUrl, + m_Project, + m_Oplog, + RawHash, + Result.Reason); } + Result.Bytes = Response.ResponsePayload; + Result.Bytes.MakeOwned(); return Result; } virtual LoadAttachmentRangesResult LoadAttachmentRanges(const IoHash& RawHash, - std::span> Ranges, - ESourceMode SourceMode) override + std::span> Ranges) override { + ZEN_ASSERT(!Ranges.empty()); LoadAttachmentRangesResult Result; - if (SourceMode != ESourceMode::kCacheOnly) - { - std::string LoadRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash); - HttpClient::Response Response = - m_Client.Download(LoadRequest, m_TempFilePath, HttpClient::Accept(ZenContentType::kCompressedBinary)); - AddStats(Response); + std::string LoadRequest = fmt::format("/{}/oplog/{}/{}"sv, m_Project, m_Oplog, RawHash); + HttpClient::Response Response = + m_Client.Download(LoadRequest, m_TempFilePath, HttpClient::Accept(ZenContentType::kCompressedBinary)); + AddStats(Response); - Result = LoadAttachmentRangesResult{ConvertResult(Response)}; - if (Result.ErrorCode) - { - Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'", - m_ProjectStoreUrl, - m_Project, - m_Oplog, - RawHash, - Result.Reason); - } - else - { - Result.Ranges = std::vector>(Ranges.begin(), Ranges.end()); - } + Result = LoadAttachmentRangesResult{ConvertResult(Response)}; + if (Result.ErrorCode) + { + Result.Reason = fmt::format("Failed fetching oplog attachment from {}/{}/{}/{}. Reason: '{}'", + m_ProjectStoreUrl, + m_Project, + m_Oplog, + RawHash, + Result.Reason); + } + else + { + Result.Ranges = std::vector>(Ranges.begin(), Ranges.end()); } return Result; } - virtual void Flush() override {} - private: void AddStats(const HttpClient::Response& Result) { diff --git a/src/zenserver/storage/buildstore/httpbuildstore.cpp b/src/zenserver/storage/buildstore/httpbuildstore.cpp index 459e044eb..38d97765b 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.cpp +++ b/src/zenserver/storage/buildstore/httpbuildstore.cpp @@ -177,6 +177,14 @@ HttpBuildStoreService::GetBlobRequest(HttpRouterRequest& Req) uint64_t RangeLength = RangeView["length"sv].AsUInt64(); OffsetAndLengthPairs.push_back(std::make_pair(RangeOffset, RangeLength)); } + if (OffsetAndLengthPairs.size() > MaxRangeCountPerRequestSupported) + { + return ServerRequest.WriteResponse(HttpResponseCode::BadRequest, + HttpContentType::kText, + fmt::format("Number of ranges ({}) for blob request exceeds maximum range count {}", + OffsetAndLengthPairs.size(), + MaxRangeCountPerRequestSupported)); + } } if (OffsetAndLengthPairs.empty()) { @@ -661,6 +669,11 @@ HttpBuildStoreService::HandleStatusRequest(HttpServerRequest& Request) ZEN_TRACE_CPU("HttpBuildStoreService::Status"); CbObjectWriter Cbo; Cbo << "ok" << true; + Cbo.BeginObject("capabilities"); + { + Cbo << "maxrangecountperrequest" << MaxRangeCountPerRequestSupported; + } + Cbo.EndObject(); // capabilities Request.WriteResponse(HttpResponseCode::OK, Cbo.Save()); } diff --git a/src/zenserver/storage/buildstore/httpbuildstore.h b/src/zenserver/storage/buildstore/httpbuildstore.h index e10986411..5fa7cd642 100644 --- a/src/zenserver/storage/buildstore/httpbuildstore.h +++ b/src/zenserver/storage/buildstore/httpbuildstore.h @@ -45,6 +45,8 @@ private: inline LoggerRef Log() { return m_Log; } + static constexpr uint32_t MaxRangeCountPerRequestSupported = 256u; + LoggerRef m_Log; void PutBlobRequest(HttpRouterRequest& Req); diff --git a/src/zenserver/storage/projectstore/httpprojectstore.cpp b/src/zenserver/storage/projectstore/httpprojectstore.cpp index 2b5474d00..0ec6faea3 100644 --- a/src/zenserver/storage/projectstore/httpprojectstore.cpp +++ b/src/zenserver/storage/projectstore/httpprojectstore.cpp @@ -13,7 +13,12 @@ #include #include #include +#include #include +#include +#include +#include +#include #include #include #include @@ -244,8 +249,22 @@ namespace { { std::shared_ptr Store; std::string Description; - double HostLatencySec = -1.0; - double CacheLatencySec = -1.0; + double LatencySec = -1.0; + uint64_t MaxRangeCountPerRequest = 1; + + struct Cache + { + std::unique_ptr Http; + std::unique_ptr Cache; + Oid BuildsId = Oid::Zero; + std::string Description; + double LatencySec = -1.0; + uint64_t MaxRangeCountPerRequest = 1; + BuildStorageCache::Statistics Stats; + bool Populate = false; + }; + + std::unique_ptr OptionalCache; }; CreateRemoteStoreResult CreateRemoteStore(LoggerRef InLog, @@ -262,9 +281,7 @@ namespace { using namespace std::literals; - std::shared_ptr RemoteStore; - double HostLatencySec = -1.0; - double CacheLatencySec = -1.0; + CreateRemoteStoreResult Result; if (CbObjectView File = Params["file"sv].AsObjectView(); File) { @@ -282,6 +299,9 @@ namespace { bool ForceDisableBlocks = File["disableblocks"sv].AsBool(false); bool ForceEnableTempBlocks = File["enabletempblocks"sv].AsBool(false); + Result.LatencySec = 0; + Result.MaxRangeCountPerRequest = 1; + FileRemoteStoreOptions Options = { RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, FolderPath, @@ -289,7 +309,7 @@ namespace { std::string(OptionalBaseName), ForceDisableBlocks, ForceEnableTempBlocks}; - RemoteStore = CreateFileRemoteStore(Log(), Options); + Result.Store = CreateFileRemoteStore(Log(), Options); } if (CbObjectView Cloud = Params["cloud"sv].AsObjectView(); Cloud) @@ -367,21 +387,32 @@ namespace { bool ForceDisableTempBlocks = Cloud["disabletempblocks"sv].AsBool(false); bool AssumeHttp2 = Cloud["assumehttp2"sv].AsBool(false); - JupiterRemoteStoreOptions Options = { - RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - Url, - std::string(Namespace), - std::string(Bucket), - Key, - BaseKey, - std::string(OpenIdProvider), - AccessToken, - AuthManager, - OidcExePath, - ForceDisableBlocks, - ForceDisableTempBlocks, - AssumeHttp2}; - RemoteStore = CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + if (JupiterEndpointTestResult TestResult = TestJupiterEndpoint(Url, AssumeHttp2, /*Verbose*/ false); TestResult.Success) + { + Result.LatencySec = TestResult.LatencySeconds; + Result.MaxRangeCountPerRequest = TestResult.MaxRangeCountPerRequest; + + JupiterRemoteStoreOptions Options = { + RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, + Url, + std::string(Namespace), + std::string(Bucket), + Key, + BaseKey, + std::string(OpenIdProvider), + AccessToken, + AuthManager, + OidcExePath, + ForceDisableBlocks, + ForceDisableTempBlocks, + AssumeHttp2}; + Result.Store = + CreateJupiterRemoteStore(Log(), Options, TempFilePath, /*Quiet*/ false, /*Unattended*/ false, /*Hidden*/ true); + } + else + { + return {nullptr, fmt::format("Unable to connect to jupiter host '{}'", Url)}; + } } if (CbObjectView Zen = Params["zen"sv].AsObjectView(); Zen) @@ -397,12 +428,13 @@ namespace { { return {nullptr, "Missing oplog"}; } + ZenRemoteStoreOptions Options = { RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, std::string(Url), std::string(Project), std::string(Oplog)}; - RemoteStore = CreateZenRemoteStore(Log(), Options, TempFilePath); + Result.Store = CreateZenRemoteStore(Log(), Options, TempFilePath); } if (CbObjectView Builds = Params["builds"sv].AsObjectView(); Builds) @@ -475,11 +507,76 @@ namespace { MemoryView MetaDataSection = Builds["metadata"sv].AsBinaryView(); IoBuffer MetaData(IoBuffer::Wrap, MetaDataSection.GetData(), MetaDataSection.GetSize()); + auto EnsureHttps = [](const std::string& Host, std::string_view PreferredProtocol) { + if (!Host.empty() && Host.find("://"sv) == std::string::npos) + { + // Assume https URL + return fmt::format("{}://{}"sv, PreferredProtocol, Host); + } + return Host; + }; + + Host = EnsureHttps(Host, "https"); + OverrideHost = EnsureHttps(OverrideHost, "https"); + ZenHost = EnsureHttps(ZenHost, "http"); + + std::function TokenProvider; + if (!OpenIdProvider.empty()) + { + TokenProvider = httpclientauth::CreateFromOpenIdProvider(AuthManager, OpenIdProvider); + } + else if (!AccessToken.empty()) + { + TokenProvider = httpclientauth::CreateFromStaticToken(AccessToken); + } + else if (!OidcExePath.empty()) + { + if (auto TokenProviderMaybe = httpclientauth::CreateFromOidcTokenExecutable(OidcExePath, + Host.empty() ? OverrideHost : Host, + /*Quiet*/ false, + /*Unattended*/ false, + /*Hidden*/ true); + TokenProviderMaybe) + { + TokenProvider = TokenProviderMaybe.value(); + } + } + + if (!TokenProvider) + { + TokenProvider = httpclientauth::CreateFromDefaultOpenIdProvider(AuthManager); + } + + BuildStorageResolveResult ResolveResult; + { + HttpClientSettings ClientSettings{.LogCategory = "httpbuildsclient", + .AccessTokenProvider = TokenProvider, + .AssumeHttp2 = AssumeHttp2, + .AllowResume = true, + .RetryCount = 2}; + + std::unique_ptr Output(CreateStandardLogOutput(Log())); + + try + { + ResolveResult = ResolveBuildStorage(*Output, + ClientSettings, + Host, + OverrideHost, + ZenHost, + ZenCacheResolveMode::Discovery, + /*Verbose*/ false); + } + catch (const std::exception& Ex) + { + return {nullptr, fmt::format("Failed resolving storage host and cache. Reason: '{}'", Ex.what())}; + } + } + Result.LatencySec = ResolveResult.Cloud.LatencySec; + Result.MaxRangeCountPerRequest = ResolveResult.Cloud.Caps.MaxRangeCountPerRequest; + BuildsRemoteStoreOptions Options = { RemoteStoreOptions{.MaxBlockSize = MaxBlockSize, .MaxChunksPerBlock = 1000, .MaxChunkEmbedSize = MaxChunkEmbedSize}, - Host, - OverrideHost, - ZenHost, std::string(Namespace), std::string(Bucket), BuildId, @@ -489,30 +586,43 @@ namespace { OidcExePath, ForceDisableBlocks, ForceDisableTempBlocks, - AssumeHttp2, - PopulateCache, MetaData, MaximumInMemoryDownloadSize}; - RemoteStore = CreateJupiterBuildsRemoteStore(Log(), - Options, - TempFilePath, - /*Quiet*/ false, - /*Unattended*/ false, - /*Hidden*/ true, - GetTinyWorkerPool(EWorkloadType::Background), - HostLatencySec, - CacheLatencySec); + Result.Store = CreateJupiterBuildsRemoteStore(Log(), ResolveResult, std::move(TokenProvider), Options, TempFilePath); + + if (!ResolveResult.Cache.Address.empty()) + { + Result.OptionalCache = std::make_unique(); + + HttpClientSettings CacheClientSettings{.LogCategory = "httpcacheclient", + .ConnectTimeout = std::chrono::milliseconds{3000}, + .Timeout = std::chrono::milliseconds{30000}, + .AssumeHttp2 = ResolveResult.Cache.AssumeHttp2, + .AllowResume = true, + .RetryCount = 0, + .MaximumInMemoryDownloadSize = MaximumInMemoryDownloadSize}; + + Result.OptionalCache->Http = std::make_unique(ResolveResult.Cache.Address, CacheClientSettings); + Result.OptionalCache->Cache = CreateZenBuildStorageCache(*Result.OptionalCache->Http, + Result.OptionalCache->Stats, + Namespace, + Bucket, + TempFilePath, + GetTinyWorkerPool(EWorkloadType::Background)); + Result.OptionalCache->BuildsId = BuildId; + Result.OptionalCache->LatencySec = ResolveResult.Cache.LatencySec; + Result.OptionalCache->MaxRangeCountPerRequest = ResolveResult.Cache.Caps.MaxRangeCountPerRequest; + Result.OptionalCache->Populate = PopulateCache; + Result.OptionalCache->Description = + fmt::format("[zenserver] {} namespace {} bucket {}", ResolveResult.Cache.Address, Namespace, Bucket); + } } - - if (!RemoteStore) + if (!Result.Store) { return {nullptr, "Unknown remote store type"}; } - return CreateRemoteStoreResult{.Store = std::move(RemoteStore), - .Description = "", - .HostLatencySec = HostLatencySec, - .CacheLatencySec = CacheLatencySec}; + return Result; } std::pair ConvertResult(const RemoteProjectStore::Result& Result) @@ -2679,38 +2789,36 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) EPartialBlockRequestMode PartialBlockRequestMode = PartialBlockRequestModeFromString(Params["partialblockrequestmode"sv].AsString("true")); - CreateRemoteStoreResult RemoteStoreResult = CreateRemoteStore(Log(), - Params, - m_AuthMgr, - MaxBlockSize, - MaxChunkEmbedSize, - GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory), - Oplog->TempPath()); + std::shared_ptr RemoteStoreResult = + std::make_shared(CreateRemoteStore(Log(), + Params, + m_AuthMgr, + MaxBlockSize, + MaxChunkEmbedSize, + GetMaxMemoryBufferSize(MaxBlockSize, BoostWorkerMemory), + Oplog->TempPath())); - if (RemoteStoreResult.Store == nullptr) + if (RemoteStoreResult->Store == nullptr) { - return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult.Description); + return HttpReq.WriteResponse(HttpResponseCode::BadRequest, HttpContentType::kText, RemoteStoreResult->Description); } - std::shared_ptr RemoteStore = std::move(RemoteStoreResult.Store); - RemoteProjectStore::RemoteStoreInfo StoreInfo = RemoteStore->GetInfo(); JobId JobId = m_JobQueue.QueueJob( fmt::format("Import oplog '{}/{}'", Project->Identifier, Oplog->OplogId()), [this, - ChunkStore = &m_CidStore, - ActualRemoteStore = std::move(RemoteStore), + RemoteStoreResult = std::move(RemoteStoreResult), Oplog, Force, IgnoreMissingAttachments, CleanOplog, PartialBlockRequestMode, - HostLatencySec = RemoteStoreResult.HostLatencySec, - CacheLatencySec = RemoteStoreResult.CacheLatencySec, BoostWorkerCount](JobContext& Context) { - Context.ReportMessage(fmt::format("Loading oplog '{}/{}' from {}", - Oplog->GetOuterProjectIdentifier(), - Oplog->OplogId(), - ActualRemoteStore->GetInfo().Description)); + Context.ReportMessage( + fmt::format("Loading oplog '{}/{}'\n Host: {}\n Cache: {}", + Oplog->GetOuterProjectIdentifier(), + Oplog->OplogId(), + RemoteStoreResult->Store->GetInfo().Description, + RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Description : "")); Ref Workers = GetThreadWorkers(BoostWorkerCount, /*SingleThreaded*/ false); @@ -2718,19 +2826,26 @@ HttpProjectService::HandleRpcRequest(HttpRouterRequest& Req) WorkerThreadPool& NetworkWorkerPool = Workers->GetNetworkPool(); Context.ReportMessage(fmt::format("{}", Workers->GetWorkersInfo())); - - RemoteProjectStore::Result Result = LoadOplog(m_CidStore, - *ActualRemoteStore, - *Oplog, - NetworkWorkerPool, - WorkerPool, - Force, - IgnoreMissingAttachments, - CleanOplog, - PartialBlockRequestMode, - HostLatencySec, - CacheLatencySec, - &Context); + RemoteProjectStore::Result Result = LoadOplog(LoadOplogContext{ + .ChunkStore = m_CidStore, + .RemoteStore = *RemoteStoreResult->Store, + .OptionalCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Cache.get() : nullptr, + .CacheBuildId = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->BuildsId : Oid::Zero, + .OptionalCacheStats = RemoteStoreResult->OptionalCache ? &RemoteStoreResult->OptionalCache->Stats : nullptr, + .Oplog = *Oplog, + .NetworkWorkerPool = NetworkWorkerPool, + .WorkerPool = WorkerPool, + .ForceDownload = Force, + .IgnoreMissingAttachments = IgnoreMissingAttachments, + .CleanOplog = CleanOplog, + .PartialBlockRequestMode = PartialBlockRequestMode, + .PopulateCache = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->Populate : false, + .StoreLatencySec = RemoteStoreResult->LatencySec, + .StoreMaxRangeCountPerRequest = RemoteStoreResult->MaxRangeCountPerRequest, + .CacheLatencySec = RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->LatencySec : -1.0, + .CacheMaxRangeCountPerRequest = + RemoteStoreResult->OptionalCache ? RemoteStoreResult->OptionalCache->MaxRangeCountPerRequest : 0, + .OptionalJobContext = &Context}); auto Response = ConvertResult(Result); ZEN_INFO("LoadOplog: Status: {} '{}'", ToString(Response.first), Response.second); if (!IsHttpSuccessCode(Response.first)) -- cgit v1.2.3 From b37b34ea6ad906f54e8104526e77ba66aed997da Mon Sep 17 00:00:00 2001 From: Stefan Boberg Date: Mon, 9 Mar 2026 17:43:08 +0100 Subject: Dashboard overhaul, compute integration (#814) - **Frontend dashboard overhaul**: Unified compute/main dashboards into a single shared UI. Added new pages for cache, projects, metrics, sessions, info (build/runtime config, system stats). Added live-update via WebSockets with pause control, sortable detail tables, themed styling. Refactored compute/hub/orchestrator pages into modular JS. - **HTTP server fixes and stats**: Fixed http.sys local-only fallback when default port is in use, implemented root endpoint redirect for http.sys, fixed Linux/Mac port reuse. Added /stats endpoint exposing HTTP server metrics (bytes transferred, request rates). Added WebSocket stats tracking. - **OTEL/diagnostics hardening**: Improved OTLP HTTP exporter with better error handling and resilience. Extended diagnostics services configuration. - **Session management**: Added new sessions service with HTTP endpoints for registering, updating, querying, and removing sessions. Includes session log file support. This is still WIP. - **CLI subcommand support**: Added support for commands with subcommands in the zen CLI tool, with improved command dispatch. - **Misc**: Exposed CPU usage/hostname to frontend, fixed JS compact binary float32/float64 decoding, limited projects displayed on front page to 25 sorted by last access, added vscode:// link support. Also contains some fixes from TSAN analysis. --- CLAUDE.md | 13 +- .../patches/0.12.1/crashpad_static_libcxx.patch | 24 + repo/packages/s/sentry-native/xmake.lua | 1 + src/zen/zen.cpp | 78 ++ src/zen/zen.h | 37 + src/zenbase/include/zenbase/refcount.h | 3 + src/zencore/include/zencore/logging/tracesink.h | 4 + src/zencore/include/zencore/sentryintegration.h | 1 + src/zencore/include/zencore/system.h | 2 + src/zencore/logging/tracesink.cpp | 4 + src/zencore/sentryintegration.cpp | 19 +- src/zencore/system.cpp | 60 +- src/zencore/testutils.cpp | 2 +- src/zenhttp/httpserver.cpp | 34 + src/zenhttp/include/zenhttp/httpclient.h | 9 + src/zenhttp/include/zenhttp/httpserver.h | 73 +- src/zenhttp/include/zenhttp/httpstats.h | 47 +- src/zenhttp/monitoring/httpstats.cpp | 195 ++++- src/zenhttp/servers/httpasio.cpp | 104 +-- src/zenhttp/servers/httpplugin.cpp | 2 + src/zenhttp/servers/httpsys.cpp | 113 ++- src/zenhttp/servers/wsasio.cpp | 18 +- src/zenhttp/servers/wsasio.h | 8 +- src/zenhttp/servers/wshttpsys.cpp | 23 +- src/zenhttp/servers/wshttpsys.h | 5 +- src/zenhttp/servers/wstest.cpp | 3 +- .../include/zenremotestore/jupiter/jupiterhost.h | 1 + .../projectstore/remoteprojectstore.cpp | 5 +- src/zenserver/compute/computeserver.cpp | 4 +- src/zenserver/compute/computeserver.h | 1 - src/zenserver/diag/diagsvcs.cpp | 31 + src/zenserver/diag/diagsvcs.h | 15 +- src/zenserver/diag/otlphttp.cpp | 59 +- src/zenserver/diag/otlphttp.h | 13 +- src/zenserver/frontend/html.zip | Bin 319315 -> 406051 bytes src/zenserver/frontend/html/banner.js | 338 +++++++++ src/zenserver/frontend/html/compute/banner.js | 321 -------- src/zenserver/frontend/html/compute/compute.html | 327 +++------ src/zenserver/frontend/html/compute/hub.html | 154 +--- src/zenserver/frontend/html/compute/nav.js | 79 -- .../frontend/html/compute/orchestrator.html | 205 +----- src/zenserver/frontend/html/index.html | 3 + src/zenserver/frontend/html/nav.js | 79 ++ src/zenserver/frontend/html/pages/cache.js | 690 ++++++++++++++++++ src/zenserver/frontend/html/pages/compute.js | 693 ++++++++++++++++++ src/zenserver/frontend/html/pages/entry.js | 4 +- src/zenserver/frontend/html/pages/hub.js | 122 ++++ src/zenserver/frontend/html/pages/info.js | 261 +++++++ src/zenserver/frontend/html/pages/map.js | 4 +- src/zenserver/frontend/html/pages/metrics.js | 232 ++++++ src/zenserver/frontend/html/pages/oplog.js | 2 +- src/zenserver/frontend/html/pages/orchestrator.js | 405 +++++++++++ src/zenserver/frontend/html/pages/page.js | 69 +- src/zenserver/frontend/html/pages/project.js | 2 +- src/zenserver/frontend/html/pages/projects.js | 447 ++++++++++++ src/zenserver/frontend/html/pages/sessions.js | 61 ++ src/zenserver/frontend/html/pages/start.js | 327 ++++++--- src/zenserver/frontend/html/pages/stat.js | 2 +- src/zenserver/frontend/html/pages/tree.js | 2 +- src/zenserver/frontend/html/pages/zcache.js | 8 +- src/zenserver/frontend/html/theme.js | 116 +++ src/zenserver/frontend/html/util/compactbinary.js | 4 +- src/zenserver/frontend/html/util/friendly.js | 21 + src/zenserver/frontend/html/util/widgets.js | 138 +++- src/zenserver/frontend/html/zen.css | 809 +++++++++++++++++---- src/zenserver/hub/zenhubserver.cpp | 2 + src/zenserver/sessions/httpsessions.cpp | 264 +++++++ src/zenserver/sessions/httpsessions.h | 55 ++ src/zenserver/sessions/sessions.cpp | 150 ++++ src/zenserver/sessions/sessions.h | 83 +++ .../storage/buildstore/httpbuildstore.cpp | 12 +- src/zenserver/storage/buildstore/httpbuildstore.h | 5 +- .../storage/cache/httpstructuredcache.cpp | 137 +++- src/zenserver/storage/cache/httpstructuredcache.h | 11 +- .../storage/projectstore/httpprojectstore.cpp | 12 +- .../storage/projectstore/httpprojectstore.h | 5 +- .../storage/workspaces/httpworkspaces.cpp | 12 +- src/zenserver/storage/workspaces/httpworkspaces.h | 5 +- src/zenserver/storage/zenstorageserver.cpp | 16 +- src/zenserver/storage/zenstorageserver.h | 4 +- src/zenserver/zenserver.cpp | 103 ++- src/zenserver/zenserver.h | 17 +- src/zenstore/cache/cachedisklayer.cpp | 4 +- src/zenstore/projectstore.cpp | 7 + src/zentelemetry/include/zentelemetry/stats.h | 1 + tsan.supp | 22 + 86 files changed, 6456 insertions(+), 1407 deletions(-) create mode 100644 repo/packages/s/sentry-native/patches/0.12.1/crashpad_static_libcxx.patch create mode 100644 src/zenserver/frontend/html/banner.js delete mode 100644 src/zenserver/frontend/html/compute/banner.js delete mode 100644 src/zenserver/frontend/html/compute/nav.js create mode 100644 src/zenserver/frontend/html/nav.js create mode 100644 src/zenserver/frontend/html/pages/cache.js create mode 100644 src/zenserver/frontend/html/pages/compute.js create mode 100644 src/zenserver/frontend/html/pages/hub.js create mode 100644 src/zenserver/frontend/html/pages/info.js create mode 100644 src/zenserver/frontend/html/pages/metrics.js create mode 100644 src/zenserver/frontend/html/pages/orchestrator.js create mode 100644 src/zenserver/frontend/html/pages/projects.js create mode 100644 src/zenserver/frontend/html/pages/sessions.js create mode 100644 src/zenserver/frontend/html/theme.js create mode 100644 src/zenserver/sessions/httpsessions.cpp create mode 100644 src/zenserver/sessions/httpsessions.h create mode 100644 src/zenserver/sessions/sessions.cpp create mode 100644 src/zenserver/sessions/sessions.h create mode 100644 tsan.supp diff --git a/CLAUDE.md b/CLAUDE.md index 405494acb..32c4b3d40 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -206,15 +206,22 @@ The codebase is organized into layered modules with clear dependencies: - clang-format enforced via pre-commit hooks - Use `std` containers; `eastl::fixed_vector` etc. for performance-critical paths where the number of elements in the container is typically small or when the container is temporary and suitable for stack allocation. +- For shared ownership, prefer intrusive reference counting via `TRefCounted` (or `RefCounted`) with `Ref` + over `std::shared_ptr`. Since the ownership policy is known at type definition time, the overhead of + `std::shared_ptr` (separate control block, etc.) is unnecessary. - Exceptions used only for unexpected errors, not flow control - Some `std` exception types like `runtime_error` are also available in the `zen` namespace and offer convenience of formatting the exception message by allowing fmt-style formatting without using `fmt::format` which can reduce clutter. To use these, please `#include ` -- Logging is done via `ZEN_DEBUG(...)`, `ZEN_INFO(...)`, `ZEN_WARN`, `ZEN_ERROR` macros - - Logging macros use `fmt::vformat` internally for message formatting. The first argument is the format string, +- Logging is done via `ZEN_DEBUG(...)`, `ZEN_INFO(...)`, `ZEN_WARN(...)`, `ZEN_ERROR(...)` macros + - Prefer these over the older `ZEN_LOG_INFO`, `ZEN_LOG_WARN`, etc. variants which require an explicit log category parameter + - Logging macros use `fmt::vformat` internally for message formatting. The first argument is the format string, which must be a string literal (turned into `std::string_view` in the macros) - - Logging channels can be overridden on a scope-by-scope basis (per-class or even per-function) by implementing + - Logging channels can be overridden on a scope-by-scope basis (per-class or even per-function) by implementing a `LoggerRef Log()` function +- Avoid repetition. If the same pattern (e.g. data structure lookups, locking + find) appears multiple times, + factor it into a helper function and call that instead. +- Avoid `auto` when the type is not explicitly visible on the right-hand side of the assignment. Spell out the type instead. - `if`/`else` should use braces even for single-statement branches **Includes:** diff --git a/repo/packages/s/sentry-native/patches/0.12.1/crashpad_static_libcxx.patch b/repo/packages/s/sentry-native/patches/0.12.1/crashpad_static_libcxx.patch new file mode 100644 index 000000000..2005ad4ec --- /dev/null +++ b/repo/packages/s/sentry-native/patches/0.12.1/crashpad_static_libcxx.patch @@ -0,0 +1,24 @@ +--- a/external/crashpad/handler/CMakeLists.txt 2026-03-09 14:47:42.109197582 +0000 ++++ b/external/crashpad/handler/CMakeLists.txt 2026-03-09 14:51:45.343538268 +0000 +@@ -120,6 +120,21 @@ + endif() + endif() + ++ if(LINUX) ++ # Statically link libc++ and libc++abi into crashpad_handler so it has ++ # no runtime dependency on libc++.so.1. This is needed when building with ++ # a toolchain that uses libc++ (e.g. UE clang) but deploys to systems ++ # where libc++.so.1 is not available. ++ # -nostdlib++ suppresses clang's automatic -lc++ addition (a linker flag, ++ # added at the end). The explicit -Bstatic libs are added via ++ # target_link_libraries so they appear after crashpad's static archives in ++ # the link order, letting the single-pass linker resolve all libc++ symbols. ++ target_link_options(crashpad_handler PRIVATE -nostdlib++) ++ target_link_libraries(crashpad_handler PRIVATE ++ -Wl,-Bstatic,-lc++,-lc++abi,-Bdynamic ++ ) ++ endif() ++ + set_property(TARGET crashpad_handler PROPERTY EXPORT_NAME crashpad_handler) + add_executable(crashpad::handler ALIAS crashpad_handler) + diff --git a/repo/packages/s/sentry-native/xmake.lua b/repo/packages/s/sentry-native/xmake.lua index ca683ee4c..77d600fab 100644 --- a/repo/packages/s/sentry-native/xmake.lua +++ b/repo/packages/s/sentry-native/xmake.lua @@ -37,6 +37,7 @@ package("sentry-native") add_versions("0.4.4", "fe6c711d42861e66e53bfd7ee0b2b226027c64446857f0d1bbb239ca824a3d8d") add_patches("0.4.4", path.join(os.scriptdir(), "patches", "0.4.4", "zlib_fix.patch"), "1a6ac711b7824112a9062ec1716a316facce5055498d1f87090d2cad031b865b") add_patches("0.7.6", path.join(os.scriptdir(), "patches", "0.7.6", "breakpad_exceptions.patch"), "7781bad0404a92252cbad39e865d17ac663eedade03cbd29c899636c7bfab1b5") + add_patches("0.12.1", path.join(os.scriptdir(), "patches", "0.12.1", "crashpad_static_libcxx.patch"), "3c2115b90179808fa639865f6eb23090e2cb6025d816ffb66c2d75c26473ec72") add_deps("cmake") diff --git a/src/zen/zen.cpp b/src/zen/zen.cpp index 7f7afa322..9a466da2e 100644 --- a/src/zen/zen.cpp +++ b/src/zen/zen.cpp @@ -194,6 +194,84 @@ ZenCmdBase::GetSubCommand(cxxopts::Options&, return argc; } +ZenSubCmdBase::ZenSubCmdBase(std::string_view Name, std::string_view Description) +: m_SubOptions(std::string(Name), std::string(Description)) +{ + m_SubOptions.add_options()("h,help", "Print help"); +} + +void +ZenCmdWithSubCommands::AddSubCommand(ZenSubCmdBase& SubCmd) +{ + m_SubCommands.push_back(&SubCmd); +} + +bool +ZenCmdWithSubCommands::OnParentOptionsParsed(const ZenCliOptions& /*GlobalOptions*/) +{ + return true; +} + +void +ZenCmdWithSubCommands::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + std::vector SubOptionPtrs; + SubOptionPtrs.reserve(m_SubCommands.size()); + for (ZenSubCmdBase* SubCmd : m_SubCommands) + { + SubOptionPtrs.push_back(&SubCmd->SubOptions()); + } + + cxxopts::Options* MatchedSubOption = nullptr; + std::vector SubCommandArguments; + int ParentArgc = GetSubCommand(Options(), argc, argv, SubOptionPtrs, MatchedSubOption, SubCommandArguments); + + if (!ParseOptions(Options(), ParentArgc, argv)) + { + return; + } + + if (MatchedSubOption == nullptr) + { + ExtendableStringBuilder<128> VerbList; + for (bool First = true; ZenSubCmdBase * SubCmd : m_SubCommands) + { + if (!First) + { + VerbList.Append(", "); + } + VerbList.Append(SubCmd->SubOptions().program()); + First = false; + } + throw OptionParseException(fmt::format("No subcommand specified. Available subcommands: {}", VerbList.ToView()), Options().help()); + } + + ZenSubCmdBase* MatchedSubCmd = nullptr; + for (ZenSubCmdBase* SubCmd : m_SubCommands) + { + if (&SubCmd->SubOptions() == MatchedSubOption) + { + MatchedSubCmd = SubCmd; + break; + } + } + ZEN_ASSERT(MatchedSubCmd != nullptr); + + // Parse subcommand args before OnParentOptionsParsed so --help on the subcommand + // works without requiring parent options like --hosturl to be populated. + if (!ParseOptions(*MatchedSubOption, gsl::narrow(SubCommandArguments.size()), SubCommandArguments.data())) + { + return; + } + + if (!OnParentOptionsParsed(GlobalOptions)) + { + return; + } + + MatchedSubCmd->Run(GlobalOptions); +} + static ReturnCode GetReturnCodeFromHttpResult(const HttpClientError& Ex) { diff --git a/src/zen/zen.h b/src/zen/zen.h index e3481beea..06e5356a6 100644 --- a/src/zen/zen.h +++ b/src/zen/zen.h @@ -79,4 +79,41 @@ class CacheStoreCommand : public ZenCmdBase virtual ZenCmdCategory& CommandCategory() const override { return g_CacheStoreCategory; } }; +// Base for individual subcommands +class ZenSubCmdBase +{ +public: + ZenSubCmdBase(std::string_view Name, std::string_view Description); + virtual ~ZenSubCmdBase() = default; + cxxopts::Options& SubOptions() { return m_SubOptions; } + virtual void Run(const ZenCliOptions& GlobalOptions) = 0; + +protected: + cxxopts::Options m_SubOptions; +}; + +// Base for commands that host subcommands - handles all dispatch boilerplate +class ZenCmdWithSubCommands : public ZenCmdBase +{ +public: + void Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) final; + +protected: + void AddSubCommand(ZenSubCmdBase& SubCmd); + virtual bool OnParentOptionsParsed(const ZenCliOptions& GlobalOptions); + +private: + std::vector m_SubCommands; +}; + +class CacheStoreCmdWithSubCommands : public ZenCmdWithSubCommands +{ + ZenCmdCategory& CommandCategory() const override { return g_CacheStoreCategory; } +}; + +class StorageCmdWithSubCommands : public ZenCmdWithSubCommands +{ + ZenCmdCategory& CommandCategory() const override { return g_StorageCategory; } +}; + } // namespace zen diff --git a/src/zenbase/include/zenbase/refcount.h b/src/zenbase/include/zenbase/refcount.h index 40ad7bca5..08bc6ae54 100644 --- a/src/zenbase/include/zenbase/refcount.h +++ b/src/zenbase/include/zenbase/refcount.h @@ -51,6 +51,9 @@ private: * NOTE: Unlike RefCounted, this class deletes the derived type when the reference count reaches zero. * It has no virtual destructor, so it's important that you either don't derive from it further, * or ensure that the derived class has a virtual destructor. + * + * This class is useful when you want to avoid adding a vtable to a class just to implement + * reference counting. */ template diff --git a/src/zencore/include/zencore/logging/tracesink.h b/src/zencore/include/zencore/logging/tracesink.h index e63d838b4..785c51e10 100644 --- a/src/zencore/include/zencore/logging/tracesink.h +++ b/src/zencore/include/zencore/logging/tracesink.h @@ -6,6 +6,8 @@ namespace zen::logging { +#if ZEN_WITH_TRACE + /** * A logging sink that forwards log messages to the trace system. * @@ -20,4 +22,6 @@ public: void SetFormatter(std::unique_ptr InFormatter) override; }; +#endif + } // namespace zen::logging diff --git a/src/zencore/include/zencore/sentryintegration.h b/src/zencore/include/zencore/sentryintegration.h index a4e33d69e..27e5a8a82 100644 --- a/src/zencore/include/zencore/sentryintegration.h +++ b/src/zencore/include/zencore/sentryintegration.h @@ -40,6 +40,7 @@ public: }; void Initialize(const Config& Conf, const std::string& CommandLine); + void Close(); void LogStartupInformation(); static void ClearCaches(); diff --git a/src/zencore/include/zencore/system.h b/src/zencore/include/zencore/system.h index fecbe2dbe..a67999e52 100644 --- a/src/zencore/include/zencore/system.h +++ b/src/zencore/include/zencore/system.h @@ -14,6 +14,7 @@ class CbWriter; std::string GetMachineName(); std::string_view GetOperatingSystemName(); +std::string GetOperatingSystemVersion(); std::string_view GetRuntimePlatformName(); // "windows", "wine", "linux", or "macos" std::string_view GetCpuName(); @@ -28,6 +29,7 @@ struct SystemMetrics uint64_t AvailVirtualMemoryMiB = 0; uint64_t PageFileMiB = 0; uint64_t AvailPageFileMiB = 0; + uint64_t UptimeSeconds = 0; }; /// Extended metrics that include CPU usage percentage, which requires diff --git a/src/zencore/logging/tracesink.cpp b/src/zencore/logging/tracesink.cpp index e3533327b..8a6f4e40c 100644 --- a/src/zencore/logging/tracesink.cpp +++ b/src/zencore/logging/tracesink.cpp @@ -7,6 +7,8 @@ #include #include +#if ZEN_WITH_TRACE + namespace zen::logging { UE_TRACE_CHANNEL_DEFINE(LogChannel) @@ -86,3 +88,5 @@ TraceSink::SetFormatter(std::unique_ptr /*InFormatter*/) } } // namespace zen::logging + +#endif diff --git a/src/zencore/sentryintegration.cpp b/src/zencore/sentryintegration.cpp index e39b8438d..8d087e8c6 100644 --- a/src/zencore/sentryintegration.cpp +++ b/src/zencore/sentryintegration.cpp @@ -128,6 +128,8 @@ namespace zen { # if ZEN_USE_SENTRY ZEN_DEFINE_LOG_CATEGORY_STATIC(LogSentry, "sentry-sdk"); +static std::atomic s_SentryLogEnabled{true}; + static void SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[maybe_unused]] void* Userdata) { @@ -147,14 +149,15 @@ SentryLogFunction(sentry_level_t Level, const char* Message, va_list Args, [[may } // SentryLogFunction can be called before the logging system is initialized - // (during sentry_init which runs before InitializeLogging). Fall back to - // console logging when the category logger is not yet available. + // (during sentry_init which runs before InitializeLogging), or after it has + // been shut down (during sentry_close on a background worker thread). Fall + // back to console logging when the category logger is not available. // // Since we want to default to WARN level but this runs before logging has // been configured, we ignore the callbacks for DEBUG/INFO explicitly here // which means users don't see every possible log message if they're trying // to configure the levels using --log-debug=sentry-sdk - if (!TheDefaultLogger) + if (!TheDefaultLogger || !s_SentryLogEnabled.load(std::memory_order_acquire)) { switch (Level) { @@ -211,12 +214,22 @@ SentryIntegration::SentryIntegration() } SentryIntegration::~SentryIntegration() +{ + Close(); +} + +void +SentryIntegration::Close() { if (m_IsInitialized && m_SentryErrorCode == 0) { logging::SetErrorLog(""); m_SentryAssert.reset(); + // Disable spdlog forwarding before sentry_close() since its background + // worker thread may still log during shutdown via SentryLogFunction + s_SentryLogEnabled.store(false, std::memory_order_release); sentry_close(); + m_IsInitialized = false; } } diff --git a/src/zencore/system.cpp b/src/zencore/system.cpp index 833d3c04b..141450b84 100644 --- a/src/zencore/system.cpp +++ b/src/zencore/system.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -135,6 +136,8 @@ GetSystemMetrics() Metrics.AvailPageFileMiB = MemStatus.ullAvailPageFile / 1024 / 1024; } + Metrics.UptimeSeconds = GetTickCount64() / 1000; + return Metrics; } #elif ZEN_PLATFORM_LINUX @@ -226,6 +229,17 @@ GetSystemMetrics() Metrics.VirtualMemoryMiB = Metrics.SystemMemoryMiB; Metrics.AvailVirtualMemoryMiB = Metrics.AvailSystemMemoryMiB; + // System uptime + if (FILE* UptimeFile = fopen("/proc/uptime", "r")) + { + double UptimeSec = 0; + if (fscanf(UptimeFile, "%lf", &UptimeSec) == 1) + { + Metrics.UptimeSeconds = static_cast(UptimeSec); + } + fclose(UptimeFile); + } + // Parse /proc/meminfo for swap/page file information Metrics.PageFileMiB = 0; Metrics.AvailPageFileMiB = 0; @@ -318,6 +332,18 @@ GetSystemMetrics() Metrics.PageFileMiB = SwapUsage.xsu_total / 1024 / 1024; Metrics.AvailPageFileMiB = (SwapUsage.xsu_total - SwapUsage.xsu_used) / 1024 / 1024; + // System uptime via boot time + { + struct timeval BootTime + { + }; + Size = sizeof(BootTime); + if (sysctlbyname("kern.boottime", &BootTime, &Size, nullptr, 0) == 0) + { + Metrics.UptimeSeconds = static_cast(time(nullptr) - BootTime.tv_sec); + } + } + return Metrics; } #else @@ -574,6 +600,38 @@ GetOperatingSystemName() return ZEN_PLATFORM_NAME; } +std::string +GetOperatingSystemVersion() +{ +#if ZEN_PLATFORM_WINDOWS + // Use RtlGetVersion to avoid the compatibility shim that GetVersionEx applies + using RtlGetVersionFn = LONG(WINAPI*)(PRTL_OSVERSIONINFOW); + RTL_OSVERSIONINFOW OsVer{.dwOSVersionInfoSize = sizeof(OsVer)}; + if (auto Fn = (RtlGetVersionFn)GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "RtlGetVersion")) + { + Fn(&OsVer); + } + return fmt::format("Windows {}.{} Build {}", OsVer.dwMajorVersion, OsVer.dwMinorVersion, OsVer.dwBuildNumber); +#elif ZEN_PLATFORM_LINUX + struct utsname Info + { + }; + if (uname(&Info) == 0) + { + return fmt::format("{} {}", Info.sysname, Info.release); + } + return "Linux"; +#elif ZEN_PLATFORM_MAC + char OsVersion[64] = ""; + size_t Size = sizeof(OsVersion); + if (sysctlbyname("kern.osproductversion", OsVersion, &Size, nullptr, 0) == 0) + { + return fmt::format("macOS {}", OsVersion); + } + return "macOS"; +#endif +} + std::string_view GetRuntimePlatformName() { @@ -608,7 +666,7 @@ Describe(const SystemMetrics& Metrics, CbWriter& Writer) Writer << "cpu_count" << Metrics.CpuCount << "core_count" << Metrics.CoreCount << "lp_count" << Metrics.LogicalProcessorCount << "total_memory_mb" << Metrics.SystemMemoryMiB << "avail_memory_mb" << Metrics.AvailSystemMemoryMiB << "total_virtual_mb" << Metrics.VirtualMemoryMiB << "avail_virtual_mb" << Metrics.AvailVirtualMemoryMiB << "total_pagefile_mb" << Metrics.PageFileMiB - << "avail_pagefile_mb" << Metrics.AvailPageFileMiB; + << "avail_pagefile_mb" << Metrics.AvailPageFileMiB << "uptime_seconds" << Metrics.UptimeSeconds; } void diff --git a/src/zencore/testutils.cpp b/src/zencore/testutils.cpp index 5bc2841ae..0cd3f8121 100644 --- a/src/zencore/testutils.cpp +++ b/src/zencore/testutils.cpp @@ -46,7 +46,7 @@ ScopedTemporaryDirectory::~ScopedTemporaryDirectory() IoBuffer CreateRandomBlob(uint64_t Size) { - static FastRandom Rand{.Seed = 0x7CEBF54E45B9F5D1}; + thread_local FastRandom Rand{.Seed = 0x7CEBF54E45B9F5D1}; return CreateRandomBlob(Rand, Size); }; diff --git a/src/zenhttp/httpserver.cpp b/src/zenhttp/httpserver.cpp index d798c46d9..9bae95690 100644 --- a/src/zenhttp/httpserver.cpp +++ b/src/zenhttp/httpserver.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -1094,6 +1095,39 @@ HttpServer::SetHttpRequestFilter(IHttpRequestFilter* RequestFilter) OnSetHttpRequestFilter(RequestFilter); } +CbObject +HttpServer::CollectStats() +{ + CbObjectWriter Cbo; + + metrics::EmitSnapshot("requests", m_RequestMeter, Cbo); + + Cbo.BeginObject("bytes"); + { + Cbo << "received" << GetTotalBytesReceived(); + Cbo << "sent" << GetTotalBytesSent(); + } + Cbo.EndObject(); + + Cbo.BeginObject("websockets"); + { + Cbo << "active_connections" << GetActiveWebSocketConnectionCount(); + Cbo << "frames_received" << m_WsFramesReceived.load(std::memory_order_relaxed); + Cbo << "frames_sent" << m_WsFramesSent.load(std::memory_order_relaxed); + Cbo << "bytes_received" << m_WsBytesReceived.load(std::memory_order_relaxed); + Cbo << "bytes_sent" << m_WsBytesSent.load(std::memory_order_relaxed); + } + Cbo.EndObject(); + + return Cbo.Save(); +} + +void +HttpServer::HandleStatsRequest(HttpServerRequest& Request) +{ + Request.WriteResponse(HttpResponseCode::OK, CollectStats()); +} + ////////////////////////////////////////////////////////////////////////// HttpRpcHandler::HttpRpcHandler() diff --git a/src/zenhttp/include/zenhttp/httpclient.h b/src/zenhttp/include/zenhttp/httpclient.h index bec4984db..1bb36a298 100644 --- a/src/zenhttp/include/zenhttp/httpclient.h +++ b/src/zenhttp/include/zenhttp/httpclient.h @@ -118,6 +118,15 @@ private: class HttpClientBase; +/** HTTP Client + * + * This is safe for use on multiple threads simultaneously, as each + * instance maintains an internal connection pool and will synchronize + * access to it as needed. + * + * Uses libcurl under the hood. We currently only use HTTP 1.1 features. + * + */ class HttpClient { public: diff --git a/src/zenhttp/include/zenhttp/httpserver.h b/src/zenhttp/include/zenhttp/httpserver.h index c1152dc3e..0e1714669 100644 --- a/src/zenhttp/include/zenhttp/httpserver.h +++ b/src/zenhttp/include/zenhttp/httpserver.h @@ -13,6 +13,8 @@ #include #include +#include + #include #include #include @@ -203,12 +205,34 @@ private: int m_UriPrefixLength = 0; }; +struct IHttpStatsProvider +{ + /** Handle an HTTP stats request, writing the response directly. + * Implementations may inspect query parameters on the request + * to include optional detailed breakdowns. + */ + virtual void HandleStatsRequest(HttpServerRequest& Request) = 0; + + /** Return the provider's current stats as a CbObject snapshot. + * Used by the WebSocket push thread to broadcast live updates + * without requiring an HttpServerRequest. Providers that do + * not override this will be skipped in WebSocket broadcasts. + */ + virtual CbObject CollectStats() { return {}; } +}; + +struct IHttpStatsService +{ + virtual void RegisterHandler(std::string_view Id, IHttpStatsProvider& Provider) = 0; + virtual void UnregisterHandler(std::string_view Id, IHttpStatsProvider& Provider) = 0; +}; + /** HTTP server * * Implements the main event loop to service HTTP requests, and handles routing * requests to the appropriate handler as registered via RegisterService */ -class HttpServer : public RefCounted +class HttpServer : public RefCounted, public IHttpStatsProvider { public: void RegisterService(HttpService& Service); @@ -235,10 +259,46 @@ public: virtual uint64_t GetTotalBytesReceived() const { return 0; } virtual uint64_t GetTotalBytesSent() const { return 0; } + /** Mark that a request has been handled. Called by server implementations. */ + void MarkRequest() { m_RequestMeter.Mark(); } + + /** Set a default redirect path for root requests */ + void SetDefaultRedirect(std::string_view Path) { m_DefaultRedirect = Path; } + + std::string_view GetDefaultRedirect() const { return m_DefaultRedirect; } + + /** Track active WebSocket connections — called by server implementations on upgrade/close. */ + void OnWebSocketConnectionOpened() { m_ActiveWebSocketConnections.fetch_add(1, std::memory_order_relaxed); } + void OnWebSocketConnectionClosed() { m_ActiveWebSocketConnections.fetch_sub(1, std::memory_order_relaxed); } + uint64_t GetActiveWebSocketConnectionCount() const { return m_ActiveWebSocketConnections.load(std::memory_order_relaxed); } + + /** Track WebSocket frame and byte counters — called by WS connection implementations per frame. */ + void OnWebSocketFrameReceived(uint64_t Bytes) + { + m_WsFramesReceived.fetch_add(1, std::memory_order_relaxed); + m_WsBytesReceived.fetch_add(Bytes, std::memory_order_relaxed); + } + void OnWebSocketFrameSent(uint64_t Bytes) + { + m_WsFramesSent.fetch_add(1, std::memory_order_relaxed); + m_WsBytesSent.fetch_add(Bytes, std::memory_order_relaxed); + } + + // IHttpStatsProvider + virtual CbObject CollectStats() override; + virtual void HandleStatsRequest(HttpServerRequest& Request) override; + private: std::vector m_KnownServices; int m_EffectivePort = 0; std::string m_ExternalHost; + metrics::Meter m_RequestMeter; + std::string m_DefaultRedirect; + std::atomic m_ActiveWebSocketConnections{0}; + std::atomic m_WsFramesReceived{0}; + std::atomic m_WsFramesSent{0}; + std::atomic m_WsBytesReceived{0}; + std::atomic m_WsBytesSent{0}; virtual void OnRegisterService(HttpService& Service) = 0; virtual int OnInitialize(int BasePort, std::filesystem::path DataDir) = 0; @@ -456,17 +516,6 @@ private: bool HandlePackageOffers(HttpService& Service, HttpServerRequest& Request, Ref& PackageHandlerRef); -struct IHttpStatsProvider -{ - virtual void HandleStatsRequest(HttpServerRequest& Request) = 0; -}; - -struct IHttpStatsService -{ - virtual void RegisterHandler(std::string_view Id, IHttpStatsProvider& Provider) = 0; - virtual void UnregisterHandler(std::string_view Id, IHttpStatsProvider& Provider) = 0; -}; - void http_forcelink(); // internal void websocket_forcelink(); // internal diff --git a/src/zenhttp/include/zenhttp/httpstats.h b/src/zenhttp/include/zenhttp/httpstats.h index e6fea6765..460315faf 100644 --- a/src/zenhttp/include/zenhttp/httpstats.h +++ b/src/zenhttp/include/zenhttp/httpstats.h @@ -3,23 +3,50 @@ #pragma once #include +#include #include +#include +#include #include +#include +#include +#include + +ZEN_THIRD_PARTY_INCLUDES_START +#include +#include +ZEN_THIRD_PARTY_INCLUDES_END namespace zen { -class HttpStatsService : public HttpService, public IHttpStatsService +class HttpStatsService : public HttpService, public IHttpStatsService, public IWebSocketHandler { public: - HttpStatsService(); + /// Construct without an io_context — optionally uses a dedicated push thread + /// for WebSocket stats broadcasting. + explicit HttpStatsService(bool EnableWebSockets = false); + + /// Construct with an external io_context — uses an asio timer instead + /// of a dedicated thread for WebSocket stats broadcasting. + /// The caller must ensure the io_context outlives this service and that + /// its run loop is active. + HttpStatsService(asio::io_context& IoContext, bool EnableWebSockets = true); + ~HttpStatsService(); + void Shutdown(); + virtual const char* BaseUri() const override; virtual void HandleRequest(HttpServerRequest& Request) override; virtual void RegisterHandler(std::string_view Id, IHttpStatsProvider& Provider) override; virtual void UnregisterHandler(std::string_view Id, IHttpStatsProvider& Provider) override; + // IWebSocketHandler + void OnWebSocketOpen(Ref Connection) override; + void OnWebSocketMessage(WebSocketConnection& Conn, const WebSocketMessage& Msg) override; + void OnWebSocketClose(WebSocketConnection& Conn, uint16_t Code, std::string_view Reason) override; + private: LoggerRef m_Log; HttpRequestRouter m_Router; @@ -28,6 +55,22 @@ private: RwLock m_Lock; std::map m_Providers; + + // WebSocket push + RwLock m_WsConnectionsLock; + std::vector> m_WsConnections; + std::atomic m_PushEnabled{false}; + + void BroadcastStats(); + + // Thread-based push (when no io_context is provided) + std::thread m_PushThread; + Event m_PushEvent; + void PushThreadFunction(); + + // Timer-based push (when an io_context is provided) + std::unique_ptr m_PushTimer; + void EnqueuePushTimer(); }; } // namespace zen diff --git a/src/zenhttp/monitoring/httpstats.cpp b/src/zenhttp/monitoring/httpstats.cpp index b097a0d3f..2370def0c 100644 --- a/src/zenhttp/monitoring/httpstats.cpp +++ b/src/zenhttp/monitoring/httpstats.cpp @@ -3,15 +3,57 @@ #include "zenhttp/httpstats.h" #include +#include +#include +#include namespace zen { -HttpStatsService::HttpStatsService() : m_Log(logging::Get("stats")) +HttpStatsService::HttpStatsService(bool EnableWebSockets) : m_Log(logging::Get("stats")) { + if (EnableWebSockets) + { + m_PushEnabled.store(true); + m_PushThread = std::thread([this] { PushThreadFunction(); }); + } +} + +HttpStatsService::HttpStatsService(asio::io_context& IoContext, bool EnableWebSockets) : m_Log(logging::Get("stats")) +{ + if (EnableWebSockets) + { + m_PushEnabled.store(true); + m_PushTimer = std::make_unique(IoContext); + EnqueuePushTimer(); + } } HttpStatsService::~HttpStatsService() { + Shutdown(); +} + +void +HttpStatsService::Shutdown() +{ + if (!m_PushEnabled.exchange(false)) + { + return; + } + + if (m_PushTimer) + { + m_PushTimer->cancel(); + m_PushTimer.reset(); + } + + if (m_PushThread.joinable()) + { + m_PushEvent.Set(); + m_PushThread.join(); + } + + m_WsConnectionsLock.WithExclusiveLock([&] { m_WsConnections.clear(); }); } const char* @@ -39,6 +81,7 @@ HttpStatsService::UnregisterHandler(std::string_view Id, IHttpStatsProvider& Pro void HttpStatsService::HandleRequest(HttpServerRequest& Request) { + ZEN_TRACE_CPU("HttpStatsService::HandleRequest"); using namespace std::literals; std::string_view Key = Request.RelativeUri(); @@ -89,4 +132,154 @@ HttpStatsService::HandleRequest(HttpServerRequest& Request) } } +////////////////////////////////////////////////////////////////////////// +// +// IWebSocketHandler +// + +void +HttpStatsService::OnWebSocketOpen(Ref Connection) +{ + ZEN_TRACE_CPU("HttpStatsService::OnWebSocketOpen"); + ZEN_INFO("Stats WebSocket client connected"); + + m_WsConnectionsLock.WithExclusiveLock([&] { m_WsConnections.push_back(std::move(Connection)); }); + + // Send initial state immediately + if (m_PushThread.joinable()) + { + m_PushEvent.Set(); + } +} + +void +HttpStatsService::OnWebSocketMessage(WebSocketConnection& /*Conn*/, const WebSocketMessage& /*Msg*/) +{ + // No client-to-server messages expected +} + +void +HttpStatsService::OnWebSocketClose(WebSocketConnection& Conn, [[maybe_unused]] uint16_t Code, [[maybe_unused]] std::string_view Reason) +{ + ZEN_TRACE_CPU("HttpStatsService::OnWebSocketClose"); + ZEN_INFO("Stats WebSocket client disconnected (code {})", Code); + + m_WsConnectionsLock.WithExclusiveLock([&] { + auto It = std::remove_if(m_WsConnections.begin(), m_WsConnections.end(), [&Conn](const Ref& C) { + return C.Get() == &Conn; + }); + m_WsConnections.erase(It, m_WsConnections.end()); + }); +} + +////////////////////////////////////////////////////////////////////////// +// +// Stats broadcast +// + +void +HttpStatsService::BroadcastStats() +{ + ZEN_TRACE_CPU("HttpStatsService::BroadcastStats"); + std::vector> Connections; + m_WsConnectionsLock.WithSharedLock([&] { Connections = m_WsConnections; }); + + if (Connections.empty()) + { + return; + } + + // Collect stats from all providers + ExtendableStringBuilder<4096> JsonBuilder; + JsonBuilder.Append("{"); + + bool First = true; + { + RwLock::SharedLockScope _(m_Lock); + for (auto& [Id, Provider] : m_Providers) + { + CbObject Stats = Provider->CollectStats(); + if (!Stats) + { + continue; + } + + if (!First) + { + JsonBuilder.Append(","); + } + First = false; + + // Emit as "provider_id": { ... } + JsonBuilder.Append("\""); + JsonBuilder.Append(Id); + JsonBuilder.Append("\":"); + + ExtendableStringBuilder<2048> StatsJson; + Stats.ToJson(StatsJson); + JsonBuilder.Append(StatsJson.ToView()); + } + } + + JsonBuilder.Append("}"); + + std::string_view Json = JsonBuilder.ToView(); + for (auto& Conn : Connections) + { + if (Conn->IsOpen()) + { + Conn->SendText(Json); + } + } +} + +////////////////////////////////////////////////////////////////////////// +// +// Thread-based push (fallback when no io_context) +// + +void +HttpStatsService::PushThreadFunction() +{ + SetCurrentThreadName("stats_ws_push"); + + while (m_PushEnabled.load()) + { + m_PushEvent.Wait(5000); + m_PushEvent.Reset(); + + if (!m_PushEnabled.load()) + { + break; + } + + BroadcastStats(); + } +} + +////////////////////////////////////////////////////////////////////////// +// +// Timer-based push (when io_context is provided) +// + +void +HttpStatsService::EnqueuePushTimer() +{ + if (!m_PushTimer) + { + return; + } + + m_PushTimer->expires_after(std::chrono::seconds(5)); + m_PushTimer->async_wait([this](const asio::error_code& Ec) { + if (Ec) + { + return; + } + + BroadcastStats(); + EnqueuePushTimer(); + }); +} + } // namespace zen diff --git a/src/zenhttp/servers/httpasio.cpp b/src/zenhttp/servers/httpasio.cpp index 2cf051d14..f5178ebe8 100644 --- a/src/zenhttp/servers/httpasio.cpp +++ b/src/zenhttp/servers/httpasio.cpp @@ -531,6 +531,8 @@ public: std::atomic m_TotalBytesReceived{0}; std::atomic m_TotalBytesSent{0}; + + HttpServer* m_HttpServer = nullptr; }; /** @@ -949,6 +951,7 @@ private: void OnDataReceived(const asio::error_code& Ec, std::size_t ByteCount); void OnResponseDataSent(const asio::error_code& Ec, std::size_t ByteCount, uint32_t RequestNumber, HttpResponse* ResponseToPop); void CloseConnection(); + void SendInlineResponse(uint32_t RequestNumber, std::string_view StatusLine, std::string_view Headers = {}, std::string_view Body = {}); HttpAsioServerImpl& m_Server; asio::streambuf m_RequestBuffer; @@ -1166,6 +1169,38 @@ HttpServerConnection::CloseConnection() } } +void +HttpServerConnection::SendInlineResponse(uint32_t RequestNumber, + std::string_view StatusLine, + std::string_view Headers, + std::string_view Body) +{ + ExtendableStringBuilder<256> ResponseBuilder; + ResponseBuilder << "HTTP/1.1 " << StatusLine << "\r\n"; + if (!Headers.empty()) + { + ResponseBuilder << Headers; + } + if (!m_RequestData.IsKeepAlive()) + { + ResponseBuilder << "Connection: close\r\n"; + } + ResponseBuilder << "\r\n"; + if (!Body.empty()) + { + ResponseBuilder << Body; + } + auto ResponseView = ResponseBuilder.ToView(); + IoBuffer ResponseData(IoBuffer::Clone, ResponseView.data(), ResponseView.size()); + auto Buffer = asio::buffer(ResponseData.GetData(), ResponseData.GetSize()); + asio::async_write( + *m_Socket.get(), + Buffer, + [Conn = AsSharedPtr(), RequestNumber, Response = std::move(ResponseData)](const asio::error_code& Ec, std::size_t ByteCount) { + Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); + }); +} + void HttpServerConnection::HandleRequest() { @@ -1204,7 +1239,9 @@ HttpServerConnection::HandleRequest() return; } - Ref WsConn(new WsAsioConnection(std::move(Conn->m_Socket), *WsHandler)); + Conn->m_Server.m_HttpServer->OnWebSocketConnectionOpened(); + Ref WsConn( + new WsAsioConnection(std::move(Conn->m_Socket), *WsHandler, Conn->m_Server.m_HttpServer)); Ref WsConnRef(WsConn.Get()); WsHandler->OnWebSocketOpen(std::move(WsConnRef)); @@ -1241,6 +1278,8 @@ HttpServerConnection::HandleRequest() { ZEN_TRACE_CPU("asio::HandleRequest"); + m_Server.m_HttpServer->MarkRequest(); + auto RemoteEndpoint = m_Socket->remote_endpoint(); bool IsLocalConnection = m_Socket->local_endpoint().address() == RemoteEndpoint.address(); @@ -1378,51 +1417,24 @@ HttpServerConnection::HandleRequest() } } - if (m_RequestData.RequestVerb() == HttpVerb::kHead) + // If a default redirect is configured and the request is for the root path, send a 302 + std::string_view DefaultRedirect = m_Server.m_HttpServer->GetDefaultRedirect(); + if (!DefaultRedirect.empty() && (m_RequestData.Url() == "/" || m_RequestData.Url().empty())) { - std::string_view Response = - "HTTP/1.1 404 NOT FOUND\r\n" - "\r\n"sv; - - if (!m_RequestData.IsKeepAlive()) - { - Response = - "HTTP/1.1 404 NOT FOUND\r\n" - "Connection: close\r\n" - "\r\n"sv; - } - - asio::async_write(*m_Socket.get(), - asio::buffer(Response), - [Conn = AsSharedPtr(), RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { - Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); - }); + ExtendableStringBuilder<128> Headers; + Headers << "Location: " << DefaultRedirect << "\r\nContent-Length: 0\r\n"; + SendInlineResponse(RequestNumber, "302 Found"sv, Headers.ToView()); + } + else if (m_RequestData.RequestVerb() == HttpVerb::kHead) + { + SendInlineResponse(RequestNumber, "404 NOT FOUND"sv); } else { - std::string_view Response = - "HTTP/1.1 404 NOT FOUND\r\n" - "Content-Length: 23\r\n" - "Content-Type: text/plain\r\n" - "\r\n" - "No suitable route found"sv; - - if (!m_RequestData.IsKeepAlive()) - { - Response = - "HTTP/1.1 404 NOT FOUND\r\n" - "Content-Length: 23\r\n" - "Content-Type: text/plain\r\n" - "Connection: close\r\n" - "\r\n" - "No suitable route found"sv; - } - - asio::async_write(*m_Socket.get(), - asio::buffer(Response), - [Conn = AsSharedPtr(), RequestNumber](const asio::error_code& Ec, std::size_t ByteCount) { - Conn->OnResponseDataSent(Ec, ByteCount, RequestNumber, /* ResponseToPop */ nullptr); - }); + SendInlineResponse(RequestNumber, + "404 NOT FOUND"sv, + "Content-Length: 23\r\nContent-Type: text/plain\r\n"sv, + "No suitable route found"sv); } } @@ -1448,8 +1460,11 @@ struct HttpAcceptor m_Acceptor.set_option(exclusive_address(true)); m_AlternateProtocolAcceptor.set_option(exclusive_address(true)); #else // ZEN_PLATFORM_WINDOWS - m_Acceptor.set_option(asio::socket_base::reuse_address(false)); - m_AlternateProtocolAcceptor.set_option(asio::socket_base::reuse_address(false)); + // Allow binding to a port in TIME_WAIT so the server can restart immediately + // after a previous instance exits. On Linux this does not allow two processes + // to actively listen on the same port simultaneously. + m_Acceptor.set_option(asio::socket_base::reuse_address(true)); + m_AlternateProtocolAcceptor.set_option(asio::socket_base::reuse_address(true)); #endif // ZEN_PLATFORM_WINDOWS m_Acceptor.set_option(asio::ip::tcp::no_delay(true)); @@ -2092,6 +2107,7 @@ HttpAsioServer::HttpAsioServer(const AsioConfig& Config) : m_InitialConfig(Config) , m_Impl(std::make_unique()) { + m_Impl->m_HttpServer = this; ZEN_DEBUG("Request object size: {} ({:#x})", sizeof(HttpRequestParser), sizeof(HttpRequestParser)); } diff --git a/src/zenhttp/servers/httpplugin.cpp b/src/zenhttp/servers/httpplugin.cpp index 021b941bd..4bf8c61bb 100644 --- a/src/zenhttp/servers/httpplugin.cpp +++ b/src/zenhttp/servers/httpplugin.cpp @@ -378,6 +378,8 @@ HttpPluginConnectionHandler::HandleRequest() { ZEN_TRACE_CPU("http_plugin::HandleRequest"); + m_Server->MarkRequest(); + HttpPluginServerRequest Request(m_RequestParser, *Service, m_RequestParser.Body()); const HttpVerb RequestVerb = Request.RequestVerb(); diff --git a/src/zenhttp/servers/httpsys.cpp b/src/zenhttp/servers/httpsys.cpp index cf639c114..dfe6bb6aa 100644 --- a/src/zenhttp/servers/httpsys.cpp +++ b/src/zenhttp/servers/httpsys.cpp @@ -451,6 +451,8 @@ public: inline uint16_t GetResponseCode() const { return m_ResponseCode; } inline int64_t GetResponseBodySize() const { return m_TotalDataSize; } + void SetLocationHeader(std::string_view Location) { m_LocationHeader = Location; } + private: eastl::fixed_vector m_HttpDataChunks; uint64_t m_TotalDataSize = 0; // Sum of all chunk sizes @@ -460,6 +462,7 @@ private: bool m_IsInitialResponse = true; HttpContentType m_ContentType = HttpContentType::kBinary; eastl::fixed_vector m_DataBuffers; + std::string m_LocationHeader; void InitializeForPayload(uint16_t ResponseCode, std::span Blobs); }; @@ -715,6 +718,15 @@ HttpMessageResponseRequest::IssueRequest(std::error_code& ErrorCode) ContentTypeHeader->pRawValue = ContentTypeString.data(); ContentTypeHeader->RawValueLength = (USHORT)ContentTypeString.size(); + // Location header (for redirects) + + if (!m_LocationHeader.empty()) + { + PHTTP_KNOWN_HEADER LocationHeader = &HttpResponse.Headers.KnownHeaders[HttpHeaderLocation]; + LocationHeader->pRawValue = m_LocationHeader.data(); + LocationHeader->RawValueLength = (USHORT)m_LocationHeader.size(); + } + std::string_view ReasonString = ReasonStringForHttpResultCode(m_ResponseCode); HttpResponse.StatusCode = m_ResponseCode; @@ -916,7 +928,10 @@ HttpAsyncWorkRequest::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesTr ZEN_UNUSED(IoResult, NumberOfBytesTransferred); - ZEN_WARN("Unexpected I/O completion during async work! IoResult: {}, NumberOfBytesTransferred: {}", IoResult, NumberOfBytesTransferred); + ZEN_WARN("Unexpected I/O completion during async work! IoResult: {} ({:#x}), NumberOfBytesTransferred: {}", + GetSystemErrorAsString(IoResult), + IoResult, + NumberOfBytesTransferred); return this; } @@ -1083,7 +1098,10 @@ HttpSysServer::InitializeServer(int BasePort) if (Result != NO_ERROR) { - ZEN_ERROR("Failed to create server session for '{}': {:#x}", WideToUtf8(WildcardUrlPath), Result); + ZEN_ERROR("Failed to create server session for '{}': {} ({:#x})", + WideToUtf8(WildcardUrlPath), + GetSystemErrorAsString(Result), + Result); return 0; } @@ -1092,7 +1110,7 @@ HttpSysServer::InitializeServer(int BasePort) if (Result != NO_ERROR) { - ZEN_ERROR("Failed to create URL group for '{}': {:#x}", WideToUtf8(WildcardUrlPath), Result); + ZEN_ERROR("Failed to create URL group for '{}': {} ({:#x})", WideToUtf8(WildcardUrlPath), GetSystemErrorAsString(Result), Result); return 0; } @@ -1116,7 +1134,9 @@ HttpSysServer::InitializeServer(int BasePort) if ((Result == ERROR_SHARING_VIOLATION)) { - ZEN_INFO("Desired port {} is in use (HttpAddUrlToUrlGroup returned: {}), retrying", EffectivePort, Result); + ZEN_INFO("Desired port {} is in use (HttpAddUrlToUrlGroup returned: {}), retrying", + EffectivePort, + GetSystemErrorAsString(Result)); Sleep(500); Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, WildcardUrlPath.c_str(), HTTP_URL_CONTEXT(0), 0); @@ -1138,7 +1158,9 @@ HttpSysServer::InitializeServer(int BasePort) { for (uint32_t Retries = 0; (Result == ERROR_SHARING_VIOLATION) && (Retries < 3); Retries++) { - ZEN_INFO("Desired port {} is in use (HttpAddUrlToUrlGroup returned: {}), retrying", EffectivePort, Result); + ZEN_INFO("Desired port {} is in use (HttpAddUrlToUrlGroup returned: {}), retrying", + EffectivePort, + GetSystemErrorAsString(Result)); Sleep(500); Result = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, WildcardUrlPath.c_str(), HTTP_URL_CONTEXT(0), 0); } @@ -1173,17 +1195,18 @@ HttpSysServer::InitializeServer(int BasePort) const std::u8string_view Hosts[] = {u8"[::1]"sv, u8"localhost"sv, u8"127.0.0.1"sv}; - ULONG InternalResult = ERROR_SHARING_VIOLATION; - for (int PortOffset = 0; (InternalResult == ERROR_SHARING_VIOLATION) && (PortOffset < 10); ++PortOffset) + bool ShouldRetryNextPort = true; + for (int PortOffset = 0; ShouldRetryNextPort && (PortOffset < 10); ++PortOffset) { - EffectivePort = BasePort + (PortOffset * 100); + EffectivePort = BasePort + (PortOffset * 100); + ShouldRetryNextPort = false; for (const std::u8string_view Host : Hosts) { WideStringBuilder<64> LocalUrlPath; LocalUrlPath << u8"http://"sv << Host << u8":"sv << int64_t(EffectivePort) << u8"/"sv; - InternalResult = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, LocalUrlPath.c_str(), HTTP_URL_CONTEXT(0), 0); + ULONG InternalResult = HttpAddUrlToUrlGroup(m_HttpUrlGroupId, LocalUrlPath.c_str(), HTTP_URL_CONTEXT(0), 0); if (InternalResult == NO_ERROR) { @@ -1191,11 +1214,25 @@ HttpSysServer::InitializeServer(int BasePort) m_BaseUris.push_back(LocalUrlPath.c_str()); } + else if (InternalResult == ERROR_SHARING_VIOLATION || InternalResult == ERROR_ACCESS_DENIED) + { + // Port may be owned by another process's wildcard registration (access denied) + // or actively in use (sharing violation) — retry on a different port + ShouldRetryNextPort = true; + } else { - break; + ZEN_WARN("Failed to register local handler '{}': {} ({:#x})", + WideToUtf8(LocalUrlPath), + GetSystemErrorAsString(InternalResult), + InternalResult); } } + + if (!m_BaseUris.empty()) + { + break; + } } } else @@ -1211,7 +1248,10 @@ HttpSysServer::InitializeServer(int BasePort) if (m_BaseUris.empty()) { - ZEN_ERROR("Failed to add base URL to URL group for '{}': {:#x}", WideToUtf8(WildcardUrlPath), Result); + ZEN_ERROR("Failed to add base URL to URL group for '{}': {} ({:#x})", + WideToUtf8(WildcardUrlPath), + GetSystemErrorAsString(Result), + Result); return 0; } @@ -1229,7 +1269,10 @@ HttpSysServer::InitializeServer(int BasePort) if (Result != NO_ERROR) { - ZEN_ERROR("Failed to create request queue for '{}': {:#x}", WideToUtf8(m_BaseUris.front()), Result); + ZEN_ERROR("Failed to create request queue for '{}': {} ({:#x})", + WideToUtf8(m_BaseUris.front()), + GetSystemErrorAsString(Result), + Result); return 0; } @@ -1241,7 +1284,10 @@ HttpSysServer::InitializeServer(int BasePort) if (Result != NO_ERROR) { - ZEN_ERROR("Failed to set server binding property for '{}': {:#x}", WideToUtf8(m_BaseUris.front()), Result); + ZEN_ERROR("Failed to set server binding property for '{}': {} ({:#x})", + WideToUtf8(m_BaseUris.front()), + GetSystemErrorAsString(Result), + Result); return 0; } @@ -1273,7 +1319,7 @@ HttpSysServer::InitializeServer(int BasePort) if (Result != NO_ERROR) { - ZEN_WARN("changing request queue length to {} failed: {}", QueueLength, Result); + ZEN_WARN("changing request queue length to {} failed: {} ({:#x})", QueueLength, GetSystemErrorAsString(Result), Result); } } @@ -1295,21 +1341,6 @@ HttpSysServer::InitializeServer(int BasePort) ZEN_INFO("Started http.sys server at '{}'", WideToUtf8(m_BaseUris.front())); } - // This is not available in all Windows SDK versions so for now we can't use recently - // released functionality. We should investigate how to get more recent SDK releases - // into the build - -# if 0 - if (HttpIsFeatureSupported(/* HttpFeatureHttp3 */ (HTTP_FEATURE_ID) 4)) - { - ZEN_DEBUG("HTTP3 is available"); - } - else - { - ZEN_DEBUG("HTTP3 is NOT available"); - } -# endif - return EffectivePort; } @@ -1695,6 +1726,8 @@ HttpSysTransaction::InvokeRequestHandler(HttpService& Service, IoBuffer Payload) { HttpSysServerRequest& ThisRequest = m_HandlerRequest.emplace(*this, Service, Payload); + m_HttpServer.MarkRequest(); + // Default request handling # if ZEN_WITH_OTEL @@ -2245,8 +2278,12 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT if (SendResult == NO_ERROR) { - Ref WsConn( - new WsHttpSysConnection(RequestQueueHandle, RequestId, *WsHandler, Transaction().Iocp())); + Transaction().Server().OnWebSocketConnectionOpened(); + Ref WsConn(new WsHttpSysConnection(RequestQueueHandle, + RequestId, + *WsHandler, + Transaction().Iocp(), + &Transaction().Server())); Ref WsConnRef(WsConn.Get()); WsHandler->OnWebSocketOpen(std::move(WsConnRef)); @@ -2255,7 +2292,7 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT return nullptr; } - ZEN_WARN("WebSocket 101 send failed: {}", SendResult); + ZEN_WARN("WebSocket 101 send failed: {} ({:#x})", GetSystemErrorAsString(SendResult), SendResult); // WebSocket upgrade failed — return nullptr since ServerRequest() // was never populated (no InvokeRequestHandler call) @@ -2330,6 +2367,18 @@ InitialRequestHandler::HandleCompletion(ULONG IoResult, ULONG_PTR NumberOfBytesT return new HttpMessageResponseRequest(Transaction(), 404, "Not found"sv); } } + else + { + // If a default redirect is configured and the request is for the root path, send a 302 + std::string_view DefaultRedirect = Transaction().Server().GetDefaultRedirect(); + std::string_view RawUrl(HttpReq->pRawUrl, HttpReq->RawUrlLength); + if (!DefaultRedirect.empty() && (RawUrl == "/" || RawUrl.empty())) + { + auto* Response = new HttpMessageResponseRequest(Transaction(), 302); + Response->SetLocationHeader(DefaultRedirect); + return Response; + } + } // Unable to route return new HttpMessageResponseRequest(Transaction(), 404, "No suitable route found"sv); diff --git a/src/zenhttp/servers/wsasio.cpp b/src/zenhttp/servers/wsasio.cpp index 3e31b58bc..b2543277a 100644 --- a/src/zenhttp/servers/wsasio.cpp +++ b/src/zenhttp/servers/wsasio.cpp @@ -4,6 +4,7 @@ #include "wsframecodec.h" #include +#include namespace zen::asio_http { @@ -16,15 +17,20 @@ WsLog() ////////////////////////////////////////////////////////////////////////// -WsAsioConnection::WsAsioConnection(std::unique_ptr Socket, IWebSocketHandler& Handler) +WsAsioConnection::WsAsioConnection(std::unique_ptr Socket, IWebSocketHandler& Handler, HttpServer* Server) : m_Socket(std::move(Socket)) , m_Handler(Handler) +, m_HttpServer(Server) { } WsAsioConnection::~WsAsioConnection() { m_IsOpen.store(false); + if (m_HttpServer) + { + m_HttpServer->OnWebSocketConnectionClosed(); + } } void @@ -101,6 +107,11 @@ WsAsioConnection::ProcessReceivedData() m_ReadBuffer.consume(Frame.BytesConsumed); + if (m_HttpServer) + { + m_HttpServer->OnWebSocketFrameReceived(Frame.BytesConsumed); + } + switch (Frame.Opcode) { case WebSocketOpcode::kText: @@ -219,6 +230,11 @@ WsAsioConnection::DoClose(uint16_t Code, std::string_view Reason) void WsAsioConnection::EnqueueWrite(std::vector Frame) { + if (m_HttpServer) + { + m_HttpServer->OnWebSocketFrameSent(Frame.size()); + } + bool ShouldFlush = false; m_WriteLock.WithExclusiveLock([&] { diff --git a/src/zenhttp/servers/wsasio.h b/src/zenhttp/servers/wsasio.h index d8ffdc00a..e8bb3b1d2 100644 --- a/src/zenhttp/servers/wsasio.h +++ b/src/zenhttp/servers/wsasio.h @@ -14,6 +14,10 @@ ZEN_THIRD_PARTY_INCLUDES_END #include #include +namespace zen { +class HttpServer; +} // namespace zen + namespace zen::asio_http { /** @@ -27,10 +31,11 @@ namespace zen::asio_http { * connection alive for the duration of the async operation. The service layer * also holds a Ref. */ + class WsAsioConnection : public WebSocketConnection { public: - WsAsioConnection(std::unique_ptr Socket, IWebSocketHandler& Handler); + WsAsioConnection(std::unique_ptr Socket, IWebSocketHandler& Handler, HttpServer* Server); ~WsAsioConnection() override; /** @@ -58,6 +63,7 @@ private: std::unique_ptr m_Socket; IWebSocketHandler& m_Handler; + zen::HttpServer* m_HttpServer; asio::streambuf m_ReadBuffer; RwLock m_WriteLock; diff --git a/src/zenhttp/servers/wshttpsys.cpp b/src/zenhttp/servers/wshttpsys.cpp index 3408b64b3..af320172d 100644 --- a/src/zenhttp/servers/wshttpsys.cpp +++ b/src/zenhttp/servers/wshttpsys.cpp @@ -7,6 +7,7 @@ # include "wsframecodec.h" # include +# include namespace zen { @@ -19,11 +20,16 @@ WsHttpSysLog() ////////////////////////////////////////////////////////////////////////// -WsHttpSysConnection::WsHttpSysConnection(HANDLE RequestQueueHandle, HTTP_REQUEST_ID RequestId, IWebSocketHandler& Handler, PTP_IO Iocp) +WsHttpSysConnection::WsHttpSysConnection(HANDLE RequestQueueHandle, + HTTP_REQUEST_ID RequestId, + IWebSocketHandler& Handler, + PTP_IO Iocp, + HttpServer* Server) : m_RequestQueueHandle(RequestQueueHandle) , m_RequestId(RequestId) , m_Handler(Handler) , m_Iocp(Iocp) +, m_HttpServer(Server) , m_ReadBuffer(8192) { m_ReadIoContext.ContextType = HttpSysIoContext::Type::kWebSocketRead; @@ -40,6 +46,11 @@ WsHttpSysConnection::~WsHttpSysConnection() { Disconnect(); } + + if (m_HttpServer) + { + m_HttpServer->OnWebSocketConnectionClosed(); + } } void @@ -174,6 +185,11 @@ WsHttpSysConnection::ProcessReceivedData() // Remove consumed bytes m_Accumulated.erase(m_Accumulated.begin(), m_Accumulated.begin() + Frame.BytesConsumed); + if (m_HttpServer) + { + m_HttpServer->OnWebSocketFrameReceived(Frame.BytesConsumed); + } + switch (Frame.Opcode) { case WebSocketOpcode::kText: @@ -250,6 +266,11 @@ WsHttpSysConnection::ProcessReceivedData() void WsHttpSysConnection::EnqueueWrite(std::vector Frame) { + if (m_HttpServer) + { + m_HttpServer->OnWebSocketFrameSent(Frame.size()); + } + bool ShouldFlush = false; { diff --git a/src/zenhttp/servers/wshttpsys.h b/src/zenhttp/servers/wshttpsys.h index d854289e0..6015e3873 100644 --- a/src/zenhttp/servers/wshttpsys.h +++ b/src/zenhttp/servers/wshttpsys.h @@ -19,6 +19,8 @@ namespace zen { +class HttpServer; + /** * WebSocket connection over an http.sys opaque-mode connection * @@ -37,7 +39,7 @@ namespace zen { class WsHttpSysConnection : public WebSocketConnection { public: - WsHttpSysConnection(HANDLE RequestQueueHandle, HTTP_REQUEST_ID RequestId, IWebSocketHandler& Handler, PTP_IO Iocp); + WsHttpSysConnection(HANDLE RequestQueueHandle, HTTP_REQUEST_ID RequestId, IWebSocketHandler& Handler, PTP_IO Iocp, HttpServer* Server); ~WsHttpSysConnection() override; /** @@ -75,6 +77,7 @@ private: HTTP_REQUEST_ID m_RequestId; IWebSocketHandler& m_Handler; PTP_IO m_Iocp; + HttpServer* m_HttpServer; // Tagged OVERLAPPED contexts for concurrent read and write HttpSysIoContext m_ReadIoContext{}; diff --git a/src/zenhttp/servers/wstest.cpp b/src/zenhttp/servers/wstest.cpp index fd023c490..2134e4ff1 100644 --- a/src/zenhttp/servers/wstest.cpp +++ b/src/zenhttp/servers/wstest.cpp @@ -767,13 +767,12 @@ namespace { void OnWsMessage(const WebSocketMessage& Msg) override { - m_MessageCount.fetch_add(1); - if (Msg.Opcode == WebSocketOpcode::kText) { std::string_view Text(static_cast(Msg.Payload.Data()), Msg.Payload.Size()); m_LastMessage = std::string(Text); } + m_MessageCount.fetch_add(1); } void OnWsClose(uint16_t Code, [[maybe_unused]] std::string_view Reason) override diff --git a/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h b/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h index bb41f9efc..caf7ecd28 100644 --- a/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h +++ b/src/zenremotestore/include/zenremotestore/jupiter/jupiterhost.h @@ -2,6 +2,7 @@ #pragma once +#include #include #include #include diff --git a/src/zenremotestore/projectstore/remoteprojectstore.cpp b/src/zenremotestore/projectstore/remoteprojectstore.cpp index d5c6286a8..4796b3f2a 100644 --- a/src/zenremotestore/projectstore/remoteprojectstore.cpp +++ b/src/zenremotestore/projectstore/remoteprojectstore.cpp @@ -2447,14 +2447,13 @@ BuildContainer(CidStore& ChunkStore, AsyncOnBlock, RemoteResult); ComposedBlocks++; + // Worker will set Blocks[BlockIndex] = Block (including ChunkRawHashes) under shared lock } else { ZEN_INFO("Bulk group {} attachments", ChunkCount); OnBlockChunks(std::move(ChunksInBlock)); - } - { - // We can share the lock as we are not resizing the vector and only touch BlockHash at our own index + // We can share the lock as we are not resizing the vector and only touch our own index RwLock::SharedLockScope _(BlocksLock); Blocks[BlockIndex].ChunkRawHashes = std::move(ChunkRawHashes); } diff --git a/src/zenserver/compute/computeserver.cpp b/src/zenserver/compute/computeserver.cpp index 802d06caf..c64f081b3 100644 --- a/src/zenserver/compute/computeserver.cpp +++ b/src/zenserver/compute/computeserver.cpp @@ -419,6 +419,8 @@ ZenComputeServer::Cleanup() m_IoRunner.join(); } + ShutdownServices(); + if (m_Http) { m_Http->Close(); @@ -570,8 +572,6 @@ ZenComputeServer::RegisterServices(const ZenComputeServerConfig& ServerConfig) ZEN_TRACE_CPU("ZenComputeServer::RegisterServices"); ZEN_UNUSED(ServerConfig); - m_Http->RegisterService(m_StatsService); - if (m_ApiService) { m_Http->RegisterService(*m_ApiService); diff --git a/src/zenserver/compute/computeserver.h b/src/zenserver/compute/computeserver.h index e4a6b01d5..8f4edc0f0 100644 --- a/src/zenserver/compute/computeserver.h +++ b/src/zenserver/compute/computeserver.h @@ -129,7 +129,6 @@ public: void Cleanup(); private: - HttpStatsService m_StatsService; GcManager m_GcManager; GcScheduler m_GcScheduler{m_GcManager}; std::unique_ptr m_CidStore; diff --git a/src/zenserver/diag/diagsvcs.cpp b/src/zenserver/diag/diagsvcs.cpp index 5fa81ff9f..dd4b8956c 100644 --- a/src/zenserver/diag/diagsvcs.cpp +++ b/src/zenserver/diag/diagsvcs.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,36 @@ HttpHealthService::HttpHealthService() Writer << "AbsLogPath"sv << m_HealthInfo.AbsLogPath.string(); Writer << "BuildVersion"sv << m_HealthInfo.BuildVersion; Writer << "HttpServerClass"sv << m_HealthInfo.HttpServerClass; + Writer << "Port"sv << m_HealthInfo.Port; + Writer << "Pid"sv << m_HealthInfo.Pid; + Writer << "IsDedicated"sv << m_HealthInfo.IsDedicated; + Writer << "StartTimeMs"sv << m_HealthInfo.StartTimeMs; + } + + Writer.BeginObject("RuntimeConfig"sv); + for (const auto& Opt : m_HealthInfo.RuntimeConfig) + { + Writer << Opt.first << Opt.second; + } + Writer.EndObject(); + + Writer.BeginObject("BuildConfig"sv); + for (const auto& Opt : m_HealthInfo.BuildOptions) + { + Writer << Opt.first << Opt.second; + } + Writer.EndObject(); + + Writer << "Hostname"sv << GetMachineName(); + Writer << "Platform"sv << GetRuntimePlatformName(); + Writer << "Arch"sv << GetCpuName(); + Writer << "OS"sv << GetOperatingSystemVersion(); + + { + auto Metrics = GetSystemMetrics(); + Writer.BeginObject("System"sv); + Describe(Metrics, Writer); + Writer.EndObject(); } HttpReq.WriteResponse(HttpResponseCode::OK, Writer.Save()); diff --git a/src/zenserver/diag/diagsvcs.h b/src/zenserver/diag/diagsvcs.h index 8cc869c83..87ce80b3c 100644 --- a/src/zenserver/diag/diagsvcs.h +++ b/src/zenserver/diag/diagsvcs.h @@ -6,6 +6,7 @@ #include #include +#include ////////////////////////////////////////////////////////////////////////// @@ -89,10 +90,16 @@ private: struct HealthServiceInfo { - std::filesystem::path DataRoot; - std::filesystem::path AbsLogPath; - std::string HttpServerClass; - std::string BuildVersion; + std::filesystem::path DataRoot; + std::filesystem::path AbsLogPath; + std::string HttpServerClass; + std::string BuildVersion; + int Port = 0; + int Pid = 0; + bool IsDedicated = false; + int64_t StartTimeMs = 0; + std::vector> BuildOptions; + std::vector> RuntimeConfig; }; /** Health monitoring endpoint diff --git a/src/zenserver/diag/otlphttp.cpp b/src/zenserver/diag/otlphttp.cpp index 1434c9331..d6e24cbe3 100644 --- a/src/zenserver/diag/otlphttp.cpp +++ b/src/zenserver/diag/otlphttp.cpp @@ -10,11 +10,18 @@ #include #include +#include + #if ZEN_WITH_OTEL namespace zen::logging { ////////////////////////////////////////////////////////////////////////// +// +// Important note: in general we cannot use ZEN_WARN/ZEN_ERROR etc in this +// file as it could cause recursive logging calls when we attempt to log +// errors from the OTLP HTTP client itself. +// OtelHttpProtobufSink::OtelHttpProtobufSink(const std::string_view& Uri) : m_OtelHttp(Uri) { @@ -35,15 +42,45 @@ OtelHttpProtobufSink::~OtelHttpProtobufSink() otel::SetTraceRecorder({}); } +void +OtelHttpProtobufSink::CheckPostResult(const HttpClient::Response& Result, const char* Endpoint) noexcept +{ + if (!Result.IsSuccess()) + { + uint32_t PrevFailures = m_ConsecutivePostFailures.fetch_add(1); + if (PrevFailures < kMaxReportedFailures) + { + fprintf(stderr, "OtelHttpProtobufSink: %s\n", Result.ErrorMessage(Endpoint).c_str()); + if (PrevFailures + 1 == kMaxReportedFailures) + { + fprintf(stderr, "OtelHttpProtobufSink: suppressing further export errors\n"); + } + } + } + else + { + m_ConsecutivePostFailures.store(0); + } +} + void OtelHttpProtobufSink::RecordSpans(zen::otel::TraceId Trace, std::span Spans) { - std::string Data = m_Encoder.FormatOtelTrace(Trace, Spans); + try + { + std::string Data = m_Encoder.FormatOtelTrace(Trace, Spans); + + IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()}; + Payload.SetContentType(ZenContentType::kProtobuf); - IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()}; - Payload.SetContentType(ZenContentType::kProtobuf); + HttpClient::Response Result = m_OtelHttp.Post("/v1/traces", Payload); - auto Result = m_OtelHttp.Post("/v1/traces", Payload); + CheckPostResult(Result, "POST /v1/traces"); + } + catch (const std::exception& Ex) + { + fprintf(stderr, "OtelHttpProtobufSink: exception exporting traces: %s\n", Ex.what()); + } } void @@ -55,22 +92,20 @@ OtelHttpProtobufSink::TraceRecorder::RecordSpans(zen::otel::TraceId Trace, std:: void OtelHttpProtobufSink::Log(const LogMessage& Msg) { + try { std::string Data = m_Encoder.FormatOtelProtobuf(Msg); IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()}; Payload.SetContentType(ZenContentType::kProtobuf); - auto Result = m_OtelHttp.Post("/v1/logs", Payload); - } + HttpClient::Response Result = m_OtelHttp.Post("/v1/logs", Payload); + CheckPostResult(Result, "POST /v1/logs"); + } + catch (const std::exception& Ex) { - std::string Data = m_Encoder.FormatOtelMetrics(); - - IoBuffer Payload{IoBuffer::Wrap, Data.data(), Data.size()}; - Payload.SetContentType(ZenContentType::kProtobuf); - - auto Result = m_OtelHttp.Post("/v1/metrics", Payload); + fprintf(stderr, "OtelHttpProtobufSink: exception exporting logs: %s\n", Ex.what()); } } void diff --git a/src/zenserver/diag/otlphttp.h b/src/zenserver/diag/otlphttp.h index 8254af04d..64b3dbc87 100644 --- a/src/zenserver/diag/otlphttp.h +++ b/src/zenserver/diag/otlphttp.h @@ -9,6 +9,8 @@ #include #include +#include + #if ZEN_WITH_OTEL namespace zen::logging { @@ -36,6 +38,7 @@ private: virtual void SetFormatter(std::unique_ptr) override {} void RecordSpans(zen::otel::TraceId Trace, std::span Spans); + void CheckPostResult(const HttpClient::Response& Result, const char* Endpoint) noexcept; // This is just a thin wrapper to call back into the sink while participating in // reference counting from the OTEL trace back-end @@ -53,9 +56,13 @@ private: OtelHttpProtobufSink* m_Sink; }; - HttpClient m_OtelHttp; - OtlpEncoder m_Encoder; - Ref m_TraceRecorder; + static constexpr uint32_t kMaxReportedFailures = 5; + + RwLock m_Lock; + std::atomic m_ConsecutivePostFailures{0}; + HttpClient m_OtelHttp; + OtlpEncoder m_Encoder; + Ref m_TraceRecorder; }; } // namespace zen::logging diff --git a/src/zenserver/frontend/html.zip b/src/zenserver/frontend/html.zip index c167cc70e..84472ff08 100644 Binary files a/src/zenserver/frontend/html.zip and b/src/zenserver/frontend/html.zip differ diff --git a/src/zenserver/frontend/html/banner.js b/src/zenserver/frontend/html/banner.js new file mode 100644 index 000000000..2e878dedf --- /dev/null +++ b/src/zenserver/frontend/html/banner.js @@ -0,0 +1,338 @@ +/** + * zen-banner.js — Zen dashboard banner Web Component + * + * Usage: + * + * + * + * + * + * + * Attributes: + * variant "full" (default) | "compact" + * cluster-status "nominal" (default) | "degraded" | "offline" + * load 0–100 integer, shown as a percentage (default: hidden) + * tagline custom tagline text (default: "Orchestrator Overview" / "Orchestrator") + * subtitle text after "ZEN" in the wordmark (default: "COMPUTE") + */ + +class ZenBanner extends HTMLElement { + + static get observedAttributes() { + return ['variant', 'cluster-status', 'load', 'tagline', 'subtitle', 'logo-src']; + } + + attributeChangedCallback() { + if (this.shadowRoot) this._render(); + } + + connectedCallback() { + if (!this.shadowRoot) this.attachShadow({ mode: 'open' }); + this._render(); + } + + // ───────────────────────────────────────────── + // Derived values + // ───────────────────────────────────────────── + + get _variant() { return this.getAttribute('variant') || 'full'; } + get _status() { return (this.getAttribute('cluster-status') || 'nominal').toLowerCase(); } + get _load() { return this.getAttribute('load'); } // null → hidden + get _tagline() { return this.getAttribute('tagline'); } // null → default + get _subtitle() { return this.getAttribute('subtitle'); } // null → "COMPUTE" + get _logoSrc() { return this.getAttribute('logo-src'); } // null → inline SVG + + get _statusColor() { + return { nominal: '#7ecfb8', degraded: '#d4a84b', offline: '#c0504d' }[this._status] ?? '#7ecfb8'; + } + + get _statusLabel() { + return { nominal: 'NOMINAL', degraded: 'DEGRADED', offline: 'OFFLINE' }[this._status] ?? 'NOMINAL'; + } + + get _loadColor() { + const v = parseInt(this._load, 10); + if (isNaN(v)) return '#7ecfb8'; + if (v >= 85) return '#c0504d'; + if (v >= 60) return '#d4a84b'; + return '#7ecfb8'; + } + + // ───────────────────────────────────────────── + // Render + // ───────────────────────────────────────────── + + _render() { + const compact = this._variant === 'compact'; + this.shadowRoot.innerHTML = ` + + ${this._html(compact)} + `; + } + + // ───────────────────────────────────────────── + // CSS + // ───────────────────────────────────────────── + + _css(compact) { + const height = compact ? '60px' : '100px'; + const padding = compact ? '0 24px' : '0 32px'; + const gap = compact ? '16px' : '24px'; + const markSize = compact ? '34px' : '52px'; + const divH = compact ? '32px' : '48px'; + const nameSize = compact ? '15px' : '22px'; + const tagSize = compact ? '9px' : '11px'; + const sc = this._statusColor; + const lc = this._loadColor; + + return ` + @import url('https://fonts.googleapis.com/css2?family=Noto+Serif+JP:wght@300;400&family=Space+Mono:wght@400;700&display=swap'); + + *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } + + :host { + display: block; + font-family: 'Space Mono', monospace; + } + + .banner { + width: 100%; + height: ${height}; + background: var(--theme_g3, #0b0d10); + border: 1px solid var(--theme_g2, #1e2330); + border-radius: 6px; + display: flex; + align-items: center; + padding: ${padding}; + gap: ${gap}; + position: relative; + overflow: hidden; + text-decoration: none; + color: inherit; + cursor: pointer; + } + + /* scan-line texture */ + .banner::before { + content: ''; + position: absolute; + inset: 0; + background: repeating-linear-gradient( + 0deg, + transparent, transparent 3px, + rgba(255,255,255,0.012) 3px, rgba(255,255,255,0.012) 4px + ); + pointer-events: none; + } + + /* ambient glow */ + .banner::after { + content: ''; + position: absolute; + right: -60px; + top: 50%; + transform: translateY(-50%); + width: 280px; + height: 280px; + background: radial-gradient(circle, rgba(130,200,180,0.06) 0%, transparent 70%); + pointer-events: none; + } + + .logo-mark { + flex-shrink: 0; + width: ${markSize}; + height: ${markSize}; + } + + .logo-mark svg, .logo-mark img { width: 100%; height: 100%; object-fit: contain; } + + .divider { + width: 1px; + height: ${divH}; + background: linear-gradient(to bottom, transparent, var(--theme_g2, #2a3040), transparent); + flex-shrink: 0; + } + + .text-block { + display: flex; + flex-direction: column; + gap: 4px; + } + + .wordmark { + font-weight: 700; + font-size: ${nameSize}; + letter-spacing: 0.12em; + color: var(--theme_bright, #e8e4dc); + text-transform: uppercase; + line-height: 1; + } + + .wordmark span { color: #7ecfb8; } + + .tagline { + font-family: 'Noto Serif JP', serif; + font-weight: 300; + font-size: ${tagSize}; + letter-spacing: 0.3em; + color: var(--theme_faint, #4a5a68); + text-transform: uppercase; + } + + .spacer { flex: 1; } + + /* ── right-side decorative circuit ── */ + .circuit { flex-shrink: 0; opacity: 0.22; } + + /* ── status cluster ── */ + .status-cluster { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 6px; + } + + .status-row { + display: flex; + align-items: center; + gap: 8px; + } + + .status-lbl { + font-size: 9px; + letter-spacing: 0.18em; + color: var(--theme_faint, #3a4555); + text-transform: uppercase; + } + + .pill { + display: flex; + align-items: center; + gap: 5px; + border-radius: 20px; + padding: 2px 10px; + font-size: 10px; + letter-spacing: 0.1em; + } + + .pill.cluster { + color: ${sc}; + background: color-mix(in srgb, ${sc} 8%, transparent); + border: 1px solid color-mix(in srgb, ${sc} 28%, transparent); + } + + .pill.load-pill { + color: ${lc}; + background: color-mix(in srgb, ${lc} 8%, transparent); + border: 1px solid color-mix(in srgb, ${lc} 28%, transparent); + } + + .dot { + width: 5px; + height: 5px; + border-radius: 50%; + animation: pulse 2.4s ease-in-out infinite; + } + + .dot.cluster { background: ${sc}; } + .dot.load-dot { background: ${lc}; animation-delay: 0.5s; } + + @keyframes pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.25; } + } + `; + } + + // ───────────────────────────────────────────── + // HTML template + // ───────────────────────────────────────────── + + _html(compact) { + const loadAttr = this._load; + const hasCluster = !compact && this.hasAttribute('cluster-status'); + const hasLoad = !compact && loadAttr !== null; + const showRight = hasCluster || hasLoad; + + const circuit = showRight ? ` + + + + + + + + ` : ''; + + const clusterRow = hasCluster ? ` +
      + Cluster +
      +
      + ${this._statusLabel} +
      +
      ` : ''; + + const loadRow = hasLoad ? ` +
      + Load +
      +
      + ${parseInt(loadAttr, 10)} % +
      +
      ` : ''; + + const rightSide = showRight ? ` + ${circuit} +
      + ${clusterRow} + ${loadRow} +
      + ` : ''; + + return ` + + `; + } + + // ───────────────────────────────────────────── + // SVG logo mark + // ───────────────────────────────────────────── + + _logoMark() { + const src = this._logoSrc; + if (src) { + return `zen`; + } + return ` + + + + + + + + + + + + + + + + + + `; + } +} + +customElements.define('zen-banner', ZenBanner); diff --git a/src/zenserver/frontend/html/compute/banner.js b/src/zenserver/frontend/html/compute/banner.js deleted file mode 100644 index 61c7ce21f..000000000 --- a/src/zenserver/frontend/html/compute/banner.js +++ /dev/null @@ -1,321 +0,0 @@ -/** - * zen-banner.js — Zen Compute dashboard banner Web Component - * - * Usage: - * - * - * - * - * - * - * Attributes: - * variant "full" (default) | "compact" - * cluster-status "nominal" (default) | "degraded" | "offline" - * load 0–100 integer, shown as a percentage (default: hidden) - * tagline custom tagline text (default: "Orchestrator Overview" / "Orchestrator") - * subtitle text after "ZEN" in the wordmark (default: "COMPUTE") - */ - -class ZenBanner extends HTMLElement { - - static get observedAttributes() { - return ['variant', 'cluster-status', 'load', 'tagline', 'subtitle']; - } - - attributeChangedCallback() { - if (this.shadowRoot) this._render(); - } - - connectedCallback() { - if (!this.shadowRoot) this.attachShadow({ mode: 'open' }); - this._render(); - } - - // ───────────────────────────────────────────── - // Derived values - // ───────────────────────────────────────────── - - get _variant() { return this.getAttribute('variant') || 'full'; } - get _status() { return (this.getAttribute('cluster-status') || 'nominal').toLowerCase(); } - get _load() { return this.getAttribute('load'); } // null → hidden - get _tagline() { return this.getAttribute('tagline'); } // null → default - get _subtitle() { return this.getAttribute('subtitle'); } // null → "COMPUTE" - - get _statusColor() { - return { nominal: '#7ecfb8', degraded: '#d4a84b', offline: '#c0504d' }[this._status] ?? '#7ecfb8'; - } - - get _statusLabel() { - return { nominal: 'NOMINAL', degraded: 'DEGRADED', offline: 'OFFLINE' }[this._status] ?? 'NOMINAL'; - } - - get _loadColor() { - const v = parseInt(this._load, 10); - if (isNaN(v)) return '#7ecfb8'; - if (v >= 85) return '#c0504d'; - if (v >= 60) return '#d4a84b'; - return '#7ecfb8'; - } - - // ───────────────────────────────────────────── - // Render - // ───────────────────────────────────────────── - - _render() { - const compact = this._variant === 'compact'; - this.shadowRoot.innerHTML = ` - - ${this._html(compact)} - `; - } - - // ───────────────────────────────────────────── - // CSS - // ───────────────────────────────────────────── - - _css(compact) { - const height = compact ? '60px' : '100px'; - const padding = compact ? '0 24px' : '0 32px'; - const gap = compact ? '16px' : '24px'; - const markSize = compact ? '34px' : '52px'; - const divH = compact ? '32px' : '48px'; - const nameSize = compact ? '15px' : '22px'; - const tagSize = compact ? '9px' : '11px'; - const sc = this._statusColor; - const lc = this._loadColor; - - return ` - @import url('https://fonts.googleapis.com/css2?family=Noto+Serif+JP:wght@300;400&family=Space+Mono:wght@400;700&display=swap'); - - *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } - - :host { - display: block; - font-family: 'Space Mono', monospace; - } - - .banner { - width: 100%; - height: ${height}; - background: #0b0d10; - border: 1px solid #1e2330; - border-radius: 6px; - display: flex; - align-items: center; - padding: ${padding}; - gap: ${gap}; - position: relative; - overflow: hidden; - } - - /* scan-line texture */ - .banner::before { - content: ''; - position: absolute; - inset: 0; - background: repeating-linear-gradient( - 0deg, - transparent, transparent 3px, - rgba(255,255,255,0.012) 3px, rgba(255,255,255,0.012) 4px - ); - pointer-events: none; - } - - /* ambient glow */ - .banner::after { - content: ''; - position: absolute; - right: -60px; - top: 50%; - transform: translateY(-50%); - width: 280px; - height: 280px; - background: radial-gradient(circle, rgba(130,200,180,0.06) 0%, transparent 70%); - pointer-events: none; - } - - .logo-mark { - flex-shrink: 0; - width: ${markSize}; - height: ${markSize}; - } - - .logo-mark svg { width: 100%; height: 100%; } - - .divider { - width: 1px; - height: ${divH}; - background: linear-gradient(to bottom, transparent, #2a3040, transparent); - flex-shrink: 0; - } - - .text-block { - display: flex; - flex-direction: column; - gap: 4px; - } - - .wordmark { - font-weight: 700; - font-size: ${nameSize}; - letter-spacing: 0.12em; - color: #e8e4dc; - text-transform: uppercase; - line-height: 1; - } - - .wordmark span { color: #7ecfb8; } - - .tagline { - font-family: 'Noto Serif JP', serif; - font-weight: 300; - font-size: ${tagSize}; - letter-spacing: 0.3em; - color: #4a5a68; - text-transform: uppercase; - } - - .spacer { flex: 1; } - - /* ── right-side decorative circuit ── */ - .circuit { flex-shrink: 0; opacity: 0.22; } - - /* ── status cluster ── */ - .status-cluster { - display: flex; - flex-direction: column; - align-items: flex-end; - gap: 6px; - } - - .status-row { - display: flex; - align-items: center; - gap: 8px; - } - - .status-lbl { - font-size: 9px; - letter-spacing: 0.18em; - color: #3a4555; - text-transform: uppercase; - } - - .pill { - display: flex; - align-items: center; - gap: 5px; - border-radius: 20px; - padding: 2px 10px; - font-size: 10px; - letter-spacing: 0.1em; - } - - .pill.cluster { - color: ${sc}; - background: color-mix(in srgb, ${sc} 8%, transparent); - border: 1px solid color-mix(in srgb, ${sc} 28%, transparent); - } - - .pill.load-pill { - color: ${lc}; - background: color-mix(in srgb, ${lc} 8%, transparent); - border: 1px solid color-mix(in srgb, ${lc} 28%, transparent); - } - - .dot { - width: 5px; - height: 5px; - border-radius: 50%; - animation: pulse 2.4s ease-in-out infinite; - } - - .dot.cluster { background: ${sc}; } - .dot.load-dot { background: ${lc}; animation-delay: 0.5s; } - - @keyframes pulse { - 0%, 100% { opacity: 1; } - 50% { opacity: 0.25; } - } - `; - } - - // ───────────────────────────────────────────── - // HTML template - // ───────────────────────────────────────────── - - _html(compact) { - const loadAttr = this._load; - const showStatus = !compact; - - const rightSide = showStatus ? ` - - - - - - - - - -
      -
      - Cluster -
      -
      - ${this._statusLabel} -
      -
      - ${loadAttr !== null ? ` -
      - Load -
      -
      - ${parseInt(loadAttr, 10)} % -
      -
      ` : ''} -
      - ` : ''; - - return ` - - `; - } - - // ───────────────────────────────────────────── - // SVG logo mark - // ───────────────────────────────────────────── - - _svgMark() { - return ` - - - - - - - - - - - - - - - - - - `; - } -} - -customElements.define('zen-banner', ZenBanner); diff --git a/src/zenserver/frontend/html/compute/compute.html b/src/zenserver/frontend/html/compute/compute.html index 1e101d839..66c20175f 100644 --- a/src/zenserver/frontend/html/compute/compute.html +++ b/src/zenserver/frontend/html/compute/compute.html @@ -5,101 +5,13 @@ Zen Compute Dashboard - - + + + + -
      - +
      + + Home Node Orchestrator @@ -369,15 +226,15 @@ -